Remove forgotten debug code

Remove forgotten debug code

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, size_t subblock, size_t hash_lane,
       size_t bw, size_t bh, size_t dx, size_t dy, size_t offset)
{
    size_t index[4];
    for (size_t i = 0; i < 4; i++) {
        size_t bpos = (hash_lane + i * offset) % 4;
        size_t x = (subblock * dy + i * dx) * bw + bpos % bw;
        size_t y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7;
    size_t hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            size_t addr_index_x = addr_index % 16;
            size_t addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
#endif
            }

            ++mem_curr;
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
#endif
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, size_t subblock, size_t hash_lane,
       size_t bw, size_t bh, size_t dx, size_t dy, size_t offset)
{
    size_t index[4];
    for (size_t i = 0; i < 4; i++) {
        size_t bpos = (hash_lane + i * offset) % 4;
        size_t x = (subblock * dy + i * dx) * bw + bpos % bw;
        size_t y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;

        if (i == 1) {
            //printf("[%i, %i] x: %2i, y: %2i\n", (uint)subblock, (uint)hash_lane, (uint)x, (uint)y);
        }
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7;
    size_t hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        //printf("[%2i, %i] pos_l: %3i\n", (uint)thread, (uint)i, (uint)pos_l);
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            size_t addr_index_x = addr_index % 16;
            size_t addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
#endif
            }

            ++mem_curr;
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
#endif
        mem_curr = mem_lane;
    }
}


Use local memory more efficiently

Use local memory more efficiently

GPU shared memory is (usually) organized into 32-bit banks, so let's store high and low 32-bits of each ulong separately, so that there are no bank conflicts when accessing the memory.

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

void g(__local uint *v_lo, __local uint *v_hi, uint4 i)
{
    ulong a, b, c, d;
    a = upsample(v_hi[i.s0], v_lo[i.s0]);
    b = upsample(v_hi[i.s1], v_lo[i.s1]);
    c = upsample(v_hi[i.s2], v_lo[i.s2]);
    d = upsample(v_hi[i.s3], v_lo[i.s3]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    v_lo[i.s0] = (uint)a;
    v_lo[i.s1] = (uint)b;
    v_lo[i.s2] = (uint)c;
    v_lo[i.s3] = (uint)d;

    v_hi[i.s0] = (uint)(a >> 32);
    v_hi[i.s1] = (uint)(b >> 32);
    v_hi[i.s2] = (uint)(c >> 32);
    v_hi[i.s3] = (uint)(d >> 32);
}

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7U;
    uint hash_lane = (thread >> 0) & 0x3U;
    uint4 u0123 = (uint4)(0U, 1U, 2U, 3U);
    uint4 index0 = 4 * u0123 + hash_lane;
    uint4 index1 = 4 * u0123 + (u0123 + hash_lane) % 4;

    __local ulong *v_lo, *v_hi;

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 16 * subblock, block->hi + 16 * subblock, index1);

    barrier(CLK_LOCAL_MEM_FENCE);

    index0 = 16 * (index0 / 2) + index0 % 2;
    index1 = 16 * (index1 / 2) + index1 % 2;

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block->lo + 2 * subblock, block->hi + 2 * subblock, index1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] = prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] = prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = ref_block->data[pos];
        next_block->lo[pos] ^= prev_block->lo[pos] ^= (uint)in;
        next_block->hi[pos] ^= prev_block->hi[pos] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        ulong in = mem_prev->data[pos];
        prev->lo[pos] = (uint)in;
        prev->hi[pos] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos = i * THREADS_PER_LANE + thread;
                    ulong in = mem_curr->data[pos];
                    curr->lo[pos] = (uint)in;
                    curr->hi[pos] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos = i * THREADS_PER_LANE + thread;
                mem_curr->data[pos] = upsample(curr->hi[pos], curr->lo[pos]);
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
            }

            ++mem_curr;
        }
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

#define G(a, b, c, d) \
    do { \
        a = F(a, b); \
        d = rotr64(d ^ a, 32); \
        c = F(c, d); \
        b = rotr64(b ^ c, 24); \
        a = F(a, b); \
        d = rotr64(d ^ a, 16); \
        c = F(c, d); \
        b = rotr64(b ^ c, 63); \
    } while (0)

struct block {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

void shuffle_block(__local struct block *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7U;
    uint hash_lane = (thread >> 0) & 0x3U;
    uint4 u0123 = (uint4)(0U, 1U, 2U, 3U);
    uint4 index0 = 4 * u0123 + hash_lane;
    uint4 index1 = 4 * u0123 + (u0123 + hash_lane) % 4;

    __local ulong *v;

    v = block->data + 16 * subblock;
    G(v[index0.s0], v[index0.s1], v[index0.s2], v[index0.s3]);
    barrier(CLK_LOCAL_MEM_FENCE);
    G(v[index1.s0], v[index1.s1], v[index1.s2], v[index1.s3]);
    barrier(CLK_LOCAL_MEM_FENCE);

    index0 = 16 * (index0 / 2) + index0 % 2;
    index1 = 16 * (index1 / 2) + index1 % 2;

    v = block->data + 2 * subblock;
    G(v[index0.s0], v[index0.s1], v[index0.s2], v[index0.s3]);
    barrier(CLK_LOCAL_MEM_FENCE);
    G(v[index1.s0], v[index1.s1], v[index1.s2], v[index1.s3]);
}

void fill_block(__global const struct block *restrict ref_block,
                __local struct block *restrict prev_block,
                __local struct block *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->data[pos] = prev_block->data[pos] ^= ref_block->data[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->data[pos] ^= prev_block->data[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block *restrict ref_block,
                    __local struct block *restrict prev_block,
                    __local struct block *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->data[pos] ^= prev_block->data[pos] ^= ref_block->data[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->data[pos] ^= prev_block->data[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(ulong thread_input,
                    __local struct block *restrict addr,
                    __local struct block *restrict tmp,
                    size_t thread)
{
    addr->data[thread] = thread_input;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->data[pos] = 0UL;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->data[thread] = addr->data[thread] ^= thread_input;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->data[pos] = addr->data[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->data[pos] ^= tmp->data[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block *memory, __local struct block *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block *restrict curr = &shared[0];
    __local struct block *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block *restrict addr = &shared[2];

    ulong thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1UL : 0UL;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0UL;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block *mem_lane = memory + lane * lane_blocks;
    __global struct block *mem_prev = mem_lane + 1;
    __global struct block *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        prev->data[pos] = mem_prev->data[pos];
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            ulong pseudo_rand;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            pseudo_rand = addr->data[addr_index];
#else
            pseudo_rand = prev->data[0];
#endif

            uint ref_lane = (uint)(pseudo_rand >> 32) % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = (uint)pseudo_rand;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos = i * THREADS_PER_LANE + thread;
                    curr->data[pos] = mem_curr->data[pos];
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos = i * THREADS_PER_LANE + thread;
                mem_curr->data[pos] = curr->data[pos];
            }

            /* swap curr and prev buffers: */
            __local struct block *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
            }

            ++mem_curr;
        }
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
        mem_curr = mem_lane;
    }
}


Use INFINITY instead of 1e38 in argmax/argmin functions of OpenCL

inline float inline_add(const float a, const float b) { return a + b; }
inline float inline_sub(const float a, const float b) { return a - b; }
inline float inline_mul(const float a, const float b) { return a * b; }
inline float inline_div(const float a, const float b) { return a / b; }

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (max_val[tid + k] > max_val[tid] \
          || (max_val[tid + k] == max_val[tid] \
              && argmax_val[tid + k] < argmax_val[tid])) { \
        max_val[tid] = max_val[tid + k]; \
        argmax_val[tid] = argmax_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMAX_KERNEL(GROUP_SIZE) \
kernel void argmax_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float max_val[GROUP_SIZE]; \
  local unsigned argmax_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  max_val[tid] = -INFINITY; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val > max_val[tid]) { \
      max_val[tid] = val; \
      argmax_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmax_val[0]; \
}

ARGMAX_KERNEL(1024)
ARGMAX_KERNEL(512)
ARGMAX_KERNEL(256)
ARGMAX_KERNEL(128)
ARGMAX_KERNEL(64)
ARGMAX_KERNEL(32)
ARGMAX_KERNEL(16)
ARGMAX_KERNEL(8)
ARGMAX_KERNEL(4)
ARGMAX_KERNEL(2)
ARGMAX_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (min_val[tid + k] < min_val[tid] \
          || (min_val[tid + k] == min_val[tid] \
              && argmin_val[tid + k] < argmin_val[tid])) { \
        min_val[tid] = min_val[tid + k]; \
        argmin_val[tid] = argmin_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMIN_KERNEL(GROUP_SIZE) \
kernel void argmin_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float min_val[GROUP_SIZE]; \
  local unsigned argmin_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  min_val[tid] = INFINITY; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val < min_val[tid]) { \
      min_val[tid] = val; \
      argmin_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmin_val[0]; \
}

ARGMIN_KERNEL(1024)
ARGMIN_KERNEL(512)
ARGMIN_KERNEL(256)
ARGMIN_KERNEL(128)
ARGMIN_KERNEL(64)
ARGMIN_KERNEL(32)
ARGMIN_KERNEL(16)
ARGMIN_KERNEL(8)
ARGMIN_KERNEL(4)
ARGMIN_KERNEL(2)
ARGMIN_KERNEL(1)

#undef REDUCE

kernel void set_identity_kernel(
    const unsigned size, const unsigned skip, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = !(i % skip);
}

kernel void pick_fw_kernel(
    const global float *px, const global unsigned *pi,
    const unsigned wx, const unsigned wy, const unsigned sx,
    const unsigned si, const unsigned sy, global float *py) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) py[oy + t] = px[ox + (t / wy) * wx + (t % wy)];
}

kernel void slice_fw_kernel(
    const global float *px, const unsigned shift, const unsigned span,
    const unsigned skip, const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[(i / span) * skip + (i % span) + shift];
}

kernel void concat_fw_kernel(
    const global float *px, const unsigned span, const unsigned skip,
    const unsigned x_size, const unsigned y_size,
    global float *py, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < y_size) py[(i / span) * skip + (i % span) + shift] = px[i % x_size];
}

inline void atomic_add_float(global float *source, const float operand) {
  union {
    unsigned u;
    float f;
  } oldval, newval;
  unsigned readback;
  oldval.f = *source;
  newval.f = oldval.f + operand;
  while ((readback = atomic_cmpxchg(
      (global unsigned *) source, oldval.u, newval.u)) != oldval.u) {
    oldval.u = readback;
    newval.f = oldval.f + operand;
  }
}

kernel void pick_bw_kernel(
    const global float *pgy, const global unsigned *pi,
    const unsigned wx, const unsigned wy,
    const unsigned sx, const unsigned si, const unsigned sy,
    global float *pgx) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) {
    atomic_add_float(pgx + ox + (t / wy) * wx + (t % wy), pgy[oy + t]);
  }
}

kernel void slice_bw_kernel(
    const global float *pgy, const unsigned wx, const unsigned wy,
    const unsigned nx, const unsigned ny,
    global float *pgx, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < wy * max(nx, ny)) {
    atomic_add_float(
        pgx + shift + ((i / wy) * wx + (i % wy)) % (wx * nx),
        pgy[i % (wy * ny)]);
  }
}

#define OPENCLDEV_KERNEL_FW_X(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const unsigned size, global float *py) { \
    const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
}

#define OPENCLDEV_KERNEL_BW_X(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const unsigned size, global float *pgx) { \
   const unsigned i = get_global_id(0); \
   if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_CONST(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const float k, \
    const unsigned size, global float *py) { \
  const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
 }

#define OPENCLDEV_KERNEL_BW_X_CONST(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const float k, const unsigned size, global float *pgx) { \
  const unsigned i = get_global_id(0); \
  if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_R(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(px[i + mbx * shift], pk[mbk * bid_y]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_L(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pk[mbk * bid_y], px[i + mbx * shift]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_AB(name, op) \
kernel void name##_fw_kernel( \
    const global float *pa, const global float *pb, const unsigned size, \
    const unsigned mba, const unsigned mbb, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pa[i + mba * shift], pb[i + mbb * shift]); \
  } \
}

OPENCLDEV_KERNEL_FW_X(negate, -px[i])
OPENCLDEV_KERNEL_FW_X(sqrt, sqrt(px[i]))
OPENCLDEV_KERNEL_FW_X(exp, exp(px[i]))
OPENCLDEV_KERNEL_FW_X(log, log(px[i]))
OPENCLDEV_KERNEL_FW_X(tanh, tanh(px[i]))
OPENCLDEV_KERNEL_FW_X(sigmoid, .5f + .5f  * tanh(.5f * px[i]))
OPENCLDEV_KERNEL_FW_X(
    softplus, max(px[i], .0f) + log(1.f + exp(-fabs(px[i]))))
OPENCLDEV_KERNEL_FW_X(sin, sin(px[i]))
OPENCLDEV_KERNEL_FW_X(cos, cos(px[i]))
OPENCLDEV_KERNEL_FW_X(tan, tan(px[i]))

OPENCLDEV_KERNEL_BW_X(sqrt, .5f * pgy[i] / py[i])
OPENCLDEV_KERNEL_BW_X(exp, py[i] * pgy[i])
OPENCLDEV_KERNEL_BW_X(log, pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X(tanh, (1.f - py[i] * py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sigmoid, py[i] * (1.f - py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(softplus, (.5f + .5f * tanh(.5f * px[i])) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sin, cos(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(cos, -sin(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(tan, (1.f + py[i] * py[i]) * pgy[i])

OPENCLDEV_KERNEL_FW_X_CONST(add_const, px[i] + k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_r, px[i] - k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_l, k - px[i])
OPENCLDEV_KERNEL_FW_X_CONST(multiply_const, px[i] * k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_r, px[i] / k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_l, k / px[i])
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_r, pow(px[i], k))
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_l, pow(k, px[i]))
OPENCLDEV_KERNEL_FW_X_CONST(prelu, max(px[i], .0f) + k * min(px[i], .0f))
OPENCLDEV_KERNEL_FW_X_CONST(
    elu, max(px[i], .0f) + k * (exp(min(px[i], .0f)) - 1.0f))

kernel void pown_fw_kernel(
    const global float *px, const int k,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = pown(px[i], k);
 }

OPENCLDEV_KERNEL_BW_X_CONST(add_const, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_r, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_l, -pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(multiply_const, k * pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_r, pgy[i] / k)
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_l, -py[i] * pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_r, k * pgy[i] * py[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_l, log(k) * pgy[i] * py[i])
OPENCLDEV_KERNEL_BW_X_CONST(
    prelu, pgy[i] * ((px[i] > .0f) + k * (px[i] <= .0f)))
OPENCLDEV_KERNEL_BW_X_CONST(
    elu, pgy[i] * ((px[i] > .0f) + (py[i] + k) * (px[i] <= .0f)))

kernel void pown_bw_kernel(
    const global float *px, const global float *py, const global float *pgy,
    const int k, const unsigned size, global float *pgx) {
  const unsigned i = get_global_id(0);
  if (i < size) pgx[i] += k * pgy[i] * py[i] / px[i];
}

OPENCLDEV_KERNEL_FW_X_SCALAR_R(add_scalar, inline_add)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(subtract_scalar_r, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(subtract_scalar_l, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(multiply_scalar, inline_mul)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(divide_scalar_r, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(divide_scalar_l, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(pow_scalar_r, pow)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(pow_scalar_l, pow)

OPENCLDEV_KERNEL_FW_AB(add, inline_add)
OPENCLDEV_KERNEL_FW_AB(subtract, inline_sub)
OPENCLDEV_KERNEL_FW_AB(multiply, inline_mul)
OPENCLDEV_KERNEL_FW_AB(divide, inline_div)
OPENCLDEV_KERNEL_FW_AB(pow, pow)

#undef OPENCLDEV_KERNEL_FW_X
#undef OPENCLDEV_KERNEL_BW_X
#undef OPENCLDEV_KERNEL_FW_X_CONST
#undef OPENCLDEV_KERNEL_BW_X_CONST
#undef CUDADEV_KERNEL_FW_X_SCALAR_R
#undef CUDADEV_KERNEL_FW_X_SCALAR_L
#undef CUDADEV_KERNEL_FW_AB

kernel void add_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, gy);
  }
}

kernel void subtract_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, -gy);
  }
}

kernel void multiply_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    atomic_add_float(pga + a_ofs, gy * pb[b_ofs]);
    atomic_add_float(pgb + b_ofs, gy * pa[a_ofs]);
  }
}

kernel void divide_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] / pb[b_ofs];
    atomic_add_float(pga + i + mba * shift, k);
    atomic_add_float(pgb + b_ofs, -k * py[y_ofs]);
  }
}

kernel void pow_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] * py[y_ofs];
    atomic_add_float(pga + a_ofs, k * pb[b_ofs] / pa[a_ofs]);
    atomic_add_float(pgb + b_ofs, k * log(pa[a_ofs]));
  }
}

kernel void transpose_fw_kernel(
    const global float *px, unsigned rows, unsigned cols, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) py[ofs + j + i * cols] = px[ofs + i + j * rows];
}

kernel void transpose_bw_kernel(
    const global float *py, const unsigned rows, const unsigned cols,
    global float *px) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) px[ofs + i + j * rows] += py[ofs + j + i * cols];
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = max(temp[tid + k], temp[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MAX_FW_KERNEL(GROUP_SIZE) \
kernel void max_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = -INFINITY; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = max(px[i * skip], temp[tid]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

MAX_FW_KERNEL(1024)
MAX_FW_KERNEL(512)
MAX_FW_KERNEL(256)
MAX_FW_KERNEL(128)
MAX_FW_KERNEL(64)
MAX_FW_KERNEL(32)
MAX_FW_KERNEL(16)
MAX_FW_KERNEL(8)
MAX_FW_KERNEL(4)
MAX_FW_KERNEL(2)
MAX_FW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) argmax_val[tid] = min(argmax_val[tid + k], argmax_val[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MAX_BW_KERNEL(GROUP_SIZE) \
kernel void max_bw_kernel_##GROUP_SIZE( \
    const global float *px, const global float *py, \
    const global float *pgy, const unsigned skip, \
    const unsigned n, global float *pgx) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  const float max_val = py[bid]; \
  local unsigned argmax_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  pgx += bid % skip + (bid / skip) * skip * n; \
  argmax_val[tid] = n; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    argmax_val[tid] \
        = px[i * skip] == max_val \
        ? min(i, argmax_val[tid]) \
        : argmax_val[tid]; \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) pgx[argmax_val[0] * skip] += pgy[bid]; \
}

MAX_BW_KERNEL(1024)
MAX_BW_KERNEL(512)
MAX_BW_KERNEL(256)
MAX_BW_KERNEL(128)
MAX_BW_KERNEL(64)
MAX_BW_KERNEL(32)
MAX_BW_KERNEL(16)
MAX_BW_KERNEL(8)
MAX_BW_KERNEL(4)
MAX_BW_KERNEL(2)
MAX_BW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = min(temp[tid + k], temp[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MIN_FW_KERNEL(GROUP_SIZE) \
kernel void min_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = INFINITY; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = min(px[i * skip], temp[tid]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

MIN_FW_KERNEL(1024)
MIN_FW_KERNEL(512)
MIN_FW_KERNEL(256)
MIN_FW_KERNEL(128)
MIN_FW_KERNEL(64)
MIN_FW_KERNEL(32)
MIN_FW_KERNEL(16)
MIN_FW_KERNEL(8)
MIN_FW_KERNEL(4)
MIN_FW_KERNEL(2)
MIN_FW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) argmin_val[tid] = min(argmin_val[tid + k], argmin_val[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MIN_BW_KERNEL(GROUP_SIZE) \
kernel void min_bw_kernel_##GROUP_SIZE( \
    const global float *px, const global float *py, \
    const global float *pgy, const unsigned skip, \
    const unsigned n, global float *pgx) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  const float min_val = py[bid]; \
  local unsigned argmin_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  pgx += bid % skip + (bid / skip) * skip * n; \
  argmin_val[tid] = n; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    argmin_val[tid] \
        = px[i * skip] == min_val \
        ? min(i, argmin_val[tid]) \
        : argmin_val[tid]; \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) pgx[argmin_val[0] * skip] += pgy[bid]; \
}

MIN_BW_KERNEL(1024)
MIN_BW_KERNEL(512)
MIN_BW_KERNEL(256)
MIN_BW_KERNEL(128)
MIN_BW_KERNEL(64)
MIN_BW_KERNEL(32)
MIN_BW_KERNEL(16)
MIN_BW_KERNEL(8)
MIN_BW_KERNEL(4)
MIN_BW_KERNEL(2)
MIN_BW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] += temp[tid + k]; \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define SUM_FW_KERNEL(GROUP_SIZE) \
kernel void sum_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = 0; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) temp[tid] += px[i * skip]; \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

SUM_FW_KERNEL(1024)
SUM_FW_KERNEL(512)
SUM_FW_KERNEL(256)
SUM_FW_KERNEL(128)
SUM_FW_KERNEL(64)
SUM_FW_KERNEL(32)
SUM_FW_KERNEL(16)
SUM_FW_KERNEL(8)
SUM_FW_KERNEL(4)
SUM_FW_KERNEL(2)
SUM_FW_KERNEL(1)

#undef REDUCE

inline float logsumexp2_fw_kernel(float a, float b) {
  return a > b
    ? a + log(1.f + exp(b - a))
    : b + log(1.f + exp(a - b));
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = logsumexp2_fw_kernel(temp[tid], temp[tid + k]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define LOGSUMEXP_FW_KERNEL(GROUP_SIZE) \
kernel void logsumexp_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = -1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = logsumexp2_fw_kernel(temp[tid], px[i * skip]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

LOGSUMEXP_FW_KERNEL(1024)
LOGSUMEXP_FW_KERNEL(512)
LOGSUMEXP_FW_KERNEL(256)
LOGSUMEXP_FW_KERNEL(128)
LOGSUMEXP_FW_KERNEL(64)
LOGSUMEXP_FW_KERNEL(32)
LOGSUMEXP_FW_KERNEL(16)
LOGSUMEXP_FW_KERNEL(8)
LOGSUMEXP_FW_KERNEL(4)
LOGSUMEXP_FW_KERNEL(2)
LOGSUMEXP_FW_KERNEL(1)

#undef REDUCE

kernel void broadcast_fw_kernel(
    const global float *px, const unsigned skip1, const unsigned skip2,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[i % skip1 + (i / skip2) * skip1];
}

kernel void batch_sum_fw_kernel(
    const global float *px, const unsigned size,
    const unsigned batch, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) {
    float temp = .0f;
    px += i;
    for (unsigned j = 0; j < batch; ++j, px += size) {
      temp += *px;
    }
    py[i] = temp;
  }
}

kernel void inplace_multiply_const_kernel(
    const float k, const unsigned size, global float *px) {
  const unsigned i = get_global_id(0);
  if (i < size) px[i] *= k;
}

kernel void inplace_add_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, px[i + mbx * shift]);
}

kernel void inplace_subtract_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, -px[i + mbx * shift]);
}

inline float inline_add(const float a, const float b) { return a + b; }
inline float inline_sub(const float a, const float b) { return a - b; }
inline float inline_mul(const float a, const float b) { return a * b; }
inline float inline_div(const float a, const float b) { return a / b; }

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (max_val[tid + k] > max_val[tid] \
          || (max_val[tid + k] == max_val[tid] \
              && argmax_val[tid + k] < argmax_val[tid])) { \
        max_val[tid] = max_val[tid + k]; \
        argmax_val[tid] = argmax_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMAX_KERNEL(GROUP_SIZE) \
kernel void argmax_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float max_val[GROUP_SIZE]; \
  local unsigned argmax_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  max_val[tid] = -1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val > max_val[tid]) { \
      max_val[tid] = val; \
      argmax_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmax_val[0]; \
}

ARGMAX_KERNEL(1024)
ARGMAX_KERNEL(512)
ARGMAX_KERNEL(256)
ARGMAX_KERNEL(128)
ARGMAX_KERNEL(64)
ARGMAX_KERNEL(32)
ARGMAX_KERNEL(16)
ARGMAX_KERNEL(8)
ARGMAX_KERNEL(4)
ARGMAX_KERNEL(2)
ARGMAX_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (min_val[tid + k] < min_val[tid] \
          || (min_val[tid + k] == min_val[tid] \
              && argmin_val[tid + k] < argmin_val[tid])) { \
        min_val[tid] = min_val[tid + k]; \
        argmin_val[tid] = argmin_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMIN_KERNEL(GROUP_SIZE) \
kernel void argmin_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float min_val[GROUP_SIZE]; \
  local unsigned argmin_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  min_val[tid] = 1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val < min_val[tid]) { \
      min_val[tid] = val; \
      argmin_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmin_val[0]; \
}

ARGMIN_KERNEL(1024)
ARGMIN_KERNEL(512)
ARGMIN_KERNEL(256)
ARGMIN_KERNEL(128)
ARGMIN_KERNEL(64)
ARGMIN_KERNEL(32)
ARGMIN_KERNEL(16)
ARGMIN_KERNEL(8)
ARGMIN_KERNEL(4)
ARGMIN_KERNEL(2)
ARGMIN_KERNEL(1)

#undef REDUCE

kernel void set_identity_kernel(
    const unsigned size, const unsigned skip, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = !(i % skip);
}

kernel void pick_fw_kernel(
    const global float *px, const global unsigned *pi,
    const unsigned wx, const unsigned wy, const unsigned sx,
    const unsigned si, const unsigned sy, global float *py) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) py[oy + t] = px[ox + (t / wy) * wx + (t % wy)];
}

kernel void slice_fw_kernel(
    const global float *px, const unsigned shift, const unsigned span,
    const unsigned skip, const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[(i / span) * skip + (i % span) + shift];
}

kernel void concat_fw_kernel(
    const global float *px, const unsigned span, const unsigned skip,
    const unsigned x_size, const unsigned y_size,
    global float *py, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < y_size) py[(i / span) * skip + (i % span) + shift] = px[i % x_size];
}

inline void atomic_add_float(global float *source, const float operand) {
  union {
    unsigned u;
    float f;
  } oldval, newval;
  unsigned readback;
  oldval.f = *source;
  newval.f = oldval.f + operand;
  while ((readback = atomic_cmpxchg(
      (global unsigned *) source, oldval.u, newval.u)) != oldval.u) {
    oldval.u = readback;
    newval.f = oldval.f + operand;
  }
}

kernel void pick_bw_kernel(
    const global float *pgy, const global unsigned *pi,
    const unsigned wx, const unsigned wy,
    const unsigned sx, const unsigned si, const unsigned sy,
    global float *pgx) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) {
    atomic_add_float(pgx + ox + (t / wy) * wx + (t % wy), pgy[oy + t]);
  }
}

kernel void slice_bw_kernel(
    const global float *pgy, const unsigned wx, const unsigned wy,
    const unsigned nx, const unsigned ny,
    global float *pgx, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < wy * max(nx, ny)) {
    atomic_add_float(
        pgx + shift + ((i / wy) * wx + (i % wy)) % (wx * nx),
        pgy[i % (wy * ny)]);
  }
}

#define OPENCLDEV_KERNEL_FW_X(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const unsigned size, global float *py) { \
    const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
}

#define OPENCLDEV_KERNEL_BW_X(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const unsigned size, global float *pgx) { \
   const unsigned i = get_global_id(0); \
   if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_CONST(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const float k, \
    const unsigned size, global float *py) { \
  const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
 }

#define OPENCLDEV_KERNEL_BW_X_CONST(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const float k, const unsigned size, global float *pgx) { \
  const unsigned i = get_global_id(0); \
  if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_R(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(px[i + mbx * shift], pk[mbk * bid_y]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_L(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pk[mbk * bid_y], px[i + mbx * shift]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_AB(name, op) \
kernel void name##_fw_kernel( \
    const global float *pa, const global float *pb, const unsigned size, \
    const unsigned mba, const unsigned mbb, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pa[i + mba * shift], pb[i + mbb * shift]); \
  } \
}

OPENCLDEV_KERNEL_FW_X(negate, -px[i])
OPENCLDEV_KERNEL_FW_X(sqrt, sqrt(px[i]))
OPENCLDEV_KERNEL_FW_X(exp, exp(px[i]))
OPENCLDEV_KERNEL_FW_X(log, log(px[i]))
OPENCLDEV_KERNEL_FW_X(tanh, tanh(px[i]))
OPENCLDEV_KERNEL_FW_X(sigmoid, .5f + .5f  * tanh(.5f * px[i]))
OPENCLDEV_KERNEL_FW_X(
    softplus, max(px[i], .0f) + log(1.f + exp(-fabs(px[i]))))
OPENCLDEV_KERNEL_FW_X(sin, sin(px[i]))
OPENCLDEV_KERNEL_FW_X(cos, cos(px[i]))
OPENCLDEV_KERNEL_FW_X(tan, tan(px[i]))

OPENCLDEV_KERNEL_BW_X(sqrt, .5f * pgy[i] / py[i])
OPENCLDEV_KERNEL_BW_X(exp, py[i] * pgy[i])
OPENCLDEV_KERNEL_BW_X(log, pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X(tanh, (1.f - py[i] * py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sigmoid, py[i] * (1.f - py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(softplus, (.5f + .5f * tanh(.5f * px[i])) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sin, cos(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(cos, -sin(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(tan, (1.f + py[i] * py[i]) * pgy[i])

OPENCLDEV_KERNEL_FW_X_CONST(add_const, px[i] + k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_r, px[i] - k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_l, k - px[i])
OPENCLDEV_KERNEL_FW_X_CONST(multiply_const, px[i] * k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_r, px[i] / k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_l, k / px[i])
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_r, pow(px[i], k))
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_l, pow(k, px[i]))
OPENCLDEV_KERNEL_FW_X_CONST(prelu, max(px[i], .0f) + k * min(px[i], .0f))
OPENCLDEV_KERNEL_FW_X_CONST(
    elu, max(px[i], .0f) + k * (exp(min(px[i], .0f)) - 1.0f))

kernel void pown_fw_kernel(
    const global float *px, const int k,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = pown(px[i], k);
 }

OPENCLDEV_KERNEL_BW_X_CONST(add_const, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_r, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_l, -pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(multiply_const, k * pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_r, pgy[i] / k)
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_l, -py[i] * pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_r, k * pgy[i] * py[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_l, log(k) * pgy[i] * py[i])
OPENCLDEV_KERNEL_BW_X_CONST(
    prelu, pgy[i] * ((px[i] > .0f) + k * (px[i] <= .0f)))
OPENCLDEV_KERNEL_BW_X_CONST(
    elu, pgy[i] * ((px[i] > .0f) + (py[i] + k) * (px[i] <= .0f)))

kernel void pown_bw_kernel(
    const global float *px, const global float *py, const global float *pgy,
    const int k, const unsigned size, global float *pgx) {
  const unsigned i = get_global_id(0);
  if (i < size) pgx[i] += k * pgy[i] * py[i] / px[i];
}

OPENCLDEV_KERNEL_FW_X_SCALAR_R(add_scalar, inline_add)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(subtract_scalar_r, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(subtract_scalar_l, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(multiply_scalar, inline_mul)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(divide_scalar_r, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(divide_scalar_l, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(pow_scalar_r, pow)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(pow_scalar_l, pow)

OPENCLDEV_KERNEL_FW_AB(add, inline_add)
OPENCLDEV_KERNEL_FW_AB(subtract, inline_sub)
OPENCLDEV_KERNEL_FW_AB(multiply, inline_mul)
OPENCLDEV_KERNEL_FW_AB(divide, inline_div)
OPENCLDEV_KERNEL_FW_AB(pow, pow)

#undef OPENCLDEV_KERNEL_FW_X
#undef OPENCLDEV_KERNEL_BW_X
#undef OPENCLDEV_KERNEL_FW_X_CONST
#undef OPENCLDEV_KERNEL_BW_X_CONST
#undef CUDADEV_KERNEL_FW_X_SCALAR_R
#undef CUDADEV_KERNEL_FW_X_SCALAR_L
#undef CUDADEV_KERNEL_FW_AB

kernel void add_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, gy);
  }
}

kernel void subtract_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, -gy);
  }
}

kernel void multiply_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    atomic_add_float(pga + a_ofs, gy * pb[b_ofs]);
    atomic_add_float(pgb + b_ofs, gy * pa[a_ofs]);
  }
}

kernel void divide_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] / pb[b_ofs];
    atomic_add_float(pga + i + mba * shift, k);
    atomic_add_float(pgb + b_ofs, -k * py[y_ofs]);
  }
}

kernel void pow_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] * py[y_ofs];
    atomic_add_float(pga + a_ofs, k * pb[b_ofs] / pa[a_ofs]);
    atomic_add_float(pgb + b_ofs, k * log(pa[a_ofs]));
  }
}

kernel void transpose_fw_kernel(
    const global float *px, unsigned rows, unsigned cols, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) py[ofs + j + i * cols] = px[ofs + i + j * rows];
}

kernel void transpose_bw_kernel(
    const global float *py, const unsigned rows, const unsigned cols,
    global float *px) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) px[ofs + i + j * rows] += py[ofs + j + i * cols];
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = max(temp[tid + k], temp[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MAX_FW_KERNEL(GROUP_SIZE) \
kernel void max_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = -INFINITY; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = max(px[i * skip], temp[tid]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

MAX_FW_KERNEL(1024)
MAX_FW_KERNEL(512)
MAX_FW_KERNEL(256)
MAX_FW_KERNEL(128)
MAX_FW_KERNEL(64)
MAX_FW_KERNEL(32)
MAX_FW_KERNEL(16)
MAX_FW_KERNEL(8)
MAX_FW_KERNEL(4)
MAX_FW_KERNEL(2)
MAX_FW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) argmax_val[tid] = min(argmax_val[tid + k], argmax_val[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MAX_BW_KERNEL(GROUP_SIZE) \
kernel void max_bw_kernel_##GROUP_SIZE( \
    const global float *px, const global float *py, \
    const global float *pgy, const unsigned skip, \
    const unsigned n, global float *pgx) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  const float max_val = py[bid]; \
  local unsigned argmax_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  pgx += bid % skip + (bid / skip) * skip * n; \
  argmax_val[tid] = n; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    argmax_val[tid] \
        = px[i * skip] == max_val \
        ? min(i, argmax_val[tid]) \
        : argmax_val[tid]; \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) pgx[argmax_val[0] * skip] += pgy[bid]; \
}

MAX_BW_KERNEL(1024)
MAX_BW_KERNEL(512)
MAX_BW_KERNEL(256)
MAX_BW_KERNEL(128)
MAX_BW_KERNEL(64)
MAX_BW_KERNEL(32)
MAX_BW_KERNEL(16)
MAX_BW_KERNEL(8)
MAX_BW_KERNEL(4)
MAX_BW_KERNEL(2)
MAX_BW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = min(temp[tid + k], temp[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MIN_FW_KERNEL(GROUP_SIZE) \
kernel void min_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = INFINITY; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = min(px[i * skip], temp[tid]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

MIN_FW_KERNEL(1024)
MIN_FW_KERNEL(512)
MIN_FW_KERNEL(256)
MIN_FW_KERNEL(128)
MIN_FW_KERNEL(64)
MIN_FW_KERNEL(32)
MIN_FW_KERNEL(16)
MIN_FW_KERNEL(8)
MIN_FW_KERNEL(4)
MIN_FW_KERNEL(2)
MIN_FW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) argmin_val[tid] = min(argmin_val[tid + k], argmin_val[tid]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define MIN_BW_KERNEL(GROUP_SIZE) \
kernel void min_bw_kernel_##GROUP_SIZE( \
    const global float *px, const global float *py, \
    const global float *pgy, const unsigned skip, \
    const unsigned n, global float *pgx) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  const float min_val = py[bid]; \
  local unsigned argmin_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  pgx += bid % skip + (bid / skip) * skip * n; \
  argmin_val[tid] = n; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    argmin_val[tid] \
        = px[i * skip] == min_val \
        ? min(i, argmin_val[tid]) \
        : argmin_val[tid]; \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) pgx[argmin_val[0] * skip] += pgy[bid]; \
}

MIN_BW_KERNEL(1024)
MIN_BW_KERNEL(512)
MIN_BW_KERNEL(256)
MIN_BW_KERNEL(128)
MIN_BW_KERNEL(64)
MIN_BW_KERNEL(32)
MIN_BW_KERNEL(16)
MIN_BW_KERNEL(8)
MIN_BW_KERNEL(4)
MIN_BW_KERNEL(2)
MIN_BW_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] += temp[tid + k]; \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define SUM_FW_KERNEL(GROUP_SIZE) \
kernel void sum_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = 0; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) temp[tid] += px[i * skip]; \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

SUM_FW_KERNEL(1024)
SUM_FW_KERNEL(512)
SUM_FW_KERNEL(256)
SUM_FW_KERNEL(128)
SUM_FW_KERNEL(64)
SUM_FW_KERNEL(32)
SUM_FW_KERNEL(16)
SUM_FW_KERNEL(8)
SUM_FW_KERNEL(4)
SUM_FW_KERNEL(2)
SUM_FW_KERNEL(1)

#undef REDUCE

inline float logsumexp2_fw_kernel(float a, float b) {
  return a > b
    ? a + log(1.f + exp(b - a))
    : b + log(1.f + exp(a - b));
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = logsumexp2_fw_kernel(temp[tid], temp[tid + k]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define LOGSUMEXP_FW_KERNEL(GROUP_SIZE) \
kernel void logsumexp_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = -1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = logsumexp2_fw_kernel(temp[tid], px[i * skip]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

LOGSUMEXP_FW_KERNEL(1024)
LOGSUMEXP_FW_KERNEL(512)
LOGSUMEXP_FW_KERNEL(256)
LOGSUMEXP_FW_KERNEL(128)
LOGSUMEXP_FW_KERNEL(64)
LOGSUMEXP_FW_KERNEL(32)
LOGSUMEXP_FW_KERNEL(16)
LOGSUMEXP_FW_KERNEL(8)
LOGSUMEXP_FW_KERNEL(4)
LOGSUMEXP_FW_KERNEL(2)
LOGSUMEXP_FW_KERNEL(1)

#undef REDUCE

kernel void broadcast_fw_kernel(
    const global float *px, const unsigned skip1, const unsigned skip2,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[i % skip1 + (i / skip2) * skip1];
}

kernel void batch_sum_fw_kernel(
    const global float *px, const unsigned size,
    const unsigned batch, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) {
    float temp = .0f;
    px += i;
    for (unsigned j = 0; j < batch; ++j, px += size) {
      temp += *px;
    }
    py[i] = temp;
  }
}

kernel void inplace_multiply_const_kernel(
    const float k, const unsigned size, global float *px) {
  const unsigned i = get_global_id(0);
  if (i < size) px[i] *= k;
}

kernel void inplace_add_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, px[i + mbx * shift]);
}

kernel void inplace_subtract_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, -px[i + mbx * shift]);
}


rewrite fullconnect layer

rewrite fullconnect layer

float sigmod(float in) {
    return 1.0f / (1.0f + exp(-in));
}
#define IN_SIZE 400
#define OUT_SIZE 120
#define BUF_SIZE 10
#ifdef __xilinx__
__attribute__((reqd_work_group_size(16, 1, 1)))
#endif
__kernel void full1(
    __global float *in,
    __global float *out,
    __global float *weight,
    __global float *offset
    ) {
#ifdef __xilinx__
    __attribute__((xcl_pipeline_workitems))
#endif
    int o = get_global_id(0);

    float inBuf[BUF_SIZE];
    float weightBuf[BUF_SIZE];

    if (o < OUT_SIZE) {
        float sum = 0;
#ifdef __xilinx__
        __attribute__((xcl_pipeline_loop))
#endif
        for (int i = 0; i < IN_SIZE; i += BUF_SIZE) {

            #ifdef __xilinx__
            __attribute__((opencl_unroll_hint))
            #endif
            for (int j = 0; j < BUF_SIZE; ++j) {
                inBuf[j] = in[i + j];
                weightBuf[j] = weight[o * IN_SIZE + i + j];
            }

            #ifdef __xilinx__
            __attribute__((opencl_unroll_hint))
            #endif
            for (int j = 0; j < BUF_SIZE; ++j) {
                sum += weightBuf[j] * inBuf[j];
            }
        }
        sum += offset[o];
        out[o] = sigmod(sum);
    }
}
#undef IN_SIZE
#undef OUT_SIZE

float sigmod(float in) {
    return 1.0f / (1.0f + exp(-in));
}
#define IN_SIZE 400
#define OUT_SIZE 120
#ifdef __xilinx__
__attribute__((reqd_work_group_size(16, 1, 1)))
#endif
__kernel void full1(
    __global float *in,
    __global float *out,
    __global float *weight,
    __global float *offset
    ) {
#ifdef __xilinx__
    __attribute__((xcl_pipeline_workitems))
#endif
    int o = get_global_id(0);
    if (o < OUT_SIZE) {
        float sum = 0;
#ifdef __xilinx__
        __attribute__((xcl_pipeline_loop))
#endif
        for (int i = 0; i < IN_SIZE; i++) {
            sum += weight[o * IN_SIZE + i] * in[i];
        }
        sum += offset[o];
        out[o] = sigmod(sum);
    }
}
#undef IN_SIZE
#undef OUT_SIZE