Simplify branching slightly.

Simplify branching slightly.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline uchar8 vp8_mbfilter(signed char mask,signed char hev,uchar8);

__inline void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
char vp8_char_clamp(int in){
    return max(min(in, 127), -128);
}
#endif

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

//OpenCL built-in functions (
size_t get_global_id(unsigned int);
size_t get_global_size(unsigned int);

__inline uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = as_char4(base) ^ (char4)0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u.s0 = add_sat(pq.s0, vp8_filter);
    u.s1 = add_sat(pq.s1, Filter.s1);
    u.s2 = sub_sat(pq.s2, Filter.s0);
    u.s3 = sub_sat(pq.s3, vp8_filter);

    return as_uchar4(u ^ (char4)0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
__inline void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    size_t thread = get_global_id(0);

    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        s_off += thread;
        //s_off += thread, so maybe it's possible to use a strided copy with local memory to load into the vector

        uchar8 data;
        data.s0 = s_base[s_off-4*p];
        data.s1 = s_base[s_off-3*p];
        data.s2 = s_base[s_off-2*p];
        data.s3 = s_base[s_off-p];
        data.s4 = s_base[s_off];
        data.s5 = s_base[s_off+p];
        data.s6 = s_base[s_off+2*p];
        data.s7 = s_base[s_off+3*p];

        char mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

        int hev = vp8_hevmask(lfi->thr[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        s_base[s_off - 2*p] = data.s2;
        s_base[s_off - p  ] = data.s3;
        s_base[s_off      ] = data.s4;
        s_base[s_off + p  ] = data.s5;
    }
}

__inline void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        size_t thread = get_global_id(0);
        s_off += p * thread;

        uchar8 data = vload8(0, &s_base[s_off-4]);

        char mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

        int hev = vp8_hevmask(lfi->thr[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        vstore4(data.s2345, 0, &s_base[s_off-2]);
    }
}

__inline void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    size_t thread = get_global_id(0);
    int s_off = offsets[8*num_blocks + block*num_planes+plane] + thread;

    uchar8 data;
    data.s0 = s_base[s_off-4*p];
    data.s1 = s_base[s_off-3*p];
    data.s2 = s_base[s_off-2*p];
    data.s3 = s_base[s_off-p];
    data.s4 = s_base[s_off];
    data.s5 = s_base[s_off+p];
    data.s6 = s_base[s_off+2*p];
    data.s7 = s_base[s_off+3*p];

    char mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    char hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    s_base[s_off - 3*p] = data.s1;
    s_base[s_off - 2*p] = data.s2;
    s_base[s_off - 1*p] = data.s3;
    s_base[s_off      ] = data.s4;
    s_base[s_off + p  ] = data.s5;
    s_base[s_off + 2*p] = data.s6;

}

__inline void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int block_offset = block*num_planes+plane;
    int s_off = offsets[block_offset];
    size_t thread = get_global_id(0);

    s_off += p * thread;

    uchar8 data = vload8(0, &s_base[s_off-4]);

    char mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    int hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    vstore8(data, 0, &s_base[s_off-4]);
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);

    for (int i = 0; i < num_levels; i++){
        int block_offset = block_offsets[priority_level];

        global int *offsets = &offsets_in[16*block_offset];
        global int *filters = &filters_in[4*block_offset];
        size_t num_blocks = priority_num_blocks[priority_level];
        int filter_level = filters[block];
        global loop_filter_info *lf_info = &lfi[filter_level];
        int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);

        if (block < num_blocks){
            //Prefetch vertical edge source pixels into global cache (horizontal isn't worth it)
            for(int pln = 0; pln < 3; pln++){
                int p = pitches[pln];
                int offset = block*3+pln;
                int s_off = offsets[offset] - 4;
                for (int thread = 0; thread < 16; thread++){
                    prefetch(&s_base[s_off+p*thread], 8);
                }
            }

            if (thread_level_filter){
                if ( filters[num_blocks*COLS_LOCATION + block] > 0 ){
                    vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            COLS_LOCATION, filter_level, num_blocks, 3,  plane, block);
                }

                //YUV planes, then 2 more passes of Y plane
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1,  plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (block < num_blocks){

            if (thread_level_filter){
                if (filters[num_blocks*ROWS_LOCATION + block] > 0){
                    vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
                        filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
                }

                //YUV planes, then 2 more passes of Y plane
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }

}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*ROWS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
            filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
        }
    }
}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*COLS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
            COLS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, plane, block);
        }
    }

}

void vp8_loop_filter_simple_horizontal_edge_worker
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    size_t num_planes = get_global_size(1);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

void vp8_loop_filter_simple_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t block = get_global_id(2);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[0];
            int block_offset = cur_iter*num_blocks + block;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[0]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
            block_offsets, priority_num_blocks
    );

    //3 Y plane iterations
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_horizontal_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 1, ROWS_LOCATION, 4, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_all_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
)
{

    for (int i = 0; i < num_levels; i++){
        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
                    block_offsets, priority_num_blocks
            );

            //3 Y plane iterations
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 1, ROWS_LOCATION, 4, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }
}

//Inline and non-kernel functions follow.
__inline uchar8 vp8_mbfilter(
    signed char mask,
    signed char hev,
    uchar8 base
)
{
    char4 u;

    char8 pq = as_char8(base);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};

    /* add outer taps if we have high edge variance */
    char vp8_filter = sub_sat(pq.s2, pq.s5);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s4 - pq.s3), -128, 127);
    vp8_filter &= mask;

    char2 filter = (char2)vp8_filter;
    filter &= hev;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat(filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    pq.s4 = sub_sat(pq.s4, filter.s0);
    pq.s3 = add_sat(pq.s3, filter.s1);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th, 2/7th, and 1/7th difference across boundary */
#if 1
    u = convert_char4(clamp(((short4)63 + (short4)filter.s1 * (short4){27, 18, 9, 0}) >> 7, -128, 127) );
#else
    u.s0 = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    u.s1 = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    u.s2 = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    u.s3 = 0;
#endif

#if 0
    char4 s;
    s = sub_sat(pq.s4567, u);
    pq.s4567 = s ^ (char4){0x80, 0x80, 0x80, 0};
    s.s3210 = add_sat(pq.s3210, u);
    pq.s0123 = s ^ (char4){0, 0x80, 0x80, 0x80};
#else
    pq.s4567 = sub_sat(pq.s4567, u.s0123);
    pq.s3210 = add_sat(pq.s3210, u.s0123);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};
#endif

    return as_uchar8(pq);
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    return any(abs_diff(pq.s03, pq.s12) > thresh) * -1;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    //Only apply the filter if the difference is LESS than 'limit'
#if 1
    char mask = (abs_diff(pq.s0, pq.s1) > limit);
    mask |= (abs_diff(pq.s1, pq.s2) > limit);
    mask |= (abs_diff(pq.s2, pq.s3) > limit);
    mask |= (abs_diff(pq.s5, pq.s4) > limit);
    mask |= (abs_diff(pq.s6, pq.s5) > limit);
    mask |= (abs_diff(pq.s7, pq.s6) > limit);
    mask |= (abs_diff(pq.s3, pq.s4) * 2 + abs_diff(pq.s2, pq.s5) / 2  > flimit * 2 + limit);
    return mask - 1;
#else
    short8 mask8 = convert_short8(abs_diff(pq.s01256732, pq.s12345645));
    mask8.s6 = mask8.s6 * 2 + mask8.s7 / 2;

    short8 limits = (short8)limit;
    limits.s6 += flimit * 2;
    mask8 = mask8 > limits;
    mask8.s7 = 0;

    return any(mask8) - 1;
#endif

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs_diff(p0, q0) * 2 + abs_diff(p1, q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

__inline void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter;
    char2 filter;

    char4 pq = (char4){*op1, *op0, *oq0, *oq1};
    pq ^= (char4)0x80;

    signed char u;

    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat((char2)vp8_filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    u = sub_sat(pq.s2, filter.s0);
    *oq0  = u ^ 0x80;

    u = add_sat(pq.s1, filter.s1);
    *op0 = u ^ 0x80;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline uchar8 vp8_mbfilter(signed char mask,signed char hev,uchar8);

__inline void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
char vp8_char_clamp(int in){
    return max(min(in, 127), -128);
}
#endif

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

//OpenCL built-in functions (
size_t get_global_id(unsigned int);
size_t get_global_size(unsigned int);

__inline uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = as_char4(base) ^ (char4)0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u.s0 = add_sat(pq.s0, vp8_filter);
    u.s1 = add_sat(pq.s1, Filter.s1);
    u.s2 = sub_sat(pq.s2, Filter.s0);
    u.s3 = sub_sat(pq.s3, vp8_filter);

    return as_uchar4(u ^ (char4)0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
__inline void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    size_t thread = get_global_id(0);

    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        s_off += thread;
        //s_off += thread, so maybe it's possible to use a strided copy with local memory to load into the vector

        uchar8 data;
        data.s0 = s_base[s_off-4*p];
        data.s1 = s_base[s_off-3*p];
        data.s2 = s_base[s_off-2*p];
        data.s3 = s_base[s_off-p];
        data.s4 = s_base[s_off];
        data.s5 = s_base[s_off+p];
        data.s6 = s_base[s_off+2*p];
        data.s7 = s_base[s_off+3*p];

        char mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

        int hev = vp8_hevmask(lfi->thr[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        s_base[s_off - 2*p] = data.s2;
        s_base[s_off - p  ] = data.s3;
        s_base[s_off      ] = data.s4;
        s_base[s_off + p  ] = data.s5;
    }
}

__inline void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        size_t thread = get_global_id(0);
        s_off += p * thread;

        uchar8 data = vload8(0, &s_base[s_off-4]);

        char mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

        int hev = vp8_hevmask(lfi->thr[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        vstore4(data.s2345, 0, &s_base[s_off-2]);
    }
}

__inline void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    size_t thread = get_global_id(0);
    int s_off = offsets[8*num_blocks + block*num_planes+plane] + thread;

    uchar8 data;
    data.s0 = s_base[s_off-4*p];
    data.s1 = s_base[s_off-3*p];
    data.s2 = s_base[s_off-2*p];
    data.s3 = s_base[s_off-p];
    data.s4 = s_base[s_off];
    data.s5 = s_base[s_off+p];
    data.s6 = s_base[s_off+2*p];
    data.s7 = s_base[s_off+3*p];

    char mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    char hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    s_base[s_off - 3*p] = data.s1;
    s_base[s_off - 2*p] = data.s2;
    s_base[s_off - 1*p] = data.s3;
    s_base[s_off      ] = data.s4;
    s_base[s_off + p  ] = data.s5;
    s_base[s_off + 2*p] = data.s6;

}

__inline void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int block_offset = block*num_planes+plane;
    int s_off = offsets[block_offset];
    size_t thread = get_global_id(0);

    s_off += p * thread;

    uchar8 data = vload8(0, &s_base[s_off-4]);

    char mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    int hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    vstore8(data, 0, &s_base[s_off-4]);
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);

    for (int i = 0; i < num_levels; i++){
        int block_offset = block_offsets[priority_level];

        global int *offsets = &offsets_in[16*block_offset];
        global int *filters = &filters_in[4*block_offset];
        size_t num_blocks = priority_num_blocks[priority_level];
        int filter_level = filters[block];
        global loop_filter_info *lf_info = &lfi[filter_level];
        int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);

        if (block < num_blocks){
            //Prefetch vertical edge source pixels into global cache (horizontal isn't worth it)
            for(int pln = 0; pln < 3; pln++){
                int p = pitches[pln];
                int offset = block*3+pln;
                int s_off = offsets[offset] - 4;
                for (int thread = 0; thread < 16; thread++){
                    prefetch(&s_base[s_off+p*thread], 8);
                }
            }

            if (thread_level_filter & (filters[num_blocks*COLS_LOCATION + block] > 0)){
                vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        COLS_LOCATION, filter_level, num_blocks, 3,  plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1,  plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (block < num_blocks){

            if (thread_level_filter & (filters[num_blocks*ROWS_LOCATION + block] > 0)){
                vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
                    filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }

}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*ROWS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
            filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
        }
    }
}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*COLS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
            COLS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, plane, block);
        }
    }

}

void vp8_loop_filter_simple_horizontal_edge_worker
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    size_t num_planes = get_global_size(1);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

void vp8_loop_filter_simple_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t block = get_global_id(2);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[0];
            int block_offset = cur_iter*num_blocks + block;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[0]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
            block_offsets, priority_num_blocks
    );

    //3 Y plane iterations
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_horizontal_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 1, ROWS_LOCATION, 4, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_all_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
)
{

    for (int i = 0; i < num_levels; i++){
        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
                    block_offsets, priority_num_blocks
            );

            //3 Y plane iterations
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 1, ROWS_LOCATION, 4, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }
}

//Inline and non-kernel functions follow.
__inline uchar8 vp8_mbfilter(
    signed char mask,
    signed char hev,
    uchar8 base
)
{
    char4 u;

    char8 pq = as_char8(base);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};

    /* add outer taps if we have high edge variance */
    char vp8_filter = sub_sat(pq.s2, pq.s5);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s4 - pq.s3), -128, 127);
    vp8_filter &= mask;

    char2 filter = (char2)vp8_filter;
    filter &= hev;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat(filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    pq.s4 = sub_sat(pq.s4, filter.s0);
    pq.s3 = add_sat(pq.s3, filter.s1);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th, 2/7th, and 1/7th difference across boundary */
#if 1
    u = convert_char4(clamp(((short4)63 + (short4)filter.s1 * (short4){27, 18, 9, 0}) >> 7, -128, 127) );
#else
    u.s0 = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    u.s1 = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    u.s2 = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    u.s3 = 0;
#endif

#if 0
    char4 s;
    s = sub_sat(pq.s4567, u);
    pq.s4567 = s ^ (char4){0x80, 0x80, 0x80, 0};
    s.s3210 = add_sat(pq.s3210, u);
    pq.s0123 = s ^ (char4){0, 0x80, 0x80, 0x80};
#else
    pq.s4567 = sub_sat(pq.s4567, u.s0123);
    pq.s3210 = add_sat(pq.s3210, u.s0123);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};
#endif

    return as_uchar8(pq);
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    return any(abs_diff(pq.s03, pq.s12) > thresh) * -1;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    //Only apply the filter if the difference is LESS than 'limit'
#if 1
    char mask = (abs_diff(pq.s0, pq.s1) > limit);
    mask |= (abs_diff(pq.s1, pq.s2) > limit);
    mask |= (abs_diff(pq.s2, pq.s3) > limit);
    mask |= (abs_diff(pq.s5, pq.s4) > limit);
    mask |= (abs_diff(pq.s6, pq.s5) > limit);
    mask |= (abs_diff(pq.s7, pq.s6) > limit);
    mask |= (abs_diff(pq.s3, pq.s4) * 2 + abs_diff(pq.s2, pq.s5) / 2  > flimit * 2 + limit);
    return mask - 1;
#else
    short8 mask8 = convert_short8(abs_diff(pq.s01256732, pq.s12345645));
    mask8.s6 = mask8.s6 * 2 + mask8.s7 / 2;

    short8 limits = (short8)limit;
    limits.s6 += flimit * 2;
    mask8 = mask8 > limits;
    mask8.s7 = 0;

    return any(mask8) - 1;
#endif

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs_diff(p0, q0) * 2 + abs_diff(p1, q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

__inline void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter;
    char2 filter;

    char4 pq = (char4){*op1, *op0, *oq0, *oq1};
    pq ^= (char4)0x80;

    signed char u;

    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat((char2)vp8_filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    u = sub_sat(pq.s2, filter.s0);
    *oq0  = u ^ 0x80;

    u = add_sat(pq.s1, filter.s1);
    *op0 = u ^ 0x80;
}


Fix opencl batch_slice_bw_kernel.

Fix opencl batch_slice_bw_kernel.

inline float inline_add(const float a, const float b) { return a + b; }
inline float inline_sub(const float a, const float b) { return a - b; }
inline float inline_mul(const float a, const float b) { return a * b; }
inline float inline_div(const float a, const float b) { return a / b; }

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (max_val[tid + k] > max_val[tid]) { \
        max_val[tid] = max_val[tid + k]; \
        argmax_val[tid] = argmax_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMAX_KERNEL(GROUP_SIZE) \
kernel void argmax_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float max_val[GROUP_SIZE]; \
  local unsigned argmax_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  max_val[tid] = -1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val > max_val[tid]) { \
      max_val[tid] = val; \
      argmax_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmax_val[0]; \
}

ARGMAX_KERNEL(1024)
ARGMAX_KERNEL(512)
ARGMAX_KERNEL(256)
ARGMAX_KERNEL(128)
ARGMAX_KERNEL(64)
ARGMAX_KERNEL(32)
ARGMAX_KERNEL(16)
ARGMAX_KERNEL(8)
ARGMAX_KERNEL(4)
ARGMAX_KERNEL(2)
ARGMAX_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (min_val[tid + k] < min_val[tid]) { \
        min_val[tid] = min_val[tid + k]; \
        argmin_val[tid] = argmin_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMIN_KERNEL(GROUP_SIZE) \
kernel void argmin_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float min_val[GROUP_SIZE]; \
  local unsigned argmin_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  min_val[tid] = 1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val < min_val[tid]) { \
      min_val[tid] = val; \
      argmin_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmin_val[0]; \
}

ARGMIN_KERNEL(1024)
ARGMIN_KERNEL(512)
ARGMIN_KERNEL(256)
ARGMIN_KERNEL(128)
ARGMIN_KERNEL(64)
ARGMIN_KERNEL(32)
ARGMIN_KERNEL(16)
ARGMIN_KERNEL(8)
ARGMIN_KERNEL(4)
ARGMIN_KERNEL(2)
ARGMIN_KERNEL(1)

#undef REDUCE

kernel void set_identity_kernel(
    const unsigned size, const unsigned skip, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = !(i % skip);
}

kernel void pick_fw_kernel(
    const global float *px, const global unsigned *pi,
    const unsigned wx, const unsigned wy, const unsigned sx,
    const unsigned si, const unsigned sy, global float *py) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) py[oy + t] = px[ox + (t / wy) * wx + (t % wy)];
}

kernel void slice_fw_kernel(
    const global float *px, const unsigned shift, const unsigned span,
    const unsigned skip, const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[(i / span) * skip + (i % span) + shift];
}

kernel void concat_fw_kernel(
    const global float *px, const unsigned span, const unsigned skip,
    const unsigned x_size, const unsigned y_size,
    global float *py, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < y_size) py[(i / span) * skip + (i % span) + shift] = px[i % x_size];
}

inline void atomic_add_float(global float *source, const float operand) {
  union {
    unsigned u;
    float f;
  } oldval, newval;
  unsigned readback;
  oldval.f = *source;
  newval.f = oldval.f + operand;
  while ((readback = atomic_cmpxchg(
      (global unsigned *) source, oldval.u, newval.u)) != oldval.u) {
    oldval.u = readback;
    newval.f = oldval.f + operand;
  }
}

kernel void pick_bw_kernel(
    const global float *pgy, const global unsigned *pi,
    const unsigned wx, const unsigned wy,
    const unsigned sx, const unsigned si, const unsigned sy,
    global float *pgx) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) {
    atomic_add_float(pgx + ox + (t / wy) * wx + (t % wy), pgy[oy + t]);
  }
}

kernel void slice_bw_kernel(
    const global float *pgy, const unsigned wx, const unsigned wy,
    const unsigned nx, const unsigned ny,
    global float *pgx, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < wy * max(nx, ny)) {
    atomic_add_float(
        pgx + shift + ((i / wy) * wx + (i % wy)) % (wx * nx),
        pgy[i % (wy * ny)]);
  }
}

#define OPENCLDEV_KERNEL_FW_X(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const unsigned size, global float *py) { \
    const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
}

#define OPENCLDEV_KERNEL_BW_X(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const unsigned size, global float *pgx) { \
   const unsigned i = get_global_id(0); \
   if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_CONST(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const float k, \
    const unsigned size, global float *py) { \
  const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
 }

#define OPENCLDEV_KERNEL_BW_X_CONST(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const float k, const unsigned size, global float *pgx) { \
  const unsigned i = get_global_id(0); \
  if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_R(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(px[i + mbx * shift], pk[mbk * bid_y]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_L(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pk[mbk * bid_y], px[i + mbx * shift]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_AB(name, op) \
kernel void name##_fw_kernel( \
    const global float *pa, const global float *pb, const unsigned size, \
    const unsigned mba, const unsigned mbb, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pa[i + mba * shift], pb[i + mbb * shift]); \
  } \
}

OPENCLDEV_KERNEL_FW_X(negate, -px[i])
OPENCLDEV_KERNEL_FW_X(sqrt, sqrt(px[i]))
OPENCLDEV_KERNEL_FW_X(exp, exp(px[i]))
OPENCLDEV_KERNEL_FW_X(log, log(px[i]))
OPENCLDEV_KERNEL_FW_X(tanh, tanh(px[i]))
OPENCLDEV_KERNEL_FW_X(sigmoid, .5f + .5f  * tanh(.5f * px[i]))
OPENCLDEV_KERNEL_FW_X(
    softplus, max(px[i], .0f) + log(1.f + exp(-fabs(px[i]))))
OPENCLDEV_KERNEL_FW_X(sin, sin(px[i]))
OPENCLDEV_KERNEL_FW_X(cos, cos(px[i]))
OPENCLDEV_KERNEL_FW_X(tan, tan(px[i]))

OPENCLDEV_KERNEL_BW_X(sqrt, .5f * pgy[i] / py[i])
OPENCLDEV_KERNEL_BW_X(exp, py[i] * pgy[i])
OPENCLDEV_KERNEL_BW_X(log, pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X(tanh, (1.f - py[i] * py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sigmoid, py[i] * (1.f - py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(softplus, (.5f + .5f * tanh(.5f * px[i])) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sin, cos(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(cos, -sin(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(tan, (1.f + py[i] * py[i]) * pgy[i])

OPENCLDEV_KERNEL_FW_X_CONST(add_const, px[i] + k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_r, px[i] - k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_l, k - px[i])
OPENCLDEV_KERNEL_FW_X_CONST(multiply_const, px[i] * k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_r, px[i] / k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_l, k / px[i])
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_r, pow(px[i], k))
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_l, pow(k, px[i]))
OPENCLDEV_KERNEL_FW_X_CONST(prelu, max(px[i], .0f) + k * min(px[i], .0f))
OPENCLDEV_KERNEL_FW_X_CONST(
    elu, max(px[i], .0f) + k * (exp(min(px[i], .0f)) - 1.0f))

kernel void pown_fw_kernel(
    const global float *px, const int k,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = pown(px[i], k);
 }

OPENCLDEV_KERNEL_BW_X_CONST(add_const, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_r, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_l, -pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(multiply_const, k * pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_r, pgy[i] / k)
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_l, -py[i] * pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_r, k * pgy[i] * py[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_l, log(k) * pgy[i] * py[i])
OPENCLDEV_KERNEL_BW_X_CONST(
    prelu, pgy[i] * ((px[i] > .0f) + k * (px[i] <= .0f)))
OPENCLDEV_KERNEL_BW_X_CONST(
    elu, pgy[i] * ((px[i] > .0f) + (py[i] + k) * (px[i] <= .0f)))

kernel void pown_bw_kernel(
    const global float *px, const global float *py, const global float *pgy,
    const int k, const unsigned size, global float *pgx) {
  const unsigned i = get_global_id(0);
  if (i < size) pgx[i] += k * pgy[i] * py[i] / px[i];
}

OPENCLDEV_KERNEL_FW_X_SCALAR_R(add_scalar, inline_add)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(subtract_scalar_r, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(subtract_scalar_l, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(multiply_scalar, inline_mul)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(divide_scalar_r, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(divide_scalar_l, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(pow_scalar_r, pow)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(pow_scalar_l, pow)

OPENCLDEV_KERNEL_FW_AB(add, inline_add)
OPENCLDEV_KERNEL_FW_AB(subtract, inline_sub)
OPENCLDEV_KERNEL_FW_AB(multiply, inline_mul)
OPENCLDEV_KERNEL_FW_AB(divide, inline_div)
OPENCLDEV_KERNEL_FW_AB(pow, pow)

#undef OPENCLDEV_KERNEL_FW_X
#undef OPENCLDEV_KERNEL_BW_X
#undef OPENCLDEV_KERNEL_FW_X_CONST
#undef OPENCLDEV_KERNEL_BW_X_CONST
#undef CUDADEV_KERNEL_FW_X_SCALAR_R
#undef CUDADEV_KERNEL_FW_X_SCALAR_L
#undef CUDADEV_KERNEL_FW_AB

kernel void add_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, gy);
  }
}

kernel void subtract_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, -gy);
  }
}

kernel void multiply_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    atomic_add_float(pga + a_ofs, gy * pb[b_ofs]);
    atomic_add_float(pgb + b_ofs, gy * pa[a_ofs]);
  }
}

kernel void divide_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] / pb[b_ofs];
    atomic_add_float(pga + i + mba * shift, k);
    atomic_add_float(pgb + b_ofs, -k * py[y_ofs]);
  }
}

kernel void pow_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] * py[y_ofs];
    atomic_add_float(pga + a_ofs, k * pb[b_ofs] / pa[a_ofs]);
    atomic_add_float(pgb + b_ofs, k * log(pa[a_ofs]));
  }
}

kernel void transpose_fw_kernel(
    const global float *px, unsigned rows, unsigned cols, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) py[ofs + j + i * cols] = px[ofs + i + j * rows];
}

kernel void transpose_bw_kernel(
    const global float *py, const unsigned rows, const unsigned cols,
    global float *px) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) px[ofs + i + j * rows] += py[ofs + j + i * cols];
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] += temp[tid + k]; \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define SUM_FW_KERNEL(GROUP_SIZE) \
kernel void sum_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = 0; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) temp[tid] += px[i * skip]; \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

SUM_FW_KERNEL(1024)
SUM_FW_KERNEL(512)
SUM_FW_KERNEL(256)
SUM_FW_KERNEL(128)
SUM_FW_KERNEL(64)
SUM_FW_KERNEL(32)
SUM_FW_KERNEL(16)
SUM_FW_KERNEL(8)
SUM_FW_KERNEL(4)
SUM_FW_KERNEL(2)
SUM_FW_KERNEL(1)

#undef REDUCE

inline float logsumexp2_fw_kernel(float a, float b) {
  return a > b
    ? a + log(1.f + exp(b - a))
    : b + log(1.f + exp(a - b));
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = logsumexp2_fw_kernel(temp[tid], temp[tid + k]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define LOGSUMEXP_FW_KERNEL(GROUP_SIZE) \
kernel void logsumexp_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = -1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = logsumexp2_fw_kernel(temp[tid], px[i * skip]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

LOGSUMEXP_FW_KERNEL(1024)
LOGSUMEXP_FW_KERNEL(512)
LOGSUMEXP_FW_KERNEL(256)
LOGSUMEXP_FW_KERNEL(128)
LOGSUMEXP_FW_KERNEL(64)
LOGSUMEXP_FW_KERNEL(32)
LOGSUMEXP_FW_KERNEL(16)
LOGSUMEXP_FW_KERNEL(8)
LOGSUMEXP_FW_KERNEL(4)
LOGSUMEXP_FW_KERNEL(2)
LOGSUMEXP_FW_KERNEL(1)

#undef REDUCE

kernel void broadcast_fw_kernel(
    const global float *px, const unsigned skip1, const unsigned skip2,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[i % skip1 + (i / skip2) * skip1];
}

kernel void batch_pick_fw_kernel(
    const global float *px, const global unsigned *pi,
    const unsigned si, const unsigned sy, global float *py) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = pi[bid_y * si] * sy;
  const unsigned oy = bid_y * sy;
  if (t < sy) py[oy + t] = px[ox + t];
}

kernel void batch_slice_fw_kernel(
    const global float *px, const unsigned shift, const unsigned size,
    global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[i + shift];
}

kernel void batch_concat_fw_kernel(
    const global float *px, const unsigned y_size,
    global float *py, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < y_size) py[i + shift] = px[i];
}

kernel void batch_sum_fw_kernel(
    const global float *px, const unsigned size,
    const unsigned batch, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) {
    float temp = .0f;
    px += i;
    for (unsigned j = 0; j < batch; ++j, px += size) {
      temp += *px;
    }
    py[i] = temp;
  }
}

kernel void batch_pick_bw_kernel(
    const global float *pgy, const global unsigned *pi,
    const unsigned si, const unsigned sy, global float *pgx) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = pi[bid_y * si] * sy;
  const unsigned oy = bid_y * sy;
  if (t < sy) atomic_add_float(pgx + ox + t, pgy[oy + t]);
}

kernel void batch_slice_bw_kernel(
    const global float *pgy, const unsigned size,
    global float *pgx, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < size) pgx[i + shift] += pgy[i];
}

kernel void inplace_multiply_const_kernel(
    const float k, const unsigned size, global float *px) {
  const unsigned i = get_global_id(0);
  if (i < size) px[i] *= k;
}

kernel void inplace_add_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, px[i + mbx * shift]);
}

kernel void inplace_subtract_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, -px[i + mbx * shift]);
}

inline float inline_add(const float a, const float b) { return a + b; }
inline float inline_sub(const float a, const float b) { return a - b; }
inline float inline_mul(const float a, const float b) { return a * b; }
inline float inline_div(const float a, const float b) { return a / b; }

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (max_val[tid + k] > max_val[tid]) { \
        max_val[tid] = max_val[tid + k]; \
        argmax_val[tid] = argmax_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMAX_KERNEL(GROUP_SIZE) \
kernel void argmax_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float max_val[GROUP_SIZE]; \
  local unsigned argmax_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  max_val[tid] = -1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val > max_val[tid]) { \
      max_val[tid] = val; \
      argmax_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmax_val[0]; \
}

ARGMAX_KERNEL(1024)
ARGMAX_KERNEL(512)
ARGMAX_KERNEL(256)
ARGMAX_KERNEL(128)
ARGMAX_KERNEL(64)
ARGMAX_KERNEL(32)
ARGMAX_KERNEL(16)
ARGMAX_KERNEL(8)
ARGMAX_KERNEL(4)
ARGMAX_KERNEL(2)
ARGMAX_KERNEL(1)

#undef REDUCE

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) { \
      if (min_val[tid + k] < min_val[tid]) { \
        min_val[tid] = min_val[tid + k]; \
        argmin_val[tid] = argmin_val[tid + k]; \
      } \
    } \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define ARGMIN_KERNEL(GROUP_SIZE) \
kernel void argmin_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, \
    const unsigned n, global unsigned *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float min_val[GROUP_SIZE]; \
  local unsigned argmin_val[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  min_val[tid] = 1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    const float val = px[i * skip]; \
    if (val < min_val[tid]) { \
      min_val[tid] = val; \
      argmin_val[tid] = i; \
    } \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = argmin_val[0]; \
}

ARGMIN_KERNEL(1024)
ARGMIN_KERNEL(512)
ARGMIN_KERNEL(256)
ARGMIN_KERNEL(128)
ARGMIN_KERNEL(64)
ARGMIN_KERNEL(32)
ARGMIN_KERNEL(16)
ARGMIN_KERNEL(8)
ARGMIN_KERNEL(4)
ARGMIN_KERNEL(2)
ARGMIN_KERNEL(1)

#undef REDUCE

kernel void set_identity_kernel(
    const unsigned size, const unsigned skip, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = !(i % skip);
}

kernel void pick_fw_kernel(
    const global float *px, const global unsigned *pi,
    const unsigned wx, const unsigned wy, const unsigned sx,
    const unsigned si, const unsigned sy, global float *py) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) py[oy + t] = px[ox + (t / wy) * wx + (t % wy)];
}

kernel void slice_fw_kernel(
    const global float *px, const unsigned shift, const unsigned span,
    const unsigned skip, const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[(i / span) * skip + (i % span) + shift];
}

kernel void concat_fw_kernel(
    const global float *px, const unsigned span, const unsigned skip,
    const unsigned x_size, const unsigned y_size,
    global float *py, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < y_size) py[(i / span) * skip + (i % span) + shift] = px[i % x_size];
}

inline void atomic_add_float(global float *source, const float operand) {
  union {
    unsigned u;
    float f;
  } oldval, newval;
  unsigned readback;
  oldval.f = *source;
  newval.f = oldval.f + operand;
  while ((readback = atomic_cmpxchg(
      (global unsigned *) source, oldval.u, newval.u)) != oldval.u) {
    oldval.u = readback;
    newval.f = oldval.f + operand;
  }
}

kernel void pick_bw_kernel(
    const global float *pgy, const global unsigned *pi,
    const unsigned wx, const unsigned wy,
    const unsigned sx, const unsigned si, const unsigned sy,
    global float *pgx) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = bid_y * sx + pi[bid_y * si] * wy;
  const unsigned oy = bid_y * sy;
  if (t < sy) {
    atomic_add_float(pgx + ox + (t / wy) * wx + (t % wy), pgy[oy + t]);
  }
}

kernel void slice_bw_kernel(
    const global float *pgy, const unsigned wx, const unsigned wy,
    const unsigned nx, const unsigned ny,
    global float *pgx, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < wy * max(nx, ny)) {
    atomic_add_float(
        pgx + shift + ((i / wy) * wx + (i % wy)) % (wx * nx),
        pgy[i % (wy * ny)]);
  }
}

#define OPENCLDEV_KERNEL_FW_X(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const unsigned size, global float *py) { \
    const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
}

#define OPENCLDEV_KERNEL_BW_X(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const unsigned size, global float *pgx) { \
   const unsigned i = get_global_id(0); \
   if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_CONST(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const float k, \
    const unsigned size, global float *py) { \
  const unsigned i = get_global_id(0); \
  if (i < size) py[i] = (op); \
 }

#define OPENCLDEV_KERNEL_BW_X_CONST(name, op) \
kernel void name##_bw_kernel( \
    const global float *px, const global float *py, const global float *pgy, \
    const float k, const unsigned size, global float *pgx) { \
  const unsigned i = get_global_id(0); \
  if (i < size) pgx[i] += (op); \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_R(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(px[i + mbx * shift], pk[mbk * bid_y]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_X_SCALAR_L(name, op) \
kernel void name##_fw_kernel( \
    const global float *px, const global float *pk, const unsigned size, \
    const unsigned mbx, const unsigned mbk, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pk[mbk * bid_y], px[i + mbx * shift]); \
  } \
}

#define OPENCLDEV_KERNEL_FW_AB(name, op) \
kernel void name##_fw_kernel( \
    const global float *pa, const global float *pb, const unsigned size, \
    const unsigned mba, const unsigned mbb, global float *py) { \
  const unsigned i = get_global_id(0); \
  const unsigned bid_y = get_group_id(1); \
  const unsigned shift = bid_y * size; \
  if (i < size) { \
    py[i + shift] = op(pa[i + mba * shift], pb[i + mbb * shift]); \
  } \
}

OPENCLDEV_KERNEL_FW_X(negate, -px[i])
OPENCLDEV_KERNEL_FW_X(sqrt, sqrt(px[i]))
OPENCLDEV_KERNEL_FW_X(exp, exp(px[i]))
OPENCLDEV_KERNEL_FW_X(log, log(px[i]))
OPENCLDEV_KERNEL_FW_X(tanh, tanh(px[i]))
OPENCLDEV_KERNEL_FW_X(sigmoid, .5f + .5f  * tanh(.5f * px[i]))
OPENCLDEV_KERNEL_FW_X(
    softplus, max(px[i], .0f) + log(1.f + exp(-fabs(px[i]))))
OPENCLDEV_KERNEL_FW_X(sin, sin(px[i]))
OPENCLDEV_KERNEL_FW_X(cos, cos(px[i]))
OPENCLDEV_KERNEL_FW_X(tan, tan(px[i]))

OPENCLDEV_KERNEL_BW_X(sqrt, .5f * pgy[i] / py[i])
OPENCLDEV_KERNEL_BW_X(exp, py[i] * pgy[i])
OPENCLDEV_KERNEL_BW_X(log, pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X(tanh, (1.f - py[i] * py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sigmoid, py[i] * (1.f - py[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(softplus, (.5f + .5f * tanh(.5f * px[i])) * pgy[i])
OPENCLDEV_KERNEL_BW_X(sin, cos(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(cos, -sin(px[i]) * pgy[i])
OPENCLDEV_KERNEL_BW_X(tan, (1.f + py[i] * py[i]) * pgy[i])

OPENCLDEV_KERNEL_FW_X_CONST(add_const, px[i] + k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_r, px[i] - k)
OPENCLDEV_KERNEL_FW_X_CONST(subtract_const_l, k - px[i])
OPENCLDEV_KERNEL_FW_X_CONST(multiply_const, px[i] * k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_r, px[i] / k)
OPENCLDEV_KERNEL_FW_X_CONST(divide_const_l, k / px[i])
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_r, pow(px[i], k))
OPENCLDEV_KERNEL_FW_X_CONST(pow_const_l, pow(k, px[i]))
OPENCLDEV_KERNEL_FW_X_CONST(prelu, max(px[i], .0f) + k * min(px[i], .0f))
OPENCLDEV_KERNEL_FW_X_CONST(
    elu, max(px[i], .0f) + k * (exp(min(px[i], .0f)) - 1.0f))

kernel void pown_fw_kernel(
    const global float *px, const int k,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = pown(px[i], k);
 }

OPENCLDEV_KERNEL_BW_X_CONST(add_const, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_r, pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(subtract_const_l, -pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(multiply_const, k * pgy[i])
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_r, pgy[i] / k)
OPENCLDEV_KERNEL_BW_X_CONST(divide_const_l, -py[i] * pgy[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_r, k * pgy[i] * py[i] / px[i])
OPENCLDEV_KERNEL_BW_X_CONST(pow_const_l, log(k) * pgy[i] * py[i])
OPENCLDEV_KERNEL_BW_X_CONST(
    prelu, pgy[i] * ((px[i] > .0f) + k * (px[i] <= .0f)))
OPENCLDEV_KERNEL_BW_X_CONST(
    elu, pgy[i] * ((px[i] > .0f) + (py[i] + k) * (px[i] <= .0f)))

kernel void pown_bw_kernel(
    const global float *px, const global float *py, const global float *pgy,
    const int k, const unsigned size, global float *pgx) {
  const unsigned i = get_global_id(0);
  if (i < size) pgx[i] += k * pgy[i] * py[i] / px[i];
}

OPENCLDEV_KERNEL_FW_X_SCALAR_R(add_scalar, inline_add)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(subtract_scalar_r, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(subtract_scalar_l, inline_sub)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(multiply_scalar, inline_mul)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(divide_scalar_r, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(divide_scalar_l, inline_div)
OPENCLDEV_KERNEL_FW_X_SCALAR_R(pow_scalar_r, pow)
OPENCLDEV_KERNEL_FW_X_SCALAR_L(pow_scalar_l, pow)

OPENCLDEV_KERNEL_FW_AB(add, inline_add)
OPENCLDEV_KERNEL_FW_AB(subtract, inline_sub)
OPENCLDEV_KERNEL_FW_AB(multiply, inline_mul)
OPENCLDEV_KERNEL_FW_AB(divide, inline_div)
OPENCLDEV_KERNEL_FW_AB(pow, pow)

#undef OPENCLDEV_KERNEL_FW_X
#undef OPENCLDEV_KERNEL_BW_X
#undef OPENCLDEV_KERNEL_FW_X_CONST
#undef OPENCLDEV_KERNEL_BW_X_CONST
#undef CUDADEV_KERNEL_FW_X_SCALAR_R
#undef CUDADEV_KERNEL_FW_X_SCALAR_L
#undef CUDADEV_KERNEL_FW_AB

kernel void add_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, gy);
  }
}

kernel void subtract_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    atomic_add_float(pga + i + mba * shift, gy);
    atomic_add_float(pgb + i + mbb * shift, -gy);
  }
}

kernel void multiply_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const float gy = pgy[i + shift];
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    atomic_add_float(pga + a_ofs, gy * pb[b_ofs]);
    atomic_add_float(pgb + b_ofs, gy * pa[a_ofs]);
  }
}

kernel void divide_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] / pb[b_ofs];
    atomic_add_float(pga + i + mba * shift, k);
    atomic_add_float(pgb + b_ofs, -k * py[y_ofs]);
  }
}

kernel void pow_bw_kernel(
    const global float *pa, const global float *pb,
    const global float *py, const global float *pgy,
    const unsigned size, const unsigned mba, const unsigned mbb,
    global float *pga, global float *pgb) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) {
    const unsigned a_ofs = i + mba * shift;
    const unsigned b_ofs = i + mbb * shift;
    const unsigned y_ofs = i + shift;
    const float k = pgy[y_ofs] * py[y_ofs];
    atomic_add_float(pga + a_ofs, k * pb[b_ofs] / pa[a_ofs]);
    atomic_add_float(pgb + b_ofs, k * log(pa[a_ofs]));
  }
}

kernel void transpose_fw_kernel(
    const global float *px, unsigned rows, unsigned cols, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) py[ofs + j + i * cols] = px[ofs + i + j * rows];
}

kernel void transpose_bw_kernel(
    const global float *py, const unsigned rows, const unsigned cols,
    global float *px) {
  const unsigned i = get_global_id(0);
  const unsigned j = get_global_id(1);
  const unsigned bid_z = get_group_id(2);
  const unsigned ofs = bid_z * rows * cols;
  if (i < rows && j < cols) px[ofs + i + j * rows] += py[ofs + j + i * cols];
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] += temp[tid + k]; \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define SUM_FW_KERNEL(GROUP_SIZE) \
kernel void sum_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = 0; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) temp[tid] += px[i * skip]; \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

SUM_FW_KERNEL(1024)
SUM_FW_KERNEL(512)
SUM_FW_KERNEL(256)
SUM_FW_KERNEL(128)
SUM_FW_KERNEL(64)
SUM_FW_KERNEL(32)
SUM_FW_KERNEL(16)
SUM_FW_KERNEL(8)
SUM_FW_KERNEL(4)
SUM_FW_KERNEL(2)
SUM_FW_KERNEL(1)

#undef REDUCE

inline float logsumexp2_fw_kernel(float a, float b) {
  return a > b
    ? a + log(1.f + exp(b - a))
    : b + log(1.f + exp(a - b));
}

#define REDUCE(k, GROUP_SIZE) \
  if (GROUP_SIZE >= k << 1) { \
    if (tid < k) temp[tid] = logsumexp2_fw_kernel(temp[tid], temp[tid + k]); \
    barrier(CLK_LOCAL_MEM_FENCE); \
  }

#define LOGSUMEXP_FW_KERNEL(GROUP_SIZE) \
kernel void logsumexp_fw_kernel_##GROUP_SIZE( \
    const global float *px, const unsigned skip, const unsigned n, \
    global float *py) { \
  const unsigned bid = get_group_id(0); \
  const unsigned tid = get_local_id(0); \
  local float temp[GROUP_SIZE]; \
  px += bid % skip + (bid / skip) * skip * n; \
  temp[tid] = -1e38; \
  for (unsigned i = tid; i < n; i += GROUP_SIZE) { \
    temp[tid] = logsumexp2_fw_kernel(temp[tid], px[i * skip]); \
  } \
  barrier(CLK_LOCAL_MEM_FENCE); \
  REDUCE(512, GROUP_SIZE) \
  REDUCE(256, GROUP_SIZE) \
  REDUCE(128, GROUP_SIZE) \
  REDUCE(64, GROUP_SIZE) \
  REDUCE(32, GROUP_SIZE) \
  REDUCE(16, GROUP_SIZE) \
  REDUCE(8, GROUP_SIZE) \
  REDUCE(4, GROUP_SIZE) \
  REDUCE(2, GROUP_SIZE) \
  REDUCE(1, GROUP_SIZE) \
  if (tid == 0) py[bid] = temp[0]; \
}

LOGSUMEXP_FW_KERNEL(1024)
LOGSUMEXP_FW_KERNEL(512)
LOGSUMEXP_FW_KERNEL(256)
LOGSUMEXP_FW_KERNEL(128)
LOGSUMEXP_FW_KERNEL(64)
LOGSUMEXP_FW_KERNEL(32)
LOGSUMEXP_FW_KERNEL(16)
LOGSUMEXP_FW_KERNEL(8)
LOGSUMEXP_FW_KERNEL(4)
LOGSUMEXP_FW_KERNEL(2)
LOGSUMEXP_FW_KERNEL(1)

#undef REDUCE

kernel void broadcast_fw_kernel(
    const global float *px, const unsigned skip1, const unsigned skip2,
    const unsigned size, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[i % skip1 + (i / skip2) * skip1];
}

kernel void batch_pick_fw_kernel(
    const global float *px, const global unsigned *pi,
    const unsigned si, const unsigned sy, global float *py) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = pi[bid_y * si] * sy;
  const unsigned oy = bid_y * sy;
  if (t < sy) py[oy + t] = px[ox + t];
}

kernel void batch_slice_fw_kernel(
    const global float *px, const unsigned shift, const unsigned size,
    global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) py[i] = px[i + shift];
}

kernel void batch_concat_fw_kernel(
    const global float *px, const unsigned y_size,
    global float *py, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < y_size) py[i + shift] = px[i];
}

kernel void batch_sum_fw_kernel(
    const global float *px, const unsigned size,
    const unsigned batch, global float *py) {
  const unsigned i = get_global_id(0);
  if (i < size) {
    float temp = .0f;
    px += i;
    for (unsigned j = 0; j < batch; ++j, px += size) {
      temp += *px;
    }
    py[i] = temp;
  }
}

kernel void batch_pick_bw_kernel(
    const global float *pgy, const global unsigned *pi,
    const unsigned si, const unsigned sy, global float *pgx) {
  const unsigned t = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned ox = pi[bid_y * si] * sy;
  const unsigned oy = bid_y * sy;
  if (t < sy) atomic_add_float(pgx + ox + t, pgy[oy + t]);
}

kernel void batch_slice_bw_kernel(
    const global float *pgy, const unsigned size,
    global float *pgx, const unsigned shift) {
  const unsigned i = get_global_id(0);
  if (i < size) atomic_add_float(pgx + i + shift, pgy[i]);
}

kernel void inplace_multiply_const_kernel(
    const float k, const unsigned size, global float *px) {
  const unsigned i = get_global_id(0);
  if (i < size) px[i] *= k;
}

kernel void inplace_add_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, px[i + mbx * shift]);
}

kernel void inplace_subtract_kernel(
    const global float *px, const unsigned size,
    const unsigned mbx, const unsigned mby, global float *py) {
  const unsigned i = get_global_id(0);
  const unsigned bid_y = get_group_id(1);
  const unsigned shift = bid_y * size;
  if (i < size) atomic_add_float(py + i + mby * shift, -px[i + mbx * shift]);
}