Further reductions in redundant calculation in kernels.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline uchar8 vp8_mbfilter(signed char mask,signed char hev,uchar8);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
char vp8_char_clamp(int in){
    return max(min(in, 127), -128);
}
#endif

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

//OpenCL built-in functions (
size_t get_global_id(unsigned int);
size_t get_global_size(unsigned int);

uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = convert_char4(base) ^ 0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u.s0 = add_sat(pq.s0, vp8_filter);
    u.s1 = add_sat(pq.s1, Filter.s1);
    u.s2 = sub_sat(pq.s2, Filter.s0);
    u.s3 = sub_sat(pq.s3, vp8_filter);

    return convert_uchar4(u ^ 0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t thread,
    size_t plane,
    size_t block
){
    if (filters[num_blocks*filter_type + block] > 0){
        if ( plane < num_planes){

            int p = pitches[plane];
            int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset];

            int  hev = 0; /* high edge variance */
            signed char mask = 0;

            s_off += thread;

            uchar8 data;
            data.s0 = s_base[s_off-4*p];
            data.s1 = s_base[s_off-3*p];
            data.s2 = s_base[s_off-2*p];
            data.s3 = s_base[s_off-p];
            data.s4 = s_base[s_off];
            data.s5 = s_base[s_off+p];
            data.s6 = s_base[s_off+2*p];
            data.s7 = s_base[s_off+3*p];

            mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

            hev = vp8_hevmask(lfi->thr[thread], data.s2345);

            data.s2345 = vp8_filter(mask, hev, data.s2345);

            s_base[s_off - 2*p] = data.s2;
            s_base[s_off - p  ] = data.s3;
            s_base[s_off      ] = data.s4;
            s_base[s_off + p  ] = data.s5;
        }
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t thread,
    size_t plane,
    size_t block
){
    if ((cur_iter == 1 || plane == 0) && filters[num_blocks*filter_type + block] > 0){

        int p = pitches[plane];
        int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        int  hev = 0; /* high edge variance */
        signed char mask = 0;

        s_off += p * thread;

        uchar8 data = vload8(0, &s_base[s_off-4]);

        mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

        hev = vp8_hevmask(lfi->thr[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        vstore4(data.s2345, 0, &s_base[s_off-2]);
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t thread,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int s_off = offsets[8*num_blocks + block*num_planes+plane] + thread;

    uchar8 data;
    data.s0 = s_base[s_off-4*p];
    data.s1 = s_base[s_off-3*p];
    data.s2 = s_base[s_off-2*p];
    data.s3 = s_base[s_off-p];
    data.s4 = s_base[s_off];
    data.s5 = s_base[s_off+p];
    data.s6 = s_base[s_off+2*p];
    data.s7 = s_base[s_off+3*p];

    char mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    char hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    s_base[s_off - 3*p] = data.s1;
    s_base[s_off - 2*p] = data.s2;
    s_base[s_off - 1*p] = data.s3;
    s_base[s_off      ] = data.s4;
    s_base[s_off + p  ] = data.s5;
    s_base[s_off + 2*p] = data.s6;

}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t thread,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int block_offset = block*num_planes+plane;
    int s_off = offsets[block_offset];

    signed char hev = 0; /* high edge variance */
    signed char mask = 0;

    s_off += p * thread;

    uchar8 data = vload8(0, &s_base[s_off-4]);

    mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    vstore8(data, 0, &s_base[s_off-4]);
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);

    for (int i = 0; i < num_levels; i++){
        int block_offset = block_offsets[priority_level];

        global int *offsets = &offsets_in[16*block_offset];
        global int *filters = &filters_in[4*block_offset];
        size_t num_blocks = priority_num_blocks[priority_level];
        int filter_level = filters[block];
        global loop_filter_info *lf_info = &lfi[filter_level];
        int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);

        if (block < num_blocks){
            //Prefetch vertical edge source pixels into global cache (horizontal isn't worth it)
            for(int pln = 0; pln < 3; pln++){
                int p = pitches[pln];
                int offset = block*3+pln;
                int s_off = offsets[offset];
                for (int thread = 0; thread < 16; thread++){
                    prefetch(&s_base[s_off+p*thread-4], 8);
                }
            }

            if (thread_level_filter & (filters[num_blocks*COLS_LOCATION + block] > 0)){
                vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        COLS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, thread, plane, block);
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, thread, plane, block);
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, thread, plane, block);
            }

        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (block < num_blocks){

            if (thread_level_filter & (filters[num_blocks*ROWS_LOCATION + block] > 0)){
                vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
                    filters, ROWS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, thread, plane, block);
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, thread, plane, block);
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, thread, plane, block);
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }

}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*ROWS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
            filters, ROWS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, thread, plane, block);
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, thread, plane, block);
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, thread, plane, block);
    }
}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*COLS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
            COLS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, thread, plane, block);
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, thread, plane, block);
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, thread, plane, block);
    }

}

void vp8_loop_filter_simple_horizontal_edge_worker
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    size_t num_planes = get_global_size(1);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

void vp8_loop_filter_simple_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t block = get_global_id(2);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[0];
            int block_offset = cur_iter*num_blocks + block;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[0]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
            block_offsets, priority_num_blocks
    );

    //3 Y plane iterations
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_horizontal_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 1, ROWS_LOCATION, 4, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_all_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
)
{

    for (int i = 0; i < num_levels; i++){
        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
                    block_offsets, priority_num_blocks
            );

            //3 Y plane iterations
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 1, ROWS_LOCATION, 4, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }
}

//Inline and non-kernel functions follow.

__inline uchar8 vp8_mbfilter(
    signed char mask,
    signed char hev,
    uchar8 base
)
{
    char4 s;
    char4 u;
    signed char vp8_filter;

    char2 filter;

    char8 pq = convert_char8(base);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s2, pq.s5);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s4 - pq.s3), -128, 127);
    vp8_filter &= mask;

    filter = (char2)vp8_filter;
    filter &= hev;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat(filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    pq.s4 = sub_sat(pq.s4, filter.s0);
    pq.s3 = add_sat(pq.s3, filter.s1);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th, 2/7th, and 1/7th difference across boundary */
    u.s0 = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    u.s1 = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    u.s2 = clamp((63 + filter.s1 * 9) >> 7, -128, 127);

    s.s012 = sub_sat(pq.s456, u.s012);
    pq.s456 = s.s012 ^ 0x80;

    s.s012 = add_sat(pq.s321, u.s012);
    pq.s321 = s.s012 ^ 0x80;

    return convert_uchar8(pq);
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    signed char hev;
    hev  = (abs_diff(pq.s0, pq.s1) > thresh) * -1;
    hev  |= (abs_diff(pq.s3, pq.s2) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    signed char mask = 0;

    //Only apply the filter if the difference is LESS than 'limit'
    mask |= (abs_diff(pq.s0, pq.s1) > limit);
    mask |= (abs_diff(pq.s1, pq.s2) > limit);
    mask |= (abs_diff(pq.s2, pq.s3) > limit);
    mask |= (abs_diff(pq.s5, pq.s4) > limit);
    mask |= (abs_diff(pq.s6, pq.s5) > limit);
    mask |= (abs_diff(pq.s7, pq.s6) > limit);
    mask |= (abs_diff(pq.s3, pq.s4) * 2 + abs_diff(pq.s2, pq.s5) / 2  > flimit * 2 + limit);
    mask *= -1;
    return ~mask;

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs_diff(p0, q0) * 2 + abs_diff(p1, q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

__inline void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter;
    char2 filter;

    char4 pq = (char4){*op1, *op0, *oq0, *oq1};
    pq ^= 0x80;

    signed char u;

    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat((char2)vp8_filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    u = sub_sat(pq.s2, filter.s0);
    *oq0  = u ^ 0x80;

    u = add_sat(pq.s1, filter.s1);
    *op0 = u ^ 0x80;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline uchar8 vp8_mbfilter(signed char mask,signed char hev,uchar8);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
char vp8_char_clamp(int in){
    return max(min(in, 127), -128);
}
#endif

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

//OpenCL built-in functions (
size_t get_global_id(unsigned int);
size_t get_global_size(unsigned int);

uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = convert_char4(base) ^ 0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u.s0 = add_sat(pq.s0, vp8_filter);
    u.s1 = add_sat(pq.s1, Filter.s1);
    u.s2 = sub_sat(pq.s2, Filter.s0);
    u.s3 = sub_sat(pq.s3, vp8_filter);

    return convert_uchar4(u ^ 0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t thread,
    size_t plane,
    size_t block
){
    size_t num_planes = 3;

    if ( cur_iter == 0 || plane == 0){
        if (cur_iter > 0){
            num_planes = 1;
        }

        int p = pitches[plane];
        int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        int  hev = 0; /* high edge variance */
        signed char mask = 0;

        global signed char *limit, *flimit, *thresh;
        global loop_filter_info *lf_info;

        lf_info = &lfi[filter_level];
        flimit = lf_info->flim;
        limit = lf_info->lim;
        thresh = lf_info->thr;

        s_off += thread;

        uchar8 data;
        data.s0 = s_base[s_off-4*p];
        data.s1 = s_base[s_off-3*p];
        data.s2 = s_base[s_off-2*p];
        data.s3 = s_base[s_off-p];
        data.s4 = s_base[s_off];
        data.s5 = s_base[s_off+p];
        data.s6 = s_base[s_off+2*p];
        data.s7 = s_base[s_off+3*p];

        mask = vp8_filter_mask(limit[thread], flimit[thread], data);

        hev = vp8_hevmask(thresh[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        s_base[s_off - 2*p] = data.s2;
        s_base[s_off - p  ] = data.s3;
        s_base[s_off      ] = data.s4;
        s_base[s_off + p  ] = data.s5;
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t thread,
    size_t plane,
    size_t block
){
    if ((cur_iter == 1 || plane == 0)){

        int p = pitches[plane];
        int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        int  hev = 0; /* high edge variance */
        signed char mask = 0;

        s_off += p * thread;

        uchar8 data = vload8(0, &s_base[s_off-4]);

        mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

        hev = vp8_hevmask(lfi->thr[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        vstore4(data.s2345, 0, &s_base[s_off-2]);
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t thread,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int s_off = offsets[8*num_blocks + block*num_planes+plane] + thread;

    uchar8 data;
    data.s0 = s_base[s_off-4*p];
    data.s1 = s_base[s_off-3*p];
    data.s2 = s_base[s_off-2*p];
    data.s3 = s_base[s_off-p];
    data.s4 = s_base[s_off];
    data.s5 = s_base[s_off+p];
    data.s6 = s_base[s_off+2*p];
    data.s7 = s_base[s_off+3*p];

    char mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    char hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    s_base[s_off - 3*p] = data.s1;
    s_base[s_off - 2*p] = data.s2;
    s_base[s_off - 1*p] = data.s3;
    s_base[s_off      ] = data.s4;
    s_base[s_off + p  ] = data.s5;
    s_base[s_off + 2*p] = data.s6;

}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t thread,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int block_offset = block*num_planes+plane;
    int s_off = offsets[block_offset];

    signed char hev = 0; /* high edge variance */
    signed char mask = 0;

    s_off += p * thread;

    uchar8 data = vload8(0, &s_base[s_off-4]);

    mask = vp8_filter_mask(lfi->lim[thread], lfi->mbflim[thread], data);

    hev = vp8_hevmask(lfi->thr[thread], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    vstore8(data, 0, &s_base[s_off-4]);
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);

    for (int i = 0; i < num_levels; i++){
        int block_offset = block_offsets[priority_level];

        global int *offsets = &offsets_in[16*block_offset];
        global int *filters = &filters_in[4*block_offset];
        size_t num_blocks = priority_num_blocks[priority_level];
        int filter_level = filters[block];
        global loop_filter_info *lf_info = &lfi[filter_level];
        int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
        int dc_diffs_filter = thread_level_filter & (filters[num_blocks*DC_DIFFS_LOCATION + block] > 0);

        if (block < num_blocks){
            //Prefetch vertical edge source pixels into global cache (horizontal isn't worth it)
            for(int pln = 0; pln < 3; pln++){
                int p = pitches[pln];
                int offset = block*3+pln;
                int s_off = offsets[offset];
                for (int thread = 0; thread < 16; thread++){
                    prefetch(&s_base[s_off+p*thread-4], 8);
                }
            }

            if (thread_level_filter & (filters[num_blocks*COLS_LOCATION + block] > 0)){
                vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        COLS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (dc_diffs_filter){
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, thread, plane, block);
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, thread, plane, block);
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, thread, plane, block);
            }

        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (block < num_blocks){

            if (thread_level_filter & (filters[num_blocks*ROWS_LOCATION + block] > 0)){
                vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
                    filters, ROWS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (dc_diffs_filter){
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
                        DC_DIFFS_LOCATION, filter_level, 0, num_blocks, thread, plane, block);
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
                        DC_DIFFS_LOCATION, filter_level, 3, num_blocks, thread, plane, block);
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
                        DC_DIFFS_LOCATION, filter_level, 4, num_blocks, thread, plane, block);
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }

}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*ROWS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
            filters, ROWS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter & (filters[num_blocks*DC_DIFFS_LOCATION + block] > 0)){
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
                DC_DIFFS_LOCATION, filter_level, 0, num_blocks, thread, plane, block);
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
                DC_DIFFS_LOCATION, filter_level, 3, num_blocks, thread, plane, block);
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
                DC_DIFFS_LOCATION, filter_level, 4, num_blocks, thread, plane, block);
    }
}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*COLS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
            COLS_LOCATION, filter_level, num_blocks, 3, thread, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter & (filters[num_blocks*DC_DIFFS_LOCATION + block] > 0)){
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, thread, plane, block);
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, thread, plane, block);
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, thread, plane, block);
    }

}

void vp8_loop_filter_simple_horizontal_edge_worker
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    size_t num_planes = get_global_size(1);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

void vp8_loop_filter_simple_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t block = get_global_id(2);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[0];
            int block_offset = cur_iter*num_blocks + block;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[0]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
            block_offsets, priority_num_blocks
    );

    //3 Y plane iterations
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_horizontal_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 1, ROWS_LOCATION, 4, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_all_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
)
{

    for (int i = 0; i < num_levels; i++){
        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
                    block_offsets, priority_num_blocks
            );

            //3 Y plane iterations
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 1, ROWS_LOCATION, 4, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }
}

//Inline and non-kernel functions follow.

__inline uchar8 vp8_mbfilter(
    signed char mask,
    signed char hev,
    uchar8 base
)
{
    char4 s;
    char4 u;
    signed char vp8_filter;

    char2 filter;

    char8 pq = convert_char8(base);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s2, pq.s5);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s4 - pq.s3), -128, 127);
    vp8_filter &= mask;

    filter = (char2)vp8_filter;
    filter &= hev;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat(filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    pq.s4 = sub_sat(pq.s4, filter.s0);
    pq.s3 = add_sat(pq.s3, filter.s1);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th, 2/7th, and 1/7th difference across boundary */
    u.s0 = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    u.s1 = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    u.s2 = clamp((63 + filter.s1 * 9) >> 7, -128, 127);

    s.s012 = sub_sat(pq.s456, u.s012);
    pq.s456 = s.s012 ^ 0x80;

    s.s012 = add_sat(pq.s321, u.s012);
    pq.s321 = s.s012 ^ 0x80;

    return convert_uchar8(pq);
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    signed char hev;
    hev  = (abs_diff(pq.s0, pq.s1) > thresh) * -1;
    hev  |= (abs_diff(pq.s3, pq.s2) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    signed char mask = 0;

    //Only apply the filter if the difference is LESS than 'limit'
    mask |= (abs_diff(pq.s0, pq.s1) > limit);
    mask |= (abs_diff(pq.s1, pq.s2) > limit);
    mask |= (abs_diff(pq.s2, pq.s3) > limit);
    mask |= (abs_diff(pq.s5, pq.s4) > limit);
    mask |= (abs_diff(pq.s6, pq.s5) > limit);
    mask |= (abs_diff(pq.s7, pq.s6) > limit);
    mask |= (abs_diff(pq.s3, pq.s4) * 2 + abs_diff(pq.s2, pq.s5) / 2  > flimit * 2 + limit);
    mask *= -1;
    return ~mask;

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs_diff(p0, q0) * 2 + abs_diff(p1, q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

__inline void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter;
    char2 filter;

    char4 pq = (char4){*op1, *op0, *oq0, *oq1};
    pq ^= 0x80;

    signed char u;

    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat((char2)vp8_filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    u = sub_sat(pq.s2, filter.s0);
    *oq0  = u ^ 0x80;

    u = add_sat(pq.s1, filter.s1);
    *op0 = u ^ 0x80;
}


Workaround for local memory bug in Backward pooling

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint
//#define MASK_Y_SHFT (sizeof(_INT_MASK_GLOBAL) / 2)
//#define MASK_X_MASK ((1 << MASK_Y_SHFT) - 1)

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLBWD_GROUP_SZ2 1

#define MLO_POOLBWD_LCL_DATA_WIDTH (MLO_POOLBWD_GROUP_SZ0 *MLO_POOLBWD_N_HORIZ_OUT_PIX + MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 2) / MLO_POOLING_STRIDE0
#define MLO_POOLBWD_LCL_DATA_HEIGHT (MLO_POOLBWD_GROUP_SZ1 *MLO_POOLBWD_N_VERT_OUT_PIX  + MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 2) / MLO_POOLING_STRIDE1

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingAveBwd(
       const __global _FLOAT * top_diff,
        __global _FLOAT * bot_diff
	   )
{
		__local _FLOAT lcl_top_diff[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLBWD_GROUP_LG2SZ1) + lcl_id0;
		int ob = get_global_id(2); // outputs * batch_sz
		int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;

		int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0)/ MLO_POOLING_STRIDE0 + 1;
		int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
		int top_off = b * MLO_POOLBWD_TOP_BATCH_STRIDE + o * MLO_POOLBWD_TOP_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				res[k][l] = 0;
			}
		}

// load tile
		for( int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
		{
			int top_y_act = top_y + tj;
			int top_y_off = top_y_act * MLO_POOLBWD_TOP_STRIDE;

			int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

			bool invisibleY = (top_y_act >= MLO_POOLBWD_TOP_HEIGHT);

			for(int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
			{

				int top_x_act = top_x + ti;

				bool invisibleX = (top_x_act >= MLO_POOLBWD_TOP_WIDTH);

				_FLOAT top_val = top_diff[top_off + top_y_off + top_x_act];

				top_val = (invisibleX || invisibleY)? 0 : top_val;

				lcl_top_diff[lcl_off_v + ti] = top_val;
#if 0
				if (lcl_id1==0&&o==0&&b==0)
				{
				  printf("K:in: %d %d %d   %f\n", top_off + top_y_off + top_x_act, top_y_act, top_x_act, top_val);
				}
#endif

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int bot_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
		int bot_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);

		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{

			int h = bot_y + k + MLO_POOLING_PAD1;
			int top_hstart = (h < MLO_POOLING_KERNEL_SZ1) ? 0 : (h - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
			int top_hend = min(h / MLO_POOLING_STRIDE1 + 1, MLO_POOLBWD_TOP_HEIGHT);

			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{

				int	w = bot_x + l + MLO_POOLING_PAD0;
				int top_wstart = (w < MLO_POOLING_KERNEL_SZ0) ? 0 : (w - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
				int top_wend = min(w / MLO_POOLING_STRIDE0 + 1, MLO_POOLBWD_TOP_WIDTH);

				for (int top_h = top_hstart; top_h < top_hend; ++top_h)
				{
				    int hstart = top_h * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
					int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLBWD_BOT_HEIGHT + MLO_POOLING_PAD1);

					for (int top_w =top_wstart; top_w < top_wend; ++top_w)
					{
        // figure out the pooling size
						int wstart = top_w * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
						int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLBWD_BOT_WIDTH + MLO_POOLING_PAD0);
						int pool_size = (hend - hstart) * (wend - wstart);
						int lcl_top_h = top_h - top_y;
						int lcl_top_w = top_w - top_x;
						_FLOAT add_val = (lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w] / (_FLOAT)pool_size);
						res[k][l] += add_val;
#if 0
				if (bot_x+l==6&&bot_y+k==0&&o==3&&b==0)
				{
				  printf("K:com: %d %d %d %d %d %d   %10.8f %10.8f %10.8f %d\n", k,l,top_h, top_w, lcl_top_h, lcl_top_w, res[k][l], add_val, lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w], pool_size);
				}
#endif
					}
				}

			}
		}

		int bot_off = b * MLO_POOLBWD_BOT_BATCH_STRIDE + o * MLO_POOLBWD_BOT_CHANNEL_STRIDE + bot_y * MLO_POOLBWD_BOT_STRIDE + bot_x;
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				if (bot_y + k < MLO_POOLBWD_BOT_HEIGHT && bot_x + l < MLO_POOLBWD_BOT_WIDTH)
				{
					bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l] = res[k][l];
#if 0
					if (lcl_id0==0&&lcl_id1==0&&o==0&&b==0)
					{
						printf("K:out: %d %d %d  %f\n", bot_off + k * MLO_POOLBWD_BOT_STRIDE +l, k, l, bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l]);
					}
#endif

				}
			}
		}

}

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingMaxBwd(
       const __global _FLOAT * top_df,
       __global _FLOAT * bot_df,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
	// BUG!!!!
	//__local _FLOAT lcl_top_df[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];
	__local _FLOAT lcl_top_df[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT + 32]; //magic number to workaround bug
	__local _INT_MASK_LOCAL lcl_mask[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

	int gid0 = get_group_id(0);
	int gid1 = get_group_id(1);
	int x = gid0 * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
	int y = gid1 * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
	int lcl_id0 = get_local_id(0);
	int lcl_id1 = get_local_id(1);
	int ob = get_global_id(2); // outputs * batch_sz
	int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
	int o = ob - b * MLO_POOLING_N_OUTPUTS;

	int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
	int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
	int top_df_off = b * MLO_POOLBWD_TOPDF_BATCH_STRIDE + o * MLO_POOLBWD_TOPDF_CHANNEL_STRIDE;

	_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];

	// load tiles
	// top df and top
	for (int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
	{
		int top_y_act = top_y + tj;
		int top_df_y_off = top_y_act * MLO_POOLBWD_TOPDF_STRIDE;

		int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

		bool visibleY = (top_y_act < MLO_POOLBWD_TOP_HEIGHT);

		for (int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
		{

			int top_x_act = top_x + ti;

			bool visible = visibleY && (top_x_act < MLO_POOLBWD_TOP_WIDTH);
			int idx = visible ? (top_df_off + top_df_y_off + top_x_act) : 0;

			_FLOAT top_df_val = top_df[idx];
			_INT_MASK_LOCAL mask_val = mask[idx];
			//top_df_val *= visible;
			mask_val = visible ? mask_val : 0xFFFFFFFF;

			int lcl_idx = lcl_off_v + ti;

#if 0
			if (ob == 0 && gid0 == 1 && gid1 == 0 && lcl_idx==0)
			{
				int inv = !visible;
				printf("ti%i tj%i topi%i lidx%i mask%i inv%i val%13.11f\n",
					ti, tj, top_x_act, lcl_idx, mask_val, inv, top_df_val);
			}
#endif

			lcl_top_df[lcl_idx] = top_df_val;
			lcl_mask[lcl_idx] = mask_val;

#if 0
			if (ob == 0 && gid0 == 1 && gid1 == 0 && lcl_idx == 0)
			{
				printf("lcl_mask[lcl_idx] = mask_val; lcl_idx=%i mask=%i lcl_mask[lcl_idx]=%i\n",
					lcl_idx, mask_val, lcl_mask[0]);
			}
#endif
		}
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0
	if (ob == 0 && gid0 == 1 && gid1 == 0 && lcl_id0 == 0 && lcl_id1 == 0)
	{
		printf("lcl_mask[0]=%i (%13.11f)\n", lcl_mask[0], lcl_mask[0]);
	}
#endif

	int bt_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
	int bt_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);
	int b_idx = bt_y * MLO_POOLBWD_BOT_WIDTH + bt_x;

	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{
		int b_y = bt_y + k;

		// top most top y that can be influenced by this bot y
		int tt_y = (b_y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1) / MLO_POOLING_STRIDE1;
		tt_y = max(0, tt_y);

		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			int	b_x = bt_x + l;
			// left most top x that can be influenced by this bot x
			int lt_x = (b_x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0) / MLO_POOLING_STRIDE0;
			lt_x = max(0, lt_x);

			// find and sum up all tops that have been influenced by particular bot
			res[k][l] = 0;

#if 0
			if (ob == 0 && b_x == 26 && b_y == 1)
			{
				printf("ltx%i tty%i bt_x%i bt_y%i top_x%i top_y%i\n",
					lt_x, tt_y, bt_x, bt_y, top_x, top_y);
			}
#endif
			for (int th = tt_y; th < tt_y + (MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 1) / MLO_POOLING_STRIDE1; ++th)
			{
				for (int tw = lt_x; tw < lt_x + (MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 1) / MLO_POOLING_STRIDE0; ++tw)
				{
					int lcl_th = th - top_y;
					int lcl_tw = tw - top_x;

					// note, that b_idx == b_y * MLO_POOLBWD_BOT_WIDTH + b_x
					// computing b_idx instead of using (b_y * MLO_POOLBWD_BOT_WIDTH + b_x) saves VGPR
					bool match = (b_idx == lcl_mask[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw]);

					_FLOAT add_val = lcl_top_df[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw] * match;
					res[k][l] += add_val;
#if 0
					if (ob == 0 && b_x == 26 && b_y == 1)
					{
						int lidx = lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw;
						int mask = lcl_mask[lidx];
						_FLOAT val = lcl_top_df[lidx];
						//printf("h=%d (%d - %i) w=%i (%i - %i) x%i y%i mx: %i my: %i topy %i\n",
						//	lcl_th, th, top_y, lcl_tw, tw, top_x, get_group_id(0), get_group_id(1), mx, my,
						//	res[k][l], top_y
						//);
						printf("th%i tw%i lh%i lw%i b_idx%i lidx%i mask%i v%13.11f\n",
							th, tw, lcl_th, lcl_tw, b_idx, lidx, mask, val);
					}
#endif
				}
			}
			b_idx++;
		}
		b_idx += MLO_POOLBWD_BOT_WIDTH - MLO_POOLBWD_N_HORIZ_OUT_PIX;
	}

	int bot_df_off = b * MLO_POOLBWD_BOTDF_BATCH_STRIDE + o * MLO_POOLBWD_BOTDF_CHANNEL_STRIDE + bt_y * MLO_POOLBWD_BOTDF_STRIDE + bt_x;
	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{
		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			if ((bt_y + k) < MLO_POOLBWD_BOT_HEIGHT && (bt_x + l) < MLO_POOLBWD_BOT_WIDTH)
			{
				bot_df[bot_df_off + k * MLO_POOLBWD_BOTDF_STRIDE + l] = res[k][l];
#if 0
				if (b == 0 && o == 0 && bt_x + l == 9 && bt_y + k == 0)
				{
					printf("K:max: %d %d   %13.11f\n",
						k, l,
						res[k][l]
					);
				}
#endif
			}
		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint
//#define MASK_Y_SHFT (sizeof(_INT_MASK_GLOBAL) / 2)
//#define MASK_X_MASK ((1 << MASK_Y_SHFT) - 1)

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLBWD_GROUP_SZ2 1

#define MLO_POOLBWD_LCL_DATA_WIDTH (MLO_POOLBWD_GROUP_SZ0 *MLO_POOLBWD_N_HORIZ_OUT_PIX + MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 2) / MLO_POOLING_STRIDE0
#define MLO_POOLBWD_LCL_DATA_HEIGHT (MLO_POOLBWD_GROUP_SZ1 *MLO_POOLBWD_N_VERT_OUT_PIX  + MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 2) / MLO_POOLING_STRIDE1

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingAveBwd(
       const __global _FLOAT * top_diff,
        __global _FLOAT * bot_diff
	   )
{
		__local _FLOAT lcl_top_diff[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLBWD_GROUP_LG2SZ1) + lcl_id0;
		int ob = get_global_id(2); // outputs * batch_sz
		int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;

		int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0)/ MLO_POOLING_STRIDE0 + 1;
		int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
		int top_off = b * MLO_POOLBWD_TOP_BATCH_STRIDE + o * MLO_POOLBWD_TOP_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				res[k][l] = 0;
			}
		}

// load tile
		for( int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
		{
			int top_y_act = top_y + tj;
			int top_y_off = top_y_act * MLO_POOLBWD_TOP_STRIDE;

			int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

			bool invisibleY = (top_y_act >= MLO_POOLBWD_TOP_HEIGHT);

			for(int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
			{

				int top_x_act = top_x + ti;

				bool invisibleX = (top_x_act >= MLO_POOLBWD_TOP_WIDTH);

				_FLOAT top_val = top_diff[top_off + top_y_off + top_x_act];

				top_val = (invisibleX || invisibleY)? 0 : top_val;

				lcl_top_diff[lcl_off_v + ti] = top_val;
#if 0
				if (lcl_id1==0&&o==0&&b==0)
				{
				  printf("K:in: %d %d %d   %f\n", top_off + top_y_off + top_x_act, top_y_act, top_x_act, top_val);
				}
#endif

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int bot_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
		int bot_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);

		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{

			int h = bot_y + k + MLO_POOLING_PAD1;
			int top_hstart = (h < MLO_POOLING_KERNEL_SZ1) ? 0 : (h - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
			int top_hend = min(h / MLO_POOLING_STRIDE1 + 1, MLO_POOLBWD_TOP_HEIGHT);

			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{

				int	w = bot_x + l + MLO_POOLING_PAD0;
				int top_wstart = (w < MLO_POOLING_KERNEL_SZ0) ? 0 : (w - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
				int top_wend = min(w / MLO_POOLING_STRIDE0 + 1, MLO_POOLBWD_TOP_WIDTH);

				for (int top_h = top_hstart; top_h < top_hend; ++top_h)
				{
				    int hstart = top_h * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
					int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLBWD_BOT_HEIGHT + MLO_POOLING_PAD1);

					for (int top_w =top_wstart; top_w < top_wend; ++top_w)
					{
        // figure out the pooling size
						int wstart = top_w * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
						int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLBWD_BOT_WIDTH + MLO_POOLING_PAD0);
						int pool_size = (hend - hstart) * (wend - wstart);
						int lcl_top_h = top_h - top_y;
						int lcl_top_w = top_w - top_x;
						_FLOAT add_val = (lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w] / (_FLOAT)pool_size);
						res[k][l] += add_val;
#if 0
				if (bot_x+l==6&&bot_y+k==0&&o==3&&b==0)
				{
				  printf("K:com: %d %d %d %d %d %d   %10.8f %10.8f %10.8f %d\n", k,l,top_h, top_w, lcl_top_h, lcl_top_w, res[k][l], add_val, lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w], pool_size);
				}
#endif
					}
				}

			}
		}

		int bot_off = b * MLO_POOLBWD_BOT_BATCH_STRIDE + o * MLO_POOLBWD_BOT_CHANNEL_STRIDE + bot_y * MLO_POOLBWD_BOT_STRIDE + bot_x;
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				if (bot_y + k < MLO_POOLBWD_BOT_HEIGHT && bot_x + l < MLO_POOLBWD_BOT_WIDTH)
				{
					bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l] = res[k][l];
#if 0
					if (lcl_id0==0&&lcl_id1==0&&o==0&&b==0)
					{
						printf("K:out: %d %d %d  %f\n", bot_off + k * MLO_POOLBWD_BOT_STRIDE +l, k, l, bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l]);
					}
#endif

				}
			}
		}

}

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingMaxBwd(
       const __global _FLOAT * top_df,
       __global _FLOAT * bot_df,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
	__local _FLOAT lcl_top_df[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];
	__local _INT_MASK_LOCAL lcl_mask[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

	int x = get_group_id(0) * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
	int y = get_group_id(1) * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
	int lcl_id0 = get_local_id(0);
	int lcl_id1 = get_local_id(1);
	int ob = get_global_id(2); // outputs * batch_sz
	int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
	int o = ob - b * MLO_POOLING_N_OUTPUTS;

	int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
	int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
	int top_df_off = b * MLO_POOLBWD_TOPDF_BATCH_STRIDE + o * MLO_POOLBWD_TOPDF_CHANNEL_STRIDE;

	_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];

	// load tiles
	// top df and top
	for (int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
	{
		int top_y_act = top_y + tj;
		int top_df_y_off = top_y_act * MLO_POOLBWD_TOPDF_STRIDE;

		int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

		bool invisibleY = (top_y_act >= MLO_POOLBWD_TOP_HEIGHT);

		for (int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
		{

			int top_x_act = top_x + ti;

			bool invisibleX = (top_x_act >= MLO_POOLBWD_TOP_WIDTH);

			_FLOAT top_df_val = top_df[top_df_off + top_df_y_off + top_x_act];
			//top_df_val = (invisibleX || invisibleY) ? 0 : top_df_val;
			_INT_MASK_LOCAL mask_val = mask[top_df_off + top_df_y_off + top_x_act];
			mask_val = (invisibleX || invisibleY) ? 0xFFFFFFFF : mask_val;

			lcl_top_df[lcl_off_v + ti] = top_df_val;
			lcl_mask[lcl_off_v + ti] = mask_val;
		}
	}

	barrier(CLK_LOCAL_MEM_FENCE);

	int bt_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
	int bt_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);

	int b_idx = bt_y * MLO_POOLBWD_BOT_WIDTH + bt_x;
	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{

		int b_y = bt_y + k;
		// top most top y that can be influenced by this bot y

		int tt_y = (b_y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1) / MLO_POOLING_STRIDE1;
		tt_y = max(0, tt_y);

		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			int	b_x = bt_x + l;
			// left most top x that can be influenced by this bot x
			int lt_x = (b_x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0) / MLO_POOLING_STRIDE0;
			lt_x = max(0, lt_x);

			// find and sum up all tops that have been influenced by particular bot
			res[k][l] = 0;

#if 0
			if (b == 0 && o == 0 && b_x == 26 && b_y == 1)
			{
				printf("gidx%i gidy%i x%i y%i bt_x%i bt_y%i top_x%i top_y%i\n",
					get_group_id(0), get_group_id(0), x, y, bt_x, bt_y, top_x, top_y);
			}
#endif
			for (int th = tt_y; th < tt_y + (MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 1) / MLO_POOLING_STRIDE1; ++th)
			{
				for (int tw = lt_x; tw < lt_x + (MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 1) / MLO_POOLING_STRIDE0; ++tw)
				{
					int lcl_th = th - top_y;
					int lcl_tw = tw - top_x;

					// note, that b_idx == b_y * MLO_POOLBWD_BOT_WIDTH + b_x
					// computing b_idx instead of using (b_y * MLO_POOLBWD_BOT_WIDTH + b_x) saves VGPR
					bool match = (b_idx == lcl_mask[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw]);

					_FLOAT add_val = lcl_top_df[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw] * match;
					res[k][l] += add_val;
#if 0
					if (b == 0 && o == 0 && b_x == 26 && b_y == 1)
					{
						int mx = lcl_mask[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw].x;
						int my = lcl_mask[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw].y;
						//printf("h=%d (%d - %i) w=%i (%i - %i) x%i y%i mx: %i my: %i topy %i\n",
						//	lcl_th, th, top_y, lcl_tw, tw, top_x, get_group_id(0), get_group_id(1), mx, my,
						//	res[k][l], top_y
						//);
						printf("gidx%i gidy%i x%i y%i bt_x%i bt_y%i top_x%i top_y%i\n",
							get_group_id(0), get_group_id(0), x, y, bt_x, bt_y, top_x, top_y);
					}
#endif
				}
			}
			b_idx++;
		}
		b_idx += MLO_POOLBWD_BOT_WIDTH - MLO_POOLBWD_N_HORIZ_OUT_PIX;
	}

	int bot_df_off = b * MLO_POOLBWD_BOTDF_BATCH_STRIDE + o * MLO_POOLBWD_BOTDF_CHANNEL_STRIDE + bt_y * MLO_POOLBWD_BOTDF_STRIDE + bt_x;
	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{
		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			if ((bt_y + k) < MLO_POOLBWD_BOT_HEIGHT && (bt_x + l) < MLO_POOLBWD_BOT_WIDTH)
			{
				bot_df[bot_df_off + k * MLO_POOLBWD_BOTDF_STRIDE + l] = res[k][l];
#if 0
				if (b == 0 && o == 0 && bt_x + l == 9 && bt_y + k == 0)
				{
					printf("K:max: %d %d   %13.11f\n",
						k, l,
						res[k][l]
					);
				}
#endif
			}
		}
	}

}