Minor optimizations to vector conversions.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline uchar8 vp8_mbfilter(signed char mask,signed char hev,uchar8);

__inline void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
char vp8_char_clamp(int in){
    return max(min(in, 127), -128);
}
#endif

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

//OpenCL built-in functions (
size_t get_global_id(unsigned int);
size_t get_global_size(unsigned int);

__inline uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = as_char4(base) ^ (char4)0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u.s0 = add_sat(pq.s0, vp8_filter);
    u.s1 = add_sat(pq.s1, Filter.s1);
    u.s2 = sub_sat(pq.s2, Filter.s0);
    u.s3 = sub_sat(pq.s3, vp8_filter);

    return as_uchar4(u ^ (char4)0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
__inline void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    size_t thread = get_global_id(0);

    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        s_off += thread;
        //s_off += thread, so maybe it's possible to use a strided copy with local memory to load into the vector

        uchar8 data;
        data.s0 = s_base[s_off-4*p];
        data.s1 = s_base[s_off-3*p];
        data.s2 = s_base[s_off-2*p];
        data.s3 = s_base[s_off-p];
        data.s4 = s_base[s_off];
        data.s5 = s_base[s_off+p];
        data.s6 = s_base[s_off+2*p];
        data.s7 = s_base[s_off+3*p];

        char mask = vp8_filter_mask(lfi->lim[thread], lfi->flim[thread], data);

        int hev = vp8_hevmask(lfi->thr[thread], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        s_base[s_off - 2*p] = data.s2;
        s_base[s_off - p  ] = data.s3;
        s_base[s_off      ] = data.s4;
        s_base[s_off + p  ] = data.s5;
    }
}

__inline void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        s_off += p * get_global_id(0);

        uchar8 data = vload8(0, &s_base[s_off-4]);

        char mask = vp8_filter_mask(lfi->lim[get_global_id(0)], lfi->flim[get_global_id(0)], data);

        int hev = vp8_hevmask(lfi->thr[get_global_id(0)], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        vstore4(data.s2345, 0, &s_base[s_off-2]);
    }
}

__inline void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int s_off = offsets[8*num_blocks + block*num_planes+plane] + get_global_id(0);

    uchar8 data;
    data.s0 = s_base[s_off-4*p];
    data.s1 = s_base[s_off-3*p];
    data.s2 = s_base[s_off-2*p];
    data.s3 = s_base[s_off-p];
    data.s4 = s_base[s_off];
    data.s5 = s_base[s_off+p];
    data.s6 = s_base[s_off+2*p];
    data.s7 = s_base[s_off+3*p];

    char mask = vp8_filter_mask(lfi->lim[get_global_id(0)], lfi->mbflim[get_global_id(0)], data);

    char hev = vp8_hevmask(lfi->thr[get_global_id(0)], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    s_base[s_off - 3*p] = data.s1;
    s_base[s_off - 2*p] = data.s2;
    s_base[s_off - 1*p] = data.s3;
    s_base[s_off      ] = data.s4;
    s_base[s_off + p  ] = data.s5;
    s_base[s_off + 2*p] = data.s6;

}

__inline void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int block_offset = block*num_planes+plane;
    int s_off = offsets[block_offset];

    s_off += p * get_global_id(0);

    uchar8 data = vload8(0, &s_base[s_off-4]);

    char mask = vp8_filter_mask(lfi->lim[get_global_id(0)], lfi->mbflim[get_global_id(0)], data);

    int hev = vp8_hevmask(lfi->thr[get_global_id(0)], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    vstore8(data, 0, &s_base[s_off-4]);
}

void vp8_loop_filter_all_edges_1_iter(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){

}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);

    for (int i = 0; i < num_levels; i++){
        int block_offset = block_offsets[priority_level];

        global int *offsets = &offsets_in[16*block_offset];
        global int *filters = &filters_in[4*block_offset];
        size_t num_blocks = priority_num_blocks[priority_level];
        int filter_level = filters[block];
        global loop_filter_info *lf_info = &lfi[filter_level];
        int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);

        if (block < num_blocks){
            //Prefetch vertical edge source pixels into global cache (horizontal isn't worth it)
            for(int pln = 0; pln < 3; pln++){
                int p = pitches[pln];
                int offset = block*3+pln;
                int s_off = offsets[offset];
                for (int thread = 0; thread < 16; thread++){
                    prefetch(&s_base[s_off+p*thread-4], 8);
                }
            }

            if (thread_level_filter & (filters[num_blocks*COLS_LOCATION + block] > 0)){
                vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        COLS_LOCATION, filter_level, num_blocks, 3,  plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1,  plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (block < num_blocks){

            if (thread_level_filter & (filters[num_blocks*ROWS_LOCATION + block] > 0)){
                vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
                    filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }

}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*ROWS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
            filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
        }
    }
}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*COLS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
            COLS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, plane, block);
        }
    }

}

void vp8_loop_filter_simple_horizontal_edge_worker
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    size_t num_planes = get_global_size(1);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

void vp8_loop_filter_simple_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t block = get_global_id(2);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[0];
            int block_offset = cur_iter*num_blocks + block;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[0]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
            block_offsets, priority_num_blocks
    );

    //3 Y plane iterations
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_horizontal_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 1, ROWS_LOCATION, 4, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_all_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
)
{

    for (int i = 0; i < num_levels; i++){
        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
                    block_offsets, priority_num_blocks
            );

            //3 Y plane iterations
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 1, ROWS_LOCATION, 4, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }
}

//Inline and non-kernel functions follow.

__inline uchar8 vp8_mbfilter(
    signed char mask,
    signed char hev,
    uchar8 base
)
{
    char4 u;

    char8 pq = as_char8(base);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};

    /* add outer taps if we have high edge variance */
    char vp8_filter = sub_sat(pq.s2, pq.s5);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s4 - pq.s3), -128, 127);
    vp8_filter &= mask;

    char2 filter = (char2)vp8_filter;
    filter &= hev;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat(filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    pq.s4 = sub_sat(pq.s4, filter.s0);
    pq.s3 = add_sat(pq.s3, filter.s1);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th, 2/7th, and 1/7th difference across boundary */
    u.s0 = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    u.s1 = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    u.s2 = clamp((63 + filter.s1 * 9) >> 7, -128, 127);

#if 0
    char4 s;
    s.s012 = sub_sat(pq.s456, u.s012);
    pq.s456 = s.s012 ^ (char3)0x80;
    s.s321 = add_sat(pq.s321, u.s012);
    pq.s321 = s.s321 ^ (char3)0x80;
#else
    pq.s456 = sub_sat(pq.s456, u.s012);
    pq.s321 = add_sat(pq.s321, u.s012);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};
#endif

    return as_uchar8(pq);
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    return any(abs_diff(pq.s03, pq.s12) > thresh) * -1;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    //Only apply the filter if the difference is LESS than 'limit'
#if 1
    char mask = (abs_diff(pq.s0, pq.s1) > limit);
    mask |= (abs_diff(pq.s1, pq.s2) > limit);
    mask |= (abs_diff(pq.s2, pq.s3) > limit);
    mask |= (abs_diff(pq.s5, pq.s4) > limit);
    mask |= (abs_diff(pq.s6, pq.s5) > limit);
    mask |= (abs_diff(pq.s7, pq.s6) > limit);
    mask |= (abs_diff(pq.s3, pq.s4) * 2 + abs_diff(pq.s2, pq.s5) / 2  > flimit * 2 + limit);
    return mask - 1;
#else
    short8 mask8 = convert_short8(abs_diff(pq.s01256732, pq.s12345645));
    mask8.s6 = mask8.s6 * 2 + mask8.s7 / 2;

    short8 limits = (short8)limit;
    limits.s6 += flimit * 2;
    mask8 = mask8 > limits;
    mask8.s7 = 0;

    return any(mask8) - 1;
#endif

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs_diff(p0, q0) * 2 + abs_diff(p1, q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

__inline void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter;
    char2 filter;

    char4 pq = (char4){*op1, *op0, *oq0, *oq1};
    pq ^= (char4)0x80;

    signed char u;

    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat((char2)vp8_filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    u = sub_sat(pq.s2, filter.s0);
    *oq0  = u ^ 0x80;

    u = add_sat(pq.s1, filter.s1);
    *op0 = u ^ 0x80;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline uchar8 vp8_mbfilter(signed char mask,signed char hev,uchar8);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
char vp8_char_clamp(int in){
    return max(min(in, 127), -128);
}
#endif

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

//OpenCL built-in functions (
size_t get_global_id(unsigned int);
size_t get_global_size(unsigned int);

uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = convert_char4(base) ^ (char4)0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u.s0 = add_sat(pq.s0, vp8_filter);
    u.s1 = add_sat(pq.s1, Filter.s1);
    u.s2 = sub_sat(pq.s2, Filter.s0);
    u.s3 = sub_sat(pq.s3, vp8_filter);

    return convert_uchar4(u ^ (char4)0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        s_off += get_global_id(0);

        uchar8 data;
        data.s0 = s_base[s_off-4*p];
        data.s1 = s_base[s_off-3*p];
        data.s2 = s_base[s_off-2*p];
        data.s3 = s_base[s_off-p];
        data.s4 = s_base[s_off];
        data.s5 = s_base[s_off+p];
        data.s6 = s_base[s_off+2*p];
        data.s7 = s_base[s_off+3*p];

        char mask = vp8_filter_mask(lfi->lim[get_global_id(0)], lfi->flim[get_global_id(0)], data);

        int hev = vp8_hevmask(lfi->thr[get_global_id(0)], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        s_base[s_off - 2*p] = data.s2;
        s_base[s_off - p  ] = data.s3;
        s_base[s_off      ] = data.s4;
        s_base[s_off + p  ] = data.s5;
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    int cur_iter,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){
    if (filters[num_blocks*filter_type + block] > 0){
        int p = pitches[plane];
        int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
        int s_off = offsets[block_offset];

        s_off += p * get_global_id(0);

        uchar8 data = vload8(0, &s_base[s_off-4]);

        char mask = vp8_filter_mask(lfi->lim[get_global_id(0)], lfi->flim[get_global_id(0)], data);

        int hev = vp8_hevmask(lfi->thr[get_global_id(0)], data.s2345);

        data.s2345 = vp8_filter(mask, hev, data.s2345);

        vstore4(data.s2345, 0, &s_base[s_off-2]);
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int s_off = offsets[8*num_blocks + block*num_planes+plane] + get_global_id(0);

    uchar8 data;
    data.s0 = s_base[s_off-4*p];
    data.s1 = s_base[s_off-3*p];
    data.s2 = s_base[s_off-2*p];
    data.s3 = s_base[s_off-p];
    data.s4 = s_base[s_off];
    data.s5 = s_base[s_off+p];
    data.s6 = s_base[s_off+2*p];
    data.s7 = s_base[s_off+3*p];

    char mask = vp8_filter_mask(lfi->lim[get_global_id(0)], lfi->mbflim[get_global_id(0)], data);

    char hev = vp8_hevmask(lfi->thr[get_global_id(0)], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    s_base[s_off - 3*p] = data.s1;
    s_base[s_off - 2*p] = data.s2;
    s_base[s_off - 1*p] = data.s3;
    s_base[s_off      ] = data.s4;
    s_base[s_off + p  ] = data.s5;
    s_base[s_off + 2*p] = data.s6;

}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int filter_level,
    size_t num_blocks,
    size_t num_planes,
    size_t plane,
    size_t block
){

    int p = pitches[plane];
    int block_offset = block*num_planes+plane;
    int s_off = offsets[block_offset];

    s_off += p * get_global_id(0);

    uchar8 data = vload8(0, &s_base[s_off-4]);

    char mask = vp8_filter_mask(lfi->lim[get_global_id(0)], lfi->mbflim[get_global_id(0)], data);

    int hev = vp8_hevmask(lfi->thr[get_global_id(0)], data.s2345);

    data = vp8_mbfilter(mask, hev, data);

    vstore8(data, 0, &s_base[s_off-4]);
}

void vp8_loop_filter_all_edges_1_iter(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){

}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets_in,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);

    for (int i = 0; i < num_levels; i++){
        int block_offset = block_offsets[priority_level];

        global int *offsets = &offsets_in[16*block_offset];
        global int *filters = &filters_in[4*block_offset];
        size_t num_blocks = priority_num_blocks[priority_level];
        int filter_level = filters[block];
        global loop_filter_info *lf_info = &lfi[filter_level];
        int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);

        if (block < num_blocks){
            //Prefetch vertical edge source pixels into global cache (horizontal isn't worth it)
            for(int pln = 0; pln < 3; pln++){
                int p = pitches[pln];
                int offset = block*3+pln;
                int s_off = offsets[offset];
                for (int thread = 0; thread < 16; thread++){
                    prefetch(&s_base[s_off+p*thread-4], 8);
                }
            }

            if (thread_level_filter & (filters[num_blocks*COLS_LOCATION + block] > 0)){
                vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        COLS_LOCATION, filter_level, num_blocks, 3,  plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
                    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1,  plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (block < num_blocks){

            if (thread_level_filter & (filters[num_blocks*ROWS_LOCATION + block] > 0)){
                vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
                    filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
            }

            //YUV planes, then 2 more passes of Y plane
            if (thread_level_filter){
                vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                        DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
                if (plane == 0){
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
                    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                            DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
                }
            }
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }

}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*ROWS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info,
            filters, ROWS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 0, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 3, num_blocks, 1, plane, block);
            vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 4, num_blocks, 1, plane, block);
        }
    }
}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    size_t thread = get_global_id(0);
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_blocks = get_global_size(2);

    int block_offset = block_offsets[priority_level];

    filters = &filters[4*block_offset];
    offsets = &offsets[16*block_offset];
    int filter_level = filters[block];
    global loop_filter_info *lf_info = &lfi[filter_level];

    int thread_level_filter = (thread<threads[plane]) & (filter_level!=0);
    int do_filter = filters[num_blocks*COLS_LOCATION + block] > 0;
    do_filter &= thread_level_filter;
    if (do_filter){
        vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
            COLS_LOCATION, filter_level, num_blocks, 3, plane, block);
    }

    //YUV planes, then 2 more passes of Y plane
    if (thread_level_filter){
        vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                DC_DIFFS_LOCATION, filter_level, 1, num_blocks, 3, plane, block);
        if (plane == 0){
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 6, num_blocks, 1, plane, block);
            vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lf_info, filters,
                    DC_DIFFS_LOCATION, filter_level, 7, num_blocks, 1, plane, block);
        }
    }

}

void vp8_loop_filter_simple_horizontal_edge_worker
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    size_t num_planes = get_global_size(1);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

void vp8_loop_filter_simple_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t block = get_global_id(2);
    size_t num_blocks = priority_num_blocks[priority_level];

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[0];
            int block_offset = cur_iter*num_blocks + block;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[0]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
            block_offsets, priority_num_blocks
    );

    //3 Y plane iterations
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
            lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_horizontal_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
){
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 1, ROWS_LOCATION, 4, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
            block_offsets, priority_num_blocks
    );
    vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
            block_offsets, priority_num_blocks
    );
}

kernel void vp8_loop_filter_simple_all_edges_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int priority_level,
    int num_levels,
    global int *block_offsets,
    global int *priority_num_blocks
)
{

    for (int i = 0; i < num_levels; i++){
        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 1, COLS_LOCATION, 0, priority_level,
                    block_offsets, priority_num_blocks
            );

            //3 Y plane iterations
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 1, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 2, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_vertical_edge_worker(s_base, offsets, pitches,
                    lfi, filters_in, 0, DC_DIFFS_LOCATION, 3, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);

        if (get_global_id(2) < priority_num_blocks[priority_level]){
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 1, ROWS_LOCATION, 4, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 5, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 6, priority_level,
                    block_offsets, priority_num_blocks
            );
            vp8_loop_filter_simple_horizontal_edge_worker(s_base, offsets, pitches, lfi,
                    filters_in, 0, DC_DIFFS_LOCATION, 7, priority_level,
                    block_offsets, priority_num_blocks
            );
        }

        barrier(CLK_GLOBAL_MEM_FENCE);
        priority_level++;
    }
}

//Inline and non-kernel functions follow.

__inline uchar8 vp8_mbfilter(
    signed char mask,
    signed char hev,
    uchar8 base
)
{
    char4 u;

    char8 pq = convert_char8(base);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};

    /* add outer taps if we have high edge variance */
    char vp8_filter = sub_sat(pq.s2, pq.s5);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s4 - pq.s3), -128, 127);
    vp8_filter &= mask;

    char2 filter = (char2)vp8_filter;
    filter &= hev;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat(filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    pq.s4 = sub_sat(pq.s4, filter.s0);
    pq.s3 = add_sat(pq.s3, filter.s1);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th, 2/7th, and 1/7th difference across boundary */
    u.s0 = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    u.s1 = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    u.s2 = clamp((63 + filter.s1 * 9) >> 7, -128, 127);

#if 0
    char4 s;
    s.s012 = sub_sat(pq.s456, u.s012);
    pq.s456 = s.s012 ^ (char3)0x80;
    s.s321 = add_sat(pq.s321, u.s012);
    pq.s321 = s.s321 ^ (char3)0x80;
#else
    pq.s456 = sub_sat(pq.s456, u.s012);
    pq.s321 = add_sat(pq.s321, u.s012);
    pq ^= (char8){0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0};
#endif

    return convert_uchar8(pq);
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
#if 1
    char hev = (abs_diff(pq.s0, pq.s1) > thresh);
    hev  |= (abs_diff(pq.s3, pq.s2) > thresh);
    return hev*-1;
#else
    return any(abs_diff(pq.s03, pq.s12) > thresh) * -1;
#endif
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    //Only apply the filter if the difference is LESS than 'limit'
#if 1
    char mask = (abs_diff(pq.s0, pq.s1) > limit);
    mask |= (abs_diff(pq.s1, pq.s2) > limit);
    mask |= (abs_diff(pq.s2, pq.s3) > limit);
    mask |= (abs_diff(pq.s5, pq.s4) > limit);
    mask |= (abs_diff(pq.s6, pq.s5) > limit);
    mask |= (abs_diff(pq.s7, pq.s6) > limit);
    mask |= (abs_diff(pq.s3, pq.s4) * 2 + abs_diff(pq.s2, pq.s5) / 2  > flimit * 2 + limit);
    return mask - 1;
#else
    short8 mask8 = convert_short8(abs_diff(pq.s01256732, pq.s12345645));
    mask8.s6 = mask8.s6 * 2 + mask8.s7 / 2;

    short8 limits = (short8)limit;
    limits.s6 += flimit * 2;
    mask8 = mask8 > limits;
    mask8.s7 = 0;

    return any(mask8) - 1;
#endif

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs_diff(p0, q0) * 2 + abs_diff(p1, q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

__inline void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter;
    char2 filter;

    char4 pq = (char4){*op1, *op0, *oq0, *oq1};
    pq ^= (char4)0x80;

    signed char u;

    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter = add_sat((char2)vp8_filter, (char2){4,3});
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    u = sub_sat(pq.s2, filter.s0);
    *oq0  = u ^ 0x80;

    u = add_sat(pq.s1, filter.s1);
    *op0 = u ^ 0x80;
}


perf tweaks

perf tweaks

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define SIGMA_NEW_BITPOS   0x1
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define SIGMA_OLD_TO_NBH_SHIFT 0x1
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

// bit flags
#define SIGMA_NEW_F			 0x1		  //position 0
#define SIGMA_OLD_F			 0x10		  //position  1
#define NBH_F				 0x20		  //position  2
#define RLC_F				 0x80		  //position  4
#define RLC_D_POSITION_F	 0x380        //positions 6-9
#define PIXEL_F				 0x7FFF0000   //positions 10-24
#define SIGN_F				 0x2000000    //position  25

#define NOT_SIGMA_NEW_F      0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD_AND_NEW_F  0x11

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)
#define NBH(pix) ((pix) & NBH_F)
#define SIGMA_OLD(pix) ((pix) & SIGMA_OLD_F)
#define SIGMA_NEW(pix) ((pix) & SIGMA_NEW_F)
#define SIGMA_OLD_AND_NEW(pix) ((pix) & SIGMA_OLD_AND_NEW_F)
#define SIGN(pix) ((pix) & SIGN_F)

#define SET_SIGMA_NEW(pix) ( (pix) |= SIGMA_NEW_F )
#define CLEAR_SIGMA_NEW(pix) ( (pix) &= NOT_SIGMA_NEW_F )

#define AMD

/**

**/

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];
	LOCAL int cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between -1 and 31 - negative one value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {

		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);

		for (int i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			int absPixel = abs(pixel);
			maxVal = max(maxVal, absPixel);
			pixel = (absPixel << PIXEL_START_BITPOS) | SIGN((pixel << INPUT_TO_SIGN_SHIFT));
			statePtr[0] = pixel;

			posIn.y++;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			}
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(int i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// i) set sigma_new for strip column (sigma_old is zero)
	LOCAL int* statePtr = state + startIndex;
	for (int i = 0; i < 4; ++i) {
		int current = statePtr[0];
		current |= BIT(current);
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// ii) set nbh for strip column, and do CUP
	statePtr = state + startIndex;
	int current			= statePtr[0];
	int top				= statePtr[TOP];
	int leftTop			= statePtr[LEFT_TOP];
	int left			= statePtr[LEFT];
	int leftBottom		= statePtr[LEFT_BOTTOM];

	int rlcCount = 0;

	// first pixel in strip column
	int nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom)) << NBH_BITPOS;
	bool doRLC = !nbh && !BIT(current);
	if (doRLC) {
		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	// next two pixels in strip column
	for (int i = 0; i < 2; ++i) {

		statePtr += STATE_BUFFER_STRIDE;
		top = current;
		current		= statePtr[0];
		leftTop		= left;
		left		= leftBottom;
		leftBottom	= statePtr[LEFT_BOTTOM];

		nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom)) << NBH_BITPOS;

		// toggle doRLC flag
		doRLC = doRLC && !nbh && !BIT(current);
		if (doRLC) {

			rlcCount++;
		} else {
		   //ZC

		   if (BIT(current)) {
		       //SC
		   }
		}
	}

	// last pixel in strip column -
	// ignore leftBottom pixel, because it is in the next
	// stripe and it hasn't been processed yet
	statePtr += STATE_BUFFER_STRIDE;
	top			= current;
	current		= statePtr[0];
	leftTop		= left;
	left		= leftBottom;

	nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left)) << NBH_BITPOS;

	doRLC = doRLC && !nbh && !BIT(current);
	if (doRLC) {

		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit planes
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (int i = 0; i < 4; ++i) {
		    int current = *statePtr;
			current |=  SIGMA_NEW(current) << SIGMA_OLD_BITPOS;
			CLEAR_SIGMA_NEW(current);

			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		int top			= statePtr[TOP];
		int leftTop		= statePtr[LEFT_TOP];
		int rightTop	= statePtr[RIGHT_TOP];
		int left		= statePtr[LEFT];
		int current		= statePtr[0];
		int right		= statePtr[RIGHT];
		int leftBottom  = statePtr[LEFT_BOTTOM];
		int bottom		= statePtr[BOTTOM];
		int rightBottom = statePtr[RIGHT_BOTTOM];

		current |=  (   SIGMA_OLD(leftTop) |
						SIGMA_OLD( top) |
						SIGMA_OLD( rightTop) |
						SIGMA_OLD( left) |
						SIGMA_OLD( right) |
						SIGMA_OLD( leftBottom) |
						SIGMA_OLD( bottom) |
						SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) &&
				NBH(current) &&
					!SIGMA_OLD(current) ) {
			SET_SIGMA_NEW(current);
			blockVote = 1;

		}
		*statePtr = current;

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			 = current;
			leftTop		 = left;
			rightTop	 = right;
			left		 = leftBottom;
			current		 = statePtr[0];
			right		 = rightBottom;
			leftBottom   = statePtr[LEFT_BOTTOM];
			bottom		 = statePtr[BOTTOM];
			rightBottom  = statePtr[RIGHT_BOTTOM];

			current |=	   ( SIGMA_OLD(leftTop) |
							 SIGMA_OLD( top) |
							 SIGMA_OLD( rightTop) |
							 SIGMA_OLD( left) |
							 SIGMA_OLD( right) |
							 SIGMA_OLD( leftBottom) |
							 SIGMA_OLD( bottom) |
							 SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) &&
					 NBH(current) &&
						 !SIGMA_OLD(current) ) {
			    SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
#ifdef AMD
		while (blockVote)
#endif
		{
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			//first pixel in strip column
			int top			= statePtr[TOP];
			int leftTop		= statePtr[LEFT_TOP];
			int left		= statePtr[LEFT];
			int current		= statePtr[0];
			int leftBottom  = statePtr[LEFT_BOTTOM];

			current  |= ( SIGMA_NEW(leftTop) |
				SIGMA_NEW( top) |
				SIGMA_NEW( left) |
				SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

			if (  BIT(current) &&
				        !SIGMA_OLD(current) &&
						    NBH(current)  &&
							    !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;

			}
			statePtr[0] = current;

			// next two pixels in strip column
			for (int i = 0; i < 2; ++i) {
				statePtr += STATE_BUFFER_STRIDE;

				top			= current;
				current		= statePtr[0];
				leftTop		= left;
				left		= leftBottom;
				leftBottom  = statePtr[LEFT_BOTTOM];

				current  |= ( SIGMA_NEW(leftTop) |
					SIGMA_NEW( top) |
					SIGMA_NEW( left) |
					SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

				if (  BIT(current) &&
				          !SIGMA_OLD(current) &&
						      NBH(current)  &&
							     !SIGMA_NEW(current)) {
					SET_SIGMA_NEW(current);
					blockVote = 1;

				}
				statePtr[0] = current;
			}

			// last pixel in strip column -
			// ignore leftBottom pixel, because it is in the next
			// stripe and it hasn't been processed yet
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			current		= statePtr[0];
			leftTop		= left;
			left		= leftBottom;

			current |= ( SIGMA_NEW(leftTop) |
						 SIGMA_NEW( top) |
						 SIGMA_NEW( left) ) << NBH_BITPOS;
			if (  BIT(current) &&
			        !SIGMA_OLD(current) &&
					    NBH(current)  &&
						    !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			statePtr[0] = current;

			localMemoryFence();

		}

		/////////////////////////
		// 5. bit plane processing
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop			= statePtr[LEFT_TOP];
		rightTop		= statePtr[RIGHT_TOP];
		left			= statePtr[LEFT];
		current			= statePtr[0];
		right			= statePtr[RIGHT];
		leftBottom		= statePtr[LEFT_BOTTOM];
		bottom			= statePtr[BOTTOM];
		rightBottom		= statePtr[RIGHT_BOTTOM];

		if (SIGMA_OLD(current)) {
			// MRC
		} else {
			int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
						SIGMA_OLD_AND_NEW( top) |
						SIGMA_OLD_AND_NEW( rightTop) |
						SIGMA_OLD_AND_NEW( left) |
						SIGMA_OLD_AND_NEW( right) |
						SIGMA_OLD_AND_NEW( leftBottom) |
						SIGMA_OLD_AND_NEW( bottom) |
						SIGMA_OLD_AND_NEW( rightBottom)  ) ;

			if (!nbh) {
				if ( !SIGMA_NEW(current) ) {
					//RLC
				}
			}
			else  {
				// ZC
				if (BIT(current)) {
				    //SC
				}
			}
		}

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			leftTop		= left;
			rightTop	= right;
			left		= leftBottom;
			current		= statePtr[0];
			right		= rightBottom;
			leftBottom  = statePtr[LEFT_BOTTOM];
			bottom		= statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			if (SIGMA_OLD(current)) {
				// MRC
			} else {
				int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
							SIGMA_OLD_AND_NEW( top) |
							SIGMA_OLD_AND_NEW( rightTop) |
							SIGMA_OLD_AND_NEW( left) |
							SIGMA_OLD_AND_NEW( right) |
							SIGMA_OLD_AND_NEW( leftBottom) |
							SIGMA_OLD_AND_NEW( bottom) |
							SIGMA_OLD_AND_NEW( rightBottom)  ) ;

				if (!nbh) {
					if ( !SIGMA_NEW(current) ) {
						//RLC
					}
				}
				else  {
					// ZC
					if (BIT(current)) {
						//SC
					}
				}
			}
		}

		localMemoryFence();
		bp--;
	}
}

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define SIGMA_NEW_BITPOS   0x1
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define SIGMA_OLD_TO_NBH_SHIFT 0x1
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

// bit flags
#define SIGMA_NEW_F			 0x1		  //position 0
#define SIGMA_OLD_F			 0x10		  //position  1
#define NBH_F				 0x20		  //position  2
#define RLC_F				 0x80		  //position  4
#define RLC_D_POSITION_F	 0x380        //positions 6-9
#define PIXEL_F				 0x7FFF0000   //positions 10-24
#define SIGN_F				 0x2000000    //position  25

#define NOT_SIGMA_NEW_F      0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD_AND_NEW_F  0x11

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)
#define NBH(pix) ((pix) & NBH_F)
#define SIGMA_OLD(pix) ((pix) & SIGMA_OLD_F)
#define SIGMA_NEW(pix) ((pix) & SIGMA_NEW_F)
#define SIGMA_OLD_AND_NEW(pix) ((pix) & SIGMA_OLD_AND_NEW_F)
#define SIGN(pix) ((pix) & SIGN_F)

#define SET_SIGMA_NEW(pix) ( (pix) |= SIGMA_NEW_F )
#define CLEAR_SIGMA_NEW(pix) ( (pix) &= NOT_SIGMA_NEW_F )

#define AMD

/**

**/

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];
	LOCAL int cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between -1 and 31 - negative one value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {

		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);

		for (int i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			int absPixel = abs(pixel);
			maxVal = max(maxVal, absPixel);
			pixel = (absPixel << PIXEL_START_BITPOS) | SIGN((pixel << INPUT_TO_SIGN_SHIFT));
			statePtr[0] = pixel;

			posIn.y++;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			}
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(int i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// i) set sigma_new for strip column (sigma_old is zero)
	LOCAL int* statePtr = state + startIndex;
	for (int i = 0; i < 4; ++i) {
		int current = statePtr[0];
		current |= BIT(current);
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// ii) set nbh for strip column, and do CUP
	statePtr = state + startIndex;
	int current			= statePtr[0];
	int top				= statePtr[TOP];
	int leftTop			= statePtr[LEFT_TOP];
	int left			= statePtr[LEFT];
	int leftBottom		= statePtr[LEFT_BOTTOM];

	int rlcCount = 0;

	// first pixel in strip column
	current |=  (BIT(top) | BIT(leftTop) | BIT(left) | BIT(leftBottom)) << NBH_BITPOS;
	statePtr[0] = current;
	bool doRLC = !NBH(current) && !BIT(current);
	if (doRLC) {
		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	// next two pixels in strip column
	for (int i = 0; i < 2; ++i) {

		statePtr += STATE_BUFFER_STRIDE;
		top = current;
		current		= statePtr[0];
		leftTop		= left;
		left		= leftBottom;
		leftBottom	= statePtr[LEFT_BOTTOM];

		current |=  (BIT(top) | BIT(leftTop) | BIT(left) | BIT(leftBottom)) << NBH_BITPOS;
		statePtr[0] = current;

		// toggle doRLC flag
		doRLC = doRLC && !NBH(current) && !BIT(current);
		if (doRLC) {

			rlcCount++;
		} else {
		   //ZC

		   if (BIT(current)) {
		       //SC
		   }
		}
	}

	// last pixel in strip column -
	// ignore leftBottom pixel, because it is in the next
	// stripe and it hasn't been processed yet
	statePtr += STATE_BUFFER_STRIDE;
	top			= current;
	current		= statePtr[0];
	leftTop		= left;
	left		= leftBottom;
	leftBottom	= statePtr[LEFT_BOTTOM];

	current |=  (BIT(top) | BIT(leftTop) | BIT(left)) << NBH_BITPOS;
	statePtr[0] = current;

	doRLC = doRLC && !NBH(current) && !BIT(current);
	if (doRLC) {

		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit plans
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (int i = 0; i < 4; ++i) {
		    int current = *statePtr;
			current |=  SIGMA_NEW(current) << SIGMA_OLD_BITPOS;
			CLEAR_SIGMA_NEW(current);

			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		int top			= statePtr[TOP];
		int leftTop		= statePtr[LEFT_TOP];
		int rightTop	= statePtr[RIGHT_TOP];
		int left		= statePtr[LEFT];
		int current		= statePtr[0];
		int right		= statePtr[RIGHT];
		int leftBottom  = statePtr[LEFT_BOTTOM];
		int bottom		= statePtr[BOTTOM];
		int rightBottom = statePtr[RIGHT_BOTTOM];

		current |=  (   SIGMA_OLD(leftTop) |
						SIGMA_OLD( top) |
						SIGMA_OLD( rightTop) |
						SIGMA_OLD( left) |
						SIGMA_OLD( right) |
						SIGMA_OLD( leftBottom) |
						SIGMA_OLD( bottom) |
						SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) &&
				NBH(current) &&
					!SIGMA_OLD(current) ) {
			SET_SIGMA_NEW(current);
			blockVote = 1;

		}
		*statePtr = current;

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			 = current;
			leftTop		 = left;
			rightTop	 = right;
			left		 = leftBottom;
			current		 = statePtr[0];
			right		 = rightBottom;
			leftBottom   = statePtr[LEFT_BOTTOM];
			bottom		 = statePtr[BOTTOM];
			rightBottom  = statePtr[RIGHT_BOTTOM];

			current |=	   ( SIGMA_OLD(leftTop) |
							 SIGMA_OLD( top) |
							 SIGMA_OLD( rightTop) |
							 SIGMA_OLD( left) |
							 SIGMA_OLD( right) |
							 SIGMA_OLD( leftBottom) |
							 SIGMA_OLD( bottom) |
							 SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) &&
					 NBH(current) &&
						 !SIGMA_OLD(current) ) {
			    SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
#ifdef AMD
		while (blockVote)
#endif
		{
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			int top			= statePtr[TOP];
			int leftTop		= statePtr[LEFT_TOP];
			int left		= statePtr[LEFT];
			int current		= statePtr[0];
			int leftBottom  = statePtr[LEFT_BOTTOM];

			// first three pixels in strip column
			for (int i = 0; i < 3; ++i) {

				current  |= ( SIGMA_NEW(leftTop) |
							  SIGMA_NEW( top) |
							  SIGMA_NEW( left) |
							  SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

				if (  BIT(current) &&
				          !SIGMA_OLD(current) &&
						      NBH(current)  &&
							     !SIGMA_NEW(current)) {
					SET_SIGMA_NEW(current);
					blockVote = 1;

				}
				statePtr[0] = current;
				statePtr += STATE_BUFFER_STRIDE;

				top			= current;
				current		= statePtr[0];
				leftTop		= left;
				left		= leftBottom;
				leftBottom  = statePtr[LEFT_BOTTOM];
			}

			// last pixel in strip column -
			// ignore leftBottom pixel, because it is in the next
			// stripe and it hasn't been processed yet
			current |= ( SIGMA_NEW(leftTop) |
						 SIGMA_NEW( top) |
						 SIGMA_NEW( left) ) << NBH_BITPOS;
			if (  BIT(current) &&
			        !SIGMA_OLD(current) &&
					    NBH(current)  &&
						    !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			statePtr[0] = current;

			localMemoryFence();

		}

		/////////////////////////
		// 5. bit plane processing
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop			= statePtr[LEFT_TOP];
		rightTop		= statePtr[RIGHT_TOP];
		left			= statePtr[LEFT];
		current			= statePtr[0];
		right			= statePtr[RIGHT];
		leftBottom		= statePtr[LEFT_BOTTOM];
		bottom			= statePtr[BOTTOM];
		rightBottom		= statePtr[RIGHT_BOTTOM];

		if (SIGMA_OLD(current)) {
			// MRC
		} else {
			int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
						SIGMA_OLD_AND_NEW( top) |
						SIGMA_OLD_AND_NEW( rightTop) |
						SIGMA_OLD_AND_NEW( left) |
						SIGMA_OLD_AND_NEW( right) |
						SIGMA_OLD_AND_NEW( leftBottom) |
						SIGMA_OLD_AND_NEW( bottom) |
						SIGMA_OLD_AND_NEW( rightBottom)  ) ;

			if (!nbh) {
				if ( !SIGMA_NEW(current) ) {
					//RLC
				}
			}
			else  {
				// ZC
				if (BIT(current)) {
				    //SC
				}
			}
		}

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			leftTop		= left;
			rightTop	= right;
			left		= leftBottom;
			current		= statePtr[0];
			right		= rightBottom;
			leftBottom  = statePtr[LEFT_BOTTOM];
			bottom		= statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			if (SIGMA_OLD(current)) {
				// MRC
			} else {
				int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
							SIGMA_OLD_AND_NEW( top) |
							SIGMA_OLD_AND_NEW( rightTop) |
							SIGMA_OLD_AND_NEW( left) |
							SIGMA_OLD_AND_NEW( right) |
							SIGMA_OLD_AND_NEW( leftBottom) |
							SIGMA_OLD_AND_NEW( bottom) |
							SIGMA_OLD_AND_NEW( rightBottom)  ) ;

				if (!nbh) {
					if ( !SIGMA_NEW(current) ) {
						//RLC
					}
				}
				else  {
					// ZC
					if (BIT(current)) {
						//SC
					}
				}
			}
		}

		localMemoryFence();
		bp--;
	}
}


r2042: Debian build

r2042: Debian build

;;;; -*- Mode: LISP; Syntax: ANSI-Common-Lisp; Base: 10 -*-
;;;; *************************************************************************
;;;; FILE IDENTIFICATION
;;;;
;;;; Name:          mysql-loader.sql
;;;; Purpose:       MySQL library loader using UFFI
;;;; Programmers:   Kevin M. Rosenberg
;;;; Date Started:  Feb 2002
;;;;
;;;; $Id: mysql-loader.cl,v 1.10 2002/05/14 16:15:24 kevin Exp $
;;;;
;;;; This file, part of CLSQL, is Copyright (c) 2002 by Kevin M. Rosenberg
;;;;
;;;; CLSQL users are granted the rights to distribute and use this software
;;;; as governed by the terms of the Lisp Lesser GNU Public License
;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL.
;;;; *************************************************************************

(declaim (optimize (debug 3) (speed 3) (safety 1) (compilation-speed 0)))
(in-package :mysql)

;;;; Modified by Kevin Rosenberg
;;;;  - probe potential directories to find library
;;;;  - Changed from CMUCL functions to UFFI to
;;;;      -- prevent library from being loaded multiple times
;;;;      -- support Allegro CL and Lispworks

(defvar *clsql-mysql-library-filename*
  (uffi:find-foreign-library
   "clsql-mysql"
   `(,(translate-logical-pathname
       "CL-LIBRARY:clsql;interfaces;clsql-uffi;")
     "/usr/share/common-lisp/source/clsql/interfaces/clsql-uffi/")))

(defvar *mysql-library-filename*
    (cond
     ((probe-file "/opt/mysql/lib/mysql/libmysqlclient.so")
      "/opt/mysql/lib/mysql/libmysqlclient.so")
     ((probe-file "/usr/local/lib/libmysqlclient.so")
      "/usr/local/lib/libmysqlclient.so")
     ((probe-file "/usr/lib/libmysqlclient.so")
      "/usr/lib/libmysqlclient.so")
     #+(or win32 mswindows)
     ((probe-file "c:/mysql/lib/opt/libmysql.dll")
      "c:/mysql/lib/opt/libmysql.dll")
     (t
      (warn "Can't find MySQL client library to load.")))
  "Location where the MySQL client library is to be found.")

(defvar *mysql-library-candidate-names*
    '("libmysqlclient" "libmysql"))

(defvar *mysql-library-candidate-directories*
    '("/opt/mysql/lib/mysql/" "/usr/local/lib/" "/usr/lib/" "/mysql/lib/opt/"))

(defvar *mysql-library-candidate-drive-letters* '("C" "D" "E"))

(defvar *mysql-supporting-libraries* '("c")
  "Used only by CMU. List of library flags needed to be passed to ld to
load the MySQL client library succesfully.  If this differs at your site,
set to the right path before compiling or loading the system.")

(defvar *mysql-library-loaded* nil
  "T if foreign library was able to be loaded successfully")

(defmethod clsql-sys:database-type-library-loaded ((database-type (eql :mysql)))
  *mysql-library-loaded*)

(defmethod clsql-sys:database-type-load-foreign ((database-type (eql :mysql)))
  (let ((mysql-path
	 (uffi:find-foreign-library *mysql-library-candidate-names*
				    *mysql-library-candidate-directories*
				    :drive-letters
				    *mysql-library-candidate-drive-letters*)))
    ;; zlib required to load mysql on CMUCL Solaris
    (uffi:load-foreign-library
     (uffi:find-foreign-library '("libz" "zlib")
				'("/usr/lib/" "/usr/local/" "/lib/")))
    (if	(and
	 (uffi:load-foreign-library mysql-path
				    :module "mysql"
				    :supporting-libraries
				    *mysql-supporting-libraries*)
	 (uffi:load-foreign-library *clsql-mysql-library-filename*
				    :module "clsql-mysql"
				    :supporting-libraries
				    (append *mysql-supporting-libraries*)))
	(setq *mysql-library-loaded* t)
      (warn "Unable to load MySQL client library ~A or CLSQL-MySQL library ~A"
	    mysql-path clsql-mysql-library-filename))))

(clsql-sys:database-type-load-foreign :mysql)
(when (clsql-sys:database-type-library-loaded :mysql)
  (clsql-sys:initialize-database-type :database-type :mysql)
  (setq clsql:*default-database-type* :mysql)
  (pushnew :mysql cl:*features*))

;;;; -*- Mode: LISP; Syntax: ANSI-Common-Lisp; Base: 10 -*-
;;;; *************************************************************************
;;;; FILE IDENTIFICATION
;;;;
;;;; Name:          mysql-loader.sql
;;;; Purpose:       MySQL library loader using UFFI
;;;; Programmers:   Kevin M. Rosenberg
;;;; Date Started:  Feb 2002
;;;;
;;;; $Id: mysql-loader.cl,v 1.9 2002/05/14 16:10:55 kevin Exp $
;;;;
;;;; This file, part of CLSQL, is Copyright (c) 2002 by Kevin M. Rosenberg
;;;;
;;;; CLSQL users are granted the rights to distribute and use this software
;;;; as governed by the terms of the Lisp Lesser GNU Public License
;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL.
;;;; *************************************************************************

(declaim (optimize (debug 3) (speed 3) (safety 1) (compilation-speed 0)))
(in-package :mysql)

;;;; Modified by Kevin Rosenberg
;;;;  - probe potential directories to find library
;;;;  - Changed from CMUCL functions to UFFI to
;;;;      -- prevent library from being loaded multiple times
;;;;      -- support Allegro CL and Lispworks

(defvar *clsql-mysql-library-filename*
  (uffi:find-foreign-library
   "clsql-mysql"
   `(,(translate-logical-pathname
       "CL-LIBRARY:clsql;interfaces;clsql-uffi;")
     "/usr/share/common-lisp/source/clsql/interfaces/clsql-uffi/")))

(defvar *mysql-library-filename*
    (cond
     ((probe-file "/opt/mysql/lib/mysql/libmysqlclient.so")
      "/opt/mysql/lib/mysql/libmysqlclient.so")
     ((probe-file "/usr/local/lib/libmysqlclient.so")
      "/usr/local/lib/libmysqlclient.so")
     ((probe-file "/usr/lib/libmysqlclient.so")
      "/usr/lib/libmysqlclient.so")
     #+(or win32 mswindows)
     ((probe-file "c:/mysql/lib/opt/libmysql.dll")
      "c:/mysql/lib/opt/libmysql.dll")
     (t
      (warn "Can't find MySQL client library to load.")))
  "Location where the MySQL client library is to be found.")

(defvar *mysql-library-candidate-names*
    '("libmysqlclient" "libmysql"))

(defvar *mysql-library-candidate-directories*
    '("/opt/mysql/lib/mysql/" "/usr/local/lib/" "/usr/lib/" "/mysql/lib/opt/"))

(defvar *mysql-library-candidate-drive-letters* '("C" "D" "E"))

(defvar *mysql-supporting-libraries* '("c")
  "Used only by CMU. List of library flags needed to be passed to ld to
load the MySQL client library succesfully.  If this differs at your site,
set to the right path before compiling or loading the system.")

(defvar *mysql-library-loaded* nil
  "T if foreign library was able to be loaded successfully")

(defmethod clsql-sys:database-type-library-loaded ((database-type (eql :mysql)))
  *mysql-library-loaded*)

(defmethod clsql-sys:database-type-load-foreign ((database-type (eql :mysql)))
  (let ((mysql-path
	 (uffi:find-foreign-library *mysql-library-candidate-names*
				    *mysql-library-candidate-directories*
				    :drive-letters
				    *mysql-library-candidate-drive-letters*)))
    ;; zlib required to load mysql on CMUCL Solaris
    (uffi:load-foreign-library
     (uffi:find-foreign-library '("libz" "zlib")
				'("/usr/lib/" "/usr/local/" "/lib/")))
    (when
	(and
	 (uffi:load-foreign-library mysql-path
				    :module "mysql"
				    :supporting-libraries
				    *mysql-supporting-libraries*)
	 (uffi:load-foreign-library *clsql-mysql-library-filename*
				    :module "clsql-mysql"
				    :supporting-libraries
				    (append *mysql-supporting-libraries*)))
      (setq *mysql-library-loaded* t))))

(clsql-sys:database-type-load-foreign :mysql)
(when (clsql-sys:database-type-library-loaded :mysql)
  (clsql-sys:initialize-database-type :database-type :mysql)
  (setq clsql:*default-database-type* :mysql)
  (pushnew :mysql cl:*features*))


Add explicit type cast for popcount() return value

/* OpenCL built-in library: popcount()

   Copyright (c) 2011 Universidad Rey Juan Carlos

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "templates.h"

#define __builtin_popcounthh(n)  ((char)__builtin_popcount((int)(n) & 0xff))
#define __builtin_popcounth(n)   ((short)__builtin_popcount((int)(n) & 0xffff))

#define __builtin_popcountuhh(n) __builtin_popcounthh(n)
#define __builtin_popcountuh(n)  __builtin_popcounth(n)
#define __builtin_popcountu(n)   __builtin_popcount(n)
#define __builtin_popcountul(n)  __builtin_popcountl(n)

DEFINE_BUILTIN_G_G(popcount)

/* OpenCL built-in library: popcount()

   Copyright (c) 2011 Universidad Rey Juan Carlos

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "templates.h"

#define __builtin_popcounthh(n)  __builtin_popcount((int)(n) & 0xff)
#define __builtin_popcounth(n)   __builtin_popcount((int)(n) & 0xffff)

#define __builtin_popcountuhh(n) __builtin_popcounthh(n)
#define __builtin_popcountuh(n)  __builtin_popcounth(n)
#define __builtin_popcountu(n)   __builtin_popcount(n)
#define __builtin_popcountul(n)  __builtin_popcountl(n)

DEFINE_BUILTIN_G_G(popcount)