Fix split horizontal/vertical filters for normal/simple filtering.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline void vp8_mbfilter(signed char mask,signed char hev,local uc*);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
#endif

char vp8_char_clamp(int in){
    if (in > 127)
        return 127;
    if (in < -128)
        return -128;

    return in;
}

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = convert_char4(base) ^ 0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    //u.s01 = add_sat(pq.s01, (char2){vp8_filter, Filter.s1});
    //u.s23 = sub_sat(pq.s23, (char2){Filter.s0, vp8_filter});
    u.s0 = clamp(pq.s0 + vp8_filter, -128, 127);
    u.s1 = clamp(pq.s1 + Filter.s1, -128, 127);
    u.s2 = clamp(pq.s2 - Filter.s0, -128, 127);
    u.s3 = clamp(pq.s3 - vp8_filter, -128, 127);

    return convert_uchar4(u ^ 0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);
    int filter_level = filters[block];

    if (filters[num_blocks*filter_type + block] > 0){
        if (cur_iter == 0 || plane == 0){
            if (cur_iter > 0){
                num_planes = 1;
            }

            if (filter_level){
                int p = pitches[plane];
                int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
                int s_off = offsets[block_offset];

                size_t i = get_global_id(0);

                int  hev = 0; /* high edge variance */
                signed char mask = 0;

                global signed char *limit, *flimit, *thresh;
                global loop_filter_info *lf_info;

                if (i < threads[plane]){
                    lf_info = &lfi[filter_level];
                    flimit = lf_info->flim;
                    limit = lf_info->lim;
                    thresh = lf_info->thr;

                    s_off += i;

                    uchar8 data;
                    data.s0 = s_base[s_off-4*p];
                    data.s1 = s_base[s_off-3*p];
                    data.s2 = s_base[s_off-2*p];
                    data.s3 = s_base[s_off-p];
                    data.s4 = s_base[s_off];
                    data.s5 = s_base[s_off+p];
                    data.s6 = s_base[s_off+2*p];
                    data.s7 = s_base[s_off+3*p];

                    mask = vp8_filter_mask(limit[i], flimit[i], data);

                    hev = vp8_hevmask(thresh[i], data.s2345);

                    data.s2345 = vp8_filter(mask, hev, data.s2345);

                    s_base[s_off - 2*p] = data.s2;
                    s_base[s_off - p  ] = data.s3;
                    s_base[s_off      ] = data.s4;
                    s_base[s_off + p  ] = data.s5;

                }
            }
        }
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);

    if ((cur_iter ==1 || plane == 0) && filters[num_blocks*filter_type + block] > 0){
        if (cur_iter > 1){
            num_planes = 1;
        }

        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset];

            int  hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i = get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->flim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;

                uchar8 data = vload8(0, &s_base[s_off-4]);

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                data.s2345 = vp8_filter(mask, hev, data.s2345);

                vstore4(data.s2345, 0, &s_base[s_off-2]);

            }
        }
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*ROWS_LOCATION + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = 8*num_blocks + block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += i;

#if 1
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4*p];
                s_data[1] = s_base[s_off - 3*p];
                s_data[2] = s_base[s_off - 2*p];
                s_data[3] = s_base[s_off - 1*p];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + p];
                s_data[6] = s_base[s_off + 2*p];
                s_data[7] = s_base[s_off + 3*p];
                uchar8 data = vload8(0, s_data);
#else
                uchar8 data;
                data.s0 = s_base[s_off-4*p];
                data.s1 = s_base[s_off-3*p];
                data.s2 = s_base[s_off-2*p];
                data.s3 = s_base[s_off-p];
                data.s4 = s_base[s_off];
                data.s5 = s_base[s_off+p];
                data.s6 = s_base[s_off+2*p];
                data.s7 = s_base[s_off+3*p];
#endif

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3*p] = s_data[1];
                s_base[s_off - 2*p] = s_data[2];
                s_base[s_off - 1*p] = s_data[3];
                s_base[s_off      ] = s_data[4];
                s_base[s_off + p  ] = s_data[5];
                s_base[s_off + 2*p] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim, //unused for normal filters
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    local unsigned char s_data[16*8*3];

    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);
}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];

                uchar8 data = vload8(0, s_data);
                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3] = s_data[1];
                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];
                s_base[s_off + 2] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){

    int block_offset = block_offsets[priority_level];
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    //Prefetch vertical edge source pixels into global cache
    int num_blocks = get_global_size(2);
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+p*thread-4], 8);
        }
    }

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

    barrier(CLK_GLOBAL_MEM_FENCE);

    //Prefetch horizontal source pixels
#if 0
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+thread-4*p], 8*p);
        }
    }
#endif

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);

}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = block_offsets[priority_level];
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

}

kernel void vp8_loop_filter_simple_horizontal_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = block_offsets[priority_level];
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

//Inline and non-kernel functions follow.

__inline void vp8_mbfilter(
    signed char mask,
    signed char hev,
    local uc *base
)
{
    signed char s, u;
    signed char vp8_filter;

    local uc *op2 = base;
    local uc *op1 = base + 1;
    local uc *op0 = base + 2;
    local uc *oq0 = base + 3;
    local uc *oq1 = base + 4;
    local uc *oq2 = base + 5;

    char2 filter;

    char4 ps = { *op0, *op1, *op2, 0 };
    ps ^= (char4){0x80, 0x80, 0x80, 0x80};

    char4 qs = { *oq0, *oq1, *oq2, 0 };
    qs ^= (char4){0x80, 0x80, 0x80, 0x80};

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps.s1 - qs.s1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (qs.s0 - ps.s0), -128, 127);
    vp8_filter &= mask;

    filter.s1 = vp8_filter;
    filter.s1 &= hev;
    filter.s0 = filter.s1;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter.s0 = clamp(filter.s0 + 4, -128, 127);
    filter.s1 = clamp(filter.s1 + 3, -128, 127);
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    qs.s0 = clamp(qs.s0 - filter.s0, -128, 127);
    ps.s0 = clamp(ps.s0 + filter.s1, -128, 127);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th difference across boundary */
    u = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    s = clamp(qs.s0 - u, -128, 127);
    *oq0 = s ^ 0x80;
    s = clamp(ps.s0 + u, -128, 127);
    *op0 = s ^ 0x80;

    /* roughly 2/7th difference across boundary */
    u = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    s = clamp(qs.s1 - u, -128, 127);
    *oq1 = s ^ 0x80;
    s = clamp(ps.s1 + u, -128, 127);
    *op1 = s ^ 0x80;

    /* roughly 1/7th difference across boundary */
    u = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    s = clamp(qs.s2 - u, -128, 127);
    *oq2 = s ^ 0x80;
    s = clamp(ps.s2 + u, -128, 127);
    *op2 = s ^ 0x80;
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    signed char hev = 0;
    hev  |= (abs(pq.s0 - pq.s1) > thresh) * -1;
    hev  |= (abs(pq.s3 - pq.s2) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    signed char mask = 0;

#if 1
    mask |= (abs(pq.s0 - pq.s1) > limit) * -1;
    mask |= (abs(pq.s1 - pq.s2) > limit) * -1;
    mask |= (abs(pq.s2 - pq.s3) > limit) * -1;
    mask |= (abs(pq.s5 - pq.s4) > limit) * -1;
    mask |= (abs(pq.s6 - pq.s5) > limit) * -1;
    mask |= (abs(pq.s7 - pq.s6) > limit) * -1;
    mask |= (abs(pq.s3 - pq.s4) * 2 + abs(pq.s2 - pq.s5) / 2  > flimit * 2 + limit) * -1;
    mask = ~mask;
    return mask;
#else
    //Only apply the filter if the difference is LESS than 'limit'
    mask |= (abs(pq.s0 - pq.s1) > limit);
    mask |= (abs(pq.s1 - pq.s2) > limit);
    mask |= (abs(pq.s2 - pq.s3) > limit);
    mask |= (abs(pq.s5 - pq.s4) > limit);
    mask |= (abs(pq.s6 - pq.s5) > limit);
    mask |= (abs(pq.s7 - pq.s6) > limit);
    mask |= (abs(pq.s3 - pq.s4) * 2 + abs(pq.s2 - pq.s5) / 2  > flimit * 2 + limit);

    return (mask != 0 ? 0 : -1);
#endif

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs(p0 - q0) * 2 + abs(p1 - q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter, Filter1, Filter2;
    signed char p1 = (signed char) * op1 ^ 0x80;
    signed char p0 = (signed char) * op0 ^ 0x80;
    signed char q0 = (signed char) * oq0 ^ 0x80;
    signed char q1 = (signed char) * oq1 ^ 0x80;
    signed char u;

    vp8_filter = clamp(p1 - q1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (q0 - p0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter1 >>= 3;
    u = clamp(q0 - Filter1, -128, 127);
    *oq0  = u ^ 0x80;

    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter2 >>= 3;
    u = clamp(p0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline void vp8_mbfilter(signed char mask,signed char hev,local uc*);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
#endif

char vp8_char_clamp(int in){
    if (in > 127)
        return 127;
    if (in < -128)
        return -128;

    return in;
}

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = convert_char4(base) ^ 0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    //u.s01 = add_sat(pq.s01, (char2){vp8_filter, Filter.s1});
    //u.s23 = sub_sat(pq.s23, (char2){Filter.s0, vp8_filter});
    u.s0 = clamp(pq.s0 + vp8_filter, -128, 127);
    u.s1 = clamp(pq.s1 + Filter.s1, -128, 127);
    u.s2 = clamp(pq.s2 - Filter.s0, -128, 127);
    u.s3 = clamp(pq.s3 - vp8_filter, -128, 127);

    return convert_uchar4(u ^ 0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);
    int filter_level = filters[block];

    if (filters[num_blocks*filter_type + block] > 0){
        if (cur_iter == 0 || plane == 0){
            if (cur_iter > 0){
                num_planes = 1;
            }

            if (filter_level){
                int p = pitches[plane];
                int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
                int s_off = offsets[block_offset];

                size_t i = get_global_id(0);

                int  hev = 0; /* high edge variance */
                signed char mask = 0;

                global signed char *limit, *flimit, *thresh;
                global loop_filter_info *lf_info;

                if (i < threads[plane]){
                    lf_info = &lfi[filter_level];
                    flimit = lf_info->flim;
                    limit = lf_info->lim;
                    thresh = lf_info->thr;

                    s_off += i;

                    uchar8 data;
                    data.s0 = s_base[s_off-4*p];
                    data.s1 = s_base[s_off-3*p];
                    data.s2 = s_base[s_off-2*p];
                    data.s3 = s_base[s_off-p];
                    data.s4 = s_base[s_off];
                    data.s5 = s_base[s_off+p];
                    data.s6 = s_base[s_off+2*p];
                    data.s7 = s_base[s_off+3*p];

                    mask = vp8_filter_mask(limit[i], flimit[i], data);

                    hev = vp8_hevmask(thresh[i], data.s2345);

                    data.s2345 = vp8_filter(mask, hev, data.s2345);

                    s_base[s_off - 2*p] = data.s2;
                    s_base[s_off - p  ] = data.s3;
                    s_base[s_off      ] = data.s4;
                    s_base[s_off + p  ] = data.s5;

                }
            }
        }
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);

    if ((cur_iter ==1 || plane == 0) && filters[num_blocks*filter_type + block] > 0){
        if (cur_iter > 1){
            num_planes = 1;
        }

        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset];

            int  hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i = get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->flim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;

                uchar8 data = vload8(0, &s_base[s_off-4]);

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                data.s2345 = vp8_filter(mask, hev, data.s2345);

                vstore4(data.s2345, 0, &s_base[s_off-2]);

            }
        }
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*ROWS_LOCATION + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = 8*num_blocks + block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += i;

#if 1
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4*p];
                s_data[1] = s_base[s_off - 3*p];
                s_data[2] = s_base[s_off - 2*p];
                s_data[3] = s_base[s_off - 1*p];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + p];
                s_data[6] = s_base[s_off + 2*p];
                s_data[7] = s_base[s_off + 3*p];
                uchar8 data = vload8(0, s_data);
#else
                uchar8 data;
                data.s0 = s_base[s_off-4*p];
                data.s1 = s_base[s_off-3*p];
                data.s2 = s_base[s_off-2*p];
                data.s3 = s_base[s_off-p];
                data.s4 = s_base[s_off];
                data.s5 = s_base[s_off+p];
                data.s6 = s_base[s_off+2*p];
                data.s7 = s_base[s_off+3*p];
#endif

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3*p] = s_data[1];
                s_base[s_off - 2*p] = s_data[2];
                s_base[s_off - 1*p] = s_data[3];
                s_base[s_off      ] = s_data[4];
                s_base[s_off + p  ] = s_data[5];
                s_base[s_off + 2*p] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim, //unused for normal filters
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = 0;
    local unsigned char s_data[16*8*3];

    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);
}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];

                uchar8 data = vload8(0, s_data);
                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3] = s_data[1];
                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];
                s_base[s_off + 2] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){

    int block_offset = block_offsets[priority_level];
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    //Prefetch vertical edge source pixels into global cache
    int num_blocks = get_global_size(2);
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+p*thread-4], 8);
        }
    }

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

    barrier(CLK_GLOBAL_MEM_FENCE);

    //Prefetch horizontal source pixels
#if 0
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+thread-4*p], 8*p);
        }
    }
#endif

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);

}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = 0;
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

}

kernel void vp8_loop_filter_simple_horizontal_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = 0;
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = 0;
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

//Inline and non-kernel functions follow.

__inline void vp8_mbfilter(
    signed char mask,
    signed char hev,
    local uc *base
)
{
    signed char s, u;
    signed char vp8_filter;

    local uc *op2 = base;
    local uc *op1 = base + 1;
    local uc *op0 = base + 2;
    local uc *oq0 = base + 3;
    local uc *oq1 = base + 4;
    local uc *oq2 = base + 5;

    char2 filter;

    char4 ps = { *op0, *op1, *op2, 0 };
    ps ^= (char4){0x80, 0x80, 0x80, 0x80};

    char4 qs = { *oq0, *oq1, *oq2, 0 };
    qs ^= (char4){0x80, 0x80, 0x80, 0x80};

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps.s1 - qs.s1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (qs.s0 - ps.s0), -128, 127);
    vp8_filter &= mask;

    filter.s1 = vp8_filter;
    filter.s1 &= hev;
    filter.s0 = filter.s1;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter.s0 = clamp(filter.s0 + 4, -128, 127);
    filter.s1 = clamp(filter.s1 + 3, -128, 127);
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    qs.s0 = clamp(qs.s0 - filter.s0, -128, 127);
    ps.s0 = clamp(ps.s0 + filter.s1, -128, 127);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th difference across boundary */
    u = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    s = clamp(qs.s0 - u, -128, 127);
    *oq0 = s ^ 0x80;
    s = clamp(ps.s0 + u, -128, 127);
    *op0 = s ^ 0x80;

    /* roughly 2/7th difference across boundary */
    u = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    s = clamp(qs.s1 - u, -128, 127);
    *oq1 = s ^ 0x80;
    s = clamp(ps.s1 + u, -128, 127);
    *op1 = s ^ 0x80;

    /* roughly 1/7th difference across boundary */
    u = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    s = clamp(qs.s2 - u, -128, 127);
    *oq2 = s ^ 0x80;
    s = clamp(ps.s2 + u, -128, 127);
    *op2 = s ^ 0x80;
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    signed char hev = 0;
    hev  |= (abs(pq.s0 - pq.s1) > thresh) * -1;
    hev  |= (abs(pq.s3 - pq.s2) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    signed char mask = 0;

#if 1
    mask |= (abs(pq.s0 - pq.s1) > limit) * -1;
    mask |= (abs(pq.s1 - pq.s2) > limit) * -1;
    mask |= (abs(pq.s2 - pq.s3) > limit) * -1;
    mask |= (abs(pq.s5 - pq.s4) > limit) * -1;
    mask |= (abs(pq.s6 - pq.s5) > limit) * -1;
    mask |= (abs(pq.s7 - pq.s6) > limit) * -1;
    mask |= (abs(pq.s3 - pq.s4) * 2 + abs(pq.s2 - pq.s5) / 2  > flimit * 2 + limit) * -1;
    mask = ~mask;
    return mask;
#else
    //Only apply the filter if the difference is LESS than 'limit'
    mask |= (abs(pq.s0 - pq.s1) > limit);
    mask |= (abs(pq.s1 - pq.s2) > limit);
    mask |= (abs(pq.s2 - pq.s3) > limit);
    mask |= (abs(pq.s5 - pq.s4) > limit);
    mask |= (abs(pq.s6 - pq.s5) > limit);
    mask |= (abs(pq.s7 - pq.s6) > limit);
    mask |= (abs(pq.s3 - pq.s4) * 2 + abs(pq.s2 - pq.s5) / 2  > flimit * 2 + limit);

    return (mask != 0 ? 0 : -1);
#endif

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs(p0 - q0) * 2 + abs(p1 - q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter, Filter1, Filter2;
    signed char p1 = (signed char) * op1 ^ 0x80;
    signed char p0 = (signed char) * op0 ^ 0x80;
    signed char q0 = (signed char) * oq0 ^ 0x80;
    signed char q1 = (signed char) * oq1 ^ 0x80;
    signed char u;

    vp8_filter = clamp(p1 - q1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (q0 - p0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter1 >>= 3;
    u = clamp(q0 - Filter1, -128, 127);
    *oq0  = u ^ 0x80;

    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter2 >>= 3;
    u = clamp(p0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
}


restore

restore

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#ifndef MLO_FILTER_STRIDE0
#define MLO_FILTER_STRIDE0 1
#endif
#ifndef MLO_FILTER_STRIDE1
#define MLO_FILTER_STRIDE1 1
#endif

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_TILE1*MLO_OUT_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_N_ALUTILES_TOTAL ((MLO_GRP_TILE0*MLO_GRP_TILE1)/(MLO_ALU_TILE_SZ))
#define MLO_N_ALUTILES_PERSTACK (MLO_N_ALUTILES_TOTAL/MLO_N_STACKS)
#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)
/*
#define MLO_N_OUT_TILES_PERSTACK (MLO_N_OUT_TILES*MLO_N_ALUTILES_PERSTACK)
#if MLO_N_OUT_TILES_PERSTACK > MLO_N_OUTPUTS
#undef MLO_N_OUT_TILES_PERSTACK
#define MLO_N_OUT_TILES_PERSTACK MLO_N_OUTPUTS
#endif
*/
#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  ((MLO_N_INPUTS+MLO_N_IN_TILES_PERSTACK-1)/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#if MLO_DIR_FORWARD == 1
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)  // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 - 1) * MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#else
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0) // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1 + MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#if MLO_DIR_FORWARD == 1
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 - 1) * MLO_FILTER_STRIDE1 + 1)
#else
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 +                        MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif

#define MLO_LCL_WEIGHTS 1

#define MLO_PADDING_SHIFT1 (MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1)
#define MLO_PADDING_SHIFT0 (MLO_FILTER_SIZE0 - MLO_FILTER_PAD0 - 1)

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void calculateXYPos(int linPos, int width, int *x, int *y)
{
	(*y) = (int)((float)linPos / (float)width + 0.00001f);
	(*x) = linPos - mul24((*y), width);
}

static inline int calculateOffset(int stride, int x, int y)
{
	int ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(int linPos,__local _FLOAT *lcl_data, int lcl_base, int lcl_height, int lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, int gbl_height, int gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug)
{
	int x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	int gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	int gbl_off = gbl_off0 + gbl_base;

	int l_x = x + lcl_x;
	int l_y = y + lcl_y;
	int lcl_off = lcl_base +
#if MLO_LARGE_MAP == 1
		linPos
#else
		mad24(l_y, lcl_stride, l_x);
#endif
	 ;

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(int lcl_id, int size, int lcl_p_stride, __local _FLOAT *lcl_data, int lcl_base, int lcl_height, int lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, int gbl_height, int gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(int i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(int lcl_id, int lcl_p_stride,
					__local _FLOAT * lcl_data,
					int lcl_off, int lcl_size, int lcl_height, int lcl_width, int lcl_stride, int lcl_bot_y, int lcl_bot_x,
					const __global _FLOAT * gbl_data,
					int gbl_off, int gbl_size, int gbl_height, int glb_width, int gbl_stride, int gbl_bot_y, int gbl_bot_x,
					int buf_block_ind, int max_n_bufs, int lcl_n_bufs,
					bool debug)
{

	for(int c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(int o_map_base,
				int in_stg_off,
				__private _FLOAT *pvt_in_stage, __local _FLOAT * lcl_indata,
				__private _FLOAT *pvt_wei_stage, __local _FLOAT * lcl_wei,
				__private _FLOAT *pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		int in_stg_off1 = in_stg_off;
		for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			int wei_stg_base_off = mad24(o_map_base, (int)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(int)MLO_FILTER_SZ));
			int in_stg_off2 = in_stg_off1;
			for(int j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(int i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#if MLO_DIR_FORWARD == 1
			for(int k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
#else
			for(int k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2 += (((k - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1%MLO_OUT_TILE1)) % MLO_FILTER_STRIDE1) ? 0 : MLO_IN_LCL_WIDTH)
			)
#endif
			{
				int k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(int i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(int o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					int wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(int i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] =  lcl_wei[wei_stg_off + i]; //(float)o_c/(float)MLO_N_OUT_TILES + (float)(i+k)/9;
					}

		// actual conv

					for( int j = 0; j < MLO_OUT_TILE1; ++j)
					{
#if MLO_DIR_FORWARD == 0
						if (((j + k + 1 - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1 % MLO_FILTER_STRIDE1)) % MLO_FILTER_STRIDE1) == 0)
#endif
						for(int i = 0; i < MLO_OUT_TILE0; ++i)
						{
							for(int l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								int l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

#if MLO_DIR_FORWARD == 1
								pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								     += pvt_in_stage[j * MLO_PVT_IN_WIDTH * MLO_FILTER_STRIDE1 + i * MLO_FILTER_STRIDE0 + l] * pvt_wei_stage[l_act];
#else
								if (((i + l + 1 - MLO_PADDING_SHIFT0 + (MLO_FILTER_SIZE0 % MLO_FILTER_STRIDE0)) % MLO_FILTER_STRIDE0) == 0)
								{
									pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
										+= pvt_in_stage[(j / MLO_FILTER_STRIDE1) * MLO_PVT_IN_WIDTH + (i + l) / MLO_FILTER_STRIDE0] * pvt_wei_stage[l_act];
								}
#endif
							}
						}

					}

				} // for(int o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(int j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(int i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

			} // for(int k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenConvUni(
       const __global _FLOAT * in,
       const __global _FLOAT * weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * bias,
#endif
 	  __global _FLOAT *out,
	   _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	int grp_id0 = get_group_id(0);
	int y_tile_blk = iDiv(grp_id0, MLO_N_OUT_TILE_BLOCKS0);
	int x_tile_blk = iMod(grp_id0, y_tile_blk, MLO_N_OUT_TILE_BLOCKS0);
	int o_pack = get_group_id(1); // block of outputs
	int b_pack = get_group_id(2); // batch block

	int lcl_id = get_local_id(0);
	int stack = iDiv(lcl_id, MLO_ALUTILES_STACK_SZ);  // stack
	int alu_stack_id = iMod(lcl_id, stack, MLO_ALUTILES_STACK_SZ);  // alu index in stack
// ALU plane inside stack
	int alu_out_plane_id = iDiv(alu_stack_id, MLO_ALU_TILE_SZ);  // alu output plane index
	int alu_out_id = iMod(alu_stack_id, alu_out_plane_id, MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
// pos inside ALU tile
	int alu_tl1 = iDiv(alu_out_id, MLO_ALU_VTILE0);
	int alu_tl0 = iMod(alu_out_id, alu_tl1, MLO_ALU_VTILE0);

	int o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	int o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	int o_map = o_map_plane + o_map_base; // output map index per ALU plane
	int b_index = b_pack * MLO_N_STACKS;

	int wave_id = iDiv(lcl_id, MLO_N_READ_PROCS);
	int wave_lcl_id = iMod(lcl_id, wave_id, MLO_N_READ_PROCS);

#if MLO_DIR_FORWARD == 1
	int x_grp = x_tile_blk * MLO_IN_TILE0 * MLO_FILTER_STRIDE0;
	int y_grp = y_tile_blk * MLO_IN_TILE1 * MLO_FILTER_STRIDE1;

	int x_in_grp = x_grp - MLO_FILTER_PAD0;
	int y_in_grp = y_grp - MLO_FILTER_PAD1;

	int x_in_lcl = alu_tl0 * MLO_OUT_TILE0 * MLO_FILTER_STRIDE0;
	int y_in_lcl = alu_tl1 * MLO_OUT_TILE1 * MLO_FILTER_STRIDE1;
#else
	int x_grp = x_tile_blk * (MLO_IN_TILE0 / MLO_FILTER_STRIDE0);
	int y_grp = y_tile_blk * (MLO_IN_TILE1 / MLO_FILTER_STRIDE1);
	int x_in_grp = x_grp - (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE0);
	int y_in_grp = y_grp - (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE1);
	int x_in_lcl = alu_tl0 * (MLO_OUT_TILE0 / MLO_FILTER_STRIDE0);
	int y_in_lcl = alu_tl1 * (MLO_OUT_TILE1 / MLO_FILTER_STRIDE1);
#endif

// base offset to read data from local input data
	int in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    int in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	int wei_off = mul24(o_map_plane, MLO_N_INPUTS * MLO_FILTER_SZ);
#else
	int wei_off = mul24(o_map_plane,MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(int i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(int i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(int ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		int in_lcl_off1 = 0;
		int in_off1 = in_off;
		for(int i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			int in_off2 = in_off1;
			int in_lcl_off2 = in_lcl_off1;
			for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				int elem_id = lcl_id;
				int lcl_p_stride = MLO_GRP_SZ0;
				int lcl_base = 0;
				int lcl_y = 0;
				int lcl_x = 0;
				int gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(int i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
			int i_b = iDiv(i, MLO_N_IN_TILES_PERSTACK);
			int i_c = iMod(i, i_b, MLO_N_IN_TILES_PERSTACK);

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			int in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			int in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			int elem_id = wave_lcl_id;
			int lcl_p_stride = MLO_N_READ_PROCS;
			int lcl_base = 0;
#if MLO_DIR_FORWARD == 1
			int lcl_y = MLO_FILTER_PAD1;
			int lcl_x = MLO_FILTER_PAD0;
#else
			int lcl_y = (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE0);
			int lcl_x = (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE1);
#endif
			int gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

		for(int i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
			int lcl_o = iDiv(i, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_i = iMod(i, lcl_o, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_we_off = wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i;
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);

			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[i] = wei;
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)
			int lcl_o = iDiv(i, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_i = iMod(i, lcl_o, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			int lcl_c = iDiv(gbl_i, MLO_FILTER_SZ);
			int lcl_i = iMod(gbl_i, lcl_c, MLO_FILTER_SZ);

			int lcl_we_off = mad24(mad24(lcl_c, (int)MLO_N_IN_TILES_PERSTACK, lcl_o), (int)MLO_FILTER_SZ, lcl_i);
			int gbl_we_off = mad24(mad24(lcl_o, (int)MLO_N_OUTPUTS, lcl_c), (int)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}
// write results out
#if MLO_DIR_FORWARD == 1
#if MLO_FILTER_STRIDE0 == 1
	int x_out_grp = x_grp;
#else
	int x_out_grp = x_tile_blk * MLO_IN_TILE0;
#endif
#if MLO_FILTER_STRIDE1 == 1
	int y_out_grp = y_grp;
#else
	int y_out_grp = y_tile_blk * MLO_IN_TILE1;
#endif
#else
	int x_out_grp = x_grp * MLO_FILTER_STRIDE0;
	int y_out_grp = y_grp * MLO_FILTER_STRIDE1;
#endif
	int x_out_lcl = alu_tl0 * MLO_OUT_TILE0;
	int y_out_lcl = alu_tl1 * MLO_OUT_TILE1;

    int out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		int out_off1 = out_off;
		for(int o = 0; o < MLO_N_OUT_TILES
						; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE
						)
		{
// over output tile

			int out_off2 = out_off1;
			for( int j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
				for(int i = 0; i < MLO_OUT_TILE0; ++i)
				{
					if (true

#if MLO_OUT_ALIGNED == 0
						&& y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT &&  x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH
#endif
#if MLO_OUTPUTS_ALIGNED == 0
						&& o_map + o < MLO_N_OUTPUTS
#endif

						)
					{
						out[out_off2 + i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_TILE0 + i]
#if MLO_CONV_BIAS
						+ bias_val = bias[o_map + o];
#endif
						 ;

					}

				}
			}

		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#ifndef MLO_FILTER_STRIDE0
#define MLO_FILTER_STRIDE0 1
#endif
#ifndef MLO_FILTER_STRIDE1
#define MLO_FILTER_STRIDE1 1
#endif

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_TILE1*MLO_OUT_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_N_ALUTILES_TOTAL ((MLO_GRP_TILE0*MLO_GRP_TILE1)/(MLO_ALU_TILE_SZ))
#define MLO_N_ALUTILES_PERSTACK (MLO_N_ALUTILES_TOTAL/MLO_N_STACKS)
#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)
/*
#define MLO_N_OUT_TILES_PERSTACK (MLO_N_OUT_TILES*MLO_N_ALUTILES_PERSTACK)
#if MLO_N_OUT_TILES_PERSTACK > MLO_N_OUTPUTS
#undef MLO_N_OUT_TILES_PERSTACK
#define MLO_N_OUT_TILES_PERSTACK MLO_N_OUTPUTS
#endif
*/
#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  ((MLO_N_INPUTS+MLO_N_IN_TILES_PERSTACK-1)/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#if MLO_DIR_FORWARD == 1
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)  // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 - 1) * MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#else
#define MLO_IN_LCL_WIDTH  ((MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0) // here we use kernel size. it's important when padding == 0  2* MLO_FILTER_PAD0
#define MLO_IN_LCL_HEIGHT ((MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1 + MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#if MLO_DIR_FORWARD == 1
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 - 1) * MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 - 1) * MLO_FILTER_STRIDE1 + 1)
#else
#define MLO_PVT_IN_WIDTH  ((MLO_OUT_TILE0 + MLO_FILTER_SIZE0 - 1 + MLO_FILTER_STRIDE0 - 1) / MLO_FILTER_STRIDE0)
#define MLO_PVT_IN_HEIGHT ((MLO_OUT_TILE1 +                        MLO_FILTER_STRIDE1 - 1) / MLO_FILTER_STRIDE1)
#endif

#define MLO_LCL_WEIGHTS 1

#define MLO_PADDING_SHIFT1 (MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1)
#define MLO_PADDING_SHIFT0 (MLO_FILTER_SIZE0 - MLO_FILTER_PAD0 - 1)

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void calculateXYPos(int linPos, int width, int *x, int *y)
{
	(*y) = (int)((float)linPos / (float)width + 0.00001f);
	(*x) = linPos - mul24((*y), width);
}

static inline int calculateOffset(int stride, int x, int y)
{
	int ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(int linPos,__local _FLOAT *lcl_data, int lcl_base, int lcl_height, int lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, int gbl_height, int gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug)
{
	int x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	int gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	int gbl_off = gbl_off0 + gbl_base;

	int l_x = x + lcl_x;
	int l_y = y + lcl_y;
	int lcl_off = lcl_base +
#if MLO_LARGE_MAP == 1
		linPos
#else
		mad24(l_y, lcl_stride, l_x);
#endif
	 ;

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(int lcl_id, int size, int lcl_p_stride, __local _FLOAT *lcl_data, int lcl_base, int lcl_height, int lcl_width, int lcl_stride, int lcl_y, int lcl_x,
					 const __global _FLOAT * gbl_data, int gbl_base, int gbl_height, int gbl_width, int gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(int i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(int lcl_id, int lcl_p_stride,
					__local _FLOAT * lcl_data,
					int lcl_off, int lcl_size, int lcl_height, int lcl_width, int lcl_stride, int lcl_bot_y, int lcl_bot_x,
					const __global _FLOAT * gbl_data,
					int gbl_off, int gbl_size, int gbl_height, int glb_width, int gbl_stride, int gbl_bot_y, int gbl_bot_x,
					int buf_block_ind, int max_n_bufs, int lcl_n_bufs,
					bool debug)
{

	for(int c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(int o_map_base,
				int in_stg_off,
				__private _FLOAT *pvt_in_stage, __local _FLOAT * lcl_indata,
				__private _FLOAT *pvt_wei_stage, __local _FLOAT * lcl_wei,
				__private _FLOAT *pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		int in_stg_off1 = in_stg_off;
		for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			int wei_stg_base_off = mad24(o_map_base, (int)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(int)MLO_FILTER_SZ));
			int in_stg_off2 = in_stg_off1;
			for(int j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(int i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#if MLO_DIR_FORWARD == 1
			for(int k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
#else
			for(int k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2 += (((k - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1%MLO_OUT_TILE1)) % MLO_FILTER_STRIDE1) ? 0 : MLO_IN_LCL_WIDTH)
			)
#endif
			{
				int k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(int i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(int o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					int wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(int i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] =  lcl_wei[wei_stg_off + i]; //(float)o_c/(float)MLO_N_OUT_TILES + (float)(i+k)/9;
					}

		// actual conv

					for( int j = 0; j < MLO_OUT_TILE1; ++j)
					{
#if MLO_DIR_FORWARD == 0
						if (((j + k + 1 - MLO_PADDING_SHIFT1 + (MLO_FILTER_SIZE1 % MLO_FILTER_STRIDE1)) % MLO_FILTER_STRIDE1) == 0)
#endif
						for(int i = 0; i < MLO_OUT_TILE0; ++i)
						{
							for(int l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								int l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

#if MLO_DIR_FORWARD == 1
								pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								     += pvt_in_stage[j * MLO_PVT_IN_WIDTH * MLO_FILTER_STRIDE1 + i * MLO_FILTER_STRIDE0 + l] * pvt_wei_stage[l_act];
#else
								if (((i + l + 1 - MLO_PADDING_SHIFT0 + (MLO_FILTER_SIZE0 % MLO_FILTER_STRIDE0)) % MLO_FILTER_STRIDE0) == 0)
								{
									pvt_accum[(o_c * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
										+= pvt_in_stage[(j / MLO_FILTER_STRIDE1) * MLO_PVT_IN_WIDTH + (i + l) / MLO_FILTER_STRIDE0] * pvt_wei_stage[l_act];
								}
#endif
							}
						}

					}

				} // for(int o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(int j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(int i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

			} // for(int k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenConvUni(
       const __global _FLOAT * in,
       const __global _FLOAT * weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * bias,
#endif
 	  __global _FLOAT *out,
	   _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	int grp_id0 = get_group_id(0);
	int y_tile_blk = iDiv(grp_id0, MLO_N_OUT_TILE_BLOCKS0);
	int x_tile_blk = iMod(grp_id0, y_tile_blk, MLO_N_OUT_TILE_BLOCKS0);
	int o_pack = get_group_id(1); // block of outputs
	int b_pack = get_group_id(2); // batch block

	int lcl_id = get_local_id(0);
	int stack = iDiv(lcl_id, MLO_ALUTILES_STACK_SZ);  // stack
	int alu_stack_id = iMod(lcl_id, stack, MLO_ALUTILES_STACK_SZ);  // alu index in stack
// ALU plane inside stack
	int alu_out_plane_id = iDiv(alu_stack_id, MLO_ALU_TILE_SZ);  // alu output plane index
	int alu_out_id = iMod(alu_stack_id, alu_out_plane_id, MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
// pos inside ALU tile
	int alu_tl1 = iDiv(alu_out_id, MLO_ALU_VTILE0);
	int alu_tl0 = iMod(alu_out_id, alu_tl1, MLO_ALU_VTILE0);

	int o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	int o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	int o_map = o_map_plane + o_map_base; // output map index per ALU plane
	int b_index = b_pack * MLO_N_STACKS;

	int wave_id = iDiv(lcl_id, MLO_N_READ_PROCS);
	int wave_lcl_id = iMod(lcl_id, wave_id, MLO_N_READ_PROCS);

#if MLO_DIR_FORWARD == 1
	int x_grp = x_tile_blk * MLO_IN_TILE0 * MLO_FILTER_STRIDE0;
	int y_grp = y_tile_blk * MLO_IN_TILE1 * MLO_FILTER_STRIDE1;

	int x_in_grp = x_grp - MLO_FILTER_PAD0;
	int y_in_grp = y_grp - MLO_FILTER_PAD1;

	int x_in_lcl = alu_tl0 * MLO_OUT_TILE0 * MLO_FILTER_STRIDE0;
	int y_in_lcl = alu_tl1 * MLO_OUT_TILE1 * MLO_FILTER_STRIDE1;
#else
	int x_grp = x_tile_blk * (MLO_IN_TILE0 / MLO_FILTER_STRIDE0);
	int y_grp = y_tile_blk * (MLO_IN_TILE1 / MLO_FILTER_STRIDE1);
	int x_in_grp = x_grp - (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE0);
	int y_in_grp = y_grp - (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE1);
	int x_in_lcl = alu_tl0 * (MLO_OUT_TILE0 / MLO_FILTER_STRIDE0);
	int y_in_lcl = alu_tl1 * (MLO_OUT_TILE1 / MLO_FILTER_STRIDE1);
#endif

// base offset to read data from local input data
	int in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    int in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	int wei_off = mul24(o_map_plane, MLO_N_INPUTS * MLO_FILTER_SZ);
#else
	int wei_off = mul24(o_map_plane,MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(int i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(int i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(int ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		int in_lcl_off1 = 0;
		int in_off1 = in_off;
		for(int i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			int in_off2 = in_off1;
			int in_lcl_off2 = in_lcl_off1;
			for(int i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				int elem_id = lcl_id;
				int lcl_p_stride = MLO_GRP_SZ0;
				int lcl_base = 0;
				int lcl_y = 0;
				int lcl_x = 0;
				int gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(int i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
			int i_b = iDiv(i, MLO_N_IN_TILES_PERSTACK);
			int i_c = iMod(i, i_b, MLO_N_IN_TILES_PERSTACK);

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			int in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			int in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			int elem_id = wave_lcl_id;
			int lcl_p_stride = MLO_N_READ_PROCS;
			int lcl_base = 0;
#if MLO_DIR_FORWARD == 1
			int lcl_y = MLO_FILTER_PAD1;
			int lcl_x = MLO_FILTER_PAD0;
#else
			int lcl_y = (MLO_FILTER_PAD1 / MLO_FILTER_STRIDE0);
			int lcl_x = (MLO_FILTER_PAD0 / MLO_FILTER_STRIDE1);
#endif
			int gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

		for(int i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
			int lcl_o = iDiv(i, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_i = iMod(i, lcl_o, (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_we_off = wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i;
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);

			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[i] = wei;
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)
			int lcl_o = iDiv(i, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			int gbl_i = iMod(i, lcl_o, (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
			int lcl_c = iDiv(gbl_i, MLO_FILTER_SZ);
			int lcl_i = iMod(gbl_i, lcl_c, MLO_FILTER_SZ);

			int lcl_we_off = mad24(mad24(lcl_c, (int)MLO_N_IN_TILES_PERSTACK, lcl_o), (int)MLO_FILTER_SZ, lcl_i);
			int gbl_we_off = mad24(mad24(lcl_o, (int)MLO_N_OUTPUTS, lcl_c), (int)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}
// write results out
#if MLO_DIR_FORWARD == 1
#if MLO_FILTER_STRIDE0 == 1
	int x_out_grp = x_grp;
#else
	int x_out_grp = x_tile_blk * MLO_IN_TILE0;
#endif
#if MLO_FILTER_STRIDE1 == 1
	int y_out_grp = y_grp;
#else
	int y_out_grp = y_tile_blk * MLO_IN_TILE1;
#endif
#else
	int x_out_grp = x_grp * MLO_FILTER_STRIDE0;
	int y_out_grp = y_grp * MLO_FILTER_STRIDE1;
#endif
	int x_out_lcl = alu_tl0 * MLO_OUT_TILE0;
	int y_out_lcl = alu_tl1 * MLO_OUT_TILE1;

    int out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		int out_off1 = out_off;
		for(int o = 0; o < MLO_N_OUT_TILES
						; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE
						)
		{
// over output tile

			int out_off2 = out_off1;
			for( int j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
				for(int i = 0; i < MLO_OUT_TILE0; ++i)
				{
					if (out_off2 + i <  MLO_OUT_BATCH_STRIDE*MLO_BATCH_SZ

#if MLO_OUT_ALIGNED == 0
						&& y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT &&  x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH
#endif
#if MLO_OUTPUTS_ALIGNED == 0
						&& o_map + o < MLO_N_OUTPUTS
#endif

						)
					{
						out[out_off2 + i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_TILE0 + i]
#if MLO_CONV_BIAS
						+ bias_val = bias[o_map + o];
#endif
						 ;

					}

				}
			}

		}
	}

}


opencl: removes status_flag.h from assign address OpenCL kernels

This suppresses errors under environments like oclgrind which, probably errenously,
advertise 64bit atomics yet do not provide the correct functions

#ifndef SIXTRACKLIB_OPENCL_KERNELS_PROCESS_ASSIGN_ADDRESS_ITEMS_CL_H__
#define SIXTRACKLIB_OPENCL_KERNELS_PROCESS_ASSIGN_ADDRESS_ITEMS_CL_H__

#if !defined( SIXTRL_NO_INCLUDES )
    #include "sixtracklib/opencl/internal/default_compile_options.h"
    #include "sixtracklib/common/definitions.h"
    #include "sixtracklib/common/control/definitions.h"
    #include "sixtracklib/common/buffer/assign_address_item_kernel_impl.h"
#endif /* !defined( SIXTRL_NO_INCLUDES ) */

__kernel void NS(AssignAddressItem_process_managed_buffer_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char const* SIXTRL_RESTRICT assign_buffer,
    NS(buffer_size_t) const assign_slot_size,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT dest_buffer,
    NS(buffer_size_t) const dest_slot_size,
    NS(arch_size_t) const dest_buffer_id,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char const* SIXTRL_RESTRICT src_buffer,
    NS(buffer_size_t) const src_slot_size,
    NS(arch_size_t) const src_buffer_id )
{
    NS(buffer_size_t) const start_idx = ( NS(buffer_size_t) )get_global_id( 0 );
    NS(buffer_size_t) const stride = ( NS(buffer_size_t) )get_global_size( 0 );

    NS(AssignAddressItem_perform_address_assignment_kernel_impl)(
        assign_buffer, assign_slot_size, start_idx, stride,
        dest_buffer, dest_slot_size, dest_buffer_id,
        src_buffer,  src_slot_size,  src_buffer_id );
}

#endif /* SIXTRACKLIB_OPENCL_KERNELS_PROCESS_ASSIGN_ADDRESS_ITEMS_CL_H__ */
/* end: /opencl/kernels/assign_address_item.cl */

#ifndef SIXTRACKLIB_OPENCL_KERNELS_PROCESS_ASSIGN_ADDRESS_ITEMS_CL_H__
#define SIXTRACKLIB_OPENCL_KERNELS_PROCESS_ASSIGN_ADDRESS_ITEMS_CL_H__

#if !defined( SIXTRL_NO_INCLUDES )
    #include "sixtracklib/opencl/internal/default_compile_options.h"
    #include "sixtracklib/common/definitions.h"
    #include "sixtracklib/common/control/definitions.h"
    #include "sixtracklib/common/buffer/assign_address_item_kernel_impl.h"
    #include "sixtracklib/opencl/internal/status_flag.h"
#endif /* !defined( SIXTRL_NO_INCLUDES ) */

__kernel void NS(AssignAddressItem_process_managed_buffer_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char const* SIXTRL_RESTRICT assign_buffer,
    NS(buffer_size_t) const assign_slot_size,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT dest_buffer,
    NS(buffer_size_t) const dest_slot_size,
    NS(arch_size_t) const dest_buffer_id,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char const* SIXTRL_RESTRICT src_buffer,
    NS(buffer_size_t) const src_slot_size,
    NS(arch_size_t) const src_buffer_id )
{
    NS(buffer_size_t) const start_idx = ( NS(buffer_size_t) )get_global_id( 0 );
    NS(buffer_size_t) const stride = ( NS(buffer_size_t) )get_global_size( 0 );

    NS(AssignAddressItem_perform_address_assignment_kernel_impl)(
        assign_buffer, assign_slot_size, start_idx, stride,
        dest_buffer, dest_slot_size, dest_buffer_id,
        src_buffer,  src_slot_size,  src_buffer_id );
}

#endif /* SIXTRACKLIB_OPENCL_KERNELS_PROCESS_ASSIGN_ADDRESS_ITEMS_CL_H__ */
/* end: /opencl/kernels/assign_address_item.cl */


Precalc hash for first letters in kernel

//
// LETTER_COUNT - will be provided during program build
//

__constant int lookup[26] = {
	0, 0, 1, 2, 3, 0, 4, 0, 5, 0, 0, 6, 7, 8, 9, 10, 0, 11, 12, 13, 14, 0, 15, 0, 0, 0 };
//a  b  c  d  e  f  g  h  i  j  k  l  m  n  o  p   q  r   s   t   u   v  w   x  y  z

// TODO remove letters array
__constant char letters[16] = {'a','c','d','e','g','i','l','m','n','o','p','r','s','t','u','w'};

long hash_step(long previous_val, const char letter){
	return previous_val * 37 + lookup[letter - 'a'];
}

__kernel
void main(__global char* target,
					// __global int* found_flag,
					long target_hash,
					int letters_from_global_id,
					int cpu_iter_i,
					int letters_to_iter_over){
	uint global_id = get_global_id(0);
	uint letters_from_cpu_iter_i = LETTER_COUNT - letters_to_iter_over - letters_from_global_id;
	char data[LETTER_COUNT]; // letters buffer

	// fill letters based on cpu_iter_i
	for (uint i = 0; i < letters_from_cpu_iter_i; i++){
		data[i] = letters[(cpu_iter_i >> (i*4)) & 0xf];
	}

	// fill letters based on global_id
	for (uint i = 0; i < letters_from_global_id; i++){
		uint ii = letters_from_cpu_iter_i + i;
		data[ii] = letters[(global_id >> (i * 4)) & 0xf];
	}

	// precalc hash for first base_idx letters
	uint base_idx = letters_from_cpu_iter_i + letters_from_global_id;
	long base_hash = 7;
	for (unsigned int i = 0; i < base_idx; i++) {
		base_hash = hash_step(base_hash, data[i]);
	}

	uint combinations = 1 << (4 * letters_to_iter_over);
	for (uint i = 0; i < combinations; i++){
		long hash_value = base_hash;
		for (uint idx = 0; idx < letters_to_iter_over; idx++){
			uint ii = base_idx + idx;
			char letter = letters[(i >> (idx * 4)) & 0xf];
			data[ii] = letter;
			hash_value = hash_step(hash_value, letter);
		}

		if( hash_value == target_hash){
			// int old_value = atomic_cmpxchg(found_flag, 0, 1); // if none found before us
			// if(old_value == 0){
				for (uint i = 0; i < LETTER_COUNT; i++){
					target[i] = data[i];
				}
			// }
			break;
		}
	}

}

//
// LETTER_COUNT - will be provided during program build
//

__constant int lookup[26] = {
	0, 0, 1, 2, 3, 0, 4, 0, 5, 0, 0, 6, 7, 8, 9, 10, 0, 11, 12, 13, 14, 0, 15, 0, 0, 0 };
//a  b  c  d  e  f  g  h  i  j  k  l  m  n  o  p   q  r   s   t   u   v  w   x  y  z

// TODO remove letters array
__constant char letters[16] = {'a','c','d','e','g','i','l','m','n','o','p','r','s','t','u','w'};

long hash(const char* c){
	long h = 7;
	for (unsigned int i = 0; i < LETTER_COUNT; i++) {
		h = (h * 37 + lookup[c[i]-'a']);
	}
	return h;
}

__kernel
void main(__global char* target,
					// __global int* found_flag,
					long target_hash,
					int letters_from_global_id,
					int cpu_iter_i,
					int letters_to_iter_over){
	uint global_id = get_global_id(0);
	uint letters_from_cpu_iter_i = LETTER_COUNT - letters_to_iter_over - letters_from_global_id;
	char data[LETTER_COUNT]; // letters buffer

	// fill letters based on cpu_iter_i
	for (uint i = 0; i < letters_from_cpu_iter_i; i++){
		data[i] = letters[(cpu_iter_i >> (i*4)) & 0xf];
	}

	// fill letters based on global_id
	for (uint i = 0; i < letters_from_global_id; i++){
		uint ii = letters_from_cpu_iter_i + i;
		data[ii] = letters[(global_id >> (i * 4)) & 0xf];
	}

	uint base_idx = letters_from_cpu_iter_i + letters_from_global_id;
	uint combinations = 1 << (4 * letters_to_iter_over);
	for (uint i = 0; i < combinations; i++){
		// fill out last letters
		for (uint idx = 0; idx < letters_to_iter_over; idx++){
			uint ii = base_idx + idx;
			data[ii] = letters[(i >> (idx * 4)) & 0xf];
		}

		// calculate hash
		long hash_value = hash(data);
		if( hash_value == target_hash){
			// int old_value = atomic_cmpxchg(found_flag, 0, 1); // if none found before us
			// if(old_value == 0){
				for (uint i = 0; i < LETTER_COUNT; i++){
					target[i] = data[i];
				}
			// }
			break;
		}
	}

}