Argument formatting changes.

Argument formatting changes.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant int bilinear_filters[8][2] = {
    { 128, 0},
    { 112, 16},
    { 96, 32},
    { 80, 48},
    { 64, 64},
    { 48, 80},
    { 32, 96},
    { 16, 112}
};

__constant short sub_pel_filters[8][8] = {
    //These were originally 8x6, but are padded for vector ops
    { 0, 0, 128, 0, 0, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0, 0, 0},
    { 2, -11, 108, 36, -8, 1, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0, 0, 0},
    { 3, -16, 77, 77, -16, 3, 0, 0}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0, 0, 0},
    { 1, -8, 36, 108, -11, 2, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0, 0, 0},
};

kernel void vp8_filter_block2d_first_pass_kernel(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we capture it now

    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

                Temp = (int)(src_ptr[src_offset - PS2]      * vp8_filter[0]) +
                   (int)(src_ptr[src_offset - (int)pixel_step] * vp8_filter[1]) +
                   (int)(src_ptr[src_offset]                * vp8_filter[2]) +
                   (int)(src_ptr[src_offset + pixel_step]   * vp8_filter[3]) +
                   (int)(src_ptr[src_offset + PS2]          * vp8_filter[4]) +
                   (int)(src_ptr[src_offset + PS3]          * vp8_filter[5]) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

            /* Normalize back to 0-255 */
            Temp = Temp >> VP8_FILTER_SHIFT;

            //Temp = (int)src_ptr[2];
            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
    }

}

kernel void vp8_filter_block2d_second_pass_kernel
(
    __global int *src_base,
    int src_offset,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    global int *src_ptr = &src_base[src_offset];
    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

kernel void vp8_filter_block2d_bil_first_pass(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
)
{
    uint tid = get_global_id(0);

    global unsigned char *src_ptr = &src_base[src_offset];

    unsigned int i, j;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    unsigned int out_row,out_offset;
    int src_increment = src_pixels_per_line - output_width;

    if (tid < output_width * output_height){
        for (i = 0; i < output_height; i++)
        {
            src_offset = i*(output_width+src_increment);
            out_row = output_width * i;
            for (j = 0; j < output_width; j++)
            {
                out_offset = out_row + j;

                /* Apply bilinear filter */
                output_ptr[out_offset] = (((int)src_ptr[src_offset]   * vp8_filter[0]) +
                                 ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                                 (VP8_FILTER_WEIGHT / 2)) >> VP8_FILTER_SHIFT;
                src_offset++;
            }
        }
    }
}

kernel void vp8_filter_block2d_bil_second_pass
(
    __global int *src_ptr,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
)
{
    uint tid = get_global_id(0);

    global unsigned char *output_ptr = &output_base[output_offset];

    unsigned int i, j;
    int Temp;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    int out_offset;
    int src_offset;
    int src_increment = src_pixels_per_line - output_width;

    if (tid < output_width*output_height){
        for (i = 0; i < output_height; i++)
        {
            src_offset = i*(output_width+src_increment);
            out_offset = i*output_pitch;
            for (j = 0; j < output_width; j++)
            {
                /* Apply filter */
                Temp = ((int)src_ptr[src_offset]         * vp8_filter[0]) +
                       ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                       (VP8_FILTER_WEIGHT / 2);
                output_ptr[out_offset++] = (unsigned int)(Temp >> VP8_FILTER_SHIFT);

                src_offset++;
            }
        }
    }
}

void vp8_filter_block2d_bil
(
    __global unsigned char *src_base,
    int src_offset,
    __global unsigned char *output_ptr,
    int output_offset,
    unsigned int src_pixels_per_line,
    unsigned int dst_pitch,
    int xoffset,
    int yoffset,
    int Width,
    int Height,
    __global int *int_mem
)
{
    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass(src_base, src_offset, int_mem, src_pixels_per_line, 1, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass(int_mem, output_ptr, output_offset, dst_pitch, Width, Width, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict4x4_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_base,
        int dst_offset,
        int dst_pitch,
        __global int *FData
)
{
    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 4, 4, FData);
}

__kernel void vp8_bilinear_predict8x8_kernel
(
    __global unsigned char *src_ptr,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *FData
)
{

    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 8, FData);

}

__kernel void vp8_bilinear_predict8x4_kernel
(
    __global unsigned char *src_ptr,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *FData
)
{

    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 4, FData);
}

__kernel void vp8_bilinear_predict16x16_kernel
(
    __global unsigned char *src_ptr,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *FData
)
{

    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 16, 16, FData);
}

//Called from reconinter_cl.c
kernel void vp8_memcpy_kernel(
    global unsigned char *src,
    int src_stride,
    global unsigned char *dst,
    int dst_stride,
    int num_bytes,
    int num_iter
){

    int i,r;
    int src_offset, dst_offset;

    r = get_global_id(1);
    if (r < get_global_size(1)){
        i = get_global_id(0);
        if (i < get_global_size(0)){
            src_offset = r*src_stride + i;
            dst_offset = r*dst_stride + i;
            dst[dst_offset] = src[src_offset];
        }
    }
}

//Not used currently.
void vp8_memset_short(
    global short *mem,
    int offset,
    short newval,
    size_t size
)
{
    int tid = get_global_id(0);

    if (tid < (size/2)){
        mem[offset+tid/2] = newval;
    }
}

//Here to EOF is old code that should be removed at end of project.
#if 0

void vp8_filter_block2d_first_pass(
    __global unsigned char *src_base,
    int src_offset,
    __private int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we capture it now
    src_offset = 0;

    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        for (i=0; i < output_width*output_height; i++){
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

                Temp = (int)(src_ptr[src_offset - PS2]      * vp8_filter[0]) +
                   (int)(src_ptr[src_offset - (int)pixel_step] * vp8_filter[1]) +
                   (int)(src_ptr[src_offset]                * vp8_filter[2]) +
                   (int)(src_ptr[src_offset + pixel_step]   * vp8_filter[3]) +
                   (int)(src_ptr[src_offset + PS2]          * vp8_filter[4]) +
                   (int)(src_ptr[src_offset + PS3]          * vp8_filter[5]) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

            /* Normalize back to 0-255 */
            Temp = Temp >> VP8_FILTER_SHIFT;

            //Temp = (int)src_ptr[2];
            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
        }
    }

    //Add a fence so that no 2nd pass stuff starts before 1st pass writes are done.
    write_mem_fence(CLK_GLOBAL_MEM_FENCE);
}

void vp8_filter_block2d_second_pass
(
    __private int *src_ptr,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

//Used?
__kernel void vp8_block_variation_kernel
(
    __global unsigned char  *src_ptr,
    int   src_pixels_per_line,
    __global int *HVar,
    __global int *VVar
)
{
    int i, j;
    __global unsigned char *Ptr = src_ptr;

    for (i = 0; i < 4; i++)
    {
        for (j = 0; j < 4; j++)
        {
            *HVar += abs((int)Ptr[j] - (int)Ptr[j+1]);
            *VVar += abs((int)Ptr[j] - (int)Ptr[j+src_pixels_per_line]);
        }

        Ptr += src_pixels_per_line;
    }
}

__kernel void vp8_sixtap_predict_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch,
    __global int *int_data
        ) {

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass_kernel(src_ptr, src_offset, int_data, src_pixels_per_line, 1, 9, 4, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass_kernel(int_data, 8, dst_ptr, dst_offset, dst_pitch, 4, 4, 4, 4, yoffset);
}

__kernel void vp8_sixtap_predict8x8_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data bufffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 13, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 8, 8, yoffset);

}

__kernel void vp8_sixtap_predict8x4_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 9, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 4, 8, yoffset);
}

__kernel void vp8_sixtap_predict16x16_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    __private int FData[21*24];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 21, 16, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[32], dst_ptr, dst_offset, dst_pitch, 16, 16, 16, 16, yoffset);

    return;
}
#endif

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant int bilinear_filters[8][2] = {
    { 128, 0},
    { 112, 16},
    { 96, 32},
    { 80, 48},
    { 64, 64},
    { 48, 80},
    { 32, 96},
    { 16, 112}
};

__constant short sub_pel_filters[8][8] = {
    //These were originally 8x6, but are padded for vector ops
    { 0, 0, 128, 0, 0, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0, 0, 0},
    { 2, -11, 108, 36, -8, 1, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0, 0, 0},
    { 3, -16, 77, 77, -16, 3, 0, 0}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0, 0, 0},
    { 1, -8, 36, 108, -11, 2, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0, 0, 0},
};

kernel void vp8_filter_block2d_first_pass_kernel(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we capture it now

    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

                Temp = (int)(src_ptr[src_offset - PS2]      * vp8_filter[0]) +
                   (int)(src_ptr[src_offset - (int)pixel_step] * vp8_filter[1]) +
                   (int)(src_ptr[src_offset]                * vp8_filter[2]) +
                   (int)(src_ptr[src_offset + pixel_step]   * vp8_filter[3]) +
                   (int)(src_ptr[src_offset + PS2]          * vp8_filter[4]) +
                   (int)(src_ptr[src_offset + PS3]          * vp8_filter[5]) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

            /* Normalize back to 0-255 */
            Temp = Temp >> VP8_FILTER_SHIFT;

            //Temp = (int)src_ptr[2];
            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
    }

}

kernel void vp8_filter_block2d_second_pass_kernel
(
    __global int *src_base,
    int src_offset,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    global int *src_ptr = &src_base[src_offset];
    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

void vp8_filter_block2d_bil_first_pass(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);

    global unsigned char *src_ptr = &src_base[src_offset];

    unsigned int i, j;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    unsigned int out_row,out_offset;
    int src_increment = src_pixels_per_line - output_width;

    if (tid < output_width * output_height){
        for (i = 0; i < output_height; i++)
        {
            src_offset = i*(output_width+src_increment);
            out_row = output_width * i;
            for (j = 0; j < output_width; j++)
            {
                out_offset = out_row + j;

                /* Apply bilinear filter */
                output_ptr[out_offset] = (((int)src_ptr[src_offset]   * vp8_filter[0]) +
                                 ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                                 (VP8_FILTER_WEIGHT / 2)) >> VP8_FILTER_SHIFT;
                src_offset++;
            }
        }
    }
}

void vp8_filter_block2d_bil_second_pass
(
    __global int *src_ptr,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {
    uint tid = get_global_id(0);

    global unsigned char *output_ptr = &output_base[output_offset];

    unsigned int i, j;
    int Temp;
    __constant int *vp8_filter = bilinear_filters[filter_offset];

    int out_offset;
    int src_offset;
    int src_increment = src_pixels_per_line - output_width;

    if (tid < output_width*output_height){
        for (i = 0; i < output_height; i++)
        {
            src_offset = i*(output_width+src_increment);
            out_offset = i*output_pitch;
            for (j = 0; j < output_width; j++)
            {
                /* Apply filter */
                Temp = ((int)src_ptr[src_offset]         * vp8_filter[0]) +
                       ((int)src_ptr[src_offset+pixel_step] * vp8_filter[1]) +
                       (VP8_FILTER_WEIGHT / 2);
                output_ptr[out_offset++] = (unsigned int)(Temp >> VP8_FILTER_SHIFT);

                src_offset++;
            }
        }
    }
}

void vp8_filter_block2d_bil
(
    __global unsigned char *src_base,
    int src_offset,
    __global unsigned char *output_ptr,
    int output_offset,
    unsigned int src_pixels_per_line,
    unsigned int dst_pitch,
    int xoffset,
    int yoffset,
    int Width,
    int Height,
    __global int *int_mem
)
{
    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass(src_base, src_offset, int_mem, src_pixels_per_line, 1, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass(int_mem, output_ptr, output_offset, dst_pitch, Width, Width, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict4x4_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_base,
        int dst_offset,
        int dst_pitch,
        __global int *FData
 ) {

    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 4, 4, FData);
}

__kernel void vp8_bilinear_predict8x8_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_base,
        int dst_offset,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 8, FData);

}

__kernel void vp8_bilinear_predict8x4_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_base,
        int dst_offset,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 8, 4, FData);
}

__kernel void vp8_bilinear_predict16x16_kernel
(
        __global unsigned char *src_ptr,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_base,
        int dst_offset,
        int dst_pitch,
        __global int *FData
        ) {

    vp8_filter_block2d_bil(src_ptr, 0, dst_base, dst_offset, src_pixels_per_line, dst_pitch, xoffset, yoffset, 16, 16, FData);
}

//Called from reconinter_cl.c
kernel void vp8_memcpy_kernel(
    global unsigned char *src,
    int src_stride,
    global unsigned char *dst,
    int dst_stride,
    int num_bytes,
    int num_iter
){

    int i,r;
    int src_offset, dst_offset;

    r = get_global_id(1);
    if (r < get_global_size(1)){
        i = get_global_id(0);
        if (i < get_global_size(0)){
            src_offset = r*src_stride + i;
            dst_offset = r*dst_stride + i;
            dst[dst_offset] = src[src_offset];
        }
    }
}

//Not used currently.
void vp8_memset_short(
    global short *mem,
    int offset,
    short newval,
    size_t size
)
{
    int tid = get_global_id(0);

    if (tid < (size/2)){
        mem[offset+tid/2] = newval;
    }
}

//Here to EOF is old code that should be removed at end of project.
#if 0

void vp8_filter_block2d_first_pass(
    __global unsigned char *src_base,
    int src_offset,
    __private int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we capture it now
    src_offset = 0;

    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        for (i=0; i < output_width*output_height; i++){
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

                Temp = (int)(src_ptr[src_offset - PS2]      * vp8_filter[0]) +
                   (int)(src_ptr[src_offset - (int)pixel_step] * vp8_filter[1]) +
                   (int)(src_ptr[src_offset]                * vp8_filter[2]) +
                   (int)(src_ptr[src_offset + pixel_step]   * vp8_filter[3]) +
                   (int)(src_ptr[src_offset + PS2]          * vp8_filter[4]) +
                   (int)(src_ptr[src_offset + PS3]          * vp8_filter[5]) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

            /* Normalize back to 0-255 */
            Temp = Temp >> VP8_FILTER_SHIFT;

            //Temp = (int)src_ptr[2];
            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
        }
    }

    //Add a fence so that no 2nd pass stuff starts before 1st pass writes are done.
    write_mem_fence(CLK_GLOBAL_MEM_FENCE);
}

void vp8_filter_block2d_second_pass
(
    __private int *src_ptr,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

//Used?
__kernel void vp8_block_variation_kernel
(
    __global unsigned char  *src_ptr,
    int   src_pixels_per_line,
    __global int *HVar,
    __global int *VVar
)
{
    int i, j;
    __global unsigned char *Ptr = src_ptr;

    for (i = 0; i < 4; i++)
    {
        for (j = 0; j < 4; j++)
        {
            *HVar += abs((int)Ptr[j] - (int)Ptr[j+1]);
            *VVar += abs((int)Ptr[j] - (int)Ptr[j+src_pixels_per_line]);
        }

        Ptr += src_pixels_per_line;
    }
}

__kernel void vp8_sixtap_predict_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch,
    __global int *int_data
        ) {

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass_kernel(src_ptr, src_offset, int_data, src_pixels_per_line, 1, 9, 4, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass_kernel(int_data, 8, dst_ptr, dst_offset, dst_pitch, 4, 4, 4, 4, yoffset);
}

__kernel void vp8_sixtap_predict8x8_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data bufffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 13, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 8, 8, yoffset);

}

__kernel void vp8_sixtap_predict8x4_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    __private int FData[13*16];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 9, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 4, 8, yoffset);
}

__kernel void vp8_sixtap_predict16x16_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    __private int FData[21*24];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 21, 16, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[32], dst_ptr, dst_offset, dst_pitch, 16, 16, 16, 16, yoffset);

    return;
}
#endif


OpenCL implementation for non-strided matrices

#define REAL float

#if GRANULARITY == 1

__kernel void gemm_batch(__global const REAL* A, __global const REAL* B, __global REAL* C, int n) {
	int matrix = get_global_id(0);
	int tid = get_local_id(0);
/* preload data */
#if CACHING_STRATEGY == 1
    int preloadStartA = get_group_id(0)*GROUP_SIZE_X*SIZE_A*SIZE_B;
    int preloadStartB = get_group_id(0)*GROUP_SIZE_X*SIZE_C*SIZE_A;
	__local REAL bufA[GROUP_SIZE_X*SIZE_A*SIZE_B];
	__local REAL bufB[GROUP_SIZE_X*SIZE_C*SIZE_A];
	for (int i = 0; i < SIZE_A*SIZE_B; i++)
		bufA[i*GROUP_SIZE_X+tid] = A[preloadStartA + i*GROUP_SIZE_X + tid];
	for (int i = 0; i < SIZE_C*SIZE_A; i++)
        bufB[i*GROUP_SIZE_X+tid] = B[preloadStartB + i*GROUP_SIZE_X + tid];
	barrier(CLK_LOCAL_MEM_FENCE);
#elif CACHING_STRATEGY == 2
    int preloadStartA = matrix*SIZE_A*SIZE_B;
    int preloadStartB = matrix*SIZE_C*SIZE_A;
    REAL bufA[SIZE_A*SIZE_B];
    REAL bufB[SIZE_C*SIZE_A];
    for (int i = 0; i < SIZE_A*SIZE_B; i++)
        bufA[i] = A[preloadStartA + i];
    for (int i = 0; i < SIZE_C*SIZE_A; i++)
        bufB[i] = B[preloadStartB + i];
#endif

/* offsets into memory */
#if CACHING_STRATEGY == 0
    int startA = matrix*SIZE_A*SIZE_B;
    int startB = matrix*SIZE_C*SIZE_A;
#else
    int startA = tid*SIZE_A*SIZE_B;
    int startB = tid*SIZE_C*SIZE_A;
#endif
    int startC = matrix*SIZE_C*SIZE_B;

/* compute multiplication */
    for (int i = 0; i < SIZE_B; i++)
        for (int j = 0; j < SIZE_C; j++) {
            REAL tmp = (REAL)0.0;
            for (int k = 0; k < SIZE_A; k++)
#if CACHING_STRATEGY == 0
                tmp += A[startA + i*SIZE_A + k] * B[startB + k*SIZE_C + j];
#endif
#if CACHING_STRATEGY == 1
                tmp += bufA[startA + i*SIZE_A + k] * bufB[startB + k*SIZE_C + j];
#endif
#if CACHING_STRATEGY == 2
                tmp += bufA[i*SIZE_A + k] * bufB[k*SIZE_C + j];
#endif
            C[startC + i*SIZE_C + j] = tmp;
        }
}

#endif

#if GRANULARITY == 2
__kernel void gemm_batch(__global const REAL* A, __global const REAL* B, __global REAL* C, int n) {
    int matrix = get_group_id(0)*MGCG_GROUP_SIZE_Y + get_local_id(1);
    int tx = get_local_id(0);
    int ty = get_local_id(1);

/* preload data */
#if CACHING_STRATEGY > 0
    int preloadStartA = get_group_id(0)*MGCG_GROUP_SIZE_Y*SIZE_A*SIZE_B;
    int preloadStartB = get_group_id(0)*MGCG_GROUP_SIZE_Y*SIZE_C*SIZE_A;
    int myOffset = ty*SIZE_C + tx;
    __local REAL bufA[MGCG_GROUP_SIZE_Y*SIZE_A*SIZE_B];
    #if CACHING_STRATEGY == 1
    __local REAL bufB[MGCG_GROUP_SIZE_Y*SIZE_C*SIZE_A];
    #endif
    for (int i = myOffset; i < SIZE_A*SIZE_B*MGCG_GROUP_SIZE_Y; i+= SIZE_C*MGCG_GROUP_SIZE_Y)
        bufA[i] = A[preloadStartA + i];
    #if CACHING_STRATEGY == 1
     for (int i = myOffset; i < SIZE_C*SIZE_A*MGCG_GROUP_SIZE_Y; i+= SIZE_C*MGCG_GROUP_SIZE_Y)
        bufB[i] = B[preloadStartB + i];
    #endif
    barrier(CLK_LOCAL_MEM_FENCE);
#endif
/* offsets into memory */
#if CACHING_STRATEGY == 0
    int startA = matrix*SIZE_A*SIZE_B;
    int startB = matrix*SIZE_C*SIZE_A;
#endif
#if CACHING_STRATEGY == 1
    int startA = ty*SIZE_A*SIZE_B;
    int startB = ty*SIZE_C*SIZE_A;
#endif
#if CACHING_STRATEGY == 2
    int startA = ty*SIZE_A*SIZE_B;
    int startB = matrix*SIZE_C*SIZE_A;
#endif
    int startC = matrix*SIZE_C*SIZE_B;

/* compute multiplication */
#if CACHING_STRATEGY == 2
    REAL tmp[SIZE_B];
    for (int i = 0; i < SIZE_B; i++)
        tmp[i] = (REAL)0.0;
    for (int i = 0; i < SIZE_A; i++) {
        REAL myB = B[startB + i*SIZE_C + tx];
        for (int j = 0; j < SIZE_B; j++)
            tmp[j] += bufA[startA + j*SIZE_A + i] * myB;
    }
    for (int i = 0; i < SIZE_B; i++)
        C[startC + i*SIZE_C + tx] = tmp[i];
#else
    for (int i = 0; i < SIZE_B; i++) {
        REAL tmp = (REAL)0.0;
        for (int k = 0; k < SIZE_A; k++)
    #if CACHING_STRATEGY == 0
            tmp += A[startA + i*SIZE_A + k] * B[startB + k*SIZE_C + tx];
    #endif
    #if CACHING_STRATEGY == 1
            tmp += bufA[startA + i*SIZE_A + k] * bufB[startB + k*SIZE_C + tx];
    #endif
        C[startC + i*SIZE_C + tx] = tmp;
    }
#endif
}
#endif

#if GRANULARITY == 3
__kernel void gemm_batch(__global const REAL* A, __global const REAL* B, __global REAL* C, int n) {
    int matrix = get_group_id(0);
    int tx = get_local_id(0);
    int ty = get_local_id(1);

/* preload data */
#if CACHING_STRATEGY == 1
    int preloadStartA = get_group_id(0)*SIZE_A*SIZE_B;
    int preloadStartB = get_group_id(0)*SIZE_C*SIZE_A;
    int myOffset = ty*SIZE_C + tx;
    __local REAL bufA[SIZE_A*SIZE_B];
    __local REAL bufB[SIZE_C*SIZE_A];
    for (int i = myOffset; i < SIZE_A*SIZE_B; i+= SIZE_C*MGCG_GROUP_SIZE_Y)
        bufA[i] = A[preloadStartA + i];
     for (int i = myOffset; i < SIZE_C*SIZE_A; i+= SIZE_C*MGCG_GROUP_SIZE_Y)
        bufB[i] = B[preloadStartB + i];
    barrier(CLK_LOCAL_MEM_FENCE);
#endif
/* offsets into memory */
#if CACHING_STRATEGY == 0
    int startA = matrix*SIZE_A*SIZE_B;
    int startB = matrix*SIZE_C*SIZE_A;
#else
    int startA = 0;
    int startB = 0;
#endif
    int startC = matrix*SIZE_C*SIZE_B;

/* compute multiplication */
    for (int i = ty; i < SIZE_B; i+= MGCG_GROUP_SIZE_Y) {
        REAL tmp = (REAL)0.0;
        for (int k = 0; k < SIZE_A; k++)
#if CACHING_STRATEGY == 0
            tmp += A[startA + i*SIZE_A + k] * B[startB + k*SIZE_C + tx];
#endif
#if CACHING_STRATEGY == 1
            tmp += bufA[startA + i*SIZE_A + k] * bufB[startB + k*SIZE_C + tx];
#endif
        C[startC + i*SIZE_C + tx] = tmp;
    }
}
#endif

#define REAL float

#if GRANULARITY == 1

__kernel void gemm_batch(__global const REAL* A, __global const REAL* B, __global const REAL* BT, __global REAL* C, int n) {
	int matrix = get_global_id(0);
	int tid = get_local_id(0);
/* preload data */
#if CACHING_STRATEGY == 1
    int preloadStartA = get_group_id(0)*GROUP_SIZE_X*SIZE_A*SIZE_B;
    int preloadStartB = get_group_id(0)*GROUP_SIZE_X*SIZE_C*SIZE_A;
	__local REAL bufA[GROUP_SIZE_X*SIZE_A*SIZE_B];
	__local REAL bufB[GROUP_SIZE_X*SIZE_C*SIZE_A];
	for (int i = 0; i < SIZE_A*SIZE_B; i++)
		bufA[i*GROUP_SIZE_X+tid] = A[preloadStartA + i*GROUP_SIZE_X + tid];
	for (int i = 0; i < SIZE_C*SIZE_A; i++)
    #if TRANSPOSED_B == 0
        bufB[i*GROUP_SIZE_X+tid] = B[preloadStartB + i*GROUP_SIZE_X + tid];
    #else
        bufB[i*GROUP_SIZE_X+tid] = BT[preloadStartB + i*GROUP_SIZE_X + tid];
    #endif
	barrier(CLK_LOCAL_MEM_FENCE);
#elif CACHING_STRATEGY == 2
    int preloadStartA = matrix*SIZE_A*SIZE_B;
    int preloadStartB = matrix*SIZE_C*SIZE_A;
    REAL bufA[SIZE_A*SIZE_B];
    REAL bufB[SIZE_C*SIZE_A];
    for (int i = 0; i < SIZE_A*SIZE_B; i++)
        bufA[i] = A[preloadStartA + i];
    for (int i = 0; i < SIZE_C*SIZE_A; i++)
    #if TRANSPOSED_B == 0
        bufB[i] = B[preloadStartB + i];
    #else
        bufB[i] = BT[preloadStartB + i];
    #endif
#endif

/* offsets into memory */
#if CACHING_STRATEGY == 0
    int startA = matrix*SIZE_A*SIZE_B;
    int startB = matrix*SIZE_C*SIZE_A;
#else
    int startA = tid*SIZE_A*SIZE_B;
    int startB = tid*SIZE_C*SIZE_A;
#endif
    int startC = matrix*SIZE_C*SIZE_B;

/* compute multiplication */
    for (int i = 0; i < SIZE_B; i++)
        for (int j = 0; j < SIZE_C; j++) {
            REAL tmp = (REAL)0.0;
            for (int k = 0; k < SIZE_A; k++)
#if CACHING_STRATEGY == 0
    #if TRANSPOSED_B == 0
                tmp += A[startA + i*SIZE_A + k] * B[startB + k*SIZE_C + j];
    #else
                tmp += A[startA + i*SIZE_A + k] * BT[startB + j*SIZE_A + k];
    #endif
#endif
#if CACHING_STRATEGY == 1
    #if TRANSPOSED_B == 0
                tmp += bufA[startA + i*SIZE_A + k] * bufB[startB + k*SIZE_C + j];
    #else
                tmp += bufA[startA + i*SIZE_A + k] * bufB[startB + j*SIZE_A + k];
    #endif
#endif
#if CACHING_STRATEGY == 2
    #if TRANSPOSED_B == 0
                tmp += bufA[i*SIZE_A + k] * bufB[k*SIZE_C + j];
    #else
                tmp += bufA[i*SIZE_A + k] * bufB[j*SIZE_A + k];
    #endif
#endif
            C[startC + i*SIZE_C + j] = tmp;
        }
}

#endif

#if GRANULARITY == 2
__kernel void gemm_batch(__global const REAL* A, __global const REAL* B, __global const REAL* BT, __global REAL* C, int n) {
    int matrix = get_group_id(0)*MG_GROUP_SIZE_Y + get_local_id(1);
    int tx = get_local_id(0);
    int ty = get_local_id(1);

/* preload data */
#if CACHING_STRATEGY == 1
    int preloadStartA = get_group_id(0)*MG_GROUP_SIZE_Y*SIZE_A*SIZE_B;
    int preloadStartB = get_group_id(0)*MG_GROUP_SIZE_Y*SIZE_C*SIZE_A;
    int myOffset = ty*SIZE_C + tx;
    __local REAL bufA[MG_GROUP_SIZE_Y*SIZE_A*SIZE_B];
    __local REAL bufB[MG_GROUP_SIZE_Y*SIZE_C*SIZE_A];
    for (int i = myOffset; i < SIZE_A*SIZE_B*MG_GROUP_SIZE_Y; i+= SIZE_C*MG_GROUP_SIZE_Y)
        bufA[i] = A[preloadStartA + i];
     for (int i = myOffset; i < SIZE_C*SIZE_A*MG_GROUP_SIZE_Y; i+= SIZE_C*MG_GROUP_SIZE_Y)
    #if TRANSPOSED_B == 0
        bufB[i] = B[preloadStartB + i];
    #else
        bufB[i] = BT[preloadStartB + i];
    #endif
    barrier(CLK_LOCAL_MEM_FENCE);
#endif

/* offsets into memory */
#if CACHING_STRATEGY == 0
    int startA = matrix*SIZE_A*SIZE_B;
    int startB = matrix*SIZE_C*SIZE_A;
#else
    int startA = ty*SIZE_A*SIZE_B;
    int startB = ty*SIZE_C*SIZE_A;
#endif
    int startC = matrix*SIZE_C*SIZE_B;

/* compute multiplication */
    for (int i = 0; i < SIZE_B; i++) {
        REAL tmp = (REAL)0.0;
        for (int k = 0; k < SIZE_A; k++)
#if CACHING_STRATEGY == 0
    #if TRANSPOSED_B == 0
            tmp += A[startA + i*SIZE_A + k] * B[startB + k*SIZE_C + tx];
    #else
            tmp += A[startA + i*SIZE_A + k] * BT[startB + tx*SIZE_A + k];
    #endif
#endif
#if CACHING_STRATEGY == 1
    #if TRANSPOSED_B == 0
            tmp += bufA[startA + i*SIZE_A + k] * bufB[startB + k*SIZE_C + tx];
    #else
            tmp += bufA[startA + i*SIZE_A + k] * bufB[startB + tx*SIZE_A + k];
    #endif
#endif
        C[startC + i*SIZE_C + tx] = tmp;
    }

}
#endif


fixed low depth problems

fixed low depth problems

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#define DBG_OUT 0

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LEFT_PAD0 (((MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / MLO_READ_UNIT) * MLO_READ_UNIT)
#define MLO_LRN_RIGHT_SIDE                                                               \
    (((MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / \
      MLO_READ_UNIT) *                                                                   \
     MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_LEFT_PAD0 + MLO_LRN_RIGHT_SIDE)
#define MLO_LCL_READ4 (MLO_LRN_LCL_DATA_WIDTH / MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

static inline int iDiv(int v, int d)
{
    int r = (int)((float)v / d + 0.00001f);
    return (r);
}

static inline int iMod(int v, int u, int d)
{
    int r = v - mul24((int)u, (int)d);
    return (r);
}
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannel_PS(const __global _FLOAT* bot,
                          __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                          __global _FLOAT* scale,
#endif
                          _FLOAT alphaoverarea,
                          _FLOAT alpha,
                          _FLOAT beta,
                          _FLOAT K)
{

    (void)alpha;
    // IT's taken from POOLING AVE with stride = 1'
    __local _FLOAT bot_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x       = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y       = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0 = get_local_id(0);
    int lcl_id1 = get_local_id(1);
    int ob      = get_global_id(2); // output * batch_sz
    int o       = iDiv(ob, MLO_LRN_BATCH_SZ);
    int b       = iMod(ob, o, MLO_LRN_BATCH_SZ);
    int bot_x   = x;
    int bot_y   = y;
    int bot_off =
        mul24(b, (int)MLO_LRN_BOT_BATCH_STRIDE) + mul24(o, (int)MLO_LRN_BOT_CHANNEL_STRIDE);

    // load tile
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int bot_y_act = bot_y + b_j - MLO_LRN_PAD1;

        bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_LRN_BOT_HEIGHT);

        int bot_y_off = mul24(bot_y_act, (int)MLO_LRN_BOT_STRIDE);

        int lcl_off_v = mul24(b_j, (int)MLO_LRN_LCL_DATA_WIDTH);

        for(int b_i = lcl_id0; b_i < MLO_LCL_READ4; b_i += MLO_LRN_GROUP_SZ0)
        {

            int bot_x_act = bot_x + (b_i * MLO_READ_UNIT) - MLO_LRN_LEFT_PAD0;

            bool invisibleX;
            for(int i = 0; i < MLO_READ_UNIT; ++i)
            {

                int bot_off_x = bot_off + bot_y_off + bot_x_act + i;

                invisibleX = (bot_x_act + i < 0) || (bot_x_act + i >= MLO_LRN_BOT_WIDTH);

                bot_off_x = (invisibleX || invisibleY) ? 0 : bot_off_x;

                _FLOAT bot_val = bot[bot_off_x];
                // since we need a sum of squares for the normalization
                // we do the square at the input
                bot_val *= bot_val;

                bot_val = (invisibleX || invisibleY) ? 0 : bot_val;

                bot_data[lcl_off_v + (b_i * MLO_READ_UNIT) + i] = bot_val;
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
    _FLOAT partial_sum_x[MLO_LRN_N_HORIZ_OUT_PIX - 1]; // horizontal partial sum
#endif
#if MLO_LRN_N_VERT_OUT_PIX > 1
    _FLOAT partial_sum_xy[MLO_LRN_N_VERT_OUT_PIX - 1]
                         [MLO_LRN_N_HORIZ_OUT_PIX]; // horizontal-vertical partial sums.
#endif
    _FLOAT accum[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; // accumulator

    int top_y = mad24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX, y);
    int top_x = mad24(lcl_id0, (int)MLO_LRN_N_HORIZ_OUT_PIX, x);

    int lcl_y = mul24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX);
    int lcl_x =
        mad24(lcl_id0, (int)(MLO_LRN_N_HORIZ_OUT_PIX), (int)(MLO_LRN_LEFT_PAD0 - MLO_LRN_PAD0));
    int lcl_off = mad24(lcl_y, MLO_LRN_LCL_DATA_WIDTH, lcl_x);

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[j][i] = 0;
        }
    }
#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX - 1; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            partial_sum_xy[j][i] = 0;
        }
    }
#endif

    // running window  summation
    _FLOAT mov_accum;
    int jj = 0;
    int ii = 0;

// first to get vertica partial sums

#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(; jj < (int)(MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            // save horizontal partial sums
            partial_sum_x[ii] = accum_tmp;
#endif
            // accumulate in vert-horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            // accumulate in vert horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            // calculate all vertical-horizontal partial sums
            partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0] +
                (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                 -
                 partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                 );
        }

        // put into accumulator[0][i]
        // whatever has been accumulated so far
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[0][i] += partial_sum_xy[jj][i];
        }
    }
#endif

    // calculate row 0 accumulators
    for(; jj < (int)MLO_LRN_KERNEL_SZ1; ++jj)
    {
        mov_accum = 0;

        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            mov_accum += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            mov_accum += accum_tmp;
        }

        accum[0][0] += mov_accum;
        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            // running horizontal window
            mov_accum += (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                          -
                          partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                          );
            accum[0][ii - MLO_LRN_KERNEL_SZ0 + 1] += mov_accum;
        }
    }

    // accumulate all other rows besides 0
    for(; jj < (int)(MLO_LRN_KERNEL_SZ1 + MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        // first running horizontal winodw as before
        mov_accum = 0;
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        // running horizontal window

        int ii1 = ii;
        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            //
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0] + accum_tmp;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_x[ii - MLO_LRN_KERNEL_SZ0];
#endif
        }

        // finally running vertical window

        for(ii = ii1; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            // finish horizontal summation
            // add/substarct vertical patial sum
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] +=
                accum[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#if MLO_LRN_N_VERT_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#endif
        }
#if MLO_LRN_N_VERT_OUT_PIX > 1
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] -= partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][0];
#endif
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum[jj - MLO_LRN_KERNEL_SZ1][0];
    }

    // normalization
    _FLOAT prv_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    _FLOAT adj_alphaoverarea = alphaoverarea;
    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX; k++)
    {

        //			int hstart = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX  + k -
        // MLO_LRN_PAD1;
        //			int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_HEIGHT +
        // MLO_LRN_PAD1);

        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX; l++)
        {

            //				int wstart = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + l -
            // MLO_LRN_PAD0;
            //				int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_WIDTH
            //+
            // MLO_LRN_PAD0);
            //				int adj_area_size = (hend - hstart) * (wend - wstart);
            //				adj_alphaoverarea = alpha / adj_area_size;

            prv_scale[k][l] = K + accum[k][l] * adj_alphaoverarea;
        }
    }

    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  top_y * MLO_LRN_TOP_STRIDE + top_x;
#if MLO_LRN_DO_SCALE
    int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    top_y * MLO_LRN_SCALE_STRIDE + top_x;
#endif

    // final output

    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX
#if MLO_OUT_VERT_ALIGNED == 0
                   &&
                   (top_y + k < MLO_LRN_TOP_HEIGHT)
#endif
            ;
        k++)
    {
        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX
#if MLO_OUT_HORIZ_ALIGNED == 0
                       &&
                       (top_x + l < MLO_LRN_TOP_WIDTH)
#endif
                ;
            l++)
        {
            _FLOAT s;
            _FLOAT bot_val;
            s = native_exp((_FLOAT)-beta * native_log(prv_scale[k][l]));
            //					s = pow(prv_scale[k][l], -beta);
            _FLOAT tmp = bot_data[lcl_off + mad24((k + MLO_LRN_PAD1),
                                                  (int)MLO_LRN_LCL_DATA_WIDTH,
                                                  (l + MLO_LRN_PAD0))];

            // do a square root to get back to the raw input
            bot_val = native_sqrt(tmp);
#if MLO_LRN_DO_SCALE
            scale[scale_off + k * MLO_LRN_SCALE_STRIDE + l] = prv_scale[k][l];
#endif
            top[top_off + k * MLO_LRN_TOP_STRIDE + l] = bot_val * s;
        }
    }
}

#if(MLO_LRN_N_INPUTS < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannels4(const __global _FLOAT* bottom,
                         __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                         __global _FLOAT* scale,
#endif
                         _FLOAT alphaoverarea,
                         _FLOAT alpha,
                         _FLOAT beta,
                         _FLOAT K)
{

    (void)alpha;
    int pix_id          = get_global_id(0); //
    int b               = get_global_id(2); // batch
    MLO_READ_TYPE accum = 0;
    MLO_READ_TYPE bot_in2[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    for(int i = 0; i < MLO_LRN_KERNEL_SZ; ++i)
    {
        bot_in2[i] = 0;
    }

    int top_off   = 0;
    int scale_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD
            ;
        c_i++)
    {
        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_LOW_CHNL_COUNT == 1
        if (c_i < MLO_LRN_N_INPUTS)
#endif
		{
#if MLO_C1x1_PIXLEFT > 0
        // if the last one
        if(pix_id == MLO_MAP_SZ4 - 1)
        {

            for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
            {
                ((_FLOAT*)&prv_in)[j] =
                    bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                           (pix_id * MLO_READ_UNIT) + j];
            }
        }
        else
#endif
        {
            prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
        }
		}

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];
        //				fma(bot_in[c_i + MLO_LRN_PAD], bot_in[c_i + MLO_LRN_PAD],
        // accum);
    }

    for(; c_i < MLO_LRN_KERNEL_SZ
            ;
        c_i++, c_o++)
    {
        MLO_READ_TYPE prv_in;
		prv_in = 0;

#if MLO_LOW_CHNL_COUNT == 1
       if (c_i < MLO_LRN_N_INPUTS)
#endif
		{

#if MLO_C1x1_PIXLEFT > 0
        // if the last one
			if(pix_id == MLO_MAP_SZ4 - 1)
			{

				for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
				{
					((_FLOAT*)&prv_in)[j] =
						bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
							   (pix_id * MLO_READ_UNIT) + j];
				}
			}
			else
#endif
		   {
			    prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
			}

		}

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, 1.f);

        MLO_READ_TYPE exp_scale = native_exp((MLO_READ_TYPE)-beta * native_log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //	MLO_READ_TYPE prv_out = native_sqrt(bot_in2[c_o]);
        MLO_READ_TYPE prv_out = bot_in2[c_o];
        prv_out               = native_sqrt(prv_out);
        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
					printf("K:o0: %d %f %f %f %f %f\n",
					top_off + j,
					top[top_off + j],
					((_FLOAT*)&prv_out)[j],
					((_FLOAT*)&exp_scale)[j],
					((_FLOAT*)&prv_scale)[j],
					((_FLOAT*)&accum)[j]
					);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS
            ;
        c_i++, c_o++)
    {

        MLO_READ_TYPE prv_in;
        prv_in = 0;

#if MLO_C1x1_PIXLEFT > 0
        // if the last one
        if(pix_id == MLO_MAP_SZ4 - 1)
        {

            for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
            {
                ((_FLOAT*)&prv_in)[j] =
                    bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                           (pix_id * MLO_READ_UNIT) + j];
            }
        }
        else
#endif
        {
            prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
        }

        MLO_READ_TYPE prv_bot_in2 = prv_in * prv_in;
        accum                     = accum + prv_bot_in2;

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        bot_in2[MLO_LRN_KERNEL_SZ - 1] = prv_bot_in2;

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, 1.f);

        MLO_READ_TYPE exp_scale = native_exp((MLO_READ_TYPE)-beta * native_log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = native_sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = native_sqrt(prv_out);
        MLO_READ_TYPE out_val = prv_out * exp_scale;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
					printf("K:o1: %d %f %f %f\n",
					top_off + j,
					top[top_off + j],
					((_FLOAT*)&prv_out)[j],
					((_FLOAT*)&exp_scale)[j]
					);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, 1.f);

        MLO_READ_TYPE exp_scale = native_exp((MLO_READ_TYPE)-beta * native_log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = native_sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = native_sqrt(prv_out);

        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if DBG_OUT
					printf("K:o2: %d %f %f %f\n",
					top_off + j,
					top[top_off + j],
					((_FLOAT*)&prv_out)[j],
					((_FLOAT*)&exp_scale)[j]
					);
#endif

#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#define MLO_LRN_GROUP_SZ2 1
#define MLO_LRN_STRIDE 1

#define MLO_LRN_LEFT_PAD0 (((MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / MLO_READ_UNIT) * MLO_READ_UNIT)
#define MLO_LRN_RIGHT_SIDE                                                               \
    (((MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX + MLO_LRN_PAD0 + MLO_READ_UNIT - 1) / \
      MLO_READ_UNIT) *                                                                   \
     MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_WIDTH (MLO_LRN_LEFT_PAD0 + MLO_LRN_RIGHT_SIDE)
#define MLO_LCL_READ4 (MLO_LRN_LCL_DATA_WIDTH / MLO_READ_UNIT)
#define MLO_LRN_LCL_DATA_HEIGHT (MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX + MLO_LRN_KERNEL_SZ - 1)
#define MLO_LRN_GROUP_SZ (MLO_LRN_GROUP_SZ2 * MLO_LRN_GROUP_SZ1 * MLO_LRN_GROUP_SZ0)
//#define MLO_LRN_PREPAD_SZ (MLO_LRN_KERNEL_SZ - 1)/2

static inline int iDiv(int v, int d)
{
    int r = (int)((float)v / d + 0.00001f);
    return (r);
}

static inline int iMod(int v, int u, int d)
{
    int r = v - mul24((int)u, (int)d);
    return (r);
}
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNWithinChannel_PS(const __global _FLOAT* bot,
                          __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                          __global _FLOAT* scale,
#endif
                          _FLOAT alphaoverarea,
                          _FLOAT alpha,
                          _FLOAT beta,
                          _FLOAT K)
{

    (void)alpha;
    // IT's taken from POOLING AVE with stride = 1'
    __local _FLOAT bot_data[MLO_LRN_LCL_DATA_WIDTH * MLO_LRN_LCL_DATA_HEIGHT];
    int x       = get_group_id(0) * MLO_LRN_GROUP_SZ0 * MLO_LRN_N_HORIZ_OUT_PIX;
    int y       = get_group_id(1) * MLO_LRN_GROUP_SZ1 * MLO_LRN_N_VERT_OUT_PIX;
    int lcl_id0 = get_local_id(0);
    int lcl_id1 = get_local_id(1);
    int ob      = get_global_id(2); // output * batch_sz
    int o       = iDiv(ob, MLO_LRN_BATCH_SZ);
    int b       = iMod(ob, o, MLO_LRN_BATCH_SZ);
    int bot_x   = x;
    int bot_y   = y;
    int bot_off =
        mul24(b, (int)MLO_LRN_BOT_BATCH_STRIDE) + mul24(o, (int)MLO_LRN_BOT_CHANNEL_STRIDE);

    // load tile
    for(int b_j = lcl_id1; b_j < MLO_LRN_LCL_DATA_HEIGHT; b_j += MLO_LRN_GROUP_SZ1)
    {
        int bot_y_act = bot_y + b_j - MLO_LRN_PAD1;

        bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_LRN_BOT_HEIGHT);

        int bot_y_off = mul24(bot_y_act, (int)MLO_LRN_BOT_STRIDE);

        int lcl_off_v = mul24(b_j, (int)MLO_LRN_LCL_DATA_WIDTH);

        for(int b_i = lcl_id0; b_i < MLO_LCL_READ4; b_i += MLO_LRN_GROUP_SZ0)
        {

            int bot_x_act = bot_x + (b_i * MLO_READ_UNIT) - MLO_LRN_LEFT_PAD0;

            bool invisibleX;
            for(int i = 0; i < MLO_READ_UNIT; ++i)
            {

                int bot_off_x = bot_off + bot_y_off + bot_x_act + i;

                invisibleX = (bot_x_act + i < 0) || (bot_x_act + i >= MLO_LRN_BOT_WIDTH);

                bot_off_x = (invisibleX || invisibleY) ? 0 : bot_off_x;

                _FLOAT bot_val = bot[bot_off_x];
                // since we need a sum of squares for the normalization
                // we do the square at the input
                bot_val *= bot_val;

                bot_val = (invisibleX || invisibleY) ? 0 : bot_val;

                bot_data[lcl_off_v + (b_i * MLO_READ_UNIT) + i] = bot_val;
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
    _FLOAT partial_sum_x[MLO_LRN_N_HORIZ_OUT_PIX - 1]; // horizontal partial sum
#endif
#if MLO_LRN_N_VERT_OUT_PIX > 1
    _FLOAT partial_sum_xy[MLO_LRN_N_VERT_OUT_PIX - 1]
                         [MLO_LRN_N_HORIZ_OUT_PIX]; // horizontal-vertical partial sums.
#endif
    _FLOAT accum[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX]; // accumulator

    int top_y = mad24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX, y);
    int top_x = mad24(lcl_id0, (int)MLO_LRN_N_HORIZ_OUT_PIX, x);

    int lcl_y = mul24(lcl_id1, (int)MLO_LRN_N_VERT_OUT_PIX);
    int lcl_x =
        mad24(lcl_id0, (int)(MLO_LRN_N_HORIZ_OUT_PIX), (int)(MLO_LRN_LEFT_PAD0 - MLO_LRN_PAD0));
    int lcl_off = mad24(lcl_y, MLO_LRN_LCL_DATA_WIDTH, lcl_x);

    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[j][i] = 0;
        }
    }
#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(int j = 0; j < MLO_LRN_N_VERT_OUT_PIX - 1; ++j)
    {
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            partial_sum_xy[j][i] = 0;
        }
    }
#endif

    // running window  summation
    _FLOAT mov_accum;
    int jj = 0;
    int ii = 0;

// first to get vertica partial sums

#if MLO_LRN_N_VERT_OUT_PIX > 1
    for(; jj < (int)(MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            // save horizontal partial sums
            partial_sum_x[ii] = accum_tmp;
#endif
            // accumulate in vert-horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            // accumulate in vert horizontal(0)
            partial_sum_xy[jj][0] += accum_tmp;
        }

        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            // calculate all vertical-horizontal partial sums
            partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                partial_sum_xy[jj][ii - MLO_LRN_KERNEL_SZ0] +
                (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                 -
                 partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                 );
        }

        // put into accumulator[0][i]
        // whatever has been accumulated so far
        for(int i = 0; i < MLO_LRN_N_HORIZ_OUT_PIX; ++i)
        {
            accum[0][i] += partial_sum_xy[jj][i];
        }
    }
#endif

    // calculate row 0 accumulators
    for(; jj < (int)MLO_LRN_KERNEL_SZ1; ++jj)
    {
        mov_accum = 0;

        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            mov_accum += accum_tmp;
        }

        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            mov_accum += accum_tmp;
        }

        accum[0][0] += mov_accum;
        // running horizontal window

        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            // running horizontal window
            mov_accum += (accum_tmp
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
                          -
                          partial_sum_x[ii - MLO_LRN_KERNEL_SZ0]
#endif
                          );
            accum[0][ii - MLO_LRN_KERNEL_SZ0 + 1] += mov_accum;
        }
    }

    // accumulate all other rows besides 0
    for(; jj < (int)(MLO_LRN_KERNEL_SZ1 + MLO_LRN_N_VERT_OUT_PIX - 1); ++jj)
    {
        // first running horizontal winodw as before
        mov_accum = 0;
        for(ii = 0; ii < (int)(MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            partial_sum_x[ii] = accum_tmp;
#endif
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        for(; ii < (int)MLO_LRN_KERNEL_SZ0; ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum_tmp;
        }
        // running horizontal window

        int ii1 = ii;
        for(; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            _FLOAT bot_val   = bot_data[lcl_off + jj * MLO_LRN_LCL_DATA_WIDTH + ii];
            _FLOAT accum_tmp = bot_val;
            //
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] =
                accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0] + accum_tmp;
#if MLO_LRN_N_HORIZ_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_x[ii - MLO_LRN_KERNEL_SZ0];
#endif
        }

        // finally running vertical window

        for(ii = ii1; ii < (int)(MLO_LRN_KERNEL_SZ0 + MLO_LRN_N_HORIZ_OUT_PIX - 1); ++ii)
        {

            // finish horizontal summation
            // add/substarct vertical patial sum
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] +=
                accum[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#if MLO_LRN_N_VERT_OUT_PIX > 1
            accum[jj - MLO_LRN_KERNEL_SZ1 + 1][ii - MLO_LRN_KERNEL_SZ0 + 1] -=
                partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][ii - MLO_LRN_KERNEL_SZ0 + 1];
#endif
        }
#if MLO_LRN_N_VERT_OUT_PIX > 1
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] -= partial_sum_xy[jj - MLO_LRN_KERNEL_SZ1][0];
#endif
        accum[jj - MLO_LRN_KERNEL_SZ1 + 1][0] += accum[jj - MLO_LRN_KERNEL_SZ1][0];
    }

    // normalization
    _FLOAT prv_scale[MLO_LRN_N_VERT_OUT_PIX][MLO_LRN_N_HORIZ_OUT_PIX];
    _FLOAT adj_alphaoverarea = alphaoverarea;
    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX; k++)
    {

        //			int hstart = y + lcl_id1 * MLO_LRN_N_VERT_OUT_PIX  + k -
        // MLO_LRN_PAD1;
        //			int hend = min(hstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_HEIGHT +
        // MLO_LRN_PAD1);

        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX; l++)
        {

            //				int wstart = x + lcl_id0 * MLO_LRN_N_HORIZ_OUT_PIX + l -
            // MLO_LRN_PAD0;
            //				int wend = min(wstart + MLO_LRN_KERNEL_SZ, MLO_LRN_BOT_WIDTH
            //+
            // MLO_LRN_PAD0);
            //				int adj_area_size = (hend - hstart) * (wend - wstart);
            //				adj_alphaoverarea = alpha / adj_area_size;

            prv_scale[k][l] = K + accum[k][l] * adj_alphaoverarea;
        }
    }

    int top_off = b * MLO_LRN_TOP_BATCH_STRIDE + o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  top_y * MLO_LRN_TOP_STRIDE + top_x;
#if MLO_LRN_DO_SCALE
    int scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    top_y * MLO_LRN_SCALE_STRIDE + top_x;
#endif

    // final output

    for(int k = 0; k < MLO_LRN_N_VERT_OUT_PIX
#if MLO_OUT_VERT_ALIGNED == 0
                   &&
                   (top_y + k < MLO_LRN_TOP_HEIGHT)
#endif
            ;
        k++)
    {
        for(int l = 0; l < MLO_LRN_N_HORIZ_OUT_PIX
#if MLO_OUT_HORIZ_ALIGNED == 0
                       &&
                       (top_x + l < MLO_LRN_TOP_WIDTH)
#endif
                ;
            l++)
        {
            _FLOAT s;
            _FLOAT bot_val;
            s = native_exp((_FLOAT)-beta * native_log(prv_scale[k][l]));
            //					s = pow(prv_scale[k][l], -beta);
            _FLOAT tmp = bot_data[lcl_off + mad24((k + MLO_LRN_PAD1),
                                                  (int)MLO_LRN_LCL_DATA_WIDTH,
                                                  (l + MLO_LRN_PAD0))];

            // do a square root to get back to the raw input
            bot_val = native_sqrt(tmp);
#if MLO_LRN_DO_SCALE
            scale[scale_off + k * MLO_LRN_SCALE_STRIDE + l] = prv_scale[k][l];
#endif
            top[top_off + k * MLO_LRN_TOP_STRIDE + l] = bot_val * s;
        }
    }
}

#if(MLO_LRN_N_INPUTS + 2 * MLO_LRN_PAD - 1 < MLO_LRN_KERNEL_SZ)
#define MLO_LOW_CHNL_COUNT 1
#else
#define MLO_LOW_CHNL_COUNT 0
#endif
__attribute__((reqd_work_group_size(MLO_LRN_GROUP_SZ0, MLO_LRN_GROUP_SZ1, MLO_LRN_GROUP_SZ2)))
__kernel void
MIOpenLRNAcrossChannels4(const __global _FLOAT* bottom,
                         __global _FLOAT* top,
#if MLO_LRN_DO_SCALE
                         __global _FLOAT* scale,
#endif
                         _FLOAT alphaoverarea,
                         _FLOAT alpha,
                         _FLOAT beta,
                         _FLOAT K)
{

    (void)alpha;
    int pix_id          = get_global_id(0); //
    int b               = get_global_id(2); // batch
    MLO_READ_TYPE accum = 0;
    MLO_READ_TYPE bot_in2[MLO_LRN_KERNEL_SZ];
    int c_i = 0, c_o = 0;
    for(int i = 0; i < MLO_LRN_KERNEL_SZ; ++i)
    {
        bot_in2[i] = 0;
    }

    int top_off   = 0;
    int scale_off = 0;

    for(c_i = 0; c_i < MLO_LRN_PAD
#if MLO_LOW_CHNL_COUNT == 1
                 &&
                 (c_i < MLO_LRN_N_INPUTS)
#endif
            ;
        c_i++)
    {

        MLO_READ_TYPE prv_in;
#if MLO_C1x1_PIXLEFT > 0
        // if the last one
        if(pix_id == MLO_MAP_SZ4 - 1)
        {
            prv_in = 0;

            for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
            {
                ((_FLOAT*)&prv_in)[j] =
                    bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                           (pix_id * MLO_READ_UNIT) + j];
            }
        }
        else
#endif
        {
            prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
        }

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];
        //				fma(bot_in[c_i + MLO_LRN_PAD], bot_in[c_i + MLO_LRN_PAD],
        // accum);
    }

    for(; c_i < MLO_LRN_KERNEL_SZ
#if MLO_LOW_CHNL_COUNT == 1
          &&
          (c_i < MLO_LRN_N_INPUTS)
#endif
            ;
        c_i++, c_o++)
    {
        MLO_READ_TYPE prv_in;
#if MLO_C1x1_PIXLEFT > 0
        // if the last one
        if(pix_id == MLO_MAP_SZ4 - 1)
        {
            prv_in = 0;

            for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
            {
                ((_FLOAT*)&prv_in)[j] =
                    bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                           (pix_id * MLO_READ_UNIT) + j];
            }
        }
        else
#endif
        {
            prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
        }

        bot_in2[c_i] = prv_in * prv_in;
        accum        = accum + bot_in2[c_i];

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, 1.f);

        MLO_READ_TYPE exp_scale = native_exp((MLO_READ_TYPE)-beta * native_log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //	MLO_READ_TYPE prv_out = native_sqrt(bot_in2[c_o]);
        MLO_READ_TYPE prv_out = bot_in2[c_o];
        prv_out               = native_sqrt(prv_out);
        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS
            ;
        c_i++, c_o++)
    {

        MLO_READ_TYPE prv_in;
#if MLO_C1x1_PIXLEFT > 0
        // if the last one
        if(pix_id == MLO_MAP_SZ4 - 1)
        {
            prv_in = 0;

            for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
            {
                ((_FLOAT*)&prv_in)[j] =
                    bottom[MLO_LRN_BOT_BATCH_STRIDE * b + MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                           (pix_id * MLO_READ_UNIT) + j];
            }
        }
        else
#endif
        {
            prv_in = *(__global MLO_READ_TYPE*)&bottom[MLO_LRN_BOT_BATCH_STRIDE * b +
                                                       MLO_LRN_BOT_CHANNEL_STRIDE * c_i +
                                                       (pix_id * MLO_READ_UNIT)];
        }

        MLO_READ_TYPE prv_bot_in2 = prv_in * prv_in;
        accum                     = accum + prv_bot_in2;

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        bot_in2[MLO_LRN_KERNEL_SZ - 1] = prv_bot_in2;

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, 1.f);

        MLO_READ_TYPE exp_scale = native_exp((MLO_READ_TYPE)-beta * native_log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = native_sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = native_sqrt(prv_out);
        MLO_READ_TYPE out_val = prv_out * exp_scale;

#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }

    for(; c_i < MLO_LRN_N_INPUTS + MLO_LRN_PAD; c_i++, c_o++)
    {

        accum = accum - bot_in2[0];
        //				fma(-bot_in[0], bot_in[0], accum);

        for(int i = 0; i < MLO_LRN_KERNEL_SZ - 1; i++)
        {
            bot_in2[i] = bot_in2[i + 1];
        }

        top_off = b * MLO_LRN_TOP_BATCH_STRIDE + c_o * MLO_LRN_TOP_CHANNEL_STRIDE +
                  (pix_id * MLO_READ_UNIT);
        scale_off = b * MLO_LRN_SCALE_BATCH_STRIDE + c_o * MLO_LRN_SCALE_CHANNEL_STRIDE +
                    (pix_id * MLO_READ_UNIT);
        MLO_READ_TYPE prv_scale = ((MLO_READ_TYPE)K + accum * (MLO_READ_TYPE)alphaoverarea);
        //				fma(accum,alphaoverarea, 1.f);

        MLO_READ_TYPE exp_scale = native_exp((MLO_READ_TYPE)-beta * native_log(prv_scale));
        //				pow(prv_scale,-beta);
        // bug
        //			MLO_READ_TYPE prv_out = native_sqrt(bot_in2[MLO_LRN_PAD]);
        MLO_READ_TYPE prv_out = bot_in2[MLO_LRN_PAD];
        prv_out               = native_sqrt(prv_out);

        MLO_READ_TYPE out_val = prv_out * exp_scale;
#if MLO_LOW_CHNL_COUNT == 1
        if(c_o < MLO_LRN_N_OUTPUTS)
#endif
        {

#if MLO_C1x1_PIXLEFT > 0

            // if the last one
            if(pix_id == MLO_MAP_SZ4 - 1)
            {
                for(int j = 0; j < MLO_C1x1_PIXLEFT; ++j)
                {
                    top[top_off + j] = ((_FLOAT*)&out_val)[j];
#if MLO_LRN_DO_SCALE
                    scale[scale_off + j] = ((_FLOAT*)&prv_scale)[j];
#endif
                }
            }
            else
#endif
            {

                *((__global MLO_READ_TYPE*)&top[top_off]) = out_val;
#if MLO_LRN_DO_SCALE
                *((__global MLO_READ_TYPE*)&scale[scale_off]) = prv_scale;
#endif
            }
        }
    }
}


r2043: Debian build

r2043: Debian build

;;;; -*- Mode: LISP; Syntax: ANSI-Common-Lisp; Base: 10 -*-
;;;; *************************************************************************
;;;; FILE IDENTIFICATION
;;;;
;;;; Name:          mysql-loader.sql
;;;; Purpose:       MySQL library loader using UFFI
;;;; Programmers:   Kevin M. Rosenberg
;;;; Date Started:  Feb 2002
;;;;
;;;; $Id: mysql-loader.cl,v 1.11 2002/05/14 16:16:54 kevin Exp $
;;;;
;;;; This file, part of CLSQL, is Copyright (c) 2002 by Kevin M. Rosenberg
;;;;
;;;; CLSQL users are granted the rights to distribute and use this software
;;;; as governed by the terms of the Lisp Lesser GNU Public License
;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL.
;;;; *************************************************************************

(declaim (optimize (debug 3) (speed 3) (safety 1) (compilation-speed 0)))
(in-package :mysql)

;;;; Modified by Kevin Rosenberg
;;;;  - probe potential directories to find library
;;;;  - Changed from CMUCL functions to UFFI to
;;;;      -- prevent library from being loaded multiple times
;;;;      -- support Allegro CL and Lispworks

(defvar *clsql-mysql-library-filename*
  (uffi:find-foreign-library
   "clsql-mysql"
   `(,(translate-logical-pathname
       "CL-LIBRARY:clsql;interfaces;clsql-uffi;")
     "/usr/share/common-lisp/source/clsql/interfaces/clsql-uffi/")))

(defvar *mysql-library-filename*
    (cond
     ((probe-file "/opt/mysql/lib/mysql/libmysqlclient.so")
      "/opt/mysql/lib/mysql/libmysqlclient.so")
     ((probe-file "/usr/local/lib/libmysqlclient.so")
      "/usr/local/lib/libmysqlclient.so")
     ((probe-file "/usr/lib/libmysqlclient.so")
      "/usr/lib/libmysqlclient.so")
     #+(or win32 mswindows)
     ((probe-file "c:/mysql/lib/opt/libmysql.dll")
      "c:/mysql/lib/opt/libmysql.dll")
     (t
      (warn "Can't find MySQL client library to load.")))
  "Location where the MySQL client library is to be found.")

(defvar *mysql-library-candidate-names*
    '("libmysqlclient" "libmysql"))

(defvar *mysql-library-candidate-directories*
    '("/opt/mysql/lib/mysql/" "/usr/local/lib/" "/usr/lib/" "/mysql/lib/opt/"))

(defvar *mysql-library-candidate-drive-letters* '("C" "D" "E"))

(defvar *mysql-supporting-libraries* '("c")
  "Used only by CMU. List of library flags needed to be passed to ld to
load the MySQL client library succesfully.  If this differs at your site,
set to the right path before compiling or loading the system.")

(defvar *mysql-library-loaded* nil
  "T if foreign library was able to be loaded successfully")

(defmethod clsql-sys:database-type-library-loaded ((database-type (eql :mysql)))
  *mysql-library-loaded*)

(defmethod clsql-sys:database-type-load-foreign ((database-type (eql :mysql)))
  (let ((mysql-path
	 (uffi:find-foreign-library *mysql-library-candidate-names*
				    *mysql-library-candidate-directories*
				    :drive-letters
				    *mysql-library-candidate-drive-letters*)))
    ;; zlib required to load mysql on CMUCL Solaris
    (uffi:load-foreign-library
     (uffi:find-foreign-library '("libz" "zlib")
				'("/usr/lib/" "/usr/local/" "/lib/")))
    (if	(and
	 (uffi:load-foreign-library mysql-path
				    :module "mysql"
				    :supporting-libraries
				    *mysql-supporting-libraries*)
	 (uffi:load-foreign-library *clsql-mysql-library-filename*
				    :module "clsql-mysql"
				    :supporting-libraries
				    (append *mysql-supporting-libraries*)))
	(setq *mysql-library-loaded* t)
      (warn "Unable to load MySQL client library ~A or CLSQL-MySQL library ~A"
	    mysql-path *clsql-mysql-library-filename*))))

(clsql-sys:database-type-load-foreign :mysql)
(when (clsql-sys:database-type-library-loaded :mysql)
  (clsql-sys:initialize-database-type :database-type :mysql)
  (setq clsql:*default-database-type* :mysql)
  (pushnew :mysql cl:*features*))

;;;; -*- Mode: LISP; Syntax: ANSI-Common-Lisp; Base: 10 -*-
;;;; *************************************************************************
;;;; FILE IDENTIFICATION
;;;;
;;;; Name:          mysql-loader.sql
;;;; Purpose:       MySQL library loader using UFFI
;;;; Programmers:   Kevin M. Rosenberg
;;;; Date Started:  Feb 2002
;;;;
;;;; $Id: mysql-loader.cl,v 1.10 2002/05/14 16:15:24 kevin Exp $
;;;;
;;;; This file, part of CLSQL, is Copyright (c) 2002 by Kevin M. Rosenberg
;;;;
;;;; CLSQL users are granted the rights to distribute and use this software
;;;; as governed by the terms of the Lisp Lesser GNU Public License
;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL.
;;;; *************************************************************************

(declaim (optimize (debug 3) (speed 3) (safety 1) (compilation-speed 0)))
(in-package :mysql)

;;;; Modified by Kevin Rosenberg
;;;;  - probe potential directories to find library
;;;;  - Changed from CMUCL functions to UFFI to
;;;;      -- prevent library from being loaded multiple times
;;;;      -- support Allegro CL and Lispworks

(defvar *clsql-mysql-library-filename*
  (uffi:find-foreign-library
   "clsql-mysql"
   `(,(translate-logical-pathname
       "CL-LIBRARY:clsql;interfaces;clsql-uffi;")
     "/usr/share/common-lisp/source/clsql/interfaces/clsql-uffi/")))

(defvar *mysql-library-filename*
    (cond
     ((probe-file "/opt/mysql/lib/mysql/libmysqlclient.so")
      "/opt/mysql/lib/mysql/libmysqlclient.so")
     ((probe-file "/usr/local/lib/libmysqlclient.so")
      "/usr/local/lib/libmysqlclient.so")
     ((probe-file "/usr/lib/libmysqlclient.so")
      "/usr/lib/libmysqlclient.so")
     #+(or win32 mswindows)
     ((probe-file "c:/mysql/lib/opt/libmysql.dll")
      "c:/mysql/lib/opt/libmysql.dll")
     (t
      (warn "Can't find MySQL client library to load.")))
  "Location where the MySQL client library is to be found.")

(defvar *mysql-library-candidate-names*
    '("libmysqlclient" "libmysql"))

(defvar *mysql-library-candidate-directories*
    '("/opt/mysql/lib/mysql/" "/usr/local/lib/" "/usr/lib/" "/mysql/lib/opt/"))

(defvar *mysql-library-candidate-drive-letters* '("C" "D" "E"))

(defvar *mysql-supporting-libraries* '("c")
  "Used only by CMU. List of library flags needed to be passed to ld to
load the MySQL client library succesfully.  If this differs at your site,
set to the right path before compiling or loading the system.")

(defvar *mysql-library-loaded* nil
  "T if foreign library was able to be loaded successfully")

(defmethod clsql-sys:database-type-library-loaded ((database-type (eql :mysql)))
  *mysql-library-loaded*)

(defmethod clsql-sys:database-type-load-foreign ((database-type (eql :mysql)))
  (let ((mysql-path
	 (uffi:find-foreign-library *mysql-library-candidate-names*
				    *mysql-library-candidate-directories*
				    :drive-letters
				    *mysql-library-candidate-drive-letters*)))
    ;; zlib required to load mysql on CMUCL Solaris
    (uffi:load-foreign-library
     (uffi:find-foreign-library '("libz" "zlib")
				'("/usr/lib/" "/usr/local/" "/lib/")))
    (if	(and
	 (uffi:load-foreign-library mysql-path
				    :module "mysql"
				    :supporting-libraries
				    *mysql-supporting-libraries*)
	 (uffi:load-foreign-library *clsql-mysql-library-filename*
				    :module "clsql-mysql"
				    :supporting-libraries
				    (append *mysql-supporting-libraries*)))
	(setq *mysql-library-loaded* t)
      (warn "Unable to load MySQL client library ~A or CLSQL-MySQL library ~A"
	    mysql-path clsql-mysql-library-filename))))

(clsql-sys:database-type-load-foreign :mysql)
(when (clsql-sys:database-type-library-loaded :mysql)
  (clsql-sys:initialize-database-type :database-type :mysql)
  (setq clsql:*default-database-type* :mysql)
  (pushnew :mysql cl:*features*))


Fixed bug that used `mv()` instead of `mv22()`.

// singular isothermal ellipsoid plus shear
// follows Schneider, Kochanek, Wambsganss (2006)

OBJECT(sie_plus_shear) = LENS;

PARAMS(sie_plus_shear) = {
    { "x" },
    { "y" },
    { "r" },
    { "q" },
    { "pa", true },
    { "g1" },
    { "g2" }
};

struct sie_plus_shear
{
    float2 x; // lens position
    mat22 m;  // rotation matrix for position angle
    mat22 w;  // inverse rotation matrix
    mat22 g;  // shear matrix

    // auxiliary
    float q2;
    float e;
    float d;
};

static float2 sie_plus_shear(constant struct sie_plus_shear* data, float2 x)
{
    float2 y;
    float r;

    // move to central coordinates
    x -= data->x;

    // rotate coordinates by position angle
    y = mv22(data->m, x);

    // SIE deflection
    r = data->e/sqrt(data->q2*y.x*y.x + y.y*y.y);
    y = data->d*(float2)(atan(y.x*r), atanh(y.y*r));

    // reverse coordinate rotation, apply shear
    return mv22(data->w, y) + mv22(data->g, x);
}

static void set_sie_plus_shear(global struct sie_plus_shear* data, float x1, float x2, float r, float q, float pa, float g1, float g2)
{
    float c = cos(pa*DEG2RAD);
    float s = sin(pa*DEG2RAD);

    // lens position
    data->x = (float2)(x1, x2);

    // rotation matrix
    data->m = (mat22)(c, s, -s, c);

    // inverse rotation matrix
    data->w = (mat22)(c, -s, s, c);

    // shear matrix
    data->g = (mat22)(g1, g2, g2, -g1);

    // auxiliary quantities
    data->q2 = q*q;
    data->e = sqrt(1 - q*q);
    data->d = r*sqrt(q)/sqrt(1 - q*q);
}

// singular isothermal ellipsoid plus shear
// follows Schneider, Kochanek, Wambsganss (2006)

OBJECT(sie_plus_shear) = LENS;

PARAMS(sie_plus_shear) = {
    { "x" },
    { "y" },
    { "r" },
    { "q" },
    { "pa", true },
    { "g1" },
    { "g2" }
};

struct sie_plus_shear
{
    float2 x; // lens position
    mat22 m;  // rotation matrix for position angle
    mat22 w;  // inverse rotation matrix
    mat22 g;  // shear matrix

    // auxiliary
    float q2;
    float e;
    float d;
};

static float2 sie_plus_shear(constant struct sie_plus_shear* data, float2 x)
{
    float2 y;
    float r;

    // move to central coordinates
    x -= data->x;

    // rotate coordinates by position angle
    y = mv22(data->m, x);

    // SIE deflection
    r = data->e/sqrt(data->q2*y.x*y.x + y.y*y.y);
    y = data->d*(float2)(atan(y.x*r), atanh(y.y*r));

    // reverse coordinate rotation, apply shear
    return mv(data->w, y) + mv22(data->g, x);
}

static void set_sie_plus_shear(global struct sie_plus_shear* data, float x1, float x2, float r, float q, float pa, float g1, float g2)
{
    float c = cos(pa*DEG2RAD);
    float s = sin(pa*DEG2RAD);

    // lens position
    data->x = (float2)(x1, x2);

    // rotation matrix
    data->m = (mat22)(c, s, -s, c);

    // inverse rotation matrix
    data->w = (mat22)(c, -s, s, c);

    // shear matrix
    data->g = (mat22)(g1, g2, g2, -g1);

    // auxiliary quantities
    data->q2 = q*q;
    data->e = sqrt(1 - q*q);
    data->d = r*sqrt(q)/sqrt(1 - q*q);
}