r172047 lacked a test (due to incomplete OpenCL support in clang). Use a modified
version of a test by Joey Gouly to use attributes to materialise the unsupported
types and test vector shifts.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@172053 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -x cl -O1 -emit-llvm  %s -o - -triple x86_64-linux-gnu | FileCheck %s
// OpenCL essentially reduces all shift amounts to the last word-size bits before evaluating.
// Test this both for variables and constants evaluated in the front-end.

//CHECK: @positiveShift32
int positiveShift32(int a,int b) {
  //CHECK: [[M32:%.+]] = and i32 %b, 31
  //CHECK-NEXT: [[C32:%.+]] = shl i32 %a, [[M32]]
  int c = a<<b;
  int d = ((int)1)<<33;
  //CHECK-NEXT: [[E32:%.+]] = add nsw i32 [[C32]], 2
  int e = c + d;
  //CHECK-NEXT: ret i32 [[E32]]
  return e;
}

//CHECK: @positiveShift64
long positiveShift64(long a,long b) {
  //CHECK: [[M64:%.+]] = and i64 %b, 63
  //CHECK-NEXT: [[C64:%.+]] = ashr i64 %a, [[M64]]
  long c = a>>b;
  long d = ((long)8)>>65;
  //CHECK-NEXT: [[E64:%.+]] = add nsw i64 [[C64]], 4
  long e = c + d;
  //CHECK-NEXT: ret i64 [[E64]]
  return e;
}

typedef __attribute__((ext_vector_type(4))) int int4;

//CHECK: @vectorVectorTest
int4 vectorVectorTest(int4 a,int4 b) {
  //CHECK: [[VM:%.+]] = and <4 x i32> %b, <i32 31, i32 31, i32 31, i32 31>
  //CHECK-NEXT: [[VC:%.+]] = shl <4 x i32> %a, [[VM]]
  int4 c = a << b;
  //CHECK-NEXT: [[VF:%.+]] = add <4 x i32> [[VC]], <i32 2, i32 4, i32 16, i32 8>
  int4 d = {1, 1, 1, 1};
  int4 e = {33, 34, -28, -29};
  int4 f = c + (d << e);
  //CHECK-NEXT: ret <4 x i32> [[VF]]
  return f;
}

//CHECK: @vectorScalarTest
int4 vectorScalarTest(int4 a,int b) {
  //CHECK: [[SP0:%.+]] = insertelement <4 x i32> undef, i32 %b, i32 0
  //CHECK: [[SP1:%.+]] = shufflevector <4 x i32> [[SP0]], <4 x i32> undef, <4 x i32> zeroinitializer
  //CHECK: [[VSM:%.+]] = and <4 x i32> [[SP1]], <i32 31, i32 31, i32 31, i32 31>
  //CHECK-NEXT: [[VSC:%.+]] = shl <4 x i32> %a, [[VSM]]
  int4 c = a << b;
  //CHECK-NEXT: [[VSF:%.+]] = add <4 x i32> [[VSC]], <i32 4, i32 4, i32 4, i32 4>
  int4 d = {1, 1, 1, 1};
  int4 f = c + (d << 34);
  //CHECK-NEXT: ret <4 x i32> [[VSF]]
  return f;
}


Add binary expression

Add binary expression


Optimize stereobm a bit. Speedup about 30% on 6730M GPU.

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Jia Haipeng, jiahaipeng95@gmail.com
//    Sen Liu, swjtuls1987@126.com
//    Peng Xiao, pengxiao@outlook.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other oclMaterials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#define ROWSperTHREAD 21     // the number of rows a thread will process
#define BLOCK_W       128    // the thread block width (464)
#define N_DISPARITIES 8

#define STEREO_MIND 0                    // The minimum d range to check
#define STEREO_DISP_STEP N_DISPARITIES   // the d step, must be <= 1 to avoid aliasing

#ifndef radius
#define radius 64
#endif

unsigned int CalcSSD(__local unsigned int *col_ssd)
{
    unsigned int cache = col_ssd[0];

#pragma unroll
    for(int i = 1; i <= (radius << 1); i++)
        cache += col_ssd[i];

    return cache;
}

uint2 MinSSD(__local unsigned int *col_ssd)
{
    unsigned int ssd[N_DISPARITIES];
    const int win_size = (radius << 1);

    //See above:  #define COL_SSD_SIZE (BLOCK_W + WIN_SIZE)
    ssd[0] = CalcSSD(col_ssd + 0 * (BLOCK_W + win_size));
    ssd[1] = CalcSSD(col_ssd + 1 * (BLOCK_W + win_size));
    ssd[2] = CalcSSD(col_ssd + 2 * (BLOCK_W + win_size));
    ssd[3] = CalcSSD(col_ssd + 3 * (BLOCK_W + win_size));
    ssd[4] = CalcSSD(col_ssd + 4 * (BLOCK_W + win_size));
    ssd[5] = CalcSSD(col_ssd + 5 * (BLOCK_W + win_size));
    ssd[6] = CalcSSD(col_ssd + 6 * (BLOCK_W + win_size));
    ssd[7] = CalcSSD(col_ssd + 7 * (BLOCK_W + win_size));

    unsigned int mssd = min(min(min(ssd[0], ssd[1]), min(ssd[4], ssd[5])), min(min(ssd[2], ssd[3]), min(ssd[6], ssd[7])));

    int bestIdx = 0;

    for (int i = 0; i < N_DISPARITIES; i++)
    {
        if (mssd == ssd[i])
            bestIdx = i;
    }

    return (uint2)(mssd, bestIdx);
}

void StepDown(int idx1, int idx2, __global unsigned char* imageL,
              __global unsigned char* imageR, int d,   __local unsigned int *col_ssd)
{
    uint8 imgR1 = convert_uint8(vload8(0, imageR + (idx1 - d - 7)));
    uint8 imgR2 = convert_uint8(vload8(0, imageR + (idx2 - d - 7)));
    uint8 diff1 = (uint8)(imageL[idx1]) - imgR1;
    uint8 diff2 = (uint8)(imageL[idx2]) - imgR2;
    uint8 res = diff2 * diff2 - diff1 * diff1;
    const int win_size = (radius << 1);
    col_ssd[0 * (BLOCK_W + win_size)] += res.s7;
    col_ssd[1 * (BLOCK_W + win_size)] += res.s6;
    col_ssd[2 * (BLOCK_W + win_size)] += res.s5;
    col_ssd[3 * (BLOCK_W + win_size)] += res.s4;
    col_ssd[4 * (BLOCK_W + win_size)] += res.s3;
    col_ssd[5 * (BLOCK_W + win_size)] += res.s2;
    col_ssd[6 * (BLOCK_W + win_size)] += res.s1;
    col_ssd[7 * (BLOCK_W + win_size)] += res.s0;
}

void InitColSSD(int x_tex, int y_tex, int im_pitch, __global unsigned char* imageL,
                __global unsigned char* imageR, int d,
                 __local unsigned int *col_ssd)
{
    uint8 leftPixel1;
    uint8 diffa = 0;
    int idx = y_tex * im_pitch + x_tex;
    const int win_size = (radius << 1);
    for(int i = 0; i < (win_size + 1); i++)
    {
        leftPixel1 = (uint8)(imageL[idx]);
        uint8 imgR = convert_uint8(vload8(0, imageR + (idx - d - 7)));
        uint8 res = leftPixel1 - imgR;
        diffa += res * res;

        idx += im_pitch;
    }
    //See above:  #define COL_SSD_SIZE (BLOCK_W + WIN_SIZE)
    col_ssd[0 * (BLOCK_W + win_size)] = diffa.s7;
    col_ssd[1 * (BLOCK_W + win_size)] = diffa.s6;
    col_ssd[2 * (BLOCK_W + win_size)] = diffa.s5;
    col_ssd[3 * (BLOCK_W + win_size)] = diffa.s4;
    col_ssd[4 * (BLOCK_W + win_size)] = diffa.s3;
    col_ssd[5 * (BLOCK_W + win_size)] = diffa.s2;
    col_ssd[6 * (BLOCK_W + win_size)] = diffa.s1;
    col_ssd[7 * (BLOCK_W + win_size)] = diffa.s0;
}

__kernel void stereoKernel(__global unsigned char *left, __global unsigned char *right,
                           __global unsigned int *cminSSDImage, int cminSSD_step,
                           __global unsigned char *disp, int disp_step,int cwidth, int cheight,
                           int img_step, int maxdisp,
                           __local unsigned int *col_ssd_cache)
{
    __local unsigned int *col_ssd = col_ssd_cache + get_local_id(0);
    __local unsigned int *col_ssd_extra = get_local_id(0) < (radius << 1) ? col_ssd + BLOCK_W : 0;

    int X = get_group_id(0) * BLOCK_W + get_local_id(0) + maxdisp + radius;

#define Y (get_group_id(1) * ROWSperTHREAD + radius)

    __global unsigned int* minSSDImage = cminSSDImage + X + Y * cminSSD_step;
    __global unsigned char* disparImage = disp + X + Y * disp_step;

    int end_row = ROWSperTHREAD < (cheight - Y) ? ROWSperTHREAD:(cheight - Y);
    int y_tex;
    int x_tex = X - radius;

    if (x_tex >= cwidth)
        return;

    for(int d = STEREO_MIND; d < maxdisp; d += STEREO_DISP_STEP)
    {
        y_tex = Y - radius;

        InitColSSD(x_tex, y_tex, img_step, left, right, d, col_ssd);
        if (col_ssd_extra > 0)
            if (x_tex + BLOCK_W < cwidth)
                InitColSSD(x_tex + BLOCK_W, y_tex, img_step, left, right, d, col_ssd_extra);

        barrier(CLK_LOCAL_MEM_FENCE); //before MinSSD function

        uint2 minSSD = MinSSD(col_ssd);
        if (X < cwidth - radius && Y < cheight - radius)
        {
            if (minSSD.x < minSSDImage[0])
            {
                disparImage[0] = (unsigned char)(d + minSSD.y);
                minSSDImage[0] = minSSD.x;
            }
        }

        for(int row = 1; row < end_row; row++)
        {
            int idx1 = y_tex * img_step + x_tex;
            int idx2 = min(y_tex + ((radius << 1) + 1), cheight - 1) * img_step + x_tex;

            barrier(CLK_LOCAL_MEM_FENCE);

            StepDown(idx1, idx2, left, right, d, col_ssd);
            if (col_ssd_extra > 0)
                if (x_tex + BLOCK_W < cwidth)
                    StepDown(idx1, idx2, left + BLOCK_W, right + BLOCK_W, d, col_ssd_extra);

            barrier(CLK_LOCAL_MEM_FENCE);

            uint2 minSSD = MinSSD(col_ssd);
            if (X < cwidth - radius && row < cheight - radius - Y)
            {
                int idx = row * cminSSD_step;
                if (minSSD.x < minSSDImage[idx])
                {
                    disparImage[disp_step * row] = (unsigned char)(d + minSSD.y);
                    minSSDImage[idx] = minSSD.x;
                }
            }

            y_tex++;
        } // for row loop
    } // for d loop
}
//////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////// Sobel Prefiler (signal channel)//////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////

__kernel void prefilter_xsobel(__global unsigned char *input, __global unsigned char *output,
                               int rows, int cols, int prefilterCap)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if(x < cols && y < rows)
    {
        int cov = input[(y-1) * cols + (x-1)] * (-1) + input[(y-1) * cols + (x+1)] * (1) +
                  input[(y)   * cols + (x-1)] * (-2) + input[(y)   * cols + (x+1)] * (2) +
                  input[(y+1) * cols + (x-1)] * (-1) + input[(y+1) * cols + (x+1)] * (1);

        cov = min(min(max(-prefilterCap, cov), prefilterCap) + prefilterCap, 255);
        output[y * cols + x] = cov & 0xFF;
    }
}

//////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////// Textureness filtering ////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////

float sobel(__global unsigned char *input, int x, int y, int rows, int cols)
{
    float conv = 0;
    int y1 = y==0? 0 : y-1;
    int x1 = x==0? 0 : x-1;
    if(x < cols && y < rows && x > 0 && y > 0)
    {
        conv = (float)input[(y1)  * cols + (x1)] * (-1) + (float)input[(y1)  * cols + (x+1)] * (1) +
               (float)input[(y)   * cols + (x1)] * (-2) + (float)input[(y)   * cols + (x+1)] * (2) +
               (float)input[(y+1) * cols + (x1)] * (-1) + (float)input[(y+1) * cols + (x+1)] * (1);

    }
    return fabs(conv);
}

float CalcSums(__local float *cols, __local float *cols_cache, int winsz)
{
    float cache = 0;
    float cache2 = 0;
    int winsz2 = winsz/2;

    int x = get_local_id(0);
    int group_size_x = get_local_size(0);

    for(int i = 1; i <= winsz2; i++)
        cache += cols[i];

    cols_cache[0] = cache;

    barrier(CLK_LOCAL_MEM_FENCE);

    if (x < group_size_x - winsz2)
        cache2 = cols_cache[winsz2];
    else
        for(int i = winsz2 + 1; i < winsz; i++)
            cache2 += cols[i];

    return cols[0] + cache + cache2;
}

#define RpT (2 * ROWSperTHREAD)  // got experimentally
__kernel void textureness_kernel(__global unsigned char *disp, int disp_rows, int disp_cols,
                                 int disp_step, __global unsigned char *input, int input_rows,
                                 int input_cols,int winsz, float threshold,
                                 __local float *cols_cache)
{
    int winsz2 = winsz/2;
    int n_dirty_pixels = (winsz2) * 2;

    int local_id_x = get_local_id(0);
    int group_size_x = get_local_size(0);
    int group_id_y = get_group_id(1);

    __local float *cols = cols_cache + group_size_x + local_id_x;
    __local float *cols_extra = local_id_x < n_dirty_pixels ? cols + group_size_x : 0;

    int x = get_global_id(0);
    int beg_row = group_id_y * RpT;
    int end_row = min(beg_row + RpT, disp_rows);

//   if (x < disp_cols)
//   {
    int y = beg_row;

    float sum = 0;
    float sum_extra = 0;

    for(int i = y - winsz2; i <= y + winsz2; ++i)
    {
        sum += sobel(input, x - winsz2, i, input_rows, input_cols);
        if (cols_extra)
            sum_extra += sobel(input, x + group_size_x - winsz2, i, input_rows, input_cols);
    }
    *cols = sum;
    if (cols_extra)
        *cols_extra = sum_extra;

    barrier(CLK_LOCAL_MEM_FENCE);

    float sum_win = CalcSums(cols, cols_cache + local_id_x, winsz) * 255;
    if (sum_win < threshold)
        disp[y * disp_step + x] = 0;

    barrier(CLK_LOCAL_MEM_FENCE);

    for(int y = beg_row + 1; y < end_row; ++y)
    {
        sum = sum - sobel(input, x - winsz2, y - winsz2 - 1, input_rows, input_cols) +
              sobel(input, x - winsz2, y + winsz2, input_rows, input_cols);
        *cols = sum;

        if (cols_extra)
        {
            sum_extra = sum_extra - sobel(input, x + group_size_x - winsz2, y - winsz2 - 1,input_rows, input_cols)
                        + sobel(input, x + group_size_x - winsz2, y + winsz2, input_rows, input_cols);
            *cols_extra = sum_extra;
        }

        barrier(CLK_LOCAL_MEM_FENCE);
        float sum_win = CalcSums(cols, cols_cache + local_id_x, winsz) * 255;
        if (sum_win < threshold)
            disp[y * disp_step + x] = 0;

        barrier(CLK_LOCAL_MEM_FENCE);
    }
    //  }
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Jia Haipeng, jiahaipeng95@gmail.com
//    Sen Liu, swjtuls1987@126.com
//    Peng Xiao, pengxiao@outlook.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other oclMaterials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#define ROWSperTHREAD 21     // the number of rows a thread will process
#define BLOCK_W       128    // the thread block width (464)
#define N_DISPARITIES 8

#define STEREO_MIND 0                    // The minimum d range to check
#define STEREO_DISP_STEP N_DISPARITIES   // the d step, must be <= 1 to avoid aliasing

#ifndef radius
#define radius 64
#endif

unsigned int CalcSSD(__local unsigned int *col_ssd)
{
    unsigned int cache = col_ssd[0];

    for(int i = 1, j = radius + 1; i <= radius; i++, j++)
        cache += col_ssd[i] + col_ssd[j];

    return cache;
}

uint2 MinSSD(__local unsigned int *col_ssd)
{
    unsigned int ssd[N_DISPARITIES];

    //See above:  #define COL_SSD_SIZE (BLOCK_W + 2 * radius)
    ssd[0] = CalcSSD(col_ssd + 0 * (BLOCK_W + 2 * radius));
    ssd[1] = CalcSSD(col_ssd + 1 * (BLOCK_W + 2 * radius));
    ssd[2] = CalcSSD(col_ssd + 2 * (BLOCK_W + 2 * radius));
    ssd[3] = CalcSSD(col_ssd + 3 * (BLOCK_W + 2 * radius));
    ssd[4] = CalcSSD(col_ssd + 4 * (BLOCK_W + 2 * radius));
    ssd[5] = CalcSSD(col_ssd + 5 * (BLOCK_W + 2 * radius));
    ssd[6] = CalcSSD(col_ssd + 6 * (BLOCK_W + 2 * radius));
    ssd[7] = CalcSSD(col_ssd + 7 * (BLOCK_W + 2 * radius));

    unsigned int mssd = min(min(min(ssd[0], ssd[1]), min(ssd[4], ssd[5])), min(min(ssd[2], ssd[3]), min(ssd[6], ssd[7])));

    int bestIdx = 0;
    for (int i = 0; i < N_DISPARITIES; i++)
    {
        if (mssd == ssd[i])
            bestIdx = i;
    }

    return (uint2)(mssd, bestIdx);
}

void StepDown(int idx1, int idx2, __global unsigned char* imageL,
              __global unsigned char* imageR, int d,   __local unsigned int *col_ssd)
{
    uint8 imgR1 = convert_uint8(vload8(0, imageR + (idx1 - d - 7)));
    uint8 imgR2 = convert_uint8(vload8(0, imageR + (idx2 - d - 7)));
    uint8 diff1 = (uint8)(imageL[idx1]) - imgR1;
    uint8 diff2 = (uint8)(imageL[idx2]) - imgR2;
    uint8 res = diff2 * diff2 - diff1 * diff1;
    col_ssd[0 * (BLOCK_W + 2 * radius)] += res.s7;
    col_ssd[1 * (BLOCK_W + 2 * radius)] += res.s6;
    col_ssd[2 * (BLOCK_W + 2 * radius)] += res.s5;
    col_ssd[3 * (BLOCK_W + 2 * radius)] += res.s4;
    col_ssd[4 * (BLOCK_W + 2 * radius)] += res.s3;
    col_ssd[5 * (BLOCK_W + 2 * radius)] += res.s2;
    col_ssd[6 * (BLOCK_W + 2 * radius)] += res.s1;
    col_ssd[7 * (BLOCK_W + 2 * radius)] += res.s0;
}

void InitColSSD(int x_tex, int y_tex, int im_pitch, __global unsigned char* imageL,
                __global unsigned char* imageR, int d,
                 __local unsigned int *col_ssd)
{
    uint8 leftPixel1;
    int idx;
    uint8 diffa = 0;

    for(int i = 0; i < (2 * radius + 1); i++)
    {
        idx = y_tex * im_pitch + x_tex;
        leftPixel1 = (uint8)(imageL[idx]);
        uint8 imgR = convert_uint8(vload8(0, imageR + (idx - d - 7)));
        uint8 res = leftPixel1 - imgR;
        diffa += res * res;

        y_tex += 1;
    }
    //See above:  #define COL_SSD_SIZE (BLOCK_W + 2 * radius)
    col_ssd[0 * (BLOCK_W + 2 * radius)] = diffa.s7;
    col_ssd[1 * (BLOCK_W + 2 * radius)] = diffa.s6;
    col_ssd[2 * (BLOCK_W + 2 * radius)] = diffa.s5;
    col_ssd[3 * (BLOCK_W + 2 * radius)] = diffa.s4;
    col_ssd[4 * (BLOCK_W + 2 * radius)] = diffa.s3;
    col_ssd[5 * (BLOCK_W + 2 * radius)] = diffa.s2;
    col_ssd[6 * (BLOCK_W + 2 * radius)] = diffa.s1;
    col_ssd[7 * (BLOCK_W + 2 * radius)] = diffa.s0;
}

__kernel void stereoKernel(__global unsigned char *left, __global unsigned char *right,
                           __global unsigned int *cminSSDImage, int cminSSD_step,
                           __global unsigned char *disp, int disp_step,int cwidth, int cheight,
                           int img_step, int maxdisp,
                           __local unsigned int *col_ssd_cache)
{
     __local unsigned int *col_ssd = col_ssd_cache + get_local_id(0);
     __local unsigned int *col_ssd_extra = get_local_id(0) < (2 * radius) ? col_ssd + BLOCK_W : 0;

    int X = get_group_id(0) * BLOCK_W + get_local_id(0) + maxdisp + radius;
    // int Y = get_group_id(1) * ROWSperTHREAD + radius;

#define Y (get_group_id(1) * ROWSperTHREAD + radius)

     __global unsigned int* minSSDImage = cminSSDImage + X + Y * cminSSD_step;
    __global unsigned char* disparImage = disp + X + Y * disp_step;

    int end_row = ROWSperTHREAD < (cheight - Y) ? ROWSperTHREAD:(cheight - Y);
    int y_tex;
    int x_tex = X - radius;

    if (x_tex >= cwidth)
        return;

    for(int d = STEREO_MIND; d < maxdisp; d += STEREO_DISP_STEP)
    {
        y_tex = Y - radius;

        InitColSSD(x_tex, y_tex, img_step, left, right, d, col_ssd);
        if (col_ssd_extra > 0)
            if (x_tex + BLOCK_W < cwidth)
                InitColSSD(x_tex + BLOCK_W, y_tex, img_step, left, right, d, col_ssd_extra);

        barrier(CLK_LOCAL_MEM_FENCE); //before MinSSD function

        uint2 minSSD = MinSSD(col_ssd);
        if (X < cwidth - radius && Y < cheight - radius)
        {
            if (minSSD.x < minSSDImage[0])
            {
                disparImage[0] = (unsigned char)(d + minSSD.y);
                minSSDImage[0] = minSSD.x;
            }
        }

        for(int row = 1; row < end_row; row++)
        {
            int idx1 = y_tex * img_step + x_tex;
            int idx2 = min(y_tex + (2 * radius + 1), cheight - 1) * img_step + x_tex;

            StepDown(idx1, idx2, left, right, d, col_ssd);
            if (col_ssd_extra > 0)
                if (x_tex + BLOCK_W < cwidth)
                    StepDown(idx1, idx2, left + BLOCK_W, right + BLOCK_W, d, col_ssd_extra);

            barrier(CLK_LOCAL_MEM_FENCE);

            uint2 minSSD = MinSSD(col_ssd);
            if (X < cwidth - radius && row < cheight - radius - Y)
            {
                int idx = row * cminSSD_step;
                if (minSSD.x < minSSDImage[idx])
                {
                    disparImage[disp_step * row] = (unsigned char)(d + minSSD.y);
                    minSSDImage[idx] = minSSD.x;
                }
            }

            y_tex++;
        } // for row loop
    } // for d loop
}
//////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////// Sobel Prefiler (signal channel)//////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////

__kernel void prefilter_xsobel(__global unsigned char *input, __global unsigned char *output,
                               int rows, int cols, int prefilterCap)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if(x < cols && y < rows)
    {
        int cov = input[(y-1) * cols + (x-1)] * (-1) + input[(y-1) * cols + (x+1)] * (1) +
                  input[(y)   * cols + (x-1)] * (-2) + input[(y)   * cols + (x+1)] * (2) +
                  input[(y+1) * cols + (x-1)] * (-1) + input[(y+1) * cols + (x+1)] * (1);

        cov = min(min(max(-prefilterCap, cov), prefilterCap) + prefilterCap, 255);
        output[y * cols + x] = cov & 0xFF;
    }
}

//////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////// Textureness filtering ////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////////

float sobel(__global unsigned char *input, int x, int y, int rows, int cols)
{
    float conv = 0;
    int y1 = y==0? 0 : y-1;
    int x1 = x==0? 0 : x-1;
    if(x < cols && y < rows && x > 0 && y > 0)
    {
        conv = (float)input[(y1)  * cols + (x1)] * (-1) + (float)input[(y1)  * cols + (x+1)] * (1) +
               (float)input[(y)   * cols + (x1)] * (-2) + (float)input[(y)   * cols + (x+1)] * (2) +
               (float)input[(y+1) * cols + (x1)] * (-1) + (float)input[(y+1) * cols + (x+1)] * (1);

    }
    return fabs(conv);
}

float CalcSums(__local float *cols, __local float *cols_cache, int winsz)
{
    float cache = 0;
    float cache2 = 0;
    int winsz2 = winsz/2;

    int x = get_local_id(0);
    int group_size_x = get_local_size(0);

    for(int i = 1; i <= winsz2; i++)
        cache += cols[i];

    cols_cache[0] = cache;

    barrier(CLK_LOCAL_MEM_FENCE);

    if (x < group_size_x - winsz2)
        cache2 = cols_cache[winsz2];
    else
        for(int i = winsz2 + 1; i < winsz; i++)
            cache2 += cols[i];

    return cols[0] + cache + cache2;
}

#define RpT (2 * ROWSperTHREAD)  // got experimentally
__kernel void textureness_kernel(__global unsigned char *disp, int disp_rows, int disp_cols,
                                 int disp_step, __global unsigned char *input, int input_rows,
                                 int input_cols,int winsz, float threshold,
                                 __local float *cols_cache)
{
    int winsz2 = winsz/2;
    int n_dirty_pixels = (winsz2) * 2;

    int local_id_x = get_local_id(0);
    int group_size_x = get_local_size(0);
    int group_id_y = get_group_id(1);

    __local float *cols = cols_cache + group_size_x + local_id_x;
    __local float *cols_extra = local_id_x < n_dirty_pixels ? cols + group_size_x : 0;

    int x = get_global_id(0);
    int beg_row = group_id_y * RpT;
    int end_row = min(beg_row + RpT, disp_rows);

//   if (x < disp_cols)
//   {
    int y = beg_row;

    float sum = 0;
    float sum_extra = 0;

    for(int i = y - winsz2; i <= y + winsz2; ++i)
    {
        sum += sobel(input, x - winsz2, i, input_rows, input_cols);
        if (cols_extra)
            sum_extra += sobel(input, x + group_size_x - winsz2, i, input_rows, input_cols);
    }
    *cols = sum;
    if (cols_extra)
        *cols_extra = sum_extra;

    barrier(CLK_LOCAL_MEM_FENCE);

    float sum_win = CalcSums(cols, cols_cache + local_id_x, winsz) * 255;
    if (sum_win < threshold)
        disp[y * disp_step + x] = 0;

    barrier(CLK_LOCAL_MEM_FENCE);

    for(int y = beg_row + 1; y < end_row; ++y)
    {
        sum = sum - sobel(input, x - winsz2, y - winsz2 - 1, input_rows, input_cols) +
              sobel(input, x - winsz2, y + winsz2, input_rows, input_cols);
        *cols = sum;

        if (cols_extra)
        {
            sum_extra = sum_extra - sobel(input, x + group_size_x - winsz2, y - winsz2 - 1,input_rows, input_cols)
                        + sobel(input, x + group_size_x - winsz2, y + winsz2, input_rows, input_cols);
            *cols_extra = sum_extra;
        }

        barrier(CLK_LOCAL_MEM_FENCE);
        float sum_win = CalcSums(cols, cols_cache + local_id_x, winsz) * 255;
        if (sum_win < threshold)
            disp[y * disp_step + x] = 0;

        barrier(CLK_LOCAL_MEM_FENCE);
    }
    //  }
}


formatting over formatted

formatting over formatted

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0  = get_global_id(0);
    uint batch_id = iDiv(gbl_id0, MLO_MAP_SZ4); // batch
    uint pos      = iMod(gbl_id0, batch_id, MLO_MAP_SZ4);

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE + pos * MLO_READ_UNIT;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    for(uint ci = 0; ci < MLO_CLOOP0; ++ci,
             // move input offset
                                      gbl_in_off += MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE,

             // move weights offset
                                      wei_off += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                                                 MLO_WEI_CHANNEL_STRIDE
#else
                                                 MLO_WEI_BSTRIDE
#endif
        )
    {
        // read weights

        for(uint o = 0, wei_off1 = wei_off; o < MLO_N_LCL_OUT_MAPS; ++o,
                 wei_off1 +=
#if MLO_DIR_FORWARD == 1
                                                                    MLO_WEI_BSTRIDE
#else
                                                                    MLO_WEI_CHANNEL_STRIDE
#endif
            )
        {
            for(uint c = 0, wei_off2 = wei_off1; c < MLO_N_LCL_IN_MAPS; ++c,
                     wei_off2 +=
#if MLO_DIR_FORWARD == 1
                                                                        MLO_WEI_CHANNEL_STRIDE
#else
                                                                        MLO_WEI_BSTRIDE
#endif
                )
            {
                weights[o][c] = wei_ptr[wei_off2];
            }
        }

        // convolve with all weights
        // read data
        for(uint j = 0, gbl_in_off1 = gbl_in_off; j < MLO_N_LCL_IN_MAPS;
            ++j, gbl_in_off1 += MLO_IN_CHANNEL_STRIDE)
        {
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                dat[j][i] = in_ptr[gbl_in_off1 + i];
            }
        }

        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
                }
            }
        }
    }

    uint gbl_out_off =
        batch_id * MLO_OUT_BATCH_STRIDE + pos * MLO_READ_UNIT + out_id * MLO_OUT_CHANNEL_STRIDE;
    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            out_ptr[gbl_out_off1 + i] = accum[o][i];
        }
    }
}

/************************************************************************
stride and padding
*************************************************************************/
__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1pquv(const __global _FLOAT* __restrict in_ptr,
                  __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
                  const __global _FLOAT* __restrict bias,
#endif
                  __global _FLOAT* __restrict out_ptr,
                  UNUSED _FLOAT dummy_val // nothing
                  )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0 = get_global_id(0);

    uint batch_id  = iDiv(gbl_id0, MLO_MAP_SZ4); // batch
    uint pos       = iMod(gbl_id0, batch_id, MLO_MAP_SZ4);
    uint pos_out_y = iDiv(pos, MLO_OUT_WIDTH4);
    uint pos_out_x = iMod(pos, pos_out_y, MLO_OUT_WIDTH4);

#if MLO_DIR_FORWARD == 1
    uint pos_in_y = pos_out_y * MLO_FILTER_STRIDE1;
    uint pos_in_x = pos_out_x * MLO_FILTER_STRIDE0;
#else
    uint pos_in_y = pos_out_y; /// MLO_FILTER_STRIDE1;   - divided already
    uint pos_in_x = pos_out_x; // MLO_FILTER_STRIDE0;  - divided already
#endif

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off =
        batch_id * MLO_IN_BATCH_STRIDE + pos_in_y * MLO_IN_STRIDE + pos_in_x * MLO_READ_UNIT;
    //	bool vis = (pos_in_y < MLO_IN_HEIGHT);
    //	gbl_in_off = (vis) ? gbl_in_off : 0;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    const __global _FLOAT* i_ptr = &in_ptr[gbl_in_off];
    __constant _FLOAT* w_ptr     = &wei_ptr[wei_off];
    for(uint ci = 0; ci < MLO_CLOOP0; ++ci)
    {

        // convolve with all weights
        // read data

        for(uint j = 0; j < MLO_N_LCL_IN_MAPS; ++j)
        {
            uint i = 0;
#if MLO_READ_UNIT > 1
            for(; i < MLO_READ_UNIT - 1; ++i)
            {
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                dat[j][i] = i_ptr[off];
            }
#endif

            for(; i < MLO_READ_UNIT; ++i)
            {
                //				vis &= (pos_in_x + i*MLO_FILTER_STRIDE0 <
                // MLO_IN_WIDTH);
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                //				off = (vis) ? off : 0;
                _FLOAT val = i_ptr[off];
                dat[j][i]  = val;
                //              dat[j][i] = (vis)? dat[j][i] : 0;
            }

            i_ptr += MLO_IN_CHANNEL_STRIDE;
        }
        // read weights
        __constant _FLOAT* w_ptr0 = w_ptr;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {

            __constant _FLOAT* w_ptr1 = w_ptr0;

            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                weights[o][c] = *w_ptr1;
                w_ptr1 +=
#if MLO_DIR_FORWARD == 1
                    MLO_WEI_CHANNEL_STRIDE
#else
                    MLO_WEI_BSTRIDE
#endif
                    ;
            }

            w_ptr0 +=
#if MLO_DIR_FORWARD == 1
                MLO_WEI_BSTRIDE
#else
                MLO_WEI_CHANNEL_STRIDE
#endif
                ;
        }

        w_ptr += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                 MLO_WEI_CHANNEL_STRIDE
#else
                 MLO_WEI_BSTRIDE
#endif
            ;
        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
#if 0
				if (get_global_id(0) == 4 && get_global_id(1) ==0 && o == 0 && i == 0)
				{
					printf((__constant char *)"K:c: %f %f %f %f\n",
					accum[o][i],
					dat[c][i] * weights[o][c],
					dat[c][i],
					weights[o][c]
					);
				}
#endif
                }
            }
        }
    }

    uint out_y = pos_out_y
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE1
#endif
        ;
    uint out_x = pos_out_x
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE0
#endif
        ;

    uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE +
                       out_y * MLO_OUT_STRIDE + out_x * MLO_READ_UNIT;

    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {

        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            uint out_off = gbl_out_off1 +
                           i
#if MLO_DIR_FORWARD == 0
                               * MLO_FILTER_STRIDE0
#endif
                ;
            out_ptr[out_off] = accum[o][i];

#if MLO_DIR_FORWARD == 0
            for(uint s = 1; s < MLO_FILTER_STRIDE0; ++s)
            {
#if MLO_HORIZ_ALIGNED == 0
                if(out_x + s < MLO_OUT_WIDTH)
#endif
                {
                    out_ptr[out_off + s] = 0;
                }
            }
#endif
        }

#if MLO_DIR_FORWARD == 0
        for(uint j = 1; j < MLO_FILTER_STRIDE1; ++j)
        {
#if MLO_VERT_ALIGNED == 0
            if(out_y + j < MLO_OUT_HEIGHT)
#endif
            {
                uint out_off = gbl_out_off1 + j * MLO_OUT_STRIDE;
                for(uint s = 0; s < MLO_READ_UNIT * MLO_FILTER_STRIDE0; ++s)
                {
#if MLO_HORIZ_ALIGNED == 0
                    if(out_x + s < MLO_OUT_WIDTH)
#endif
                    {
                        out_ptr[out_off + s] = 0;
                    }
                }
            }
        }
#endif
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0  = get_global_id(0);
    uint batch_id = iDiv(gbl_id0, MLO_MAP_SZ4); // batch
    uint pos      = iMod(gbl_id0, batch_id, MLO_MAP_SZ4);

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE + pos * MLO_READ_UNIT;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    for(uint ci = 0; ci < MLO_CLOOP0; ++ci,
             // move input offset
                                      gbl_in_off += MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE,

             // move weights offset
                                      wei_off += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                                                 MLO_WEI_CHANNEL_STRIDE
#else
                                                 MLO_WEI_BSTRIDE
#endif
        )
    {
        // read weights

        for(uint o = 0, wei_off1 = wei_off; o < MLO_N_LCL_OUT_MAPS; ++o,
                 wei_off1 +=
#if MLO_DIR_FORWARD == 1
                                                                    MLO_WEI_BSTRIDE
#else
                                                                    MLO_WEI_CHANNEL_STRIDE
#endif
            )
        {
            for(uint c = 0, wei_off2 = wei_off1; c < MLO_N_LCL_IN_MAPS; ++c,
                     wei_off2 +=
#if MLO_DIR_FORWARD == 1
                                                                        MLO_WEI_CHANNEL_STRIDE
#else
                                                                        MLO_WEI_BSTRIDE
#endif
                )
            {
                weights[o][c] = wei_ptr[wei_off2];
            }
        }

        // convolve with all weights
        // read data
        for(uint j = 0, gbl_in_off1 = gbl_in_off; j < MLO_N_LCL_IN_MAPS;
            ++j, gbl_in_off1 += MLO_IN_CHANNEL_STRIDE)
        {
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                dat[j][i] = in_ptr[gbl_in_off1 + i];
            }
        }

        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
                }
            }
        }
    }

    uint gbl_out_off =
        batch_id * MLO_OUT_BATCH_STRIDE + pos * MLO_READ_UNIT + out_id * MLO_OUT_CHANNEL_STRIDE;
    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            out_ptr[gbl_out_off1 + i] = accum[o][i];
        }
    }
}

/************************************************************************
stride and padding
*************************************************************************/
__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1pquv(const __global _FLOAT* __restrict in_ptr,
                  __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
                  const __global _FLOAT* __restrict bias,
#endif
                  __global _FLOAT* __restrict out_ptr,
                  UNUSED _FLOAT dummy_val // nothing
                  )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0 = get_global_id(0);

    uint batch_id  = iDiv(gbl_id0, MLO_MAP_SZ4); // batch
    uint pos       = iMod(gbl_id0, batch_id, MLO_MAP_SZ4);
    uint pos_out_y = iDiv(pos, MLO_OUT_WIDTH4);
    uint pos_out_x = iMod(pos, pos_out_y, MLO_OUT_WIDTH4);

#if MLO_DIR_FORWARD == 1
    uint pos_in_y = pos_out_y * MLO_FILTER_STRIDE1;
    uint pos_in_x = pos_out_x * MLO_FILTER_STRIDE0;
#else
    uint pos_in_y = pos_out_y; /// MLO_FILTER_STRIDE1;   - divided already
    uint pos_in_x = pos_out_x; // MLO_FILTER_STRIDE0;  - divided already
#endif

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off =
        batch_id * MLO_IN_BATCH_STRIDE + pos_in_y * MLO_IN_STRIDE + pos_in_x * MLO_READ_UNIT;
    //	bool vis = (pos_in_y < MLO_IN_HEIGHT);
    //	gbl_in_off = (vis) ? gbl_in_off : 0;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    const __global _FLOAT* i_ptr = &in_ptr[gbl_in_off];
    __constant _FLOAT* w_ptr     = &wei_ptr[wei_off];
    for(uint ci = 0; ci < MLO_CLOOP0; ++ci)
    {

        // convolve with all weights
        // read data

        for(uint j = 0; j < MLO_N_LCL_IN_MAPS; ++j)
        {
            uint i = 0;
#if MLO_READ_UNIT > 1
            for(; i < MLO_READ_UNIT - 1; ++i)
            {
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                dat[j][i] = i_ptr[off];
            }
#endif

            for(; i < MLO_READ_UNIT; ++i)
            {
                //				vis &= (pos_in_x + i*MLO_FILTER_STRIDE0 <
                //MLO_IN_WIDTH);
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                //				off = (vis) ? off : 0;
                _FLOAT val = i_ptr[off];
                dat[j][i]  = val;
                //              dat[j][i] = (vis)? dat[j][i] : 0;
            }

            i_ptr += MLO_IN_CHANNEL_STRIDE;
        }
        // read weights
        __constant _FLOAT* w_ptr0 = w_ptr;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {

            __constant _FLOAT* w_ptr1 = w_ptr0;

            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                weights[o][c] = *w_ptr1;
                w_ptr1 +=
#if MLO_DIR_FORWARD == 1
                    MLO_WEI_CHANNEL_STRIDE
#else
                    MLO_WEI_BSTRIDE
#endif
                    ;
            }

            w_ptr0 +=
#if MLO_DIR_FORWARD == 1
                MLO_WEI_BSTRIDE
#else
                MLO_WEI_CHANNEL_STRIDE
#endif
                ;
        }

        w_ptr += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                 MLO_WEI_CHANNEL_STRIDE
#else
                 MLO_WEI_BSTRIDE
#endif
            ;
        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
#if 0
				if (get_global_id(0) == 4 && get_global_id(1) ==0 && o == 0 && i == 0)
				{
					printf((__constant char *)"K:c: %f %f %f %f\n",
					accum[o][i],
					dat[c][i] * weights[o][c],
					dat[c][i],
					weights[o][c]
					);
				}
#endif
                }
            }
        }
    }

    uint out_y = pos_out_y
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE1
#endif
        ;
    uint out_x = pos_out_x
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE0
#endif
        ;

    uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE +
                       out_y * MLO_OUT_STRIDE + out_x * MLO_READ_UNIT;

    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {

        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            uint out_off = gbl_out_off1 +
                           i
#if MLO_DIR_FORWARD == 0
                               * MLO_FILTER_STRIDE0
#endif
                ;
            out_ptr[out_off] = accum[o][i];

#if MLO_DIR_FORWARD == 0
            for(uint s = 1; s < MLO_FILTER_STRIDE0; ++s)
            {
#if MLO_HORIZ_ALIGNED == 0
                if(out_x + s < MLO_OUT_WIDTH)
#endif
                {
                    out_ptr[out_off + s] = 0;
                }
            }
#endif
        }

#if MLO_DIR_FORWARD == 0
        for(uint j = 1; j < MLO_FILTER_STRIDE1; ++j)
        {
#if MLO_VERT_ALIGNED == 0
            if(out_y + j < MLO_OUT_HEIGHT)
#endif
            {
                uint out_off = gbl_out_off1 + j * MLO_OUT_STRIDE;
                for(uint s = 0; s < MLO_READ_UNIT * MLO_FILTER_STRIDE0; ++s)
                {
#if MLO_HORIZ_ALIGNED == 0
                    if(out_x + s < MLO_OUT_WIDTH)
#endif
                    {
                        out_ptr[out_off + s] = 0;
                    }
                }
            }
        }
#endif
    }
}


foud "to ggod to be true" problem

foud "to ggod to be true" problem

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{

				for (int f_s = 0; f_s < MLO_FILTER_STRIDE1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_OUT_STACKS);
						int t0 = iMod(w, k, MLO_OUT_STACKS);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}

					}

					int n_reads = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						int t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = c_scan* MLO_FILTER_STRIDE1 + f_s;

						if (ob*MLO_OUT_EXTENT1*MLO_FILTER_STRIDE1 + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								for (; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = 0;
								}

							}
							else
#endif
							{
								//					*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}
							int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
							for (int i = 0; i < MLO_READ_UNIT; ++i)
							{
								bot_mem[lcl_off + i] = in_rd_data[i];
							}

						}
					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
						// select all vertical scans that matches the vertical filter tap
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)];
						// read input values for this filter phase
						for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
						{
							for (int i = 0; i < ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0); ++i)
							{
								in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + i]
									= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row* MLO_N_FILTER_SPLITS1 + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i];
							}

						}
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_FILTER_STRIDE0];
						// first 2 splits
						int l;
						for (l = 0; l < MLO_N_FILTER_SPLITS0 - 1; ++l)
						{
// read all weights
							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + l*MLO_FILTER_STRIDE0 + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l
// 3d
						{

							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);
			int out_y = ob*MLO_OUT_EXTENT1;

			for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && (out_y + ex_row) < MLO_OUT_HEIGHT && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// filter size for all filters with small n of input maps (first layer)
// split a long filter by stride

#define MLO_N_FILTER_SPLITS1 ((MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1 - 1)/ MLO_FILTER_STRIDE1)
#ifndef MLO_OUT_PIX_TILE0
#define MLO_N_FILTER_SPLITS0 ((MLO_FILTER_SIZE0 + MLO_FILTER_STRIDE0 - 1)/ MLO_FILTER_STRIDE0)
#define MLO_OUT_PIX_TILE0 MLO_N_FILTER_SPLITS0
#endif
// processing arrangement
// generate full output width
// extent1 == MLO_GRP_SZ / MLO_PROCESING_WIDTH
#ifndef MLO_OUT_EXTENT1
#define MLO_PROCESSING_WIDTH  ((MLO_OUT_WIDTH + MLO_OUT_PIX_TILE0 - 1) / MLO_OUT_PIX_TILE0)
#define MLO_OUT_EXTENT1 (MLO_GRP_SZ / MLO_PROCESSING_WIDTH)
#endif

#define MLO_WEI_LCL_WIDTH MLO_FILTER_SIZE0 //(MLO_N_FILTER_SPLITS0*MLO_FILTER_STRIDE0)
#define MLO_WEI_EXTENT1 MLO_N_FILTER_SPLITS1
#define MLO_WEI_SZ (MLO_WEI_EXTENT1*MLO_WEI_LCL_WIDTH)
// LDS size
#define MLO_WEI_LCL_SZ (MLO_WEI_SZ * MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

#define MLO_IN_LCL_HEIGHT (MLO_OUT_EXTENT1 + MLO_N_FILTER_SPLITS1 - 1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
// LDS size
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_IN_LCL_SZ* MLO_N_LCL_IN_MAPS)

//#if (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#define MLO_LCL_MEM_SZ (MLO_WEI_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
//#else
//#define MLO_LCL_MEM_SZ (MLO_N_PARTIAL_SUMS *MLO_OUT_WIDTH*MLO_OUT_EXTENT1)
#//endif

// number of loops to flush put full output map
#define MLO_N_OUT_BLKS 1   //((MLO_OUT_HEIGHT + (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1) -1) / (MLO_OUT_PIX_TILE1*MLO_N_OUT_FOLDS1))

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

static inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

static inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

static inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

static inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(_FLOAT *sum, _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void  Kahan_summation_tricked(_FLOAT *sum, _FLOAT * c, _FLOAT v, _FLOAT mod)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) * mod - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

static inline void Kahan_summation2(_FLOAT *sum, _FLOAT *c, _FLOAT *v, int n)
{
	for (int i = 0; i < n; ++i)
	{
		_FLOAT y = v[i] - c[i];    //So far, so good: c is zero.
		_FLOAT t = sum[i] + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
		c[i] = (t - sum[i]) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
		sum[i] = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// process 3 output pixel per wk-item, 19 wk-items per output scan,
// 13 output sacn-line per group of 256
// read (13+2) input scan-lines 4 scan-lines apart from 2 batches
// convolve with 3 filters rows 4 rowes apart from 4(8) filter banks.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvFwd(
	const __global _FLOAT * bot,
	const __global _FLOAT * weights,
#if MLO_CONV_BIAS == 1
	const __global _FLOAT * bias,
#endif
	__global _FLOAT *top,
	_FLOAT padding_val
)
{

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * bot_mem = lcl_mem;
	__local _FLOAT * wei_mem = lcl_mem + MLO_TOTAL_IN_LCL_SZ;

	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int ob = get_group_id(0);  // output map extent id

	int k_idx = get_group_id(1) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // input map index based

	int c_idx = 0;

	int ib_idx = get_group_id(2)*MLO_N_LCL_BATCHS; // batch idx

	int ib = ib_idx;

	int gbl_in_off = /*c_idx * MLO_IN_CHANNEL_STRIDE + */ib * MLO_IN_BATCH_STRIDE;
	int gbl_wei_off = k_idx * MLO_WEI_BATCH_STRIDE;

#define MLO_ACCUM_SZ (MLO_OUT_PIX_TILE1 * MLO_OUT_PIX_TILE0 * MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS*MLO_N_LCL_BATCHS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_MEM_SZ); i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// processing arrangement
	int ex_row = iDiv(lcl_id, MLO_PROCESSING_WIDTH);
//
	int ex_col = iMod(lcl_id, ex_row, MLO_PROCESSING_WIDTH);
	int ex_pix = ex_col * MLO_OUT_PIX_TILE0;

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		b += MLO_N_LCL_BATCHS,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off0 = gbl_in_off;

		// generate pixels all MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS output maps
//		for (int ob = 0; ob < MLO_N_OUT_BLKS; ++ob, in_y0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1), gbl_in_scan_off0 += (MLO_OUT_PIX_TILE1 *MLO_FILTER_STRIDE1*MLO_N_OUT_FOLDS1) * MLO_IN_CHANNEL_STRIDE, out_y += MLO_OUT_PIX_TILE1 *MLO_N_OUT_FOLDS1)
		{

			for (int i = 0; i < MLO_ACCUM_SZ; ++i)
			{
				pvt_accum[i] = 0;
			}

			// all input maps
			for (int c = 0, gbl_in_scan_off = gbl_in_scan_off0; c < MLO_N_INPUTS; ++c, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)
			{

				for (int f_s = 0; f_s < MLO_FILTER_STRIDE1; ++f_s)
				{

					barrier(CLK_LOCAL_MEM_FENCE);

					// read weights by stride
					for (int w = lcl_id; w < MLO_WEI_LCL_SZ; w += MLO_GRP_SZ)
					{
						int k = iDiv(w, MLO_OUT_STACKS);
						int t0 = iMod(w, k, MLO_OUT_STACKS);
						int j = iDiv(t0, MLO_FILTER_SIZE0);
						int i = iMod(t0, j, MLO_FILTER_SIZE0);
						int wei_off = gbl_wei_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;

						if ((j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i < MLO_WEI_CHANNEL_STRIDE)
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = weights[wei_off + (j*MLO_FILTER_STRIDE1 + f_s)*MLO_FILTER_SIZE0 + i];
						}
						else
						{
							wei_mem[k*MLO_WEI_SZ + j*MLO_WEI_LCL_WIDTH + i] = 0;
						}

					}

					int n_reads = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? MLO_IN_LCL_HEIGHT - 1 : MLO_IN_LCL_HEIGHT;
					int lcl_scan = (ob == 0 && (f_s < MLO_FILTER_PAD1)) ? 1 : 0;

					// fetch input by stride
					for (int p4 = lcl_id, c_scan = 0;  p4 < MLO_N_IN_HORIZ_READS * n_reads * MLO_N_LCL_BATCHS;
						p4 += MLO_GRP_SZ)
					{
						int b = iDiv(p4, MLO_N_IN_HORIZ_READS * n_reads);
						int t0 = iMod(p4, b, MLO_N_IN_HORIZ_READS * n_reads);
						c_scan = iDiv(t0, MLO_N_IN_HORIZ_READS);
						int c_pix4 = iMod(t0, c_scan, MLO_N_IN_HORIZ_READS);
						int in_scan = c_scan* MLO_FILTER_STRIDE1 + f_s;

						if (ob*MLO_OUT_EXTENT1*MLO_FILTER_STRIDE1 + in_scan < MLO_IN_HEIGHT)
						{

							int gbl_off = gbl_in_scan_off + b*MLO_IN_BATCH_STRIDE + in_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
							// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
							if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
							{
								int i = 0;
								for (; i < MLO_IN_N_PIXS_OFF; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
								for (; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = 0;
								}

							}
							else
#endif
							{
								//					*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];

								for (int i = 0; i < MLO_READ_UNIT; ++i)
								{
									in_rd_data[i] = bot[gbl_off + i];
								}
							}
							int lcl_off = (lcl_scan + c_scan)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
							for (int i = 0; i < MLO_READ_UNIT; ++i)
							{
								bot_mem[lcl_off + i] = in_rd_data[i];
							}

						}
					}

					barrier(CLK_LOCAL_MEM_FENCE);

					// convolution
					// along vertical filter
					for (int m = 0; m < MLO_N_FILTER_SPLITS1; ++m)
					{
						// select all vertical scans that matches the vertical filter tap
						__private _FLOAT in_vals[MLO_N_LCL_BATCHS * ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0)];
						// read input values for this filter phase
						for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
						{
							for (int i = 0; i < ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0); ++i)
							{
								in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + i]
									= bot_mem[bb*MLO_IN_LCL_SZ + (ex_row* MLO_N_FILTER_SPLITS1 + m) * MLO_IN_LCL_WIDTH + ex_pix*MLO_FILTER_STRIDE0 + i];
							}

						}
// only for 11
						__private _FLOAT wei_vals[MLO_N_LCL_OUT_MAPS*MLO_FILTER_STRIDE0];
						// first 2 splits
						int l;
						for (l = 0; l < MLO_N_FILTER_SPLITS0 - 1; ++l)
						{

							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l
// 3d
						{

							for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
							{
								for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
								{
									wei_vals[k*MLO_FILTER_STRIDE0 + i]
										= wei_mem[k*MLO_WEI_SZ + m*MLO_WEI_LCL_WIDTH + i];
								}
							}

							// convolve
							for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
							{

								for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
								{
									for (int n = 0; n < MLO_OUT_PIX_TILE0; ++n)
									{

										for (int i = 0; i < MLO_FILTER_STRIDE0 - 1; ++i)
										{
											_FLOAT in_val = in_vals[bb* ((MLO_OUT_PIX_TILE0 - 1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + l*MLO_FILTER_STRIDE0 + i];
											_FLOAT wei_val = wei_vals[k*MLO_FILTER_STRIDE0 + i];
											pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + n]
												+= wei_val * in_val;
										}
									}
								}
							} // b
						} // l

					} // m

				} // f_s

			} // c

			barrier(CLK_LOCAL_MEM_FENCE);
			int out_y = ob*MLO_OUT_EXTENT1;

			for (int bb = 0; bb < MLO_N_LCL_BATCHS; ++bb)
			{
				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					// write out
					// inputs are outputs
					int out_off = (ib + b + bb) * MLO_OUT_BATCH_STRIDE + (k_idx + k) * MLO_OUT_CHANNEL_STRIDE + (out_y + ex_row) *MLO_OUT_STRIDE + ex_pix;
					for (int i = 0; i < MLO_OUT_PIX_TILE0 && (out_y + ex_row) < MLO_OUT_HEIGHT && ex_pix + i < MLO_OUT_WIDTH; ++i)
					{
						top[out_off + i] = pvt_accum[(bb*MLO_N_LCL_OUT_MAPS + k) * MLO_OUT_PIX_TILE0 + i];
					}
				}
			}

		} // ob

	} // b
}