tweak kernel

tweak kernel

// Copyright Hugh Perkins 2014 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// every plane is independent
// every example is independent
// so, globalid can be: [n][plane][outputRow][outputCol]
kernel void propagateNaive( const int batchSize, global const float *input, global int *selectors, global float *output ) {
    const int globalId = get_global_id(0);

    const int intraBoardOffset = globalId % gOutputBoardSizeSquared;
    const int outputRow = intraBoardOffset / gOutputBoardSize;
    const int outputCol = intraBoardOffset % gOutputBoardSize;

    const int board2dIdx = globalId / gOutputBoardSizeSquared;
    const int plane = board2dIdx % gNumPlanes;
    const int n = board2dIdx / gNumPlanes;

    if( n >= batchSize ) {
        return;
    }

    const int inputRow = outputRow * gPoolingSize;
    const int inputCol = outputCol * gPoolingSize;
    const int inputBoardOffset = ( n * gNumPlanes + plane ) * gInputBoardSizeSquared;
    int selector = 0;
    int poolOffset = inputBoardOffset + inputRow * gInputBoardSize + inputCol;
    float maxValue = input[ poolOffset ];
    for( int dRow = 0; dRow < gPoolingSize; dRow++ ) {
        for( int dCol = 0; dCol < gPoolingSize; dCol++ ) {
            float thisValue = input[ poolOffset + dRow * gInputBoardSize + dCol ];
            if( thisValue > maxValue ) {
                maxValue = thisValue;
                selector = dRow * gPoolingSize + dCol;
            }
        }
    }
    output[ globalId ] = maxValue;
    selectors[ globalId ] = selector;
}

// Copyright Hugh Perkins 2014 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// every plane is independent
// every example is independent
// so, globalid can be: [n][plane][outputRow][outputCol]
kernel void propagateNaive( global const float *input, global int *selectors, global float *output ) {
    const int globalId = get_global_id(0);

    const int intraBoardOffset = globalId % gOutputBoardSizeSquared;
    const int outputRow = intraBoardOffset / gOutputBoardSize;
    const int outputCol = intraBoardOffset % gOutputBoardSize;

    const int board2dIdx = globalId / gOutputBoardSizeSquared;
    const int plane = board2dIdx % gNumPlanes;
    const int n = board2dIdx / gNumPlanes;

    const int inputRow = outputRow * gPoolingSize;
    const int inputCol = outputCol * gPoolingSize;
    const int inputBoardOffset = ( n * gNumPlanes + plane ) * gInputBoardSizeSquared;
    int selector = 0;
    int poolOffset = inputBoardOffset + inputRow * gInputBoardSize + inputCol;
    float maxValue = input[ poolOffset ];
    for( int dRow = 0; dRow < gPoolingSize; dRow++ ) {
        for( int dCol = 0; dCol < gPoolingSize; dCol++ ) {
            float thisValue = input[ poolOffset + dRow * gInputBoardSize + dCol ];
            if( thisValue > maxValue ) {
                maxValue = thisValue;
                selector = dRow * gPoolingSize + dCol;
            }
        }
    }
    output[ globalId ] = maxValue;
    selectors[ globalId ] = selector;
}


fix code in the kernel

fix code in the kernel

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the copyright holders or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#ifdef DOUBLE_SUPPORT
#ifdef cl_amd_fp64
#pragma OPENCL EXTENSION cl_amd_fp64:enable
#elif defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#endif
#endif

#if ddepth == 0
#define MIN_VAL 0
#define MAX_VAL 255
#elif ddepth == 1
#define MIN_VAL -128
#define MAX_VAL 127
#elif ddepth == 2
#define MIN_VAL 0
#define MAX_VAL 65535
#elif ddepth == 3
#define MIN_VAL -32768
#define MAX_VAL 32767
#elif ddepth == 4
#define MIN_VAL INT_MIN
#define MAX_VAL INT_MAX
#elif ddepth == 5
#define MIN_VAL (-FLT_MAX)
#define MAX_VAL FLT_MAX
#elif ddepth == 6
#define MIN_VAL (-DBL_MAX)
#define MAX_VAL DBL_MAX
#else
#error "Unsupported depth"
#endif

#define noconvert

#ifdef OCL_CV_REDUCE_SUM
#define INIT_VALUE 0
#define PROCESS_ELEM(acc, value) acc += value
#elif defined(OCL_CV_REDUCE_MAX)
#define INIT_VALUE MIN_VAL
#define PROCESS_ELEM(acc, value) acc = value > acc ? value : acc
#elif defined(OCL_CV_REDUCE_MIN)
#define INIT_VALUE MAX_VAL
#define PROCESS_ELEM(acc, value) acc = value < acc ? value : acc
#elif defined(OCL_CV_REDUCE_AVG)
#error "This operation should be implemented through OCL_CV_REDUCE_SUM"
#else
#error "No operation is specified"
#endif

#ifndef BUF_COLS
#define BUF_COLS  32
#endif

__kernel void reduce_horz_pre(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * bufptr, int buf_step, int buf_offset)
{
    int x = get_global_id(0);
    int y = get_global_id(1);
    if (x < BUF_COLS)
    {
        int src_index = mad24(y, src_step, mad24(x, (int)sizeof(srcT) * cn, src_offset));
        int buf_index = mad24(y, buf_step, mad24(x, (int)sizeof(dstT) * cn, buf_offset));

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * buf = (__global dstT *)(bufptr + buf_index);
        dstT tmp[cn] = { INIT_VALUE };

        int src_step_mul = BUF_COLS * cn;
        for (int idx = x; idx < cols; idx += BUF_COLS, src += src_step_mul)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            buf[c] = tmp[c];
    }
}

__kernel void reduce(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * dstptr, int dst_step, int dst_offset)
{
#if dim == 0 // reduce to a single row
    int x = get_global_id(0);
    if (x < cols)
    {
        int src_index = mad24(x, (int)sizeof(srcT) * cn, src_offset);
        __global dstT * dst = (__global dstT *)(dstptr + dst_offset) + x * cn;
        dstT tmp[cn] = { INIT_VALUE };

        for (int y = 0; y < rows; ++y, src_index += src_step)
        {
            __global const srcT * src = (__global const srcT *)(srcptr + src_index);
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#elif dim == 1 // reduce to a single column
    int y = get_global_id(0);
    if (y < rows)
    {
        int src_index = mad24(y, src_step, src_offset);
        int dst_index = mad24(y, dst_step, dst_offset);

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * dst = (__global dstT *)(dstptr + dst_index);
        dstT tmp[cn] = { INIT_VALUE };

        for (int x = 0; x < cols; ++x, src += cn)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#else
#error "Dims must be either 0 or 1"
#endif
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the copyright holders or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#ifdef DOUBLE_SUPPORT
#ifdef cl_amd_fp64
#pragma OPENCL EXTENSION cl_amd_fp64:enable
#elif defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#endif
#endif

#if ddepth == 0
#define MIN_VAL 0
#define MAX_VAL 255
#elif ddepth == 1
#define MIN_VAL -128
#define MAX_VAL 127
#elif ddepth == 2
#define MIN_VAL 0
#define MAX_VAL 65535
#elif ddepth == 3
#define MIN_VAL -32768
#define MAX_VAL 32767
#elif ddepth == 4
#define MIN_VAL INT_MIN
#define MAX_VAL INT_MAX
#elif ddepth == 5
#define MIN_VAL (-FLT_MAX)
#define MAX_VAL FLT_MAX
#elif ddepth == 6
#define MIN_VAL (-DBL_MAX)
#define MAX_VAL DBL_MAX
#else
#error "Unsupported depth"
#endif

#define noconvert

#ifdef OCL_CV_REDUCE_SUM
#define INIT_VALUE 0
#define PROCESS_ELEM(acc, value) acc += value
#elif defined(OCL_CV_REDUCE_MAX)
#define INIT_VALUE MIN_VAL
#define PROCESS_ELEM(acc, value) acc = value > acc ? value : acc
#elif defined(OCL_CV_REDUCE_MIN)
#define INIT_VALUE MAX_VAL
#define PROCESS_ELEM(acc, value) acc = value < acc ? value : acc
#elif defined(OCL_CV_REDUCE_AVG)
#error "This operation should be implemented through OCL_CV_REDUCE_SUM"
#else
#error "No operation is specified"
#endif

#ifndef BUF_COLS
#define BUF_COLS  32
#endif

__kernel void reduce_horz_pre(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * bufptr, int buf_step, int buf_offset)
{
    int x = get_global_id(0);
    int y = get_global_id(1);
    if (x < BUF_COLS)
    {
        int src_index = mad24(y, src_step, mad24(x, (int)sizeof(srcT) * cn, src_offset));
        int buf_index = mad24(y, buf_step, mad24(x, (int)sizeof(dstT) * cn, buf_offset));

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * buf = (__global dstT *)(bufptr + buf_index);
        dstT tmp[cn] = { INIT_VALUE };

        int src_step_mul = BUF_COLS * cn;
        for (int x = 0; x < cols; x += BUF_COLS, src += src_step_mul)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            buf[c] = tmp[c];
    }
}

__kernel void reduce(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * dstptr, int dst_step, int dst_offset)
{
#if dim == 0 // reduce to a single row
    int x = get_global_id(0);
    if (x < cols)
    {
        int src_index = mad24(x, (int)sizeof(srcT) * cn, src_offset);
        __global dstT * dst = (__global dstT *)(dstptr + dst_offset) + x * cn;
        dstT tmp[cn] = { INIT_VALUE };

        for (int y = 0; y < rows; ++y, src_index += src_step)
        {
            __global const srcT * src = (__global const srcT *)(srcptr + src_index);
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#elif dim == 1 // reduce to a single column
    int y = get_global_id(0);
    if (y < rows)
    {
        int src_index = mad24(y, src_step, src_offset);
        int dst_index = mad24(y, dst_step, dst_offset);

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * dst = (__global dstT *)(dstptr + dst_index);
        dstT tmp[cn] = { INIT_VALUE };

        for (int x = 0; x < cols; ++x, src += cn)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#else
#error "Dims must be either 0 or 1"
#endif
}


Removed an unnecessary test file.

Removed an unnecessary test file.

-- example of static and dynamic type differing for a dispatch

Class Book inherits IO {
    title : String;
    author : String;

    initBook(title_p : String, author_p : String) : Book {
        {
            title <- title_p;
            author <- author_p;
            self;
        }
    };

    print() : Book {
        {
            out_string("title:      ").out_string(title).out_string("\n");
            out_string("author:     ").out_string(author).out_string("\n");
            self;
        }
    };
};

Class Article inherits Book {
    per_title : String;

    initArticle(title_p : String, author_p : String,
		per_title_p : String) : Article {
        {
            initBook(title_p, author_p);
            per_title <- per_title_p;
            self;
        }
    };

    print() : Book {
        {
	    self@Book.print();
            out_string("periodical:  ").out_string(per_title).out_string("\n");
            self;
        }
    };
};

Class BookList inherits IO {
    (* Since abort "returns" type Object, we have to add
       an expression of type Bool here to satisfy the typechecker.
       This code is unreachable, since abort() halts the program.
    *)
    isNil() : Bool { { abort(); true; } };

    cons(hd : Book) : Cons {
        (let new_cell : Cons <- new Cons in
            new_cell.init(hd,self)
        )
    };

    (* Since abort "returns" type Object, we have to add
       an expression of type Book here to satisfy the typechecker.
       This code is unreachable, since abort() halts the program.
    *)
    car() : Book { { abort(); new Book; } };

    (* Since abort "returns" type Object, we have to add
       an expression of type BookList here to satisfy the typechecker.
       This code is unreachable, since abort() halts the program.
    *)
    cdr() : BookList { { abort(); new BookList; } };

    print_list() : Object { abort() };
};

Class Cons inherits BookList {
    xcar : Book;  -- We keep the car and cdr in attributes.
    xcdr : BookList; -- Because methods and features must have different names,
    -- we use xcar and xcdr for the attributes and reserve
    -- car and cdr for the features.

    isNil() : Bool { false };

    init(hd : Book, tl : BookList) : Cons {
        {
            xcar <- hd;
            xcdr <- tl;
            self;
        }
    };

    car() : Book { xcar };

    cdr() : BookList { xcdr };

    print_list() : Object {
        {
            case xcar.print() of
                dummy : Book => out_string("- dynamic type was Book -\n");
                dummy : Article => out_string("- dynamic type was Article -\n");
            esac;
            xcdr.print_list();
        }
    };
};

Class Nil inherits BookList {
    isNil() : Bool { true };

    print_list() : Object { true };
};

Class Main {

    books : BookList;

    main() : Object {
        (let a_book : Book <-
            (new Book).initBook("Compilers, Principles, Techniques, and Tools",
                                "Aho, Sethi, and Ullman")
        in
            (let an_article : Article <-
                (new Article).initArticle("The Top 100 CD_ROMs",
                                          "Ulanoff",
                                          "PC Magazine")
            in
                {
                    books <- (new Nil).cons(a_book).cons(an_article);
                    books.print_list();
                }
            )  -- end let an_article
        )  -- end let a_book
    };
};


Fixed type conversion bug and now use standard functino.

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_SHIFT 0
#define PKT_DEST_SHIFT 2
#define PKT_ARG_SHIFT  10
#define PKT_SUB_SHIFT  14
#define PKT_TYPE_MASK 0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK 0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK  0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK  0xFFC000 // 00000000111111111100000000000000

/* Packet Types */
#define ERROR     0
#define REFERENCE 1
#define DATA      2
#define REQUEST   3

typedef uint2 packet;

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

void transferRQ(__global uint2 *rq,  __global uint2 *q, int n);
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n);
uint q_get_tail_index(size_t id, size_t gid, __global uint2 *q, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global uint2 *q, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global uint2 *q, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_empty(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_full(size_t id, size_t gid,__global uint2 *q, int n);
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n);
bool q_write(uint2 value, size_t id, __global uint2 *q, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void vm(__global uint2 *q,
		 __global uint2 *rq,
		 int n,
		 __global int *state)
{
  size_t gid = get_global_id(0);

  packet p = pkt_create(ERROR, 7, 0, 0, 0);
  if (*state == WRITE) {
    transferRQ(rq, q, n);
  } else {
    q_write(p, gid, q, n);
  }

  *state = COMPLETE;
}

/**************************/
/**** Packet Functions ****/
/**************************/

uint pkt_get_type(packet p) {
  return (p.x & PKT_TYPE_MASK) >> PKT_TYPE_SHIFT;
}

uint pkt_get_dest(packet p) {
  return (p.x & PKT_DEST_MASK) >> PKT_DEST_SHIFT;
}

uint pkt_get_arg(packet p) {
  return (p.x & PKT_ARG_MASK) >> PKT_ARG_SHIFT;
}

uint pkt_get_sub(packet p) {
  return (p.x & PKT_SUB_MASK) >> PKT_SUB_SHIFT;
}

uint pkt_get_payload(packet p) {
  return p.y;
}

void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_SHIFT) & PKT_TYPE_MASK);
}

void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_SHIFT) & PKT_DEST_MASK);
}

void pkt_set_arg(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_SHIFT) & PKT_ARG_MASK);
}

void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_SHIFT) & PKT_SUB_MASK);
}

void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p = 0;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue specifided by 'id' */
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  return indices.x;
}

/* Returns the array index of the tail element of the queue specified by 'id'. */
uint q_get_tail_index(size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  return indices.y;
}

/* Set the array index of the head element of the queue specified by 'id'. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  indices.x = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the array index of the tail element of the queue specified by 'id'. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  indices.y = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the type of the operation last performed on the queue specified by 'id'. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint2 *q, int n) {
  q[id * n + gid].y = type;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == READ;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y != READ;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_read(id, gid, q, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_write(id, gid, q, n);
}

/* Read the value located at the head index into 'result' from the queue specified by 'id'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(id, gid, q, n)) {
    return false;
  }

  int index = q_get_head_index(id, gid, q, n);
  *result = q[(n*n) + (gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(READ, id, gid, q, n);
  return true;
}

/* Write a value into the tail index of the queue specified by 'id'.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, q, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, q, n);
  q[(n*n) + (gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(WRITE, id, gid, q, n);
  return true;
}

void transferRQ(__global uint2 *rq,  __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (!q_is_empty(i, gid, rq, n)) {
      q_read(&packet, i, rq, n);
      q_write(packet, i, q, n);
    }
  }
}

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_SHIFT 0
#define PKT_DEST_SHIFT 2
#define PKT_ARG_SHIFT  10
#define PKT_SUB_SHIFT  14
#define PKT_TYPE_MASK 0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK 0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK  0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK  0xFFC000 // 00000000111111111100000000000000

/* Packet Types */
#define ERROR     0
#define REFERENCE 1
#define DATA      2
#define REQUEST   3

typedef uint2 packet;

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

void transferRQ(__global uint2 *rq,  __global uint2 *q, int n);
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n);
uint q_get_tail_index(size_t id, size_t gid, __global uint2 *q, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global uint2 *q, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global uint2 *q, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_empty(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_full(size_t id, size_t gid,__global uint2 *q, int n);
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n);
bool q_write(uint2 value, size_t id, __global uint2 *q, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void vm(__global uint2 *q,
		 __global uint2 *rq,
		 int n,
		 __global int *state)
{
  size_t gid = get_global_id(0);

  packet p = pkt_create(ERROR, 7, 0, 0, 0);
  if (*state == WRITE) {
    transferRQ(rq, q, n);
  } else {
    q_write(p, gid, q, n);
  }

  *state = COMPLETE;
}

/**************************/
/**** Packet Functions ****/
/**************************/

uint pkt_get_type(packet p) {
  return (p.x & PKT_TYPE_MASK) >> PKT_TYPE_SHIFT;
}

uint pkt_get_dest(packet p) {
  return (p.x & PKT_DEST_MASK) >> PKT_DEST_SHIFT;
}

uint pkt_get_arg(packet p) {
  return (p.x & PKT_ARG_MASK) >> PKT_ARG_SHIFT;
}

uint pkt_get_sub(packet p) {
  return (p.x & PKT_SUB_MASK) >> PKT_SUB_SHIFT;
}

uint pkt_get_payload(packet p) {
  return p.y;
}

void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_SHIFT) & PKT_TYPE_MASK);
}

void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_SHIFT) & PKT_DEST_MASK);
}

void pkt_set_arg(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_SHIFT) & PKT_ARG_MASK);
}

void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_SHIFT) & PKT_SUB_MASK);
}

void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p = 0;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue specifided by 'id' */
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  return indices.x;
}

/* Returns the array index of the tail element of the queue specified by 'id'. */
uint q_get_tail_index(size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  return indices.y;
}

/* Set the array index of the head element of the queue specified by 'id'. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  indices.x = index;
  q[id * n + gid].x = (uint) indices;
}

/* Set the array index of the tail element of the queue specified by 'id'. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = q[id * n + gid].x;
  indices.y = index;
  q[id * n + gid].x = (uint) indices;
}

/* Set the type of the operation last performed on the queue specified by 'id'. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint2 *q, int n) {
  q[id * n + gid].y = type;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == READ;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y != READ;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_read(id, gid, q, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_write(id, gid, q, n);
}

/* Read the value located at the head index into 'result' from the queue specified by 'id'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(id, gid, q, n)) {
    return false;
  }

  int index = q_get_head_index(id, gid, q, n);
  *result = q[(n*n) + (gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(READ, id, gid, q, n);
  return true;
}

/* Write a value into the tail index of the queue specified by 'id'.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, q, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, q, n);
  q[(n*n) + (gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(WRITE, id, gid, q, n);
  return true;
}

void transferRQ(__global uint2 *rq,  __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (!q_is_empty(i, gid, rq, n)) {
      q_read(&packet, i, rq, n);
      q_write(packet, i, q, n);
    }
  }
}


removed unrolls

removed unrolls

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	uint m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	uint p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	uint m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	uint p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ  && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
#endif

	for (uint b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		uint bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					const __global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
		else
#endif
		{
			// check
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
			{
				// reading in order per group and jump over maps been read
				// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}

		} // if (last_pixel)

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				uint m = iDiv(p, MLO_MAP_WK_SZ);
				uint pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				uint m = ((uint)p / MLO_MAP_WK_SZ);
				uint pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				uint top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif

				const __global _FLOAT * top_df1 = &top_df[top_off1];

#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (uint k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			uint wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	uint gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	uint gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (uint pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		uint p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		uint b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		uint p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		uint gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		uint gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif

				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;

#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

		// processing
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	uint m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	uint p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	uint m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	uint p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ  && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
#endif

	for (uint b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		uint bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					const __global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
		else
#endif
		{
			// check
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
			{
				// reading in order per group and jump over maps been read
				// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}

		} // if (last_pixel)

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll 4
			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				uint m = iDiv(p, MLO_MAP_WK_SZ);
				uint pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				uint m = ((uint)p / MLO_MAP_WK_SZ);
				uint pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				uint top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif

				const __global _FLOAT * top_df1 = &top_df[top_off1];

#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

#pragma unroll 4
			for (uint k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			uint wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	uint gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	uint gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (uint pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		uint p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		uint b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		uint p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		uint gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		uint gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif

				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;

#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

//#pragma unroll 2
		// processing
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
//#pragma unroll 4
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}