Update passthrough kernel to include PAPhysicsForce arg

/* particles - simulator - passthrough.cl
 *
 * Copyright 2010 Tim Horton. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    1. Redistributions of source code must retain the above copyright notice,
 *       this list of conditions and the following disclaimer.
 *
 *    2. Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY TIM HORTON "AS IS" AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
 * EVENT SHALL TIM HORTON OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "libparticles/PATypes.h"

__kernel void passthrough(__global PAPhysicsParticle * input,
                          __global PAPhysicsParticle * output,
                          __global PAPhysicsNewtonian * newtonIn,
                          __global PAPhysicsNewtonian * newtonOut,
                          __global PAPhysicsForce * force,
                          const unsigned int count)
{
    int id = get_global_id(0);

    if(id > count)
        return;

    output[id].enabled = input[id].enabled;
    output[id].x = input[id].x;
    output[id].y = input[id].y;
    output[id].z = input[id].z;
}

/* particles - simulator - passthrough.cl
 *
 * Copyright 2010 Tim Horton. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    1. Redistributions of source code must retain the above copyright notice,
 *       this list of conditions and the following disclaimer.
 *
 *    2. Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY TIM HORTON "AS IS" AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
 * EVENT SHALL TIM HORTON OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "libparticles/PATypes.h"

__kernel void passthrough(__global PAPhysicsParticle * input,
                          __global PAPhysicsParticle * output,
                          __global PAPhysicsNewtonian * newtonIn,
                          __global PAPhysicsNewtonian * newtonOut,
                          const unsigned int count)
{
    int id = get_global_id(0);

    if(id > count)
        return;

    output[id].enabled = input[id].enabled;
    output[id].x = input[id].x;
    output[id].y = input[id].y;
    output[id].z = input[id].z;
}


Refined unroll in MIOpenConvDirUniC for different configs

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_PIX_TILE1*MLO_OUT_PIX_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)

#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  (MLO_N_INPUTS/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#define MLO_IN_LCL_WIDTH (MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1)  // here we use kernel size. it's important when padding == 0
#define MLO_IN_LCL_HEIGHT (MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1)
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#define MLO_PVT_IN_WIDTH (MLO_FILTER_SIZE0 + MLO_OUT_PIX_TILE0 - 1)
#define MLO_PVT_IN_HEIGHT (MLO_OUT_PIX_TILE1)

#define MLO_LCL_WEIGHTS 1

#if defined(__AMDGCN__)
extern uint __llvm_amdgcn_readfirstlane(uint) __asm("llvm.amdgcn.readfirstlane");
#define uniform(x) __llvm_amdgcn_readfirstlane(x)
#else
#define uniform(x) (x)
#endif

static inline void calculateXYPos(uint linPos, uint width, uint * __restrict x, uint * __restrict y)
{

	(*y) = (uint)((float)linPos * (1.0f / (float)width) + 0.00001f);

	(*x) = linPos - (*y) * width;
}

static inline uint calculateOffset(uint stride, uint x, uint y)
{
	uint ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(uint linPos,__local _FLOAT *lcl_data, uint lcl_base, UNUSED uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 UNUSED bool debug)
{
	uint x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	uint gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	uint gbl_off = gbl_off0 + gbl_base;

#if MLO_LARGE_MAP == 1
	uint lcl_off = lcl_base + linPos;
	(void)lcl_stride;
	(void)lcl_x;
	(void)lcl_y;
#else
	uint l_x = x + lcl_x;
	uint l_y = y + lcl_y;
	uint lcl_off = lcl_base + mad24(l_y, lcl_stride, l_x);
#endif

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#else
	 (void)gbl_width;
	 (void)gbl_height;
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(uint lcl_id, uint size, uint lcl_p_stride, __local _FLOAT *lcl_data, uint lcl_base, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(uint i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(uint lcl_id, uint lcl_p_stride,
					__local _FLOAT * lcl_data,
					uint lcl_off, uint lcl_size, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_bot_y, uint lcl_bot_x,
					const __global _FLOAT * gbl_data,
					uint gbl_off, uint gbl_size, uint gbl_height, uint glb_width, uint gbl_stride, int gbl_bot_y, int gbl_bot_x,
					uint buf_block_ind, uint max_n_bufs, uint lcl_n_bufs,
					bool debug)
{

	for(uint c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(uint o_map_base,
				uint in_stg_off,
				 _FLOAT * pvt_in_stage, __local _FLOAT * __restrict lcl_indata,
				 _FLOAT * pvt_wei_stage, __local _FLOAT * __restrict lcl_wei,
				 _FLOAT * pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		uint in_stg_off1 = in_stg_off;
		for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			uint wei_stg_base_off = mad24(o_map_base, (uint)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(uint)MLO_FILTER_SZ));
			uint in_stg_off2 = in_stg_off1;
			for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#ifdef __AMDGCN__
			#pragma unroll
#endif
			for(uint k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
			{
				uint k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(uint i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					uint wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(uint i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] = lcl_wei[wei_stg_off + i];
					}

		// actual conv

					for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j)
					{
						for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
						{
							for(uint l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								uint l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

								pvt_accum[(o_c * MLO_OUT_PIX_TILE1 + j) * MLO_OUT_PIX_TILE0 + i]
									 += pvt_in_stage[j * MLO_PVT_IN_WIDTH + i + l] * pvt_wei_stage[l_act];

							}

						}
					}

				} // for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

//				mem_fence(CLK_LOCAL_MEM_FENCE);

			} // for(uint k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenConvUniC(
       const __global _FLOAT * __restrict in,
       const __global _FLOAT * __restrict weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * __restrict bias,
#endif
	__global _FLOAT * __restrict out,
	UNUSED _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	uint grp_id0 = get_group_id(0);
#if MLO_OUT_WIDTH == MLO_IN_TILE0
	uint y_tile_blk = grp_id0;
	uint x_tile_blk = 0;
#else
#if MLO_N_OUT_TILE_BLOCKS0 & (MLO_N_OUT_TILE_BLOCKS0 - 1)
	uint y_tile_blk = (uint)((float)grp_id0 * (1.0f / (float) MLO_N_OUT_TILE_BLOCKS0) + 0.00001f);
	int x_tile_blk = grp_id0 - mul24(y_tile_blk, (uint)MLO_N_OUT_TILE_BLOCKS0);
#else
	uint y_tile_blk = grp_id0 / MLO_N_OUT_TILE_BLOCKS0;
	int x_tile_blk = grp_id0 & (MLO_N_OUT_TILE_BLOCKS0 - 1);
#endif
#endif
	uint o_pack = get_group_id(1); // block of outputs
	uint b_pack = get_group_id(2); // batch block

	uint lcl_id = get_local_id(0);
#if MLO_ALUTILES_STACK_SZ & (MLO_ALUTILES_STACK_SZ - 1)
	uint stack = (uint)((float)lcl_id * (1.0f / (float)MLO_ALUTILES_STACK_SZ) + 0.00001f);  // stack
	uint alu_stack_id = lcl_id - mul24(stack, (uint)MLO_ALUTILES_STACK_SZ);  // alu index in stack
#else
	uint stack = lcl_id / MLO_ALUTILES_STACK_SZ;  // stack
	uint alu_stack_id = lcl_id & (MLO_ALUTILES_STACK_SZ - 1);  // alu index in stack
#if MLO_ALUTILES_STACK_SZ >= 64
	stack = uniform(stack);
#endif
#endif
// ALU plane inside stack
#if MLO_ALU_TILE_SZ & (MLO_ALU_TILE_SZ - 1)
	uint alu_out_plane_id = (uint)((float)alu_stack_id * (1.0f / (float)MLO_ALU_TILE_SZ) + 0.00001f);  // alu output plane index
	uint alu_out_id = alu_stack_id - mul24(alu_out_plane_id, (uint)MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
#else
	uint alu_out_plane_id = alu_stack_id / MLO_ALU_TILE_SZ;  // alu output plane index
	uint alu_out_id = alu_stack_id & (MLO_ALU_TILE_SZ - 1); // alu index inside an ALU output plane
#endif
// pos inside ALU tile
#if MLO_ALU_VTILE0 & (MLO_ALU_VTILE0 - 1)
	uint alu_tl1 = (uint)((float)alu_out_id * (1.0f / (float)MLO_ALU_VTILE0) + 0.00001f);
	uint alu_tl0 = alu_out_id - mul24(alu_tl1, (uint)MLO_ALU_VTILE0);
#else
	uint alu_tl1 = alu_out_id / MLO_ALU_VTILE0;
	uint alu_tl0 = alu_out_id & (MLO_ALU_VTILE0 - 1);
#endif

	uint o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	uint o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	uint o_map = o_map_plane + o_map_base; // output map index per ALU plane
	uint b_index = b_pack * MLO_N_STACKS;

#if MLO_LARGE_MAP != 1
#if MLO_GRP_SZ <= MLO_N_READ_PROCS
	uint wave_id = 0;
	uint wave_lcl_id = lcl_id;
#elif MLO_N_READ_PROCS & (MLO_N_READ_PROCS - 1)
	uint wave_id = (uint)((float)lcl_id * (1.0f / (float)MLO_N_READ_PROCS) + 0.00001f);
	uint wave_lcl_id = lcl_id - mul24(wave_id, (uint)MLO_N_READ_PROCS);
#else
	uint wave_id = (uint)((uint)lcl_id / MLO_N_READ_PROCS);
	uint wave_lcl_id = lcl_id & (MLO_N_READ_PROCS - 1);
#if MLO_N_READ_PROCS >= 64
        wave_id = uniform(wave_id);
#endif
#endif
#endif

	int x_grp = x_tile_blk * MLO_IN_TILE0;
	uint y_grp = y_tile_blk * MLO_IN_TILE1;

// TO DO: scale
#if MLO_LARGE_MAP == 1
	int x_in_grp = x_grp - MLO_FILTER_PAD0;
	int y_in_grp = y_grp - MLO_FILTER_PAD1;
#endif

	uint x_in_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_in_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

// base offset to read data from local input data
	uint in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    uint in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	uint wei_off = mul24(o_map_plane, (uint)(MLO_N_INPUTS * MLO_FILTER_SZ));
#else
	uint wei_off = mul24(o_map_plane,(uint)MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(uint i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(uint i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(uint ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		uint in_lcl_off1 = 0;
		uint in_off1 = in_off;
		for(uint i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			uint in_off2 = in_off1;
			uint in_lcl_off2 = in_lcl_off1;
			for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				uint elem_id = lcl_id;
				uint lcl_p_stride = MLO_GRP_SZ0;
				uint lcl_base = 0;
				uint lcl_y = 0;
				uint lcl_x = 0;
				uint gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(uint i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
		//(MLO_N_STACKS * MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_IN_TILES_PERSTACK & (MLO_N_IN_TILES_PERSTACK - 1)
			uint i_b = (uint)((float)i * (1.0f / (float)MLO_N_IN_TILES_PERSTACK) + 0.00001f);
			uint i_c = i - mul24(i_b, (uint)MLO_N_IN_TILES_PERSTACK);
#else
			uint i_b = (uint)i / MLO_N_IN_TILES_PERSTACK;
			uint i_c = i & (MLO_N_IN_TILES_PERSTACK - 1);
#endif

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			uint in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			uint in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			uint elem_id = wave_lcl_id;
			uint lcl_p_stride = MLO_N_READ_PROCS;
			uint lcl_base = 0;
			uint lcl_y = MLO_FILTER_PAD1;
			uint lcl_x = MLO_FILTER_PAD0;
			uint gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

#if (MLO_WEIGHTS_SZ >= MLO_GRP_SZ) && defined(__AMDGCN__)
#if MLO_WEIGHTS_SZ/MLO_GRP_SZ > 4
#pragma unroll
#else
#pragma unroll (MLO_WEIGHTS_SZ/MLO_GRP_SZ)
#endif
#endif
		for(uint i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
#if (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = (uint)((float)i * (1.0f / (float)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ)) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
			if((wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i) < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ))
				lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i];
			else
				lcl_wei[i] = weights[0];
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)

#if (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = (uint)((float)i * (1.0f / (float)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ)) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
#if MLO_FILTER_SZ & (MLO_FILTER_SZ - 1)
			uint lcl_c = (uint)((float)gbl_i * (1.0f / (float)MLO_FILTER_SZ) + 0.00001f);
			uint lcl_i = gbl_i - mul24(lcl_c, (uint)MLO_FILTER_SZ);
#else
			uint lcl_c = gbl_i / MLO_FILTER_SZ;
			uint lcl_i = gbl_i & (MLO_FILTER_SZ - 1);
#endif

			uint lcl_we_off = mad24(mad24(lcl_c, (uint)MLO_N_IN_TILES_PERSTACK, lcl_o), (uint)MLO_FILTER_SZ, lcl_i);
			uint gbl_we_off = mad24(mad24(lcl_o, (uint)MLO_N_OUTPUTS, lcl_c), (uint)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
		if (lcl_id < MLO_ACTIVE_ALUS)
#endif
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}

#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
	if (lcl_id >= MLO_ACTIVE_ALUS)
	{
		return;
	}
#endif
// write results out
	uint x_out_grp = x_grp;
	uint y_out_grp = y_grp;
	uint x_out_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_out_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

    uint out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		uint out_off1 = out_off;
		for(uint o = 0; o < MLO_N_OUT_TILES
						; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE
						)
		{
// over output tile
			_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS
			bias_val = bias[o_map + o];
#endif
			uint out_off2 = out_off1;
			for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
				__global _FLOAT *out_p = &out[out_off2];
				for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
				{
					if ( true
#if 1 //MLO_OUT_ALIGNED == 0
					&& y_out_lcl + j < MLO_OUT_TILE1 && y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT &&  x_out_lcl + i < MLO_OUT_TILE0 && x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH
#endif
#if MLO_OUTPUTS_ALIGNED == 0
						&& o_map + o < MLO_N_OUTPUTS
#endif
					)
					{
						out_p[i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i] + bias_val;
#if 0
						if ( out_off2 + i == 1 /*y_out_grp + y_out_lcl + j == 2 && x_out_grp + x_out_lcl + i == 0*/)
						{
							printf("K:out: %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d    %f %f %f\n",
								MLO_OUT_TILE1,
								MLO_OUT_TILE0,
								grp_id0,
								lcl_id,
								alu_out_plane_id,
								alu_out_id,
								b_index + stack,
								o,
								o_map,
								out_off,
								out_off1,
								out_off2,
								y_out_grp,
								y_out_lcl,
								x_out_grp,
								x_out_lcl,
								j,
								i,
								pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i],
								 bias_val,
								out[out_off2 + i]
								);
						}
#endif
					}

				}
			}

		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_FILTER_SZ (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

#define MLO_GRP_SZ0  (MLO_GRP_TILE0*MLO_GRP_TILE1)
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1
#define MLO_GRP_SZ (MLO_GRP_SZ0*MLO_GRP_SZ1*MLO_GRP_SZ2)
#define MLO_N_PROC_WAVES ((MLO_GRP_SZ + MLO_N_READ_PROCS - 1)/MLO_N_READ_PROCS)
#define MLO_OUT_TILE_SZ (MLO_OUT_PIX_TILE1*MLO_OUT_PIX_TILE0)
#define MLO_ALU_TILE_SZ (MLO_ALU_VTILE1*MLO_ALU_VTILE0)

#if MLO_IN_TILE0 < MLO_OUT_WIDTH || MLO_IN_TILE1 < MLO_OUT_HEIGHT
#define MLO_LARGE_MAP 1
#else
#define MLO_LARGE_MAP 0
#endif

#if (MLO_IN_WIDTH == MLO_OUT_WIDTH && (MLO_IN_WIDTH / MLO_IN_TILE0 ) * MLO_IN_TILE0 == MLO_IN_WIDTH && MLO_IN_HEIGHT == MLO_OUT_HEIGHT && (MLO_IN_HEIGHT / MLO_IN_TILE1 ) * MLO_IN_TILE1 == MLO_IN_HEIGHT)
#define MLO_OUT_ALIGNED 1
#else
#define MLO_OUT_ALIGNED 0
#endif

#define MLO_ALUTILES_STACK_SZ (MLO_N_ALUTILES_PERSTACK*MLO_ALU_TILE_SZ)
#define MLO_N_IN_TILES_TOTAL (MLO_N_IN_TILES_PERSTACK*MLO_N_STACKS)

#define MLO_N_OUT_TILE_BLOCKS0 ((MLO_OUT_WIDTH+MLO_IN_TILE0-1)/MLO_IN_TILE0)
#define MLO_N_OUT_TILE_BLOCKS1 ((MLO_OUT_HEIGHT+MLO_IN_TILE1-1)/MLO_IN_TILE1)
#define MLO_N_IN_PACKS  (MLO_N_INPUTS/MLO_N_IN_TILES_PERSTACK)

#define MLO_N_IN_READ (MLO_N_IN_PACKS*MLO_N_IN_TILES_PERSTACK)
#if MLO_N_IN_READ == MLO_N_INPUTS
#define MLO_INPUTS_ALIGNED 1
#else
#define MLO_INPUTS_ALIGNED 0
#endif

#define MLO_N_OUT_PACKS  (MLO_N_OUTPUTS/MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_OUT_PACKS*MLO_N_OUT_TILES_PERSTACK == MLO_N_OUTPUTS && MLO_N_OUT_TILES_PERSTACK != MLO_N_OUTPUTS
#define MLO_OUTPUTS_ALIGNED 1
#else
#define MLO_OUTPUTS_ALIGNED 0
#endif

#define MLO_N_BATCH_PACKS (MLO_BATCH_SZ/MLO_N_STACKS)
#if MLO_N_BATCH_PACKS*MLO_N_STACKS == MLO_BATCH_SZ && MLO_N_STACKS != MLO_BATCH_SZ
#define MLO_BATCH_ALIGNED 1
#else
#define MLO_BATCH_ALIGNED 0
#endif

#define MLO_IN_LCL_WIDTH (MLO_IN_TILE0 + MLO_FILTER_SIZE0 - 1)  // here we use kernel size. it's important when padding == 0
#define MLO_IN_LCL_HEIGHT (MLO_IN_TILE1 + MLO_FILTER_SIZE1 - 1)
#define MLO_IN_LCL_TILE_SZ (MLO_IN_LCL_WIDTH*MLO_IN_LCL_HEIGHT)
#define MLO_IN_LCL_PERSTACK_SZ (MLO_IN_LCL_TILE_SZ*MLO_N_IN_TILES_PERSTACK)
#define MLO_IN_LCL_SZ (MLO_IN_LCL_PERSTACK_SZ*MLO_N_STACKS)

#define MLO_WEIGHTS_SZ (MLO_N_OUT_TILES_PERSTACK*MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ)

#define MLO_PVT_ACCUM_DATA_SZ (MLO_N_OUT_TILES * MLO_OUT_TILE_SZ)
#define MLO_PVT_IN_WIDTH (MLO_FILTER_SIZE0 + MLO_OUT_PIX_TILE0 - 1)
#define MLO_PVT_IN_HEIGHT (MLO_OUT_PIX_TILE1)

#define MLO_LCL_WEIGHTS 1

#if defined(__AMDGCN__)
extern uint __llvm_amdgcn_readfirstlane(uint) __asm("llvm.amdgcn.readfirstlane");
#define uniform(x) __llvm_amdgcn_readfirstlane(x)
#else
#define uniform(x) (x)
#endif

static inline void calculateXYPos(uint linPos, uint width, uint * __restrict x, uint * __restrict y)
{

	(*y) = (uint)((float)linPos * (1.0f / (float)width) + 0.00001f);

	(*x) = linPos - (*y) * width;
}

static inline uint calculateOffset(uint stride, uint x, uint y)
{
	uint ret = y * stride + x;
	return(ret);
}

static inline void readDataElem(uint linPos,__local _FLOAT *lcl_data, uint lcl_base, UNUSED uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 UNUSED bool debug)
{
	uint x, y;
	calculateXYPos(linPos, lcl_width, &x, &y);
	int g_x = x + gbl_x;
	int g_y = y + gbl_y;
	uint gbl_off0 = calculateOffset(gbl_stride, g_x, g_y);
	uint gbl_off = gbl_off0 + gbl_base;

#if MLO_LARGE_MAP == 1
	uint lcl_off = lcl_base + linPos;
	(void)lcl_stride;
	(void)lcl_x;
	(void)lcl_y;
#else
	uint l_x = x + lcl_x;
	uint l_y = y + lcl_y;
	uint lcl_off = lcl_base + mad24(l_y, lcl_stride, l_x);
#endif

#if MLO_LARGE_MAP == 1
	 vis &= (g_x >= 0 && g_x < gbl_width && g_y >= 0 && g_y < gbl_height);
#else
	 (void)gbl_width;
	 (void)gbl_height;
#endif
	 gbl_off = (vis) ? gbl_off : 0;
	 _FLOAT gbl_val = gbl_data[gbl_off];
	 gbl_val = (vis) ? gbl_val : 0;

	lcl_data[lcl_off] = gbl_val;

}

static inline void readData(uint lcl_id, uint size, uint lcl_p_stride, __local _FLOAT *lcl_data, uint lcl_base, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_y, uint lcl_x,
					 const __global _FLOAT * gbl_data, uint gbl_base, uint gbl_height, uint gbl_width, uint gbl_stride, int gbl_y, int gbl_x,
					 bool vis,
					 bool debug
					 )
{

	for(uint i = lcl_id; i < size; i+= lcl_p_stride)
	{
		readDataElem(i, lcl_data, lcl_base, lcl_height, lcl_width, lcl_stride, lcl_y, lcl_x,
					 gbl_data, gbl_base, gbl_height, gbl_width, gbl_stride, gbl_y, gbl_x,
					 vis,
					 debug);
	}

}

static inline void loadData(uint lcl_id, uint lcl_p_stride,
					__local _FLOAT * lcl_data,
					uint lcl_off, uint lcl_size, uint lcl_height, uint lcl_width, uint lcl_stride, uint lcl_bot_y, uint lcl_bot_x,
					const __global _FLOAT * gbl_data,
					uint gbl_off, uint gbl_size, uint gbl_height, uint glb_width, uint gbl_stride, int gbl_bot_y, int gbl_bot_x,
					uint buf_block_ind, uint max_n_bufs, uint lcl_n_bufs,
					bool debug)
{

	for(uint c = 0; c < lcl_n_bufs;
		 ++c, lcl_off += lcl_size, gbl_off += gbl_size )
	{
		bool vis = (buf_block_ind + c <  max_n_bufs);
		readData(lcl_id, lcl_size, lcl_p_stride,
				 lcl_data, lcl_off, lcl_height, lcl_width, lcl_stride, lcl_bot_y, lcl_bot_x,
				 gbl_data, gbl_off, gbl_height, glb_width, gbl_stride, gbl_bot_y, gbl_bot_x,
				 vis,
				 (debug));
	}

}

static inline void Conv(uint o_map_base,
				uint in_stg_off,
				 _FLOAT * pvt_in_stage, __local _FLOAT * __restrict lcl_indata,
				 _FLOAT * pvt_wei_stage, __local _FLOAT * __restrict lcl_wei,
				 _FLOAT * pvt_accum
				 )
{
// convolution

		// over all inputs in stack
		uint in_stg_off1 = in_stg_off;
		for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_TILE_SZ)
		{
        // preload input
			uint wei_stg_base_off = mad24(o_map_base, (uint)(MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ), mul24(i_c,(uint)MLO_FILTER_SZ));
			uint in_stg_off2 = in_stg_off1;
			for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j, in_stg_off2+=MLO_IN_LCL_WIDTH
					)
			{
				for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
				{
					pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = lcl_indata[in_stg_off2 + i];
				}
			}

		// over filter rows
#ifdef __AMDGCN__
			#pragma unroll
#endif
			for(uint k = 0; k < MLO_FILTER_SIZE1; ++k, in_stg_off2+=MLO_IN_LCL_WIDTH
			)
			{
				uint k_act = 0;
#if MLO_DIR_FORWARD==1
				k_act = k;
#else
// load filter in reverse order
				k_act = MLO_FILTER_SIZE1 - 1 - k;
#endif
		// load next input row
				for(uint i_pvt = 0; i_pvt < MLO_PVT_IN_WIDTH; ++i_pvt)
				{
					pvt_in_stage[(MLO_PVT_IN_HEIGHT-1)*MLO_PVT_IN_WIDTH + i_pvt] = lcl_indata[in_stg_off2 + i_pvt];
				}

		// over all outputs
				for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)
				{
					uint wei_stg_off = wei_stg_base_off + o_c * MLO_N_IN_TILES_PERSTACK*MLO_FILTER_SZ + k_act * MLO_FILTER_SIZE0;
					for(uint i = 0; i < MLO_FILTER_SIZE0; ++i)
					{
						pvt_wei_stage[i] = lcl_wei[wei_stg_off + i];
					}

		// actual conv

					for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j)
					{
						for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
						{
							for(uint l = 0; l < MLO_FILTER_SIZE0; ++l)
							{

								uint l_act = 0;
#if MLO_DIR_FORWARD==1
								l_act = l;

#else
// in reverse horizontal and vertical orders
								l_act = MLO_FILTER_SIZE0 - 1 - l;

#endif

								pvt_accum[(o_c * MLO_OUT_PIX_TILE1 + j) * MLO_OUT_PIX_TILE0 + i]
									 += pvt_in_stage[j * MLO_PVT_IN_WIDTH + i + l] * pvt_wei_stage[l_act];

							}

						}
					}

				} // for(uint o_c = 0; o_c < MLO_N_OUT_TILES; ++o_c)

               // move data up
				for(uint j = 0; j < MLO_PVT_IN_HEIGHT-1; ++j)
				{
					for(uint i = 0; i < MLO_PVT_IN_WIDTH; ++i)
					{
						pvt_in_stage[j*MLO_PVT_IN_WIDTH + i] = pvt_in_stage[(j+1)*MLO_PVT_IN_WIDTH + i];
					}
				}

//				mem_fence(CLK_LOCAL_MEM_FENCE);

			} // for(uint k = 0; k < MLO_FILER_SIZE1; ++k,in_stg_off2+=MLO_IN_LCL_WIDTH)

		} // for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_stg_off1 += MLO_IN_LCL_PERSTACK_SZ)

}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenConvUniC(
       const __global _FLOAT * __restrict in,
       const __global _FLOAT * __restrict weights,
#if MLO_CONV_BIAS
       const __global _FLOAT * __restrict bias,
#endif
	__global _FLOAT * __restrict out,
	UNUSED _FLOAT padding_val
	   )
{
	__local _FLOAT lcl_indata[MLO_IN_LCL_SZ];
	__local _FLOAT lcl_wei[MLO_WEIGHTS_SZ];
	__private _FLOAT pvt_accum[MLO_PVT_ACCUM_DATA_SZ];
	__private _FLOAT pvt_in_stage[MLO_PVT_IN_HEIGHT * MLO_PVT_IN_WIDTH];
	__private _FLOAT pvt_wei_stage[MLO_FILTER_SIZE0];

	uint grp_id0 = get_group_id(0);
#if MLO_OUT_WIDTH == MLO_IN_TILE0
	uint y_tile_blk = grp_id0;
	uint x_tile_blk = 0;
#else
#if MLO_N_OUT_TILE_BLOCKS0 & (MLO_N_OUT_TILE_BLOCKS0 - 1)
	uint y_tile_blk = (uint)((float)grp_id0 * (1.0f / (float) MLO_N_OUT_TILE_BLOCKS0) + 0.00001f);
	int x_tile_blk = grp_id0 - mul24(y_tile_blk, (uint)MLO_N_OUT_TILE_BLOCKS0);
#else
	uint y_tile_blk = grp_id0 / MLO_N_OUT_TILE_BLOCKS0;
	int x_tile_blk = grp_id0 & (MLO_N_OUT_TILE_BLOCKS0 - 1);
#endif
#endif
	uint o_pack = get_group_id(1); // block of outputs
	uint b_pack = get_group_id(2); // batch block

	uint lcl_id = get_local_id(0);
#if MLO_ALUTILES_STACK_SZ & (MLO_ALUTILES_STACK_SZ - 1)
	uint stack = (uint)((float)lcl_id * (1.0f / (float)MLO_ALUTILES_STACK_SZ) + 0.00001f);  // stack
	uint alu_stack_id = lcl_id - mul24(stack, (uint)MLO_ALUTILES_STACK_SZ);  // alu index in stack
#else
	uint stack = lcl_id / MLO_ALUTILES_STACK_SZ;  // stack
	uint alu_stack_id = lcl_id & (MLO_ALUTILES_STACK_SZ - 1);  // alu index in stack
#if MLO_ALUTILES_STACK_SZ >= 64
	stack = uniform(stack);
#endif
#endif
// ALU plane inside stack
#if MLO_ALU_TILE_SZ & (MLO_ALU_TILE_SZ - 1)
	uint alu_out_plane_id = (uint)((float)alu_stack_id * (1.0f / (float)MLO_ALU_TILE_SZ) + 0.00001f);  // alu output plane index
	uint alu_out_id = alu_stack_id - mul24(alu_out_plane_id, (uint)MLO_ALU_TILE_SZ); // alu index inside an ALU output plane
#else
	uint alu_out_plane_id = alu_stack_id / MLO_ALU_TILE_SZ;  // alu output plane index
	uint alu_out_id = alu_stack_id & (MLO_ALU_TILE_SZ - 1); // alu index inside an ALU output plane
#endif
// pos inside ALU tile
#if MLO_ALU_VTILE0 & (MLO_ALU_VTILE0 - 1)
	uint alu_tl1 = (uint)((float)alu_out_id * (1.0f / (float)MLO_ALU_VTILE0) + 0.00001f);
	uint alu_tl0 = alu_out_id - mul24(alu_tl1, (uint)MLO_ALU_VTILE0);
#else
	uint alu_tl1 = alu_out_id / MLO_ALU_VTILE0;
	uint alu_tl0 = alu_out_id & (MLO_ALU_VTILE0 - 1);
#endif

	uint o_map_plane = o_pack * MLO_N_OUT_TILES_PERSTACK; // first output maps index per full ALU plane stack
	uint o_map_base = alu_out_plane_id*MLO_N_OUT_TILES;  // local output map offset
	uint o_map = o_map_plane + o_map_base; // output map index per ALU plane
	uint b_index = b_pack * MLO_N_STACKS;

#if MLO_LARGE_MAP != 1
#if MLO_GRP_SZ <= MLO_N_READ_PROCS
	uint wave_id = 0;
	uint wave_lcl_id = lcl_id;
#elif MLO_N_READ_PROCS & (MLO_N_READ_PROCS - 1)
	uint wave_id = (uint)((float)lcl_id * (1.0f / (float)MLO_N_READ_PROCS) + 0.00001f);
	uint wave_lcl_id = lcl_id - mul24(wave_id, (uint)MLO_N_READ_PROCS);
#else
	uint wave_id = (uint)((uint)lcl_id / MLO_N_READ_PROCS);
	uint wave_lcl_id = lcl_id & (MLO_N_READ_PROCS - 1);
#if MLO_N_READ_PROCS >= 64
        wave_id = uniform(wave_id);
#endif
#endif
#endif

	int x_grp = x_tile_blk * MLO_IN_TILE0;
	uint y_grp = y_tile_blk * MLO_IN_TILE1;

// TO DO: scale
#if MLO_LARGE_MAP == 1
	int x_in_grp = x_grp - MLO_FILTER_PAD0;
	int y_in_grp = y_grp - MLO_FILTER_PAD1;
#endif

	uint x_in_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_in_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

// base offset to read data from local input data
	uint in_stg_off = stack*MLO_IN_LCL_PERSTACK_SZ + (y_in_lcl) * MLO_IN_LCL_WIDTH + x_in_lcl;

    uint in_off = b_index * MLO_IN_BATCH_STRIDE;

#if MLO_DIR_FORWARD==1
	uint wei_off = mul24(o_map_plane, (uint)(MLO_N_INPUTS * MLO_FILTER_SZ));
#else
	uint wei_off = mul24(o_map_plane,(uint)MLO_FILTER_SZ);
#endif

#if MLO_LARGE_MAP == 0
	for(uint i = lcl_id; i < MLO_IN_LCL_SZ;  i += MLO_GRP_SZ)
	{
		lcl_indata[i] = 0;
	}
#endif

	for(uint i = 0; i < MLO_PVT_ACCUM_DATA_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for(uint ic = 0; ic < MLO_N_INPUTS; ic += MLO_N_IN_TILES_PERSTACK, in_off += MLO_IN_CHANNEL_STRIDE*MLO_N_IN_TILES_PERSTACK,
				wei_off += MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ
#if MLO_DIR_FORWARD==0
							* MLO_N_OUTPUTS
#endif
	)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

// small map has been read in full continiously into the lDS buffer within padded rect,
// padding has been done on initilization.
// large map calculates padding on the fly and fills it with 0.

#if 1 // all inputs

#if MLO_LARGE_MAP == 1
		uint in_lcl_off1 = 0;
		uint in_off1 = in_off;
		for(uint i_b = 0; i_b < MLO_N_STACKS; ++i_b, in_off1 += MLO_IN_BATCH_STRIDE, in_lcl_off1 += MLO_IN_LCL_PERSTACK_SZ)
		{
			bool vis = true;
#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

// over all inputs in stack
			uint in_off2 = in_off1;
			uint in_lcl_off2 = in_lcl_off1;
			for(uint i_c = 0; i_c < MLO_N_IN_TILES_PERSTACK; ++i_c, in_off2 += MLO_IN_CHANNEL_STRIDE, in_lcl_off2 += MLO_IN_LCL_TILE_SZ)
			{
#if MLO_INPUTS_ALIGNED == 0
				vis &= (ic + i_c < MLO_N_INPUTS);
#endif

				uint elem_id = lcl_id;
				uint lcl_p_stride = MLO_GRP_SZ0;
				uint lcl_base = 0;
				uint lcl_y = 0;
				uint lcl_x = 0;
				uint gbl_base = 0;

				readData(elem_id, (MLO_IN_LCL_HEIGHT * MLO_IN_LCL_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_LCL_HEIGHT, MLO_IN_LCL_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_in_grp, x_in_grp,
						vis,
						true
					 );
			}

		}
#else
		for(uint i = wave_id; i < MLO_N_IN_TILES_TOTAL;  i += MLO_N_PROC_WAVES)
		{
		//(MLO_N_STACKS * MLO_N_OUT_TILES_PERSTACK)
#if MLO_N_IN_TILES_PERSTACK & (MLO_N_IN_TILES_PERSTACK - 1)
			uint i_b = (uint)((float)i * (1.0f / (float)MLO_N_IN_TILES_PERSTACK) + 0.00001f);
			uint i_c = i - mul24(i_b, (uint)MLO_N_IN_TILES_PERSTACK);
#else
			uint i_b = (uint)i / MLO_N_IN_TILES_PERSTACK;
			uint i_c = i & (MLO_N_IN_TILES_PERSTACK - 1);
#endif

			bool vis = true;

#if MLO_BATCH_ALIGNED == 0
			vis &= (b_index + i_b < MLO_BATCH_SZ);
#endif

#if MLO_INPUTS_ALIGNED == 0
			vis &= (ic + i_c < MLO_N_INPUTS);
#endif
			uint in_off2 = in_off + i_b * MLO_IN_BATCH_STRIDE + i_c * MLO_IN_CHANNEL_STRIDE;
			uint in_lcl_off2 = i_b * MLO_IN_LCL_PERSTACK_SZ + i_c * MLO_IN_LCL_TILE_SZ;

			uint elem_id = wave_lcl_id;
			uint lcl_p_stride = MLO_N_READ_PROCS;
			uint lcl_base = 0;
			uint lcl_y = MLO_FILTER_PAD1;
			uint lcl_x = MLO_FILTER_PAD0;
			uint gbl_base = 0;

			readData(elem_id, (MLO_IN_HEIGHT * MLO_IN_WIDTH), lcl_p_stride,
						&lcl_indata[in_lcl_off2], lcl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_LCL_WIDTH, lcl_y, lcl_x,
						&in[in_off2], gbl_base, MLO_IN_HEIGHT, MLO_IN_WIDTH, MLO_IN_STRIDE, y_grp, x_grp,
						vis,
						true
					 );
		}
#endif

// read inputs and weights
// put weights into LDS

#if 1  // only weights

		for(uint i = lcl_id; i < MLO_WEIGHTS_SZ; i += MLO_GRP_SZ)
		{
#if MLO_DIR_FORWARD==1
// here is [tops][bottoms]
#if (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = (uint)((float)i * (1.0f / (float)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ)) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_IN_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
			if((wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i) < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ))
				lcl_wei[i] = weights[wei_off + lcl_o * MLO_N_INPUTS * MLO_FILTER_SZ + gbl_i];
			else
				lcl_wei[i] = weights[0];
#else
// outputs are botoms(inputs))
// inputs are tops(outputs)

#if (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1)
			uint lcl_o = (uint)((float)i * (1.0f / (float)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ)) + 0.00001f);
			uint gbl_i = i - mul24(lcl_o, (uint)(MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ));
#else
			uint lcl_o = i / (MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ);
			uint gbl_i = i & ((MLO_N_OUT_TILES_PERSTACK * MLO_FILTER_SZ) - 1);
#endif
#if MLO_FILTER_SZ & (MLO_FILTER_SZ - 1)
			uint lcl_c = (uint)((float)gbl_i * (1.0f / (float)MLO_FILTER_SZ) + 0.00001f);
			uint lcl_i = gbl_i - mul24(lcl_c, (uint)MLO_FILTER_SZ);
#else
			uint lcl_c = gbl_i / MLO_FILTER_SZ;
			uint lcl_i = gbl_i & (MLO_FILTER_SZ - 1);
#endif

			uint lcl_we_off = mad24(mad24(lcl_c, (uint)MLO_N_IN_TILES_PERSTACK, lcl_o), (uint)MLO_FILTER_SZ, lcl_i);
			uint gbl_we_off = mad24(mad24(lcl_o, (uint)MLO_N_OUTPUTS, lcl_c), (uint)MLO_FILTER_SZ, wei_off + lcl_i);
			bool within_range = gbl_we_off < (MLO_N_OUTPUTS*MLO_N_INPUTS*MLO_FILTER_SZ);
			gbl_we_off = (within_range) ? gbl_we_off : 0;
			_FLOAT wei = weights[gbl_we_off];
			wei = (within_range) ? wei : 0;
			lcl_wei[lcl_we_off]
				= wei;

#endif

		}

#endif

// over all batch stacks

#endif  // all input

		barrier(CLK_LOCAL_MEM_FENCE);

// convolution
#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
		if (lcl_id < MLO_ACTIVE_ALUS)
#endif
		Conv(o_map_base,
			in_stg_off,
			pvt_in_stage, lcl_indata,
			pvt_wei_stage, lcl_wei,
			pvt_accum
			);

//		barrier(CLK_LOCAL_MEM_FENCE);
	}

#if MLO_GRP_SZ > MLO_ACTIVE_ALUS
	if (lcl_id >= MLO_ACTIVE_ALUS)
	{
		return;
	}
#endif
// write results out
	uint x_out_grp = x_grp;
	uint y_out_grp = y_grp;
	uint x_out_lcl = alu_tl0 * MLO_OUT_PIX_TILE0;
	uint y_out_lcl = alu_tl1 * MLO_OUT_PIX_TILE1;

    uint out_off = (b_index + stack) * MLO_OUT_BATCH_STRIDE + o_map * MLO_OUT_CHANNEL_STRIDE + (y_out_grp + y_out_lcl) * MLO_OUT_STRIDE + x_out_grp + x_out_lcl;
// over all local stacks
#if MLO_BATCH_ALIGNED == 0
	if (b_index + stack < MLO_BATCH_SZ)
#endif
	{

// over all local outputs
		uint out_off1 = out_off;
		for(uint o = 0; o < MLO_N_OUT_TILES
						; ++o, out_off1 += MLO_OUT_CHANNEL_STRIDE
						)
		{
// over output tile
			_FLOAT  bias_val = 0;
#if MLO_CONV_BIAS
			bias_val = bias[o_map + o];
#endif
			uint out_off2 = out_off1;
			for( uint j = 0; j < MLO_OUT_PIX_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
				__global _FLOAT *out_p = &out[out_off2];
				for(uint i = 0; i < MLO_OUT_PIX_TILE0; ++i)
				{
					if ( true
#if 1 //MLO_OUT_ALIGNED == 0
					&& y_out_lcl + j < MLO_OUT_TILE1 && y_out_grp + y_out_lcl + j < MLO_OUT_HEIGHT &&  x_out_lcl + i < MLO_OUT_TILE0 && x_out_grp + x_out_lcl + i < MLO_OUT_WIDTH
#endif
#if MLO_OUTPUTS_ALIGNED == 0
						&& o_map + o < MLO_N_OUTPUTS
#endif
					)
					{
						out_p[i] = pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i] + bias_val;
#if 0
						if ( out_off2 + i == 1 /*y_out_grp + y_out_lcl + j == 2 && x_out_grp + x_out_lcl + i == 0*/)
						{
							printf("K:out: %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d    %f %f %f\n",
								MLO_OUT_TILE1,
								MLO_OUT_TILE0,
								grp_id0,
								lcl_id,
								alu_out_plane_id,
								alu_out_id,
								b_index + stack,
								o,
								o_map,
								out_off,
								out_off1,
								out_off2,
								y_out_grp,
								y_out_lcl,
								x_out_grp,
								x_out_lcl,
								j,
								i,
								pvt_accum[o*MLO_OUT_TILE_SZ + j * MLO_OUT_PIX_TILE0 + i],
								 bias_val,
								out[out_off2 + i]
								);
						}
#endif
					}

				}
			}

		}
	}

}


OpenCL implementation of seven-XOR PRNG

OpenCL implementation of seven-XOR PRNG

/******************************************************************************
 * @file     prngcl_xor7.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of seven-XOR pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   Francois Panneton and Pierre L'ecuyer,
 *   "On the xorshift random number generators",
 *   ACM Transactions on Modeling and Computer Simulation (TOMACS), 15, 4 (2005), 346--361.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, 2014 Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_XOR7_CL
#define PRNGCL_XOR7_CL

#include "prngcl_common.cl"

#define XOR7_m_FP   (4294967296.0f)
#define XOR7_min    1
#define XOR7_max    4294967295
#define XOR7_min_FP (1.0/4294967296.0)
#define XOR7_max_FP (4294967295.0/4294967296.0)
#define XOR7_k      (2.3283064365386962890625E-10) // 1/(2^32-1)
#define XOR7_left   (XOR7_min_FP+XOR7_k*XOR7_max_FP)
#define XOR7_right  (XOR7_max_FP+XOR7_k*XOR7_min_FP)

__attribute__((always_inline)) void
xor7_step(uint4* seed1,uint4* seed2)
{
    uint t, y;
        t = (*seed2).w;     t = t ^ (t<<13);    y = t ^ (t<<9);
        t = (*seed2).x;     y^= t ^ (t<<7);
        t = (*seed1).w;     y^= t ^ (t>>3);
        t = (*seed1).y;     y^= t ^ (t>>10);
        t = (*seed1).x;     t = t ^ (t>>7);     y^= t ^ (t<<24);

        (*seed1).xyz = (*seed1).yzw;
        (*seed1).w   = (*seed2).x;
        (*seed2).xyz = (*seed2).yzw;
        (*seed2).w   = y;
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double
xor7_step_double(uint4* seed1,uint4* seed2)
{
    hgpu_double result;
    uint rnd1, rnd2;
    uint4 sed1 = (*seed1);
    uint4 sed2 = (*seed2);
    xor7_step(&sed1,&sed2);
    rnd1 = sed2.w;

#ifndef PRNG_SKIP_CHECK
    bool flag = true;
    while (flag) {
#endif
        xor7_step(&sed1,&sed2);
        rnd2 = sed2.w;
        result = hgpu_uint_to_double(rnd1,rnd2,XOR7_min,XOR7_max,XOR7_k);

#ifndef PRNG_SKIP_CHECK
        if ((result>=XOR7_left) && (result<XOR7_right))
            flag = false;
        else
            rnd1 = rnd2;
    }
#endif
    (*seed1) = sed1;
    (*seed2) = sed2;
    return result;
}
#endif

__kernel void
xor7(__global uint4* seed_table,
     __global hgpu_float4* randoms,
       const uint N)
{
    uint giddst = GID;
#ifdef PRECISION_DOUBLE  // if double precision is defined
    hgpu_double4 result;
#else
    float4 result;
    float4 normal = (float4) XOR7_m_FP;
#endif
    uint4 seed1 = seed_table[GID];
    uint4 seed2 = seed_table[GID + GID_SIZE];
    for (uint i = 0; i < N; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        result.x = xor7_step_double(&seed1,&seed2);
        result.y = xor7_step_double(&seed1,&seed2);
        result.z = xor7_step_double(&seed1,&seed2);
        result.w = xor7_step_double(&seed1,&seed2);
        randoms[giddst] = result;
#else
        xor7_step(&seed1,&seed2);
            result.x = (float) seed2.w;
        xor7_step(&seed1,&seed2);
            result.y = (float) seed2.w;
        xor7_step(&seed1,&seed2);
            result.z = (float) seed2.w;
        xor7_step(&seed1,&seed2);
            result.w = (float) seed2.w;
        randoms[giddst] = result / normal;
#endif
        giddst += GID_SIZE;
    }
    seed_table[GID] = seed1;
    seed_table[GID + GID_SIZE] = seed2;
}

#endif

/******************************************************************************
 * @file     prngcl_xor7.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of seven-XOR pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   Francois Panneton and Pierre L'ecuyer,
 *   "On the xorshift random number generators",
 *   ACM Transactions on Modeling and Computer Simulation (TOMACS), 15, 4 (2005), 346--361.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_XOR7_CL
#define PRNGCL_XOR7_CL

#include "prngcl_common.cl"

#define XOR7_m_FP   (4294967296.0f)
#define XOR7_min    1
#define XOR7_max    4294967295
#define XOR7_min_FP (1.0/4294967296.0)
#define XOR7_max_FP (4294967295.0/4294967296.0)
#define XOR7_k      (2.3283064365386962890625E-10) // 1/(2^32-1)
#define XOR7_left   (XOR7_min_FP+XOR7_k*XOR7_max_FP)
#define XOR7_right  (XOR7_max_FP+XOR7_k*XOR7_min_FP)

__attribute__((always_inline)) void
xor7_step(uint4* seed1,uint4* seed2)
{
    uint t, y;
        t = (*seed2).w;     t = t ^ (t<<13);    y = t ^ (t<<9);
        t = (*seed2).x;     y^= t ^ (t<<7);
        t = (*seed1).w;     y^= t ^ (t>>3);
        t = (*seed1).y;     y^= t ^ (t>>10);
        t = (*seed1).x;     t = t ^ (t>>7);     y^= t ^ (t<<24);

        (*seed1).xyz = (*seed1).yzw;
        (*seed1).w   = (*seed2).x;
        (*seed2).xyz = (*seed2).yzw;
        (*seed2).w   = y;
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double
xor7_step_double(uint4* seed1,uint4* seed2)
{
    hgpu_double result;
    uint rnd1, rnd2;
    uint4 sed1 = (*seed1);
    uint4 sed2 = (*seed2);
    bool flag = true;
    xor7_step(&sed1,&sed2);
    rnd1 = sed2.w;
    while (flag) {
        xor7_step(&sed1,&sed2);
        rnd2 = sed2.w;
        result = hgpu_uint_to_double(rnd1,rnd2,XOR7_min,XOR7_max,XOR7_k);
        if ((result>=XOR7_left) && (result<XOR7_right))
            flag = false;
        else
            rnd1 = rnd2;
    }
    (*seed1) = sed1;
    (*seed2) = sed2;
    return result;
}
#endif

__kernel void
xor7(__global uint4* seed_table,
     __global hgpu_float4* randoms,
       const uint N)
{
    uint giddst = GID;
#ifdef PRECISION_DOUBLE  // if double precision is defined
    hgpu_double4 result;
#else
    float4 result;
    float4 normal = (float4) XOR7_m_FP;
#endif
    uint4 seed1 = seed_table[GID];
    uint4 seed2 = seed_table[GID + GID_SIZE];
    for (uint i = 0; i < N; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        result.x = xor7_step_double(&seed1,&seed2);
        result.y = xor7_step_double(&seed1,&seed2);
        result.z = xor7_step_double(&seed1,&seed2);
        result.w = xor7_step_double(&seed1,&seed2);
        randoms[giddst] = result;
#else
        xor7_step(&seed1,&seed2);
            result.x = (float) seed2.w;
        xor7_step(&seed1,&seed2);
            result.y = (float) seed2.w;
        xor7_step(&seed1,&seed2);
            result.z = (float) seed2.w;
        xor7_step(&seed1,&seed2);
            result.w = (float) seed2.w;
        randoms[giddst] = result / normal;
#endif
        giddst += GID_SIZE;
    }
    seed_table[GID] = seed1;
    seed_table[GID + GID_SIZE] = seed2;
}

#endif


make debayering consistent at edges (#24437)

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.82717181, -0.31231438, 0.07307673),
  (half3)(-0.5743977, 1.36858544, -0.53183455),
  (half3)(-0.25277411, -0.05627105, 1.45875782),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.0025;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 9 - 100*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb) {
  half3 ret = (0,0,0);
  half cpx = 0.01;
  ret += (half)rgb.x * color_correction[0];
  ret += (half)rgb.y * color_correction[1];
  ret += (half)rgb.z * color_correction[2];
  ret.x = mf(ret.x, cpx);
  ret.y = mf(ret.y, cpx);
  ret.z = mf(ret.z, cpx);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

inline half val_from_10(const uchar * source, int gx, int gy, half black_level) {
  // parse 12bit
  int start = gy * FRAME_STRIDE + (3 * (gx / 2));
  int offset = gx % 2;
  uint major = (uint)source[start + offset] << 4;
  uint minor = (source[start + 2] >> (4 * offset)) & 0xf;
  half pv = (half)((major + minor)/4);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        float black_level
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global, black_level);
  cached[localOffset] = pv;

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_global >= 1 && x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global, black_level);
  } else if (x_global < RGB_WIDTH - 1 && x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global, black_level);
  }

  if (y_global >= 1 && y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1, black_level);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1, black_level);
    }
  } else if (y_global < RGB_HEIGHT - 1 && y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1, black_level);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1, black_level);
    }
  }

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);
}

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.82717181, -0.31231438, 0.07307673),
  (half3)(-0.5743977, 1.36858544, -0.53183455),
  (half3)(-0.25277411, -0.05627105, 1.45875782),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.0025;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 9 - 100*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb) {
  half3 ret = (0,0,0);
  half cpx = 0.01;
  ret += (half)rgb.x * color_correction[0];
  ret += (half)rgb.y * color_correction[1];
  ret += (half)rgb.z * color_correction[2];
  ret.x = mf(ret.x, cpx);
  ret.y = mf(ret.y, cpx);
  ret.z = mf(ret.z, cpx);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

inline half val_from_10(const uchar * source, int gx, int gy, half black_level) {
  // parse 12bit
  int start = gy * FRAME_STRIDE + (3 * (gx / 2));
  int offset = gx % 2;
  uint major = (uint)source[start + offset] << 4;
  uint minor = (source[start + 2] >> (4 * offset)) & 0xf;
  half pv = (half)((major + minor)/4);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        float black_level
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global, black_level);
  cached[localOffset] = pv;

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global, black_level);
  } else if (x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global, black_level);
  }

  if (y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1, black_level);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1, black_level);
    }
  } else if (y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1, black_level);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1, black_level);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);
}


adding couple of more sizes

adding couple of more sizes


5/3 filter now working with floating point data

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns
Right (odd) boundary column

 **/

#define BOUNDARY_X 2

#define VERTICAL_STRIDE 64  // WIN_SIZE_X/2

// two vertical neighbours: pointer diff:
#define BUFFER_SIZE            512	// VERTICAL_STRIDE * WIN_SIZE_Y

#define CHANNEL_BUFFER_SIZE     1024            // BUFFER_SIZE + BUFFER_SIZE

#define CHANNEL_BUFFER_SIZE_X2  2048
#define CHANNEL_BUFFER_SIZE_X3  3072

#define PIXEL_BUFFER_SIZE   4096

#define HORIZONTAL_ODD_TO_PREVIOUS_EVEN -512
#define HORIZONTAL_ODD_TO_NEXT_EVEN     -511

#define HORIZONTAL_EVEN_TO_PREVIOUS_ODD  511
#define HORIZONTAL_EVEN_TO_NEXT_ODD      512

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

inline int getCorrectedGlobalIdX() {
      return getGlobalId(0) - 2 * BOUNDARY_X * getGroupId(0);
}

// read pixel from local buffer
float4 readPixel( LOCAL float*  restrict  src) {
	return (float4)(*src, *(src+CHANNEL_BUFFER_SIZE),  *(src+CHANNEL_BUFFER_SIZE_X2),  *(src+CHANNEL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(float4 pix, LOCAL float*  restrict  dest) {
	*dest = pix.x;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.y;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.z;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.w;
}

// write column to destination
void writeColumnToOutput(LOCAL float* restrict currentScratch, __write_only image2d_t odata, int firstY, int inputX, int height, int halfHeight){

	int2 posOut = {inputX, firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even row

		//only need to check evens, since even point will be the first out of bound point
	    if (posOut.y >= halfHeight)
			break;

		write_imagef(odata, posOut,readPixel(currentScratch));

		// odd row
		currentScratch += VERTICAL_STRIDE ;
		posOut.y+= halfHeight;

		write_imagef(odata, posOut,readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE;
		posOut.y -= (halfHeight - 1);
	}
}

// initial scratch offset when transforming vertically
inline int getScratchOffset(){
   return (getLocalId(0)>> 1) + (getLocalId(0)&1) * BUFFER_SIZE;
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	int inputX = getCorrectedGlobalIdX();

	int outputX = inputX;
	outputX = (outputX >> 1) + (outputX & 1)*( width >> 1);

	const int halfWinSizeX = WIN_SIZE_X >> 1;
    const unsigned int halfHeight = height >> 1;
	LOCAL float scratch[PIXEL_BUFFER_SIZE];
	const float yDelta = 1.0/(height-1);
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)

	// read -1 point
	float2 posIn = (float2)(inputX, firstY - 1) /  (float2)(width-1, height-1);
	float4 minusOne = read_imagef(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	float4 current = read_imagef(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= ( read_imagef(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) /2;
	for (int i = 0; i < steps; ++i) {

		// 1. read from source image, transform columns, and store in local scratch
		LOCAL float* currentScratch = scratch + getScratchOffset();
		for (int j = 0; j < WIN_SIZE_Y; j+=2) {

			///////////////////////////////////////////////////////////////////////////////////////////
			// fetch next two pixels, then transform and write current (odd) and last (even)

			// read current plus one (odd) point
			posIn.y += yDelta;
			if (posIn.y > 1 + yDelta)
				break;
			float4 currentPlusOne = read_imagef(idata, sampler, posIn);

			// read current plus two (even) point
			posIn.y += yDelta;
			float4 currentPlusTwo = read_imagef(idata, sampler, posIn);

			// transform current plus one (odd) point
			currentPlusOne -= (current + currentPlusTwo) /2;  // F.4, page 118, ITU-T Rec. T.800 final draft

			// transform current (even) point
			current += (minusOne + currentPlusOne + 0.0078125f) /4; // F.3, page 118, ITU-T Rec. T.800 final draft

			//write current (even)
			writePixel(current, currentScratch);

			//write odd
			currentScratch += VERTICAL_STRIDE;
			writePixel(currentPlusOne, currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE;

			//update registers
			minusOne = currentPlusOne;
			current = currentPlusTwo;
		}

		//4. transform horizontally
		currentScratch = scratch + getScratchOffset();

		localMemoryFence();
		//odd columns (skip right odd boundary column)
		if ( (getLocalId(0)&1) && (getLocalId(0) != WIN_SIZE_X-1) ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 currentOdd = readPixel(currentScratch);
				float4 prevEven = readPixel(currentScratch + HORIZONTAL_ODD_TO_PREVIOUS_EVEN);
				float4 nextEven = readPixel(currentScratch + HORIZONTAL_ODD_TO_NEXT_EVEN);
				currentOdd -= ((prevEven + nextEven) /2);
				writePixel( currentOdd, currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}

		currentScratch = scratch + getScratchOffset();
		localMemoryFence();
		//even columns (skip left and right even boundary columns)
		if ( !(getLocalId(0)&1) && (getLocalId(0) != 0) && (getLocalId(0) != WIN_SIZE_X-2)  ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 currentEven = readPixel(currentScratch);
				float4 prevOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_PREVIOUS_ODD);
				float4 nextOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_NEXT_ODD);
				currentEven += (prevOdd + nextOdd + 0.0078125f) /4;
				writePixel( currentEven, currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}
		localMemoryFence();

		//5. write local buffer column to destination image
		// (only write non-boundary columns that are within the image bounds)
		if ((getLocalId(0) >= BOUNDARY_X) && ( getLocalId(0) < WIN_SIZE_X - BOUNDARY_X) && (inputX < width) && inputX >= 0) {
			writeColumnToOutput(scratch + getScratchOffset(), odata, firstY, outputX, height, halfHeight);

		}
		// move to next step
		firstY += WIN_SIZE_Y;
	}
}

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns
Right (odd) boundary column

 **/

#define BOUNDARY_X 2

#define VERTICAL_STRIDE 64  // WIN_SIZE_X/2

// two vertical neighbours: pointer diff:
#define BUFFER_SIZE            512	// VERTICAL_STRIDE * WIN_SIZE_Y

#define CHANNEL_BUFFER_SIZE     1024            // BUFFER_SIZE + BUFFER_SIZE

#define CHANNEL_BUFFER_SIZE_X2  2048
#define CHANNEL_BUFFER_SIZE_X3  3072

#define PIXEL_BUFFER_SIZE   4096

#define HORIZONTAL_ODD_TO_PREVIOUS_EVEN -512
#define HORIZONTAL_ODD_TO_NEXT_EVEN     -511

#define HORIZONTAL_EVEN_TO_PREVIOUS_ODD  511
#define HORIZONTAL_EVEN_TO_NEXT_ODD      512

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

inline int getCorrectedGlobalIdX() {
      return getGlobalId(0) - 2 * BOUNDARY_X * getGroupId(0);
}

// read pixel from local buffer
float4 readPixel( LOCAL float*  restrict  src) {
	return (float4)(*src, *(src+CHANNEL_BUFFER_SIZE),  *(src+CHANNEL_BUFFER_SIZE_X2),  *(src+CHANNEL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(float4 pix, LOCAL float*  restrict  dest) {
	*dest = pix.x;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.y;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.z;
	dest += CHANNEL_BUFFER_SIZE;
	*dest = pix.w;
}

// write column to destination
void writeColumnToOutput(LOCAL float* restrict currentScratch, __write_only image2d_t odata, int firstY, int inputX, int height, int halfHeight){

	int2 posOut = {inputX, firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even row

		//only need to check evens, since even point will be the first out of bound point
	    if (posOut.y >= halfHeight)
			break;

		write_imagef(odata, posOut,readPixel(currentScratch));

		// odd row
		currentScratch += VERTICAL_STRIDE ;
		posOut.y+= halfHeight;

		write_imagef(odata, posOut,readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE;
		posOut.y -= (halfHeight - 1);
	}
}

// initial scratch offset when transforming vertically
inline int getScratchOffset(){
   return (getLocalId(0)>> 1) + (getLocalId(0)&1) * BUFFER_SIZE;
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	int inputX = getCorrectedGlobalIdX();

	int outputX = inputX;
	outputX = (outputX >> 1) + (outputX & 1)*( width >> 1);

	const int halfWinSizeX = WIN_SIZE_X >> 1;
    const unsigned int halfHeight = height >> 1;
	LOCAL float scratch[PIXEL_BUFFER_SIZE];
	const float yDelta = 1.0/(height-1);
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)

	// read -1 point
	float2 posIn = (float2)(inputX, firstY - 1) /  (float2)(width-1, height-1);
	float4 minusOne = read_imagef(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	float4 current = read_imagef(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= ( read_imagef(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) /2;
	for (int i = 0; i < steps; ++i) {

		// 1. read from source image, transform columns, and store in local scratch
		LOCAL float* currentScratch = scratch + getScratchOffset();
		for (int j = 0; j < WIN_SIZE_Y; j+=2) {

			///////////////////////////////////////////////////////////////////////////////////////////
			// fetch next two pixels, then transform and write current (odd) and last (even)

			// read current plus one (odd) point
			posIn.y += yDelta;
			if (posIn.y > 1 + yDelta)
				break;
			float4 currentPlusOne = read_imagef(idata, sampler, posIn);

			// read current plus two (even) point
			posIn.y += yDelta;
			float4 currentPlusTwo = read_imagef(idata, sampler, posIn);

			// transform current plus one (odd) point
			currentPlusOne -= (current + currentPlusTwo) /2;  // F.4, page 118, ITU-T Rec. T.800 final draft

			// transform current (even) point
			current += (minusOne + currentPlusOne + 2) /4; // F.3, page 118, ITU-T Rec. T.800 final draft

			//write current (even)
			writePixel(current, currentScratch);

			//write odd
			currentScratch += VERTICAL_STRIDE;
			writePixel(currentPlusOne, currentScratch);

			//advance scratch pointer
			currentScratch += VERTICAL_STRIDE;

			//update registers
			minusOne = currentPlusOne;
			current = currentPlusTwo;
		}

		//4. transform horizontally
		currentScratch = scratch + getScratchOffset();

		localMemoryFence();
		//odd columns (skip right odd boundary column)
		if ( (getLocalId(0)&1) && (getLocalId(0) != WIN_SIZE_X-1) ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 currentOdd = readPixel(currentScratch);
				float4 prevEven = readPixel(currentScratch + HORIZONTAL_ODD_TO_PREVIOUS_EVEN);
				float4 nextEven = readPixel(currentScratch + HORIZONTAL_ODD_TO_NEXT_EVEN);
				currentOdd -= ((prevEven + nextEven) /2);
				writePixel( currentOdd, currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}

		currentScratch = scratch + getScratchOffset();
		localMemoryFence();
		//even columns (skip left and right even boundary columns)
		if ( !(getLocalId(0)&1) && (getLocalId(0) != 0) && (getLocalId(0) != WIN_SIZE_X-2)  ) {
			for (int j = 0; j < WIN_SIZE_Y; j++) {
				float4 currentEven = readPixel(currentScratch);
				float4 prevOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_PREVIOUS_ODD);
				float4 nextOdd = readPixel(currentScratch + HORIZONTAL_EVEN_TO_NEXT_ODD);
				currentEven += (prevOdd + nextOdd + 2) /4;
				writePixel( currentEven, currentScratch);
				currentScratch += VERTICAL_STRIDE;
			}
		}
		localMemoryFence();

		//5. write local buffer column to destination image
		// (only write non-boundary columns that are within the image bounds)
		if ((getLocalId(0) >= BOUNDARY_X) && ( getLocalId(0) < WIN_SIZE_X - BOUNDARY_X) && (inputX < width) && inputX >= 0) {
			writeColumnToOutput(scratch + getScratchOffset(), odata, firstY, outputX, height, halfHeight);

		}
		// move to next step
		firstY += WIN_SIZE_Y;
	}
}