[OpenCL] Remove redundant test for OpenCL header file
Differential Revision: https://reviews.llvm.org/D26302

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@286114 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s

// CHECK: _Z16convert_char_rtec
// CHECK-NOT: _Z3ctzc
// CHECK20: _Z3ctzc
// CHECK20-NOT: _Z16convert_char_rtec
// CHECK-MOD: Reading modules

// Test including the default header as a module.
// The module should be compiled only once and loaded from cache afterwards.
// Change the directory mode to read only to make sure no new modules are created.
// Check time report to make sure module is used.

// ===
// Clear current directory.
// RUN: rm -rf %t
// RUN: mkdir -p %t

// ===
// Compile for OpenCL 1.0 for the first time. A module should be generated.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 1.0 for the second time. The module should not be re-created.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t/opencl_c.pcm
// RUN: mv %t/opencl_c.pcm %t/1_0.pcm

// ===
// Compile for OpenCL 2.0 for the first time. The module should change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: not diff %t/1_0.pcm %t/opencl_c.pcm
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 2.0 for the second time. The module should not change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s

// Check cached module works for different OpenCL versions.
// RUN: rm -rf %t
// RUN: mkdir -p %t
// RUN: %clang_cc1 -triple spir64-unknown-unknown -emit-llvm -o - -cl-std=CL1.2 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t
// RUN: %clang_cc1 -triple spir64-unknown-unknown -emit-llvm -o - -cl-std=CL1.2 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t

char f(char x) {
#if __OPENCL_C_VERSION__ != CL_VERSION_2_0
  return convert_char_rte(x);
#ifdef NO_HEADER
  //expected-warning@-2{{implicit declaration of function 'convert_char_rte' is invalid in C99}}
#endif //NO_HEADER

#else //__OPENCL_C_VERSION__
  return ctz(x);
#endif //__OPENCL_C_VERSION__
}

// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s

// RUN: %clang_cc1 -triple x86_64-unknown-unknown -finclude-default-header -emit-llvm -o - %s | FileCheck %s
// CHECK: _Z16convert_char_rtec
// CHECK-NOT: _Z3ctzc
// CHECK20: _Z3ctzc
// CHECK20-NOT: _Z16convert_char_rtec
// CHECK-MOD: Reading modules

// Test including the default header as a module.
// The module should be compiled only once and loaded from cache afterwards.
// Change the directory mode to read only to make sure no new modules are created.
// Check time report to make sure module is used.

// ===
// Clear current directory.
// RUN: rm -rf %t
// RUN: mkdir -p %t

// ===
// Compile for OpenCL 1.0 for the first time. A module should be generated.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 1.0 for the second time. The module should not be re-created.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t/opencl_c.pcm
// RUN: mv %t/opencl_c.pcm %t/1_0.pcm

// ===
// Compile for OpenCL 2.0 for the first time. The module should change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: not diff %t/1_0.pcm %t/opencl_c.pcm
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 2.0 for the second time. The module should not change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s

// Check cached module works for different OpenCL versions.
// RUN: rm -rf %t
// RUN: mkdir -p %t
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0  -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t

char f(char x) {
#if __OPENCL_C_VERSION__ != CL_VERSION_2_0
  return convert_char_rte(x);
#ifdef NO_HEADER
  //expected-warning@-2{{implicit declaration of function 'convert_char_rte' is invalid in C99}}
#endif //NO_HEADER

#else //__OPENCL_C_VERSION__
  return ctz(x);
#endif //__OPENCL_C_VERSION__
}


now finished coding

now finished coding

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// number of filter taps in the processing wk_item
#define MLO_WEI_WKITEM 4
// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be mdae in the inner loop
#define MLO_OUT_WEI_SCAN_BLK ((MLO_OUT_HORIX_PIX_SZ + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIX_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)
#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIX_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

// extended scan to deal with overshot in the inner loop
#define MLO_OUT_HORIX_PIX_EXT_SZ (MLO_OUT_WEI_SCAN_BLK * MLO_N_WEI_BLK)
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIX_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#define MLO_N_IN_HORIZ_READS ((((MLO_OUT_HORIX_PIX_SZ-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u,(int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm
// data layout.
// registers:
// per wk_item:
// MLO_READ_UNIT x MLO_N_LCL_OUT_MAPS of dy
// group:
// MLO_N_OUT_BLK_GRP blocks of MLO_N_OUT_BLK_GRP_PIX_SZ of dy
//
// LDS:
// data buffer
// MLO_N_LCL_IN_MAPS blocks of MLO_IN_LCL_SZ of x

// dw local buffer keeps 1 set of dweghts calculated at each iteration
	MLO_TEMP_WEI_BUFFER_SZ

//
// inputs:
// read into LDS
// MLO_N_IN_BLK_GRP groups
// output
// fetch
// MLO_N_OUT_BLK_GRP groups

// alg
// for each batch
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for k in filter size1
//				for l in filter size 0
//					at for each out/in pixel pair op,ip in wk_item
//                  calculate the contribution to a dweights by formula ip.x (input pixel.x) = (op.x * stride.x - pad .x + l) % (filter size 0), ip.y = (op.y * stride.y - pad.y + k) % filter size1,
//					wei[k,l] = in[ip.y, ip.x]*op[op.y, op.x]
//                  lcl_wei[MLO_OUT_BLK_GRP_PIX_SZ * out.grp + [op.y, op.x]] = we[k,l]
//                  accumulate the weights in the wk_item with wei_accum +=  lcl_wei[accum_id], accum_id = MLO_OUT_BLK_GRP*MLO_N_LCL_OUT_MAPS*map_id + k*(filter size 0) + l
//	write out accululate weights

// group layout
// 0 - input channel index
// 1 - output channel index
// 2 - 1

// loop over batches
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
// input/output tiles

	__local _FLOAT lcl[(MLO_TOTAL_IN_LCL_SZ + MLO_TOTAL_OUT_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();

	int c_idx_base = get_group_id(1); // input map index base

// register memory
	int o_idx_base = get_group_id(2); // output map index base

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	int lcl_id = get_local_id(0);

// output global read
//	int o_blk_idx = iDiv(lcl_id, MLO_OUT_LCL_SZ);
//	int o_pix_X4 = iMod(lcl_id, o_blk_idx, MLO_OUT_LCL_SZ);
	int o_scan = iDiv(lcl_id, MLO_N_OUT_HORIZ_READS);
	int o_pix4 = iMod(lcl_id, o_scan, MLO_N_OUT_HORIZ_READS);

// input global read
	int i_blk_idx = 0; //iDiv(lcl_id, MLO_IN_LCL_SZ);
	//int c_pix_X4 = iMod(lcl_id, i_blk_idx, MLO_IN_LCL_SZ);
	int c_scan0 = iDiv(lcl_id, MLO_N_IN_HORIZ_READS);
	int c_pix4 = iMod(lcl_id,c_scan0, MLO_N_IN_HORIZ_READS);

	int w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	int w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
	int w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
	int w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);

// FILTR SPECIFIC !!!!!
	int last_tap_to_remove = (w_x0 > 0)? 0 : 1;

// mask to shut down the overshot filter tap
	__constant int filter_mask0 = ~(1 << MLO_FILTER_SIZE0);
	int working_mask = 0;
// filled with 1 evry stride bit
	for(int i = 0; i < MLO_WEI_WKITEM; ++i)
	{
		working_mask |= (1 << (i*MLO_FILTER_STRIDE0));
	}
// add strting point
	working_mask <<= w_x0;
// clamp to the filter size
	working_mask &= filter_mask0;
// move back to the relative start

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE;

	for (int i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

// zero out LDS
	for (int i = 0; i <  (MLO_TOTAL_IN_LCL_SZ + MLO_TOTAL_OUT_LCL_SZ) ; i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);
// zero out pvt accum
	// over all batches

#if 0
	if ( lcl_id == 0 && c_idx ==0 && o_idx == 0)
	{
		printf("K:s:%d %d\n",
			MLO_TOTAL_IN_LCL_SZ*4,
			MLO_TOTAL_OUT_LCL_SZ
		);
	}
#endif
#if 1
	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;
// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1
// LDS UPDATE !!!!
// input scan
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off = 0;
		int gbl_out_scan_off = 0;

		for(int c_scan = c_scan0; c_scan < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; c_scan += MLO_IN_VERT_READS)
		{
// TODO :: multiply on right mask if needed
			*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
// TO DO:: CHECK HEIGHT
// TO DO:: CHECK PADDING
			*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
		}

		in_y +=  MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1;

// TO DO: HANDLE PADDING
// over all out blocks
		for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			gbl_in_scan_off = gbl_in_off + mul24(in_y, MLO_IN_STRIDE);

// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
               // TODO:: HANDLE multiple INPUTS
			   // overshot i shadled by out put

			for (int c_scan = c_scan0;i_blk_idx < MLO_N_LCL_IN_MAPS && in_y + c_scan < MLO_IN_HEIGHT && c_scan <  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1); c_scan += MLO_IN_VERT_READS)
			{

					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
					*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

			}

			// ouput scan
			__private _FLOAT out_rd_data[MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT];

			gbl_out_scan_off = gbl_out_off + mul24(out_y, MLO_OUT_STRIDE);

// over all outputs groups

			int gbl_out_scan_off1 = gbl_out_scan_off;
			for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og, gbl_out_scan_off1 += MLO_OUT_LCL_SZ*MLO_OUT_CHANNEL_STRIDE )
			{

// fetch output

				for(int o = 0; o < MLO_N_LCL_OUT_MAPS
						&&  ( o_idx + o < MLO_N_OUTPUTS && out_y + o_scan < MLO_OUT_HEIGHT && o_scan < MLO_N_ALIGNED_OUT_SCAN_BLK)
						; ++o)
				{
						*(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT]
							= *(MLO_READ_TYPE*)&top_df[gbl_out_scan_off + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT];
#if MLO_OUT_SCAN_NOT_DIVBY4
						if (o_pix4 == (MLO_N_OUT_HORIZ_READS-1))
						{
							for(int i = MLO_READ_UNIT-1; i >= MLO_READ_UNIT - MLO_OUT_N_PIXS_OFF; --i)
							{
	 							out_rd_data[o*MLO_READ_UNIT + i] = 0;
							}
						}
#endif
// write with MLO_OUT_HORIX_PIX_EXT_SZ stride
						*(MLO_READ_TYPE*)&lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIX_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT];
				}

// process
// algorithm

#if 1
				for(int j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
				{

					_FLOAT i_vals[MLO_WEI_WKITEM];
					for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
					{
							int w_x = w_x0 + w*MLO_FILTER_STRIDE0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[w] = i_val;

					}

// if we overshoot the scanline
// out data will be 0 by initial setting

					for(int i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
					{

//						for(int w = 0; w < MLO_WEI_WKITEM; ++w)

						{
							int w_x = w_x0 + (MLO_WEI_WKITEM-1)* MLO_FILTER_STRIDE0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[(MLO_WEI_WKITEM-1)] = i_val;

						}

						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
							_FLOAT o_val
								= lcl_top[o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIX_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

#if 1
							for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w )
							{

								_FLOAT i_val = i_vals[w];
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;
							}

// set input data to zero if MLO_FILTER_SIZE0 is not multiple of MLO_WEI_WKITEM and it's over MLO_FILTER_SIZE0

							_FLOAT i_val = i_vals[MLO_WEI_WKITEM - 1];
							int ii_val = as_uint(i_val) & last_tap_to_remove;
							i_val = as_float(ii_val);
							pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + MLO_WEI_WKITEM - 1] += i_val * o_val;

#else
							uint running_mask = working_mask;
							for(int w = 0; w < MLO_WEI_WKITEM; ++w, (running_mask >>= MLO_FILTER_STRIDE0) )
							{
// set input data to zero if MLO_FILTER_SIZE0 is not multiple of MLO_WEI_WKITEM and it's over MLO_FILTER_SIZE0

								_FLOAT i_val = i_vals[w];
								int ii_val = as_uint(i_val) & (running_mask & 1);
								i_val = as_float(ii_val);
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;
							}

#endif

						}

						for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
						{
							i_vals[w] = i_vals[w + 1];

						}

					}
				}
#else
						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
							for(int w = 0; w < MLO_WEI_WKITEM; ++w)
							{
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = lcl_top[(og * MLO_N_LCL_OUT_MAPS + o)] * lcl_bot[w];
							}
						}
#endif

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

			barrier(CLK_LOCAL_MEM_FENCE);

// move data up
			for(int c_scan = c_scan0; c_scan < MLO_FILTER_SIZE1 -1; c_scan += MLO_IN_VERT_READS)
			{
					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&lcl_bot[(c_scan +  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1))*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT];
					*(MLO_READ_TYPE*)&lcl_bot[c_scan*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

			}

		} // for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (int b = 0;

#endif

	int o = iDiv(lcl_id, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_i = iMod(lcl_id, o, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
	int wei_i_x = iMod(wei_i,wei_i_y, (MLO_FILTER_SIZE0));

	int wei_lcl_off = 0;
	_FLOAT final_sum = 0;

#if 1
// save in lcl and orgnize in a proper order
// outputs
//	filter size1
//	  filter size0

	for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for(int o = 0;  /*w_blk_idx < MLO_N_WEI_BLK && */o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for(; w < MLO_WEI_WKITEM; ++w)
			{
// save "virtual" filter table
				wei_lcl_off = ((o * MLO_N_LCL_OUT_MAPS *  MLO_N_WEI_BLK  + w_blk_idx) * MLO_FILTER_SIZE1  + w_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + w_x0 + w * MLO_FILTER_STRIDE0;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];

			}

		}

#if 1
// read into real filter table
		if (lcl_id >= (og * MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) && lcl_id < ((og + 1) * MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
		{
			for(int i = 0; i < MLO_N_WEI_BLK; ++i)
			{
				final_sum += lcl_bot[(((o - og * MLO_N_LCL_OUT_MAPS) * MLO_N_WEI_BLK  + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i];
			}
		}
#endif

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
#else

	for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for(int o = 0; w_blk_idx < MLO_N_WEI_BLK && o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for(; w < MLO_WEI_WKITEM; ++w)
			{
				wei_lcl_off = og + (o * MLO_N_LCL_OUT_MAPS *  MLO_N_WEI_BLK  + w_blk_idx) * (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0) + w_y * MLO_FILTER_SIZE0 + w_x0 + w * MLO_FILTER_STRIDE0;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];

			}
		}

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

#endif
// send it out
	int wei_df_off = mul24(o_idx, (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
						+ mul24(c_idx, (int)MLO_WEI_CHANNEL_STRIDE);
// write out from 0th wk_item
	if (lcl_id < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
	{

#if 1

		weights_df[wei_df_off + o * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //
#else
		_FLOAT t_accum = 0;

		for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
		{
			for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
			{
				for(int w = 0; w < MLO_WEI_WKITEM; ++w)
				{
					t_accum += pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
				}
			}
		}

		weights_df[lcl_id] = t_accum;
#endif

	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

// number of filter taps in the processing wk_item
#define MLO_WEI_WKITEM 4
// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be mdae in the inner loop
#define MLO_OUT_WEI_SCAN_BLK ((MLO_OUT_HORIX_PIX_SZ + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIX_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)
#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIX_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

// extended scan to deal with overshot in the inner loop
#define MLO_OUT_HORIX_PIX_EXT_SZ (MLO_OUT_WEI_SCAN_BLK * MLO_N_WEI_BLK)
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIX_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
#define MLO_N_IN_HORIZ_READS ((((MLO_OUT_HORIX_PIX_SZ-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0) + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u,(int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{

				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm
// data layout.
// registers:
// per wk_item:
// MLO_READ_UNIT x MLO_N_LCL_OUT_MAPS of dy
// group:
// MLO_N_OUT_BLK_GRP blocks of MLO_N_OUT_BLK_GRP_PIX_SZ of dy
//
// LDS:
// data buffer
// MLO_N_LCL_IN_MAPS blocks of MLO_IN_LCL_SZ of x

// dw local buffer keeps 1 set of dweghts calculated at each iteration
	MLO_TEMP_WEI_BUFFER_SZ

//
// inputs:
// read into LDS
// MLO_N_IN_BLK_GRP groups
// output
// fetch
// MLO_N_OUT_BLK_GRP groups

// alg
// for each batch
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for k in filter size1
//				for l in filter size 0
//					at for each out/in pixel pair op,ip in wk_item
//                  calculate the contribution to a dweights by formula ip.x (input pixel.x) = (op.x * stride.x - pad .x + l) % (filter size 0), ip.y = (op.y * stride.y - pad.y + k) % filter size1,
//					wei[k,l] = in[ip.y, ip.x]*op[op.y, op.x]
//                  lcl_wei[MLO_OUT_BLK_GRP_PIX_SZ * out.grp + [op.y, op.x]] = we[k,l]
//                  accumulate the weights in the wk_item with wei_accum +=  lcl_wei[accum_id], accum_id = MLO_OUT_BLK_GRP*MLO_N_LCL_OUT_MAPS*map_id + k*(filter size 0) + l
//	write out accululate weights

// group layout
// 0 - input channel index
// 1 - output channel index
// 2 - 1

// loop over batches
**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
// input/output tiles

	__local _FLOAT lcl[(MLO_TOTAL_IN_LCL_SZ + MLO_TOTAL_OUT_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();

	int c_idx_base = get_group_id(1); // input map index base

// register memory
	int o_idx_base = get_group_id(2); // output map index base

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	int lcl_id = get_local_id(0);

// output global read
//	int o_blk_idx = iDiv(lcl_id, MLO_OUT_LCL_SZ);
//	int o_pix_X4 = iMod(lcl_id, o_blk_idx, MLO_OUT_LCL_SZ);
	int o_scan = iDiv(lcl_id, MLO_N_OUT_HORIZ_READS);
	int o_pix4 = iMod(lcl_id, o_scan, MLO_N_OUT_HORIZ_READS);

// input global read
	int i_blk_idx = 0; //iDiv(lcl_id, MLO_IN_LCL_SZ);
	//int c_pix_X4 = iMod(lcl_id, i_blk_idx, MLO_IN_LCL_SZ);
	int c_scan0 = iDiv(lcl_id, MLO_N_IN_HORIZ_READS);
	int c_pix4 = iMod(lcl_id,c_scan0, MLO_N_IN_HORIZ_READS);

	int w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	int w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
	int w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
	int w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);

// FILTR SPECIFIC !!!!!
	int last_tap_to_remove = (w_x0 > 0)? 0 : 1;

// mask to shut down the overshot filter tap
	__constant int filter_mask0 = ~(1 << MLO_FILTER_SIZE0);
	int working_mask = 0;
// filled with 1 evry stride bit
	for(int i = 0; i < MLO_WEI_WKITEM; ++i)
	{
		working_mask |= (1 << (i*MLO_FILTER_STRIDE0));
	}
// add strting point
	working_mask <<= w_x0;
// clamp to the filter size
	working_mask &= filter_mask0;
// move back to the relative start

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE;

	for (int i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

// zero out LDS
	for (int i = 0; i <  (MLO_TOTAL_IN_LCL_SZ + MLO_TOTAL_OUT_LCL_SZ) ; i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);
// zero out pvt accum
	// over all batches

#if 0
	if ( lcl_id == 0 && c_idx ==0 && o_idx == 0)
	{
		printf("K:s:%d %d\n",
			MLO_TOTAL_IN_LCL_SZ*4,
			MLO_TOTAL_OUT_LCL_SZ
		);
	}
#endif
#if 1
	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE
		)
	{
		int in_y = 0;
		int out_y = 0;
// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1
// LDS UPDATE !!!!
// input scan
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		int gbl_in_scan_off = 0;
		int gbl_out_scan_off = 0;

		for(int c_scan = c_scan0; c_scan < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; c_scan += MLO_IN_VERT_READS)
		{
// TODO :: multiply on right mask if needed
			*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
// TO DO:: CHECK HEIGHT
// TO DO:: CHECK PADDING
			*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;
		}

		in_y +=  MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1;

// TO DO: HANDLE PADDING
// over all out blocks
		for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			gbl_in_scan_off = gbl_in_off + mul24(in_y, MLO_IN_STRIDE);

// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
               // TODO:: HANDLE multiple INPUTS
			   // overshot i shadled by out put

			for (int c_scan = c_scan0;i_blk_idx < MLO_N_LCL_IN_MAPS && in_y + c_scan < MLO_IN_HEIGHT && c_scan <  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1); c_scan += MLO_IN_VERT_READS)
			{

					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&bot[gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
					*(MLO_READ_TYPE*)&lcl_bot[(c_scan + MLO_FILTER_SIZE1 - 1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

			}

			// ouput scan
			__private _FLOAT out_rd_data[MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT];

			gbl_out_scan_off = gbl_out_off + mul24(out_y, MLO_OUT_STRIDE);

// over all outputs groups

			int gbl_out_scan_off1 = gbl_out_scan_off;
			for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og, gbl_out_scan_off1 += MLO_OUT_LCL_SZ*MLO_OUT_CHANNEL_STRIDE )
			{

// fetch output

				for(int o = 0; o < MLO_N_LCL_OUT_MAPS
						&&  ( o_idx + o < MLO_N_OUTPUTS && out_y + o_scan < MLO_OUT_HEIGHT && o_scan < MLO_N_ALIGNED_OUT_SCAN_BLK)
						; ++o)
				{
						*(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT]
							= *(MLO_READ_TYPE*)&top_df[gbl_out_scan_off + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT];
#if MLO_OUT_SCAN_NOT_DIVBY4
						if (o_pix4 == (MLO_N_OUT_HORIZ_READS-1))
						{
							for(int i = MLO_READ_UNIT-1; i >= MLO_READ_UNIT - MLO_OUT_N_PIXS_OFF; --i)
							{
	 							out_rd_data[o*MLO_READ_UNIT + i] = 0;
							}
						}
#endif
// write with MLO_OUT_HORIX_PIX_EXT_SZ stride
						*(MLO_READ_TYPE*)&lcl_top[o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIX_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)&out_rd_data[o*MLO_READ_UNIT];
				}

// process
// algorithm

#if 1
				for(int j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
				{

					_FLOAT i_vals[MLO_WEI_WKITEM];
					for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
					{
							int w_x = w_x0 + w*MLO_FILTER_STRIDE0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[w] = i_val;

					}

// if we overshoot the scanline
// out data will be 0 by initial setting

					for(int i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
					{

//						for(int w = 0; w < MLO_WEI_WKITEM; ++w)

						{
							int w_x = w_x0 + (MLO_WEI_WKITEM-1)* MLO_FILTER_STRIDE0;
							int i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[(MLO_WEI_WKITEM-1)] = i_val;

						}

						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
							_FLOAT o_val
								= lcl_top[o*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIX_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

#if 1
							for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w )
							{

								_FLOAT i_val = i_vals[w];
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;
							}

// set input data to zero if MLO_FILTER_SIZE0 is not multiple of MLO_WEI_WKITEM and it's over MLO_FILTER_SIZE0

							_FLOAT i_val = i_vals[MLO_WEI_WKITEM - 1];
							int ii_val = as_uint(i_val) & last_tap_to_remove;
							i_val = as_float(ii_val);
							pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + MLO_WEI_WKITEM - 1] += i_val * o_val;

#else
							uint running_mask = working_mask;
							for(int w = 0; w < MLO_WEI_WKITEM; ++w, (running_mask >>= MLO_FILTER_STRIDE0) )
							{
// set input data to zero if MLO_FILTER_SIZE0 is not multiple of MLO_WEI_WKITEM and it's over MLO_FILTER_SIZE0

								_FLOAT i_val = i_vals[w];
								int ii_val = as_uint(i_val) & (running_mask & 1);
								i_val = as_float(ii_val);
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;
							}

#endif

						}

						for(int w = 0; w < MLO_WEI_WKITEM - 1; ++w)
						{
							i_vals[w] = i_vals[w + 1];

						}

					}
				}
#else
						for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
						{
							for(int w = 0; w < MLO_WEI_WKITEM; ++w)
							{
								pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] = lcl_top[(og * MLO_N_LCL_OUT_MAPS + o)] * lcl_bot[w];
							}
						}
#endif

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

			barrier(CLK_LOCAL_MEM_FENCE);

// move data up
			for(int c_scan = c_scan0; c_scan < MLO_FILTER_SIZE1 -1; c_scan += MLO_IN_VERT_READS)
			{
					*(MLO_READ_TYPE*)in_rd_data = *(MLO_READ_TYPE*)&lcl_bot[(c_scan +  (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1))*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT];
					*(MLO_READ_TYPE*)&lcl_bot[c_scan*(MLO_IN_LCL_WIDTH) + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT] = *(MLO_READ_TYPE*)in_rd_data;

			}

		} // for (int ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (int b = 0;

#endif

	int o = iDiv(lcl_id, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_i = iMod(lcl_id, o, (MLO_FILTER_SIZE1*MLO_FILTER_SIZE0));
	int wei_lcl_off = 0;
	_FLOAT final_sum = 0;

#if 1
// save in lcl and orgnize in a proper order
// outputs
//	filter size1
//	  filter size0

	for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for(int o = 0;  /*w_blk_idx < MLO_N_WEI_BLK && */o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for(; w < MLO_WEI_WKITEM; ++w)
			{
				wei_lcl_off = (o * MLO_N_LCL_OUT_MAPS *  MLO_N_WEI_BLK  + w_blk_idx) * (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0) + w_y * MLO_FILTER_SIZE0 + w_x0 + w * MLO_FILTER_STRIDE0;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];

			}

		}

#if 1
		if (lcl_id >= (og * MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0) && lcl_id < ((og + 1) * MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
		{
			for(int i = 0; i < MLO_N_WEI_BLK; ++i)
			{
				final_sum += lcl_bot[((o - og * MLO_N_LCL_OUT_MAPS) * MLO_N_WEI_BLK  + i) * (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0) + wei_i];
			}
		}
#endif

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
#else

	for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		for(int o = 0; w_blk_idx < MLO_N_WEI_BLK && o < MLO_N_LCL_OUT_MAPS; ++o)
		{
			int w = 0;
			for(; w < MLO_WEI_WKITEM; ++w)
			{
				wei_lcl_off = og + (o * MLO_N_LCL_OUT_MAPS *  MLO_N_WEI_BLK  + w_blk_idx) * (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0) + w_y * MLO_FILTER_SIZE0 + w_x0 + w * MLO_FILTER_STRIDE0;
				lcl_bot[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];

			}
		}

	} // for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

#endif
// send it out
	int wei_df_off = mul24(o_idx, (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
						+ mul24(c_idx, (int)MLO_WEI_CHANNEL_STRIDE);
// write out from 0th wk_item
	if (lcl_id < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0))
	{

#if 1

		weights_df[wei_df_off + o * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //
#else
		_FLOAT t_accum = 0;

		for(int og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
		{
			for(int o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
			{
				for(int w = 0; w < MLO_WEI_WKITEM; ++w)
				{
					t_accum += pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];
				}
			}
		}

		weights_df[lcl_id] = t_accum;
#endif

	}

}


switched euler fluid equations into the more bureaucratic api.  many more to go.

// Roe solver:

<?
-- for cartesian, use the normal in cartesian coordinates
-- however that 'x' is still being used for who knows what
--  ... and could be used for grid-metric-based norms
-- but you can't use 'cartesianFromCoord' because we're using the cartesian basis already
-- for coordinate or coordinate-orthonormal, use the coordinate-aligned normal
local gridIsCartesian = require 'coord.cartesian'.is(solver.coord)
local nonAxisAlignedNormals = solver.coord.vectorComponent == 'cartesian' and not gridIsCartesian
?>

//call this, initializing nL to the normalBasisForSide<?=side?>
// this calculates the rest based on the x coordinate
// TODO put this somewhere above where display functions can use it
void calcNormalBasisForGrid(
	real3x3* nL,
	real3x3* nU,
	real *nLen,
	real3 x
) {
	// if we are non-cartesian grid with cartesian components then the normal of the non-cartesian grid needs to be in cartesian components as well ...
<? if nonAxisAlignedNormals then ?>
	nL.x = coord_cartesianFromCoord(nL.x, xInt);
	nL.y = coord_cartesianFromCoord(nL.y, xInt);
	nL.z = coord_cartesianFromCoord(nL.z, xInt);
<? end ?>
<? if gridIsCartesian then -- or if the metric is identity ... cylinder surface ?>
	*nU = *nL;
	*nLen = 1;
<? else ?>
	//if we have a non-cartesian grid + cartesian component normal ... or if we have a non cartesian component normal ... then raising is different than the lowered version
	*nU = (real3x3){
		.x = coord_raise(nL->x, xInt),
		.y = coord_raise(nL->y, xInt),
		.z = coord_raise(nL->z, xInt),
	};
	*nLen = sqrt(real3_dot(nL->x, nU->x));
<? end ?>
}

//TODO entropy fix ... for the Euler equations at least
kernel void calcFlux(
	constant <?=solver.solver_t?>* solver,
	global <?=eqn.cons_t?>* fluxBuf,
	const global <?=solver.getULRArg?>,
	realparam dt
) {
	typedef <?=eqn.cons_t?> cons_t;
	typedef <?=eqn.eigen_t?> eigen_t;
	typedef <?=eqn.waves_t?> waves_t;

	SETBOUNDS(numGhost,numGhost-1);
	real3 xR = cell_x(i);
	int indexR = index;
	<? for side=0,solver.dim-1 do ?>{
		const int side = <?=side?>;
		int indexL = index - solver->stepsize.s<?=side?>;

		real dx = solver->grid_dx.s<?=side?>;

		real3 xL = xR;
		xL.s<?=side?> -= dx;

		real3 xInt = xR;
		xInt.s<?=side?> -= .5 * dx;

		//this is used for the flux limiter
		//should it be using the coordinate dx or the grid dx?
		//real dt_dx = dt / cell_dx<?=side?>(xInt);
		real dt_dx = dt / dx;

		real3 xIntL = xInt;
		xIntL.s<?=side?> -= dx;

		real3 xIntR = xInt;
		xIntR.s<?=side?> += dx;

<?=solver:getULRCode():gsub('\t', '\t\t')?>

		// here is where parallel propagator comes into play
		cons_t pUL = cons_parallelPropagate<?=side?>(*UL, xL, .5 * dx);
		cons_t pUR = cons_parallelPropagate<?=side?>(*UR, xR, -.5 * dx);

		int indexInt = side + dim * index;

		real3x3 nL = normalBasisForSide<?=side?>;
		real3x3 nU;
		real nLen;
		calcNormalBasisForGrid(&nL, &nU, &nLen, xInt);

		eigen_t eig = eigen_forInterface(solver, pUL, pUR, xInt, nL, nU, nLen);

<? if nonAxisAlignedNormals then ?>
<?=eqn:eigenWaveCodePrefixForNormal('nL', 'nU', 'nLen', 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? else ?>
<?=eqn:eigenWaveCodePrefixForSide(side, 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? end ?>

		waves_t fluxEig;
<? if not eqn.roeUseFluxFromCons then
?>		cons_t UAvg;
		for (int j = 0; j < numIntStates; ++j) {
			UAvg.ptr[j] = .5 * (pUL.ptr[j] + pUR.ptr[j]);
		}

<? 	if nonAxisAlignedNormals then ?>
#error here
		fluxEig = eigen_leftTransformForNormal(solver, eig, UAvg, xInt, nL, nU, nLen);
<?	else ?>
		fluxEig = eigen_leftTransformForSide<?=side?>(solver, eig, UAvg, xInt);
<?	end
end
?>
		cons_t deltaU;
<? if solver.fluxLimiter > 1 then
?>		int indexR2 = indexR + solver->stepsize.s<?=side?>;
		int indexL2 = indexL - solver->stepsize.s<?=side?>;
		<?=solver:getULRCode{indexL = 'indexL2', indexR = 'indexL', suffix='_L'}?>
		<?=solver:getULRCode{indexL = 'indexR', indexR = 'indexR2', suffix='_R'}?>

		// here is where parallel propagator comes into play
		cons_t pUL_L = cons_parallelPropagate<?=side?>(*UL_L, xIntL, 1.5 * dx);		//xIntL2?
		cons_t pUL_R = cons_parallelPropagate<?=side?>(*UL_R, xIntL, .5 * dx);
		cons_t pUR_L = cons_parallelPropagate<?=side?>(*UR_L, xIntR, -.5 * dx);
		cons_t pUR_R = cons_parallelPropagate<?=side?>(*UR_R, xIntR, -1.5 * dx);	// xIntR2?

		cons_t deltaUL, deltaUR;
<? end
?>
		for (int j = 0; j < numStates; ++j) {
			deltaU.ptr[j] = pUR.ptr[j] - pUL.ptr[j];
<? if solver.fluxLimiter > 1 then
?>			deltaUL.ptr[j] = pUR_L.ptr[j] - pUL_L.ptr[j];
			deltaUR.ptr[j] = pUR_R.ptr[j] - pUL_R.ptr[j];
<? end
?>		}

<? if nonAxisAlignedNormals then ?>
		waves_t deltaUEig = eigen_leftTransformForNormal(solver, eig, deltaU, xInt, nL, nU, nLen);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForNormal(solver, eigL, deltaUL, xIntL, nL, nU, nLen);
		waves_t deltaUEigR = eigen_leftTransformForNormal(solver, eigR, deltaUR, xIntR, nL, nU, nLen);
<? 	end ?>
<? else ?>
		waves_t deltaUEig = eigen_leftTransformForSide<?=side?>(solver, eig, deltaU, xInt);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForSide<?=side?>(solver, eigL, deltaUL, xIntL);
		waves_t deltaUEigR = eigen_leftTransformForSide<?=side?>(solver, eigR, deltaUR, xIntR);
<? 	end ?>
<? end ?>

		<? for j=0,eqn.numWaves-1 do ?>{
			const int j = <?=j?>;
			real lambda = <?=eqn:eigenWaveCode(side, 'eig', 'xInt', j)?>;

<? if not eqn.roeUseFluxFromCons then
?>			fluxEig.ptr[j] *= lambda;
<? else
?>			fluxEig.ptr[j] = 0.;
<? end
?>			real sgnLambda = lambda >= 0 ? 1 : -1;

<? if solver.fluxLimiter > 1 then
?>			real rEig;
			if (deltaUEig.ptr[j] == 0) {
				rEig = 0;
			} else {
				if (lambda >= 0) {
					rEig = deltaUEigL.ptr[j] / deltaUEig.ptr[j];
				} else {
					rEig = deltaUEigR.ptr[j] / deltaUEig.ptr[j];
				}
			}
			real phi = fluxLimiter(rEig);
<? end
?>
			fluxEig.ptr[j] -= .5 * lambda * deltaUEig.ptr[j] * (sgnLambda
<? if solver.fluxLimiter > 1 then
?>				+ phi * (lambda * dt_dx - sgnLambda)
<? end
?>			);
		}<? end ?>

		global cons_t* flux = fluxBuf + indexInt;

<? if nonAxisAlignedNormals then ?>
		*flux = eigen_rightTransformForNormal(solver, eig, fluxEig, xInt, nL, nU, nLen);
<? else ?>
		*flux = eigen_rightTransformForSide<?=side?>(solver, eig, fluxEig, xInt);
<? end ?>

<? if eqn.roeUseFluxFromCons then ?>
<? 	if nonAxisAlignedNormals then ?>
		cons_t FL = fluxFromConsForNormal(solver, *UL, xL, nL, nU, nLen);
		cons_t FR = fluxFromConsForNormal(solver, *UR, xR, nL, nU, nLen);
<?	else ?>
		cons_t FL = fluxFromConsForSide<?=side?>(solver, *UL, xL);
		cons_t FR = fluxFromConsForSide<?=side?>(solver, *UR, xR);
<?	end ?>

		cons_t pFL = cons_parallelPropagate<?=side?>(FL, xL, .5 * dx);
		cons_t pFR = cons_parallelPropagate<?=side?>(FR, xR, -.5 * dx);

		for (int j = 0; j < numIntStates; ++j) {
			flux->ptr[j] += .5 * (pFL.ptr[j] + pFR.ptr[j]);
		}
<? end
?>	}<? end ?>
}

// Roe solver:

<?
-- for cartesian, use the normal in cartesian coordinates
-- however that 'x' is still being used for who knows what
--  ... and could be used for grid-metric-based norms
-- but you can't use 'cartesianFromCoord' because we're using the cartesian basis already
-- for coordinate or coordinate-orthonormal, use the coordinate-aligned normal
local gridIsCartesian = require 'coord.cartesian'.is(solver.coord)
local nonAxisAlignedNormals = solver.coord.vectorComponent == 'cartesian' and not gridIsCartesian
?>

//call this, initializing nL to the normalBasisForSide<?=side?>
// this calculates the rest based on the x coordinate
// TODO put this somewhere above where display functions can use it
void calcNormalBasisForGrid(
	real3x3* nL,
	real3x3* nU,
	real *nLen,
	real3 x
) {
	// if we are non-cartesian grid with cartesian components then the normal of the non-cartesian grid needs to be in cartesian components as well ...
<? if nonAxisAlignedNormals then ?>
	nL.x = coord_cartesianFromCoord(nL.x, xInt);
	nL.y = coord_cartesianFromCoord(nL.y, xInt);
	nL.z = coord_cartesianFromCoord(nL.z, xInt);
<? end ?>
<? if gridIsCartesian then -- or if the metric is identity ... cylinder surface ?>
	*nU = *nL;
	*nLen = 1;
<? else ?>
	//if we have a non-cartesian grid + cartesian component normal ... or if we have a non cartesian component normal ... then raising is different than the lowered version
	*nU = (real3x3){
		.x = coord_raise(nL->x, xInt),
		.y = coord_raise(nL->y, xInt),
		.z = coord_raise(nL->z, xInt),
	};
	*nLen = sqrt(real3_dot(nL->x, nU->x));
<? end ?>
}

//TODO entropy fix ... for the Euler equations at least
kernel void calcFlux(
	constant <?=solver.solver_t?>* solver,
	global <?=eqn.cons_t?>* fluxBuf,
	const global <?=solver.getULRArg?>,
	realparam dt
) {
	typedef <?=eqn.cons_t?> cons_t;
	typedef <?=eqn.eigen_t?> eigen_t;
	typedef <?=eqn.waves_t?> waves_t;

	SETBOUNDS(numGhost,numGhost-1);
	real3 xR = cell_x(i);
	int indexR = index;
	<? for side=0,solver.dim-1 do ?>{
		const int side = <?=side?>;
		int indexL = index - solver->stepsize.s<?=side?>;

		real dx = solver->grid_dx.s<?=side?>;

		real3 xL = xR;
		xL.s<?=side?> -= dx;

		real3 xInt = xR;
		xInt.s<?=side?> -= .5 * dx;

		//this is used for the flux limiter
		//should it be using the coordinate dx or the grid dx?
		//real dt_dx = dt / cell_dx<?=side?>(xInt);
		real dt_dx = dt / dx;

		real3 xIntL = xInt;
		xIntL.s<?=side?> -= dx;

		real3 xIntR = xInt;
		xIntR.s<?=side?> += dx;

<?=solver:getULRCode():gsub('\t', '\t\t')?>

		// here is where parallel propagator comes into play
		cons_t pUL = cons_parallelPropagate<?=side?>(*UL, xL, .5 * dx);
		cons_t pUR = cons_parallelPropagate<?=side?>(*UR, xR, -.5 * dx);

		int indexInt = side + dim * index;

		real3x3 nL = normalBasisForSide<?=side?>;
		real3x3 nU;
		real nLen;
		calcNormalBasisForGrid(&nL, &nU, &nLen, xInt);

		eigen_t eig = eigen_forInterface(solver, pUL, pUR, xInt, nL, nU, nLen);

<? if nonAxisAlignedNormals then ?>
<?=eqn:eigenWaveCodePrefixForNormal('nL', 'nU', 'nLen', 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? else ?>
<?=eqn:eigenWaveCodePrefixForSide(side, 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? end ?>

		waves_t fluxEig;
<? if not eqn.roeUseFluxFromCons then
?>		cons_t UAvg;
		for (int j = 0; j < numIntStates; ++j) {
			UAvg.ptr[j] = .5 * (pUL.ptr[j] + pUR.ptr[j]);
		}

<? 	if nonAxisAlignedNormals then ?>
		fluxEig = eigen_leftTransformForNormal(solver, eig, UAvg, xInt, nL, nU, nLen);
<?	else ?>
		fluxEig = eigen_leftTransformForSide<?=side?>(solver, eig, UAvg, xInt);
<?	end
end
?>
		cons_t deltaU;
<? if solver.fluxLimiter > 1 then
?>		int indexR2 = indexR + solver->stepsize.s<?=side?>;
		int indexL2 = indexL - solver->stepsize.s<?=side?>;
		<?=solver:getULRCode{indexL = 'indexL2', indexR = 'indexL', suffix='_L'}?>
		<?=solver:getULRCode{indexL = 'indexR', indexR = 'indexR2', suffix='_R'}?>

		// here is where parallel propagator comes into play
		cons_t pUL_L = cons_parallelPropagate<?=side?>(*UL_L, xIntL, 1.5 * dx);		//xIntL2?
		cons_t pUL_R = cons_parallelPropagate<?=side?>(*UL_R, xIntL, .5 * dx);
		cons_t pUR_L = cons_parallelPropagate<?=side?>(*UR_L, xIntR, -.5 * dx);
		cons_t pUR_R = cons_parallelPropagate<?=side?>(*UR_R, xIntR, -1.5 * dx);	// xIntR2?

		cons_t deltaUL, deltaUR;
<? end
?>
		for (int j = 0; j < numStates; ++j) {
			deltaU.ptr[j] = pUR.ptr[j] - pUL.ptr[j];
<? if solver.fluxLimiter > 1 then
?>			deltaUL.ptr[j] = pUR_L.ptr[j] - pUL_L.ptr[j];
			deltaUR.ptr[j] = pUR_R.ptr[j] - pUL_R.ptr[j];
<? end
?>		}

<? if nonAxisAlignedNormals then ?>
		waves_t deltaUEig = eigen_leftTransformForNormal(solver, eig, deltaU, xInt, nL, nU, nLen);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForNormal(solver, eigL, deltaUL, xIntL, nL, nU, nLen);
		waves_t deltaUEigR = eigen_leftTransformForNormal(solver, eigR, deltaUR, xIntR, nL, nU, nLen);
<? 	end ?>
<? else ?>
		waves_t deltaUEig = eigen_leftTransformForSide<?=side?>(solver, eig, deltaU, xInt);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForSide<?=side?>(solver, eigL, deltaUL, xIntL);
		waves_t deltaUEigR = eigen_leftTransformForSide<?=side?>(solver, eigR, deltaUR, xIntR);
<? 	end ?>
<? end ?>

		<? for j=0,eqn.numWaves-1 do ?>{
			const int j = <?=j?>;
			real lambda = <?=eqn:eigenWaveCode(side, 'eig', 'xInt', j)?>;

<? if not eqn.roeUseFluxFromCons then
?>			fluxEig.ptr[j] *= lambda;
<? else
?>			fluxEig.ptr[j] = 0.;
<? end
?>			real sgnLambda = lambda >= 0 ? 1 : -1;

<? if solver.fluxLimiter > 1 then
?>			real rEig;
			if (deltaUEig.ptr[j] == 0) {
				rEig = 0;
			} else {
				if (lambda >= 0) {
					rEig = deltaUEigL.ptr[j] / deltaUEig.ptr[j];
				} else {
					rEig = deltaUEigR.ptr[j] / deltaUEig.ptr[j];
				}
			}
			real phi = fluxLimiter(rEig);
<? end
?>
			fluxEig.ptr[j] -= .5 * lambda * deltaUEig.ptr[j] * (sgnLambda
<? if solver.fluxLimiter > 1 then
?>				+ phi * (lambda * dt_dx - sgnLambda)
<? end
?>			);
		}<? end ?>

		global cons_t* flux = fluxBuf + indexInt;

<? if nonAxisAlignedNormals then ?>
		*flux = eigen_rightTransformForNormal(solver, eig, fluxEig, xInt, nL, nU, nLen);
<? else ?>
		*flux = eigen_rightTransformForSide<?=side?>(solver, eig, fluxEig, xInt);
<? end ?>

<? if eqn.roeUseFluxFromCons then ?>
<? 	if nonAxisAlignedNormals then ?>
		cons_t FL = fluxFromConsForNormal(solver, *UL, xL, nL, nU, nLen);
		cons_t FR = fluxFromConsForNormal(solver, *UR, xR, nL, nU, nLen);
<?	else ?>
		cons_t FL = fluxFromConsForSide<?=side?>(solver, *UL, xL);
		cons_t FR = fluxFromConsForSide<?=side?>(solver, *UR, xR);
<?	end ?>

		cons_t pFL = cons_parallelPropagate<?=side?>(FL, xL, .5 * dx);
		cons_t pFR = cons_parallelPropagate<?=side?>(FR, xR, -.5 * dx);

		for (int j = 0; j < numIntStates; ++j) {
			flux->ptr[j] += .5 * (pFL.ptr[j] + pFR.ptr[j]);
		}
<? end
?>	}<? end ?>
}


simplifying

simplifying

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define SIGMA_NEW_BITPOS   0x1
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define SIGMA_OLD_TO_NBH_SHIFT 0x1
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

#define SIGMA_NEW			 0x1		  //position 0
#define NOT_SIGMA_NEW        0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD			 0x10		  //position  1
#define NBH					 0x20		  //position  2
#define RLC					 0x80		  //position  4
#define RLC_D_POSITION		 0x380        //positions 6-9
#define PIXEL				 0x7FFF0000   //positions 10-24
#define SIGN				 0x2000000    //position  25

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)

/**

**/

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];
	LOCAL int cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between one and 32 - zero value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {
		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);
		for (int i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			pixel = (abs(pixel) << PIXEL_START_BITPOS) | ((pixel << INPUT_TO_SIGN_SHIFT) & SIGN);
			posIn.y++;
			maxVal = max(maxVal, (pixel>> PIXEL_START_BITPOS)&0x7FFF);
			statePtr[0] = pixel;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			 for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			 }
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(int i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	//Algorithm 13 Clean-up pass on the MSB
	//1: set sigmanew := bit value
	//2: set rlcNbh :=  SUM(sigmanew) of preceding column and upper position
	//3: if rlcNbh = 0 and y-dimension of current position is a multiple of 4
	//then
	//4: execute RLC operation
	//5: end if
	//6: if rlcNbh = 1 then
	//7: execute ZC operation
	//8: end if
	//9: if sigmanew = 1 then
	//10: execute SC operation
	//11: end if

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// i) set sigma_new for strip column (sigma_old is zero)
	LOCAL int* statePtr = state + startIndex;
	for (int i = 0; i < 4; ++i) {
		int current = statePtr[0];
		current |= BIT(current);
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// ii) set rlc nbh for strip column
	statePtr = state + startIndex;
	int current = statePtr[0];
	int top = statePtr[TOP];
	int leftTop = statePtr[LEFT_TOP];
	int left = statePtr[LEFT];
	int leftBottom = statePtr[LEFT_BOTTOM];

	int rlcCount = 0;

	current |=  (BIT(top) | BIT(leftTop) | BIT(left) | BIT(leftBottom)) << NBH_BITPOS;
	statePtr[0] = current;

	// set doRLC flag
	bool doRLC = !(current & NBH) && !BIT(current);

	if (doRLC) {
		rlcCount++;
	} else {
		if (current & NBH) {
			//ZC
		}
		if (BIT(current)) {
		    //SC
		}
	}

	for (int i = 0; i < 3; ++i) {

		statePtr += STATE_BUFFER_STRIDE;
		top = current;
		current = statePtr[0];
		leftTop = left;
		left = leftBottom;
		leftBottom = statePtr[LEFT_BOTTOM];

		// toggle doRLC flag
		doRLC = doRLC && !(current & NBH) && !BIT(current);
		if (doRLC) {

			rlcCount++;
		} else {
		   if (current & NBH) {
				//ZC
		   }
		   if (BIT(current)) {
		       //SC
		   }
		}

	}
	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit plans
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		//Algorithm 15 Bit-plane Preprocessing
		//1: //First step
		//2: set sigma_old |= sigma_new
		//3: set nbh |=  old_sigma from neighbouring positions
		//4: if sigma_old = 0 AND nbh != 0 AND bit value = 1 then
		//5: set sigma_new := 1
		//6: end if
		//7: if there is some sigma_new = 1 then
		//8: blockVote := true
		//9: end if
		//10: //Second step
		//11: while blockVote = true do
		//12: set blockVote := false //reset the voting variable
		//13: set nbh :=  sigma_new from the four previous positions
		//14: if sigma_old = 0 AND nbh != 0 AND bit value = 1 then
		//15: set sigma_new = 1
		//16: end if
		//17: if there is some sigma_new = 1 found in this iteration then
		//18: set blockVote := true
		//19: end if
		//20: end while

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (int i = 0; i < 4; ++i) {
		    int current = *statePtr;
			current |= ((current) & SIGMA_NEW) << SIGMA_OLD_BITPOS;
			current &= NOT_SIGMA_NEW;  // clear sigma new
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		int top = statePtr[TOP];
		int leftTop = statePtr[LEFT_TOP];
		int rightTop = statePtr[RIGHT_TOP];
		int left = statePtr[LEFT];
		int current = statePtr[0];
		int right = statePtr[RIGHT];
		int leftBottom = statePtr[LEFT_BOTTOM];
		int bottom = statePtr[BOTTOM];
		int rightBottom = statePtr[RIGHT_BOTTOM];

		current |= ((leftTop & SIGMA_OLD) |
							( top & SIGMA_OLD) |
							( rightTop &  SIGMA_OLD) |
							( left & SIGMA_OLD) |
							( right & SIGMA_OLD) |
							( leftBottom & SIGMA_OLD) |
							( bottom & SIGMA_OLD) |
							( rightBottom & SIGMA_OLD)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) && (current & NBH) && !(current & SIGMA_OLD) ) {
			current |= SIGMA_NEW;
			blockVote = 1;

		}
		*statePtr = current;

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			current |= ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) && (current & NBH) && !(current & SIGMA_OLD) ) {
			    current |= SIGMA_NEW;
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
#ifdef AMD
		while (blockVote)
#endif
		{
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			int top = statePtr[TOP];
			int leftTop = statePtr[LEFT_TOP];
			int left = statePtr[LEFT];
			int current = statePtr[0];
			int leftBottom = statePtr[LEFT_BOTTOM];

			for (int i = 0; i < 3; ++i) {

				current  |= ((leftTop & SIGMA_NEW) |
							( top & SIGMA_NEW) |
							( left & SIGMA_NEW) |
							( leftBottom & SIGMA_NEW) ) << NBH_BITPOS;

				if (  BIT(current) && !(current & SIGMA_OLD) &&  (current & NBH)  && !(current & SIGMA_NEW)) {
					current |= SIGMA_NEW;
					blockVote = 1;

				}
				statePtr[0] = current;
				statePtr += STATE_BUFFER_STRIDE;
				top = current;
				current = statePtr[0];
				leftTop = left;
				left = leftBottom;
				leftBottom = statePtr[LEFT_BOTTOM];
			}
			current |= ((leftTop & SIGMA_NEW) |
						( top & SIGMA_NEW) |
						( left & SIGMA_NEW) ) << NBH_BITPOS; // ignore leftBottom pixel, because it is in the next
						                                     // stripe and it hasn't been processed yet

			if (  BIT(current) && !(current & SIGMA_OLD) &&  (current & NBH)  && !(current & SIGMA_NEW)) {
				current |= SIGMA_NEW;
				blockVote = 1;

			}
			statePtr[0] = current;
			localMemoryFence();

		}

		/////////////////////////
		// 5. Significance
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop = statePtr[LEFT_TOP];
		rightTop = statePtr[RIGHT_TOP];
		left = statePtr[LEFT];
		current = statePtr[0];
		right = statePtr[RIGHT];
		leftBottom = statePtr[LEFT_BOTTOM];
		bottom = statePtr[BOTTOM];
		rightBottom = statePtr[RIGHT_BOTTOM];
		for (int i = 0; i < 4; ++i) {

			int nbh = ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << NBH_BITPOS;

			current |= nbh;
			if ( !(current & NBH) && nbh && BIT(current) ) {
			    current |= SIGMA_NEW;

			}
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];
		}
		localMemoryFence();

		//////////////////////////
		// 6. Magnitude Refinement
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop = statePtr[LEFT_TOP];
		rightTop = statePtr[RIGHT_TOP];
		left = statePtr[LEFT];
		current = statePtr[0];
		right = statePtr[RIGHT];
		leftBottom = statePtr[LEFT_BOTTOM];
		bottom = statePtr[BOTTOM];
		rightBottom = statePtr[RIGHT_BOTTOM];
		for (int i = 0; i < 4; ++i) {

			int nbh = ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << NBH_BITPOS;

			current |= nbh;
			if ( !(current & NBH) && nbh && BIT(current) ) {
			    current |= SIGMA_NEW;

			}
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];
		}
		localMemoryFence();

		///////////////////////
		// 7. cleanup
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop = statePtr[LEFT_TOP];
		rightTop = statePtr[RIGHT_TOP];
		left = statePtr[LEFT];
		current = statePtr[0];
		right = statePtr[RIGHT];
		leftBottom = statePtr[LEFT_BOTTOM];
		bottom = statePtr[BOTTOM];
		rightBottom = statePtr[RIGHT_BOTTOM];
		for (int i = 0; i < 4; ++i) {

			int nbh = ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << NBH_BITPOS;

			current |= nbh;
			if ( !(current & NBH) && nbh && BIT(current) ) {
			    current |= SIGMA_NEW;

			}
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];
		}
		localMemoryFence();

		bp--;
	}
}

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define SIGMA_NEW_BITPOS   0x1
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define SIGMA_OLD_TO_NBH_SHIFT 0x1
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

#define SIGMA_NEW			 0x1		  //position 0
#define NOT_SIGMA_NEW        0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD			 0x10		  //position  1
#define NBH					 0x20		  //position  2
#define RLC					 0x80		  //position  4
#define RLC_D_POSITION		 0x380        //positions 6-9
#define PIXEL				 0x7FFF0000   //positions 10-24
#define SIGN				 0x2000000    //position  25

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)

/**

**/

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];
	LOCAL int cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between one and 32 - zero value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {
		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);
		for (int i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			pixel = (abs(pixel) << PIXEL_START_BITPOS) | ((pixel << INPUT_TO_SIGN_SHIFT) & SIGN);
			posIn.y++;
			maxVal = max(maxVal, (pixel>> PIXEL_START_BITPOS)&0x7FFF);
			statePtr[0] = pixel;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			 for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			 }
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(int i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	//Algorithm 13 Clean-up pass on the MSB
	//1: set sigmanew := bit value
	//2: set rlcNbh :=  SUM(sigmanew) of preceding column and upper position
	//3: if rlcNbh = 0 and y-dimension of current position is a multiple of 4
	//then
	//4: execute RLC operation
	//5: end if
	//6: if rlcNbh = 1 then
	//7: execute ZC operation
	//8: end if
	//9: if sigmanew = 1 then
	//10: execute SC operation
	//11: end if

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// set sigma_new for strip column (sigma_old is zero)
	LOCAL int* statePtr = state + startIndex;
	for (int i = 0; i < 4; ++i) {
		int current = statePtr[0];
		current |= BIT(current);
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// set rlc nbh for strip column
	statePtr = state + startIndex;
	int current = statePtr[0];
	int top = statePtr[TOP];
	int leftTop = statePtr[LEFT_TOP];
	int left = statePtr[LEFT];
	int leftBottom = statePtr[LEFT_BOTTOM];

	int rlcCount = 0;
	bool doRLC = false;

	for (int i = 0; i < 4; ++i) {
		current |=  (BIT(top) | BIT(leftTop) | BIT(left) | BIT(leftBottom)) << NBH_BITPOS;
		statePtr[0] = current;

		// toggle doRLC flag
		int currentBit = BIT(current);
		if (i == 0 && !(current & NBH) && !currentBit ) {
			doRLC = true;
		} else if (doRLC && ( (current & NBH) || currentBit) ) {
			doRLC = false;
		}

		if (doRLC) {

			rlcCount++;
		} else {
		   if (current & NBH) {
				//ZC
		   }
		   if (currentBit) {
		       //SC

		   }

		}

		statePtr += STATE_BUFFER_STRIDE;
		top = current;
		current = statePtr[0];
		leftTop = left;
		left = leftBottom;
		leftBottom = statePtr[LEFT_BOTTOM];
	}
	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit plans
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		//Algorithm 15 Bit-plane Preprocessing
		//1: //First step
		//2: set sigma_old |= sigma_new
		//3: set nbh |=  old_sigma from neighbouring positions
		//4: if sigma_old = 0 AND nbh != 0 AND bit value = 1 then
		//5: set sigma_new := 1
		//6: end if
		//7: if there is some sigma_new = 1 then
		//8: blockVote := true
		//9: end if
		//10: //Second step
		//11: while blockVote = true do
		//12: set blockVote := false //reset the voting variable
		//13: set nbh :=  sigma_new from the four previous positions
		//14: if sigma_old = 0 AND nbh != 0 AND bit value = 1 then
		//15: set sigma_new = 1
		//16: end if
		//17: if there is some sigma_new = 1 found in this iteration then
		//18: set blockVote := true
		//19: end if
		//20: end while

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (int i = 0; i < 4; ++i) {
		    int current = *statePtr;
			current |= ((current) & SIGMA_NEW) << SIGMA_OLD_BITPOS;
			current &= NOT_SIGMA_NEW;  // clear sigma new
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		int top = statePtr[TOP];
		int leftTop = statePtr[LEFT_TOP];
		int rightTop = statePtr[RIGHT_TOP];
		int left = statePtr[LEFT];
		int current = statePtr[0];
		int right = statePtr[RIGHT];
		int leftBottom = statePtr[LEFT_BOTTOM];
		int bottom = statePtr[BOTTOM];
		int rightBottom = statePtr[RIGHT_BOTTOM];

		current |= ((leftTop & SIGMA_OLD) |
							( top & SIGMA_OLD) |
							( rightTop &  SIGMA_OLD) |
							( left & SIGMA_OLD) |
							( right & SIGMA_OLD) |
							( leftBottom & SIGMA_OLD) |
							( bottom & SIGMA_OLD) |
							( rightBottom & SIGMA_OLD)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) && (current & NBH) && !(current & SIGMA_OLD) ) {
			current |= SIGMA_NEW;
			blockVote = 1;

		}
		*statePtr = current;

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			current |= ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) && (current & NBH) && !(current & SIGMA_OLD) ) {
			    current |= SIGMA_NEW;
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
		while (blockVote) {
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			int top = statePtr[TOP];
			int leftTop = statePtr[LEFT_TOP];
			int left = statePtr[LEFT];
			int current = statePtr[0];
			int leftBottom = statePtr[LEFT_BOTTOM];

			for (int i = 0; i < 3; ++i) {

				current  |= ((leftTop & SIGMA_NEW) |
							( top & SIGMA_NEW) |
							( left & SIGMA_NEW) |
							( leftBottom & SIGMA_NEW) ) << NBH_BITPOS;

				if (  BIT(current) && !(current & SIGMA_OLD) &&  (current & NBH)  && !(current & SIGMA_NEW)) {
					current |= SIGMA_NEW;
					blockVote = 1;

				}
				statePtr[0] = current;
				statePtr += STATE_BUFFER_STRIDE;
				top = current;
				current = statePtr[0];
				leftTop = left;
				left = leftBottom;
				leftBottom = statePtr[LEFT_BOTTOM];
			}
			current |= ((leftTop & SIGMA_NEW) |
						( top & SIGMA_NEW) |
						( left & SIGMA_NEW) ) << NBH_BITPOS; // ignore leftBottom pixel, because it is in the next
						                                     // stripe and it hasn't been processed yet

			if (  BIT(current) && !(current & SIGMA_OLD) &&  (current & NBH)  && !(current & SIGMA_NEW)) {
				current |= SIGMA_NEW;
				blockVote = 1;

			}
			statePtr[0] = current;
			localMemoryFence();

		}

		/////////////////////////
		// 5. Significance
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop = statePtr[LEFT_TOP];
		rightTop = statePtr[RIGHT_TOP];
		left = statePtr[LEFT];
		current = statePtr[0];
		right = statePtr[RIGHT];
		leftBottom = statePtr[LEFT_BOTTOM];
		bottom = statePtr[BOTTOM];
		rightBottom = statePtr[RIGHT_BOTTOM];
		for (int i = 0; i < 4; ++i) {

			int nbh = ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << NBH_BITPOS;

			current |= nbh;
			if ( !(current & NBH) && nbh && BIT(current) ) {
			    current |= SIGMA_NEW;

			}
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];
		}
		localMemoryFence();

		//////////////////////////
		// 6. Magnitude Refinement
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop = statePtr[LEFT_TOP];
		rightTop = statePtr[RIGHT_TOP];
		left = statePtr[LEFT];
		current = statePtr[0];
		right = statePtr[RIGHT];
		leftBottom = statePtr[LEFT_BOTTOM];
		bottom = statePtr[BOTTOM];
		rightBottom = statePtr[RIGHT_BOTTOM];
		for (int i = 0; i < 4; ++i) {

			int nbh = ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << NBH_BITPOS;

			current |= nbh;
			if ( !(current & NBH) && nbh && BIT(current) ) {
			    current |= SIGMA_NEW;

			}
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];
		}
		localMemoryFence();

		///////////////////////
		// 7. cleanup
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop = statePtr[LEFT_TOP];
		rightTop = statePtr[RIGHT_TOP];
		left = statePtr[LEFT];
		current = statePtr[0];
		right = statePtr[RIGHT];
		leftBottom = statePtr[LEFT_BOTTOM];
		bottom = statePtr[BOTTOM];
		rightBottom = statePtr[RIGHT_BOTTOM];
		for (int i = 0; i < 4; ++i) {

			int nbh = ((leftTop & SIGMA_OLD) |
							 ( top & SIGMA_OLD) |
							 ( rightTop &  SIGMA_OLD) |
							 ( left & SIGMA_OLD) |
							 ( right & SIGMA_OLD) |
							 ( leftBottom & SIGMA_OLD) |
							 ( bottom & SIGMA_OLD) |
							 ( rightBottom & SIGMA_OLD)  ) << NBH_BITPOS;

			current |= nbh;
			if ( !(current & NBH) && nbh && BIT(current) ) {
			    current |= SIGMA_NEW;

			}
			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;
			top = current;
			leftTop = left;
			rightTop = right;
			left = leftBottom;
			current = statePtr[0];
			right = rightBottom;
			leftBottom = statePtr[LEFT_BOTTOM];
			bottom = statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];
		}
		localMemoryFence();

		bp--;
	}
}