Attempting to fix lit test test/Headers/opencl-c-header.cl on cygwin.

"chmod u-x *" does not work for lit on cygwin.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@273948 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s

// RUN: %clang_cc1 -triple x86_64-unknown-unknown -finclude-default-header -emit-llvm -o - %s | FileCheck %s
// CHECK: _Z16convert_char_rtec
// CHECK-NOT: _Z3ctzc
// CHECK20: _Z3ctzc
// CHECK20-NOT: _Z16convert_char_rtec
// CHECK-MOD: Reading modules

// Test including the default header as a module.
// The module should be compiled only once and loaded from cache afterwards.
// Change the directory mode to read only to make sure no new modules are created.
// Check time report to make sure module is used.

// ===
// Clear current directory.
// RUN: rm -rf %t
// RUN: mkdir -p %t

// ===
// Compile for OpenCL 1.0 for the first time. A module should be generated.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 1.0 for the second time. The module should not be re-created.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t/opencl_c.pcm
// RUN: mv %t/opencl_c.pcm %t/1_0.pcm

// ===
// Compile for OpenCL 2.0 for the first time. The module should change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: not diff %t/1_0.pcm %t/opencl_c.pcm
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 2.0 for the second time. The module should not change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s

// Check cached module works for different OpenCL versions.
// RUN: rm -rf %t
// RUN: mkdir -p %t
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0  -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t

char f(char x) {
#if __OPENCL_C_VERSION__ != CL_VERSION_2_0
  return convert_char_rte(x);
#ifdef NO_HEADER
  //expected-warning@-2{{implicit declaration of function 'convert_char_rte' is invalid in C99}}
#endif //NO_HEADER

#else //__OPENCL_C_VERSION__
  return ctz(x);
#endif //__OPENCL_C_VERSION__
}

// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple spir64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple amdgcn-unknown-amdhsa -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple ppc64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple powerpc-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple nvptx-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s | FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.1| FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -cl-std=CL1.2| FileCheck %s
// RUN: %clang_cc1 -triple nvptx64-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -fblocks -emit-llvm -o - %s -cl-std=CL2.0| FileCheck --check-prefix=CHECK20 %s

// RUN: %clang_cc1 -triple x86_64-unknown-unknown -finclude-default-header -emit-llvm -o - %s | FileCheck %s
// CHECK: _Z16convert_char_rtec
// CHECK-NOT: _Z3ctzc
// CHECK20: _Z3ctzc
// CHECK20-NOT: _Z16convert_char_rtec
// CHECK-MOD: Reading modules

// Test including the default header as a module.
// The module should be compiled only once and loaded from cache afterwards.
// Change the directory mode to read only to make sure no new modules are created.
// Check time report to make sure module is used.

// ===
// Clear current directory.
// RUN: rm -rf %t
// RUN: mkdir -p %t

// ===
// Compile for OpenCL 1.0 for the first time. A module should be generated.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 1.0 for the second time. The module should not be re-created.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t/opencl_c.pcm
// RUN: mv %t/opencl_c.pcm %t/1_0.pcm

// ===
// Compile for OpenCL 2.0 for the first time. The module should change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: not diff %t/1_0.pcm %t/opencl_c.pcm
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 2.0 for the second time. The module should not change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s

// Check cached module works for different OpenCL versions.
// RUN: rm -rf %t
// RUN: mkdir -p %t
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0  -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t/*
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t/*

char f(char x) {
#if __OPENCL_C_VERSION__ != CL_VERSION_2_0
  return convert_char_rte(x);
#ifdef NO_HEADER
  //expected-warning@-2{{implicit declaration of function 'convert_char_rte' is invalid in C99}}
#endif //NO_HEADER

#else //__OPENCL_C_VERSION__
  return ctz(x);
#endif //__OPENCL_C_VERSION__
}


more math error fixes

more math error fixes

/*
Baumgarte & Shapiro "Numerical Relativity: Solving Einstein's Equations on the Computer" 2010
Alcubierre "Introduction to Numerical Relativity" 2008
*/

<?
local table = require 'ext.table'
local from3x3to6_table = {
	{1, 2, 3},
	{2, 4, 5},
	{3, 5, 6},
}
local function from3x3to6(i,j)
	return from3x3to6_table[i][j]
end
local from6to3x3_table = {{1,1},{1,2},{1,3},{2,2},{2,3},{3,3}}
local function from6to3x3(i)
	return table.unpack(from6to3x3_table[i])
end

local function sym(a,b)
	assert(a >= 1 and a <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	assert(b >= 1 and b <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	if a > b then a,b = b,a end
	return xNames[a]..xNames[b]
end
?>

kernel void constrainU(
	global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* U = UBuf + index;

	/*
	det(gammaBar_ij)
	= det(gamma^-1/3 gamma_ij)
	= gamma^-1 gamma
	= 1

	det(a * 1/det(a)^(1/n) )
	det(a) * 1/det(a)^(1/n)^n
	det(a) * 1/det(a)
	1
	*/
	real det_gammaBar = sym3_det(U->gammaBar_ll);
	real _1_cbrt_det_gammaBar = 1./cbrt(det_gammaBar);
<? for ij,xij in ipairs(symNames) do
?>	U->gammaBar_ll.<?=xij?> *= _1_cbrt_det_gammaBar;
<? end
?>
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	/*
	tr(A_ij)
	= tr(K_ij - 1/3 gamma_ij K)
	= gamma^ij K_ij - 1/3 gamma^ij gamma_ij K
	= K - 1/3 3 K
	= 0

	tr(ATilde_ij) = 3 exp(-4 phi) tr(A_ij)
	= 3 exp(-4 phi) * 0
	= 0
	*/
	real tr_ATilde = sym3_dot(gammaBar_uu, U->ATilde_ll);
<? for ij,xij in ipairs(symNames) do
?>	U->ATilde_ll.<?=xij?> -= U->gammaBar_ll.<?=xij?> * tr_ATilde * (1./3.);
<? end
?>
}

kernel void calcDeriv(
	global <?=eqn.cons_t?>* derivBuf,
	const global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* deriv = derivBuf + index;
	const global <?=eqn.cons_t?>* U = UBuf + index;

	const global <?=eqn.cons_t?>* Up[dim];
	const global <?=eqn.cons_t?>* Um[dim];
	for (int i = 0; i < dim; ++i) {
		Up[i] = U + stepsize[i];
		Um[i] = U - stepsize[i];
	}

	real3 partial_alpha_l;			//alpha_,i = partial_alpha_l.i
	real3 partial_phi_l;			//phi_,i = partial_phi_l.i
	real3 partial_K_l;				//K,i = partial_K_l.i
	real3 partial_beta_ul[3];		//beta^i_,j = partial_beta_ul[j].i
	real3 partial_connBar_ul[3];	//connBar^i_,j = partial_connBar_ul[j].i
	real tr_partial_beta = 0.;		//beta^i_,i
	sym3 partial_gammaBar_lll[3];	//gammaBar_ij,k = partial_gammaBar[k].ij
	sym3 partial_ATilde_lll[3];		//ATilde_ij,k = partial_ATilde_lll[k].ij
<? for i=1,solver.dim do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = (Up[<?=i-1?>]->alpha - Um[<?=i-1?>]->alpha) / (2. * grid_dx<?=i-1?>);
	partial_phi_l.<?=xi?> = (Up[<?=i-1?>]->phi - Um[<?=i-1?>]->phi) / (2. * grid_dx<?=i-1?>);
	partial_beta_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->beta_u, Um[<?=i-1?>]->beta_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_connBar_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->connBar_u, Um[<?=i-1?>]->connBar_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_gammaBar_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->gammaBar_ll, Um[<?=i-1?>]->gammaBar_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_ATilde_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->ATilde_ll, Um[<?=i-1?>]->ATilde_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_K_l.<?=xi?> = (Up[<?=i-1?>]->K - Um[<?=i-1?>]->K) / (2. * grid_dx<?=i-1?>);
	tr_partial_beta += partial_beta_ul[<?=i-1?>].<?=xi?>;
<? end
for i=solver.dim+1,3 do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = 0;
	partial_phi_l.<?=xi?> = 0;
	partial_beta_ul[<?=i-1?>] = _real3(0,0,0);
	partial_connBar_ul[<?=i-1?>] = _real3(0,0,0);
	partial_gammaBar_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_ATilde_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_K_l.<?=xi?> = 0;
<? end
?>

	//alpha_,ij = partial2_alpha_ll.ij
	sym3 partial2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_alpha_ll.<?=xij?> = (Up[<?=i-1?>]->alpha - 2. * U->alpha + Um[<?=i-1?>]->alpha) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_alpha_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>
	//phi_,ij = partial2_phi_ll.ij
	sym3 partial2_phi_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_phi_ll.<?=xij?> = (Up[<?=i-1?>]->phi - 2. * U->phi + Um[<?=i-1?>]->phi) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_phi_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>

	real exp_4phi = exp(4. * U->phi);
	real exp_neg4phi = 1. / exp_4phi;

	//gamma_ij = exp(4 phi) gammaBar_ij
	sym3 gamma_ll = sym3_scale(U->gammaBar_ll, exp_4phi);

	//gammaBar^ij = inv gammaBar_ij
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	//gamma^ij = inv gamma_ij
	//sym3 gamma_uu = sym3_inv(gamma_ll);
	//gamma^ij = exp(-4 phi) gammaBar^ij
	sym3 gamma_uu = sym3_scale(gammaBar_uu, exp_neg4phi);

	sym3 connBar_lll[3];	//connBar_ijk = connBar_lll[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	connBar_lll[<?=i-1?>].<?=xjk?> = .5 * (partial_gammaBar_lll[<?=k-1?>].<?=sym(i,j)?> + partial_gammaBar_lll[<?=j-1?>].<?=sym(i,k)?> - partial_gammaBar_lll[<?=i-1?>].<?=xjk?>);
<?	end
end
?>
	sym3 connBar_ull[3];	//connBar^i_jk = connBar_ull[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
?>	connBar_ull[<?=i-1?>].<?=xjk?> = 0. <?
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(i,l)?> * connBar_lll[<?=l-1?>].<?=xjk?><?
		end
?>;
<?	end
end
?>

	sym3 conn_ull[3];	//conn^i_jk = conn_ull[i].jk
<? for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	conn_ull[<?=i-1?>].<?=xjk?> = connBar_ull[<?=i-1?>].<?=xjk?><?
		if i==j then
?> + 2 * partial_phi_l.<?=xNames[k]?><?
		end
		if i==k then
?> + 2 * partial_phi_l.<?=xNames[j]?><?
		end
		for l,xl in ipairs(xNames) do
?> - 2 * U->gammaBar_ll.<?=xjk?> * gammaBar_uu.<?=sym(i,l)?> * partial_phi_l.<?=xl?><?
		end
?>;
<?	end
end
?>

	//D2_alpha_ll.ij = D_i D_j alpha = partial_i partial_j alpha - conn^k_ij partial_k alpha
	sym3 D2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	D2_alpha_ll.<?=xij?> = partial2_alpha_ll.<?=xij?><?
	for k,xk in ipairs(xNames) do
?> - conn_ull[<?=k-1?>].<?=xij?> * partial_alpha_l.<?=xk?><?
	end ?>;
<? end
?>

	//Q = f(alpha) K
	real Q = calc_f(U->alpha) * U->K;

	//d/dt alpha = -alpha^2 Q = alpha,t + alpha,i beta^i
	//alpha,t = -alpha^2 Q + alpha,i beta^i
	deriv->alpha += -U->alpha * U->alpha * Q + real3_dot(partial_alpha_l, U->beta_u);

	//manuall update elsewhere?
	//deriv->beta += _real3(0,0,0);

	//B&S 11.50
	//phi,t = -1/6 alpha K + beta^i phi,i + 1/6 beta^i_,i
	deriv->phi += -U->alpha * U->K / 6. + real3_dot(U->beta_u, partial_phi_l) + tr_partial_beta / 6.;

	//B&S 11.51
	//gammaBar_ij,t = -2 alpha ATilde_ij + beta^k gammaBar_ij,k + gammaBar_ik beta^k_,j + gammaBar_kj beta^k_,i - 2/3 gammaBar_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
?>	deriv->gammaBar_ll.<?=xij?> += -2 * U->alpha * U->ATilde_ll.<?=xij?>	//-2 alpha ATilde_ij
<? 	for k,xk in ipairs(xNames) do
?>		+ partial_gammaBar_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>		//+ beta^k gammaBar_ij,k
		+ U->gammaBar_ll.<?=sym(k,j)?> * partial_beta_ul[<?=i-1?>].<?=xk?>	//+ gammaBar_jk beta^k_,i
		+ U->gammaBar_ll.<?=sym(k,i)?> * partial_beta_ul[<?=j-1?>].<?=xk?> 	//+ gammaBar_ik beta^k_,j
<? 	end
?>		- 2./3. * U->gammaBar_ll.<?=xij?> * tr_partial_beta;				//- 2/3 gammaBar_ij beta^k_,k
<? end
?>
	mat3 ATilde_ul = sym3_sym3_mul(gammaBar_uu, U->ATilde_ll);		//ATilde^i_j = gammaBar^kl ATilde_kj
	sym3 ATilde_uu = mat3_sym3_to_sym3_mul(ATilde_ul, gammaBar_uu);	//ATilde^ij = gammaBar^ik ATilde_kl gammaBar^lj
	real tr_ATilde_sq = sym3_dot(U->ATilde_ll, ATilde_uu);			//tr_ATilde_sq := tr(ATilde^2) = ATilde_ij ATilde^ji

	real S = sym3_dot(U->S_ll, gamma_uu);

	//B&S 11.52
	//K_,t = -gamma^ij D_ij alpha + alpha (ATilde_ij ATilde^ij + K^2 / 3) + 4 pi alpha (rho + S) + beta^i K_,i
	deriv->K += -sym3_dot(gamma_uu, D2_alpha_ll)
		+ U->alpha * (tr_ATilde_sq + U->K * U->K / 3.)
		+ 4. * M_PI * U->alpha * (U->rho + S)
		+ real3_dot(U->beta_u, partial_K_l);

	sym3 partial2_gammaBar_llll[6];	//partial2_gammaBar_llll[kl].ij = gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_add(
			sym3_scale(U->gammaBar_ll, -2.),
			sym3_add(
				Up[<?=i-1?>]->gammaBar_ll,
				Um[<?=i-1?>]->gammaBar_ll)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_sub(
			sym3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll),
			sym3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>
	sym3 partial2_gammaBar_ll;	//partial2_gammaBar_ll.ij = gammaBar^kl gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
?>	partial2_gammaBar_ll.<?=xij?> = 0. <?
	for k,xk in ipairs(xNames) do
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(k,l)?> * partial2_gammaBar_llll[<?=from3x3to6(k,l)-1?>].<?=xij?><?
		end
	end
?>;
<? end
?>

	//B&S 11.54
	//Alcubierre eqn 2.8.17
	sym3 RBar_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	RBar_ll.<?=xij?> = -.5 * partial2_gammaBar_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		+ .5 * U->gammaBar_ll.<?=sym(k,i)?> * partial_connBar_ul[<?=j-1?>].<?=xk?>
		+ .5 * U->gammaBar_ll.<?=sym(k,j)?> * partial_connBar_ul[<?=i-1?>].<?=xk?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=i-1?>].<?=sym(j,k)?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=j-1?>].<?=sym(i,k)?>
<?		for l,xl in ipairs(xNames) do
			for m,xm in ipairs(xNames) do
?>		+ gammaBar_uu.<?=sym(k,m)?> * (
			+ connBar_ull[<?=k-1?>].<?=sym(l,i)?> * connBar_lll[<?=j-1?>].<?=sym(k,m)?>
			+ connBar_ull[<?=k-1?>].<?=sym(l,j)?> * connBar_lll[<?=i-1?>].<?=sym(k,m)?>
			+ connBar_ull[<?=k-1?>].<?=sym(i,m)?> * connBar_lll[<?=k-1?>].<?=sym(l,j)?>
		)
<?			end
		end
	end
?>	;
<? end
?>

	sym3 DBar2_phi_ll;
<? for ij,xij in ipairs(symNames) do
?>	DBar2_phi_ll.<?=xij?> = partial2_phi_ll.<?=xij?> <?
	for k,xk in ipairs(xNames) do
?> - connBar_ull[<?=k?>].<?=xij?> * partial_phi_l.<?=xk?><?
	end
?>;
<? end
?>

	real tr_DBar2_phi = sym3_dot(gammaBar_uu, DBar2_phi_ll);

	real DBar_phi_norm = real3_weighted_norm(partial_phi_l, gammaBar_uu);

	//Baumgarte & Shapiro p.57 eqn 3.10
	//R_ll(i,j) := R_ij = RBar_ij - 2 (DBar_i DBar_j ln(psi) + gammaBar_ij gammaBar^lm DBar_l DBar_m ln(psi)) + 4((DBar_i ln(psi)) (DBar_j ln(psi)) - gammaBar_ij gammaBar^lm (DBar_l ln(psi)) (DBar_m ln(psi)))
	//Then Baumgarte & Shapiro on p.390 say RPhi_ij is the same as p.57 substituting phi for ln(psi)
	// ... but I thought phi was ln(psi)?  Then why would you need to separate R_ij = RBar_ij + RPhi_ij ?  I thought the substitution showed that R_ij was RPhi_ij?
	//phi = ln(psi), so DBar_i ln psi = partial_phi_i
	sym3 RPhi_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	RPhi_ll.<?=xij?> = RBar_ll.<?=xij?> - 2. * (DBar2_phi_ll.<?=xij?> + gammaBar_uu.<?=xij?> * tr_DBar2_phi) + 4. * (partial_phi_l.<?=xi?> * partial_phi_l.<?=xj?> - U->gammaBar_ll.<?=xij?> * DBar_phi_norm);
<? end
?>
	sym3 R_ll = sym3_add(RPhi_ll, RBar_ll);

	//traceless portion of -D^2 alpha + alpha (R_ij - 8 pi S_ij)
	sym3 tracelessPart_ll = sym3_sub(
		sym3_scale(
			sym3_add(R_ll, sym3_scale(U->S_ll, -8. * M_PI)),
			U->alpha),
		D2_alpha_ll);
	real tr_tracelessPart = sym3_dot(gamma_uu, tracelessPart_ll);
	tracelessPart_ll = sym3_sub(tracelessPart_ll, sym3_scale(gamma_ll, -1./3. * tr_tracelessPart));

#if 1
//something is asymmetric
	//B&S 11.53
	//ATilde_ij,t =
	//	exp(-4phi) (-(D_ij alpha)^TF + alpha (R_ij^TF - 8 pi S_ij^TF) )
	//	+ alpha (K ATilde_ij - 2 ATilde_il ATilde^l_j)
	//	+ beta^k ATilde_ij,k
	//	+ ATilde_ik beta^k_,j
	//	+ ATidle_kj beta^k_,i
	//	- 2/3 ATilde_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	deriv->ATilde_ll.<?=xij?> += exp_neg4phi * tracelessPart_ll.<?=xij?>
		+ U->alpha * U->K * U->ATilde_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		- 2. * U->alpha * U->ATilde_ll.<?=sym(i,k)?> * ATilde_ul.<?=xk?>.<?=xj?>
		+ partial_ATilde_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>
		+ U->ATilde_ll.<?=sym(i,k)?> * partial_beta_ul[<?=j-1?>].<?=xk?>
		+ U->ATilde_ll.<?=sym(j,k)?> * partial_beta_ul[<?=i-1?>].<?=xk?>
<?	end
?>		- 2./3. * U->ATilde_ll.<?=xij?> * tr_partial_beta;
<? end
?>
#endif

	real3 partial2_beta_ull[6];	//partial2_beta_ull[jk].i = beta^i_,jk
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_add(
			real3_scale(U->beta_u, -2.),
			real3_add(
				Up[<?=i-1?>]->beta_u,
				Um[<?=i-1?>]->beta_u)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_sub(
			real3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u),
			real3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>

	//connBar^i is the connection function / connection coefficient iteration with Hamiltonian constraint baked in (Baumgarte & Shapiro p.389, Alcubierre p.86).
	//B&S 11.55
	//partial_t connBar^i =
	//	-2 ATilde^ij alpha_,j
	//	+ 2 alpha (
	//		connBar^i_jk ATilde^kj
	//		- 2/3 gammaBar^ij K_,j
	//		- 8 pi gammaBar^ij S_j
	//		+ 6 ATilde^ij phi_,j
	//	)
	//	+ beta^j connBar^i_,j
	//	- connBar^j beta^i_,j
	//	+ 2/3 connBar^i beta^j_,j
	//	+ 1/3 gammaBar^ki beta^j_,jk
	//	+ gammaBar^kj beta^i_,jk
<?
for i,xi in ipairs(xNames) do
?>	deriv->connBar_u.<?=xi?> +=
		2./3. * U->connBar_u.<?=xi?> * tr_partial_beta
<?	for j,xj in ipairs(xNames) do
		local xij = sym(i,j)
		local jj = from3x3to6(j,j)
?>		- 2. * U->ATilde_ll.<?=xij?> * partial_alpha_l.<?=xj?>
		+ 2. * U->alpha * (
			-2./3. * gammaBar_uu.<?=xij?> * partial_K_l.<?=xj?>
			- 8. * M_PI * U->gammaBar_ll.<?=xij?> * U->S_u.<?=xj?>
			+ 6. * ATilde_uu.<?=xij?> * partial_phi_l.<?=xj?>)
		+ U->beta_u.<?=xi?> * partial_connBar_ul[<?=j-1?>].<?=xi?>
		- U->connBar_u.<?=xj?> * partial_beta_ul[<?=j-1?>].<?=xi?>
<?		for k,xk in ipairs(xNames) do
			local xik = sym(i,k)
			local jk = from3x3to6(j,k)
			local xjk = symNames[jk]
?>		+ 2. * U->alpha * connBar_ull[<?=i-1?>].<?=xjk?> * ATilde_uu.<?=xjk?>
		+ 1./3. * gammaBar_uu.<?=xik?> * partial2_beta_ull[<?=jk-1?>].<?=xj?>
		+ gammaBar_uu.<?=xjk?> * partial2_beta_ull[<?=jk-1?>].<?=xi?>
<?		end
	end
?>	;
<? end
?>
}

/*
Baumgarte & Shapiro "Numerical Relativity: Solving Einstein's Equations on the Computer" 2010
Alcubierre "Introduction to Numerical Relativity" 2008
*/

<?
local table = require 'ext.table'
local from3x3to6_table = {
	{1, 2, 3},
	{2, 4, 5},
	{3, 5, 6},
}
local function from3x3to6(i,j)
	return from3x3to6_table[i][j]
end
local from6to3x3_table = {{1,1},{1,2},{1,3},{2,2},{2,3},{3,3}}
local function from6to3x3(i)
	return table.unpack(from6to3x3_table[i])
end

local function sym(a,b)
	assert(a >= 1 and a <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	assert(b >= 1 and b <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	if a > b then a,b = b,a end
	return xNames[a]..xNames[b]
end
?>

kernel void constrainU(
	global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* U = UBuf + index;

	/*
	det(gammaBar_ij)
	= det(gamma^-1/3 gamma_ij)
	= gamma^-1 gamma
	= 1

	det(a * 1/det(a)^(1/n) )
	det(a) * 1/det(a)^(1/n)^n
	det(a) * 1/det(a)
	1
	*/
	real det_gammaBar = sym3_det(U->gammaBar_ll);
	real _1_cbrt_det_gammaBar = 1./cbrt(det_gammaBar);
<? for ij,xij in ipairs(symNames) do
?>	U->gammaBar_ll.<?=xij?> *= _1_cbrt_det_gammaBar;
<? end
?>
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	/*
	tr(A_ij)
	= tr(K_ij - 1/3 gamma_ij K)
	= gamma^ij K_ij - 1/3 gamma^ij gamma_ij K
	= K - 1/3 3 K
	= 0

	tr(ATilde_ij) = 3 exp(-4 phi) tr(A_ij)
	= 3 exp(-4 phi) * 0
	= 0
	*/
	real tr_ATilde = sym3_dot(gammaBar_uu, U->ATilde_ll);
<? for ij,xij in ipairs(symNames) do
?>	U->ATilde_ll.<?=xij?> -= U->gammaBar_ll.<?=xij?> * tr_ATilde * (1./3.);
<? end
?>
}

kernel void calcDeriv(
	global <?=eqn.cons_t?>* derivBuf,
	const global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* deriv = derivBuf + index;
	const global <?=eqn.cons_t?>* U = UBuf + index;

	const global <?=eqn.cons_t?>* Up[dim];
	const global <?=eqn.cons_t?>* Um[dim];
	for (int i = 0; i < dim; ++i) {
		Up[i] = U + stepsize[i];
		Um[i] = U - stepsize[i];
	}

	real3 partial_alpha_l;			//alpha_,i = partial_alpha_l.i
	real3 partial_phi_l;			//phi_,i = partial_phi_l.i
	real3 partial_K_l;				//K,i = partial_K_l.i
	real3 partial_beta_ul[3];		//beta^i_,j = partial_beta_ul[j].i
	real3 partial_connBar_ul[3];	//connBar^i_,j = partial_connBar_ul[j].i
	real tr_partial_beta = 0.;		//beta^i_,i
	sym3 partial_gammaBar_lll[3];	//gammaBar_ij,k = partial_gammaBar[k].ij
	sym3 partial_ATilde_lll[3];		//ATilde_ij,k = partial_ATilde_lll[k].ij
<? for i=1,solver.dim do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = (Up[<?=i-1?>]->alpha - Um[<?=i-1?>]->alpha) / (2. * grid_dx<?=i-1?>);
	partial_phi_l.<?=xi?> = (Up[<?=i-1?>]->phi - Um[<?=i-1?>]->phi) / (2. * grid_dx<?=i-1?>);
	partial_beta_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->beta_u, Um[<?=i-1?>]->beta_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_connBar_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->connBar_u, Um[<?=i-1?>]->connBar_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_gammaBar_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->gammaBar_ll, Um[<?=i-1?>]->gammaBar_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_ATilde_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->ATilde_ll, Um[<?=i-1?>]->ATilde_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_K_l.<?=xi?> = (Up[<?=i-1?>]->K - Um[<?=i-1?>]->K) / (2. * grid_dx<?=i-1?>);
	tr_partial_beta += partial_beta_ul[<?=i-1?>].<?=xi?>;
<? end
for i=solver.dim+1,3 do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = 0;
	partial_phi_l.<?=xi?> = 0;
	partial_beta_ul[<?=i-1?>] = _real3(0,0,0);
	partial_connBar_ul[<?=i-1?>] = _real3(0,0,0);
	partial_gammaBar_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_ATilde_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_K_l.<?=xi?> = 0;
<? end
?>

	//alpha_,ij = partial2_alpha_ll.ij
	sym3 partial2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_alpha_ll.<?=xij?> = (Up[<?=i-1?>]->alpha - 2. * U->alpha + Um[<?=i-1?>]->alpha) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_alpha_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>
	//phi_,ij = partial2_phi_ll.ij
	sym3 partial2_phi_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_phi_ll.<?=xij?> = (Up[<?=i-1?>]->phi - 2. * U->phi + Um[<?=i-1?>]->phi) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_phi_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>

	real exp_4phi = exp(4. * U->phi);
	real exp_neg4phi = 1. / exp_4phi;

	//gamma_ij = exp(4 phi) gammaBar_ij
	sym3 gamma_ll = sym3_scale(U->gammaBar_ll, exp_4phi);

	//gammaBar^ij = inv gammaBar_ij
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	//gamma^ij = inv gamma_ij
	//sym3 gamma_uu = sym3_inv(gamma_ll);
	//gamma^ij = exp(-4 phi) gammaBar^ij
	sym3 gamma_uu = sym3_scale(gammaBar_uu, exp_neg4phi);

	sym3 connBar_lll[3];	//connBar_ijk = connBar_lll[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	connBar_lll[<?=i-1?>].<?=xjk?> = .5 * (partial_gammaBar_lll[<?=k-1?>].<?=sym(i,j)?> + partial_gammaBar_lll[<?=j-1?>].<?=sym(i,k)?> - partial_gammaBar_lll[<?=i-1?>].<?=xjk?>);
<?	end
end
?>
	sym3 connBar_ull[3];	//connBar^i_jk = connBar_ull[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
?>	connBar_ull[<?=i-1?>].<?=xjk?> = 0. <?
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(i,l)?> * connBar_lll[<?=l-1?>].<?=xjk?><?
		end
?>;
<?	end
end
?>

	sym3 conn_ull[3];	//conn^i_jk = conn_ull[i].jk
<? for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	conn_ull[<?=i-1?>].<?=xjk?> = connBar_ull[<?=i-1?>].<?=xjk?><?
		if i==j then
?> + 2 * partial_phi_l.<?=xNames[k]?><?
		end
		if i==k then
?> + 2 * partial_phi_l.<?=xNames[j]?><?
		end
		for l,xl in ipairs(xNames) do
?> - 2 * U->gammaBar_ll.<?=xjk?> * gammaBar_uu.<?=sym(i,l)?> * partial_phi_l.<?=xl?><?
		end
?>;
<?	end
end
?>

	//D2_alpha_ll.ij = D_i D_j alpha = partial_i partial_j alpha - conn^k_ij partial_k alpha
	sym3 D2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	D2_alpha_ll.<?=xij?> = partial2_alpha_ll.<?=xij?><?
	for k,xk in ipairs(xNames) do
?> - conn_ull[<?=k-1?>].<?=xij?> * partial_alpha_l.<?=xk?><?
	end ?>;
<? end
?>

	//Q = f(alpha) K
	real Q = calc_f(U->alpha) * U->K;

	//d/dt alpha = -alpha^2 Q = alpha,t + alpha,i beta^i
	//alpha,t = -alpha^2 Q + alpha,i beta^i
	deriv->alpha += -U->alpha * U->alpha * Q + real3_dot(partial_alpha_l, U->beta_u);

	//manuall update elsewhere?
	//deriv->beta += _real3(0,0,0);

	//B&S 11.50
	//phi,t = -1/6 alpha K + beta^i phi,i + 1/6 beta^i_,i
	deriv->phi += -U->alpha * U->K / 6. + real3_dot(U->beta_u, partial_phi_l) + tr_partial_beta / 6.;

	//B&S 11.51
	//gammaBar_ij,t = -2 alpha ATilde_ij + beta^k gammaBar_ij,k + gammaBar_ik beta^k_,j + gammaBar_kj beta^k_,i - 2/3 gammaBar_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
?>	deriv->gammaBar_ll.<?=xij?> += -2 * U->alpha * U->ATilde_ll.<?=xij?>	//-2 alpha ATilde_ij
<? 	for k,xk in ipairs(xNames) do
?>		+ partial_gammaBar_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>		//+ beta^k gammaBar_ij,k
		+ U->gammaBar_ll.<?=sym(i,k)?> * partial_beta_ul[<?=j-1?>].<?=xk?> 	//+ gammaBar_ik beta^k_,j
		+ U->gammaBar_ll.<?=sym(j,k)?> * partial_beta_ul[<?=i-1?>].<?=xk?>	//+ gammaBar_jk beta^k_,i
<? 	end
?>		- 2./3. * U->gammaBar_ll.<?=sym(i,j)?> * tr_partial_beta;		//- 2/3 gammaBar_ij beta^k_,k
<? end
?>
	mat3 ATilde_ul = sym3_sym3_mul(gammaBar_uu, U->ATilde_ll);		//ATilde^i_j = gammaBar^kl ATilde_kj
	sym3 ATilde_uu = mat3_sym3_to_sym3_mul(ATilde_ul, gammaBar_uu);	//ATilde^ij = gammaBar^ik ATilde_kl gammaBar^lj
	real tr_ATilde_sq = sym3_dot(U->ATilde_ll, ATilde_uu);			//tr_ATilde_sq := tr(ATilde^2) = ATilde_ij ATilde^ji

	real S = sym3_dot(U->S_ll, gamma_uu);

	//B&S 11.52
	//K_,t = -gamma^ij D_ij alpha + alpha (ATilde_ij ATilde^ij + K^2 / 3) + 4 pi alpha (rho + S) + beta^i K_,i
	deriv->K += -sym3_dot(gamma_uu, D2_alpha_ll)
		+ U->alpha * (tr_ATilde_sq + U->K * U->K / 3.)
		+ 4. * M_PI * U->alpha * (U->rho + S)
		+ real3_dot(U->beta_u, partial_K_l);

	sym3 partial2_gammaBar_llll[6];	//partial2_gammaBar_llll[kl].ij = gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_add(
			sym3_scale(U->gammaBar_ll, -2.),
			sym3_add(
				Up[<?=i-1?>]->gammaBar_ll,
				Um[<?=i-1?>]->gammaBar_ll)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_sub(
			sym3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll),
			sym3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>
	sym3 partial2_gammaBar_ll;	//partial2_gammaBar_ll.ij = gammaBar^kl gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
?>	partial2_gammaBar_ll.<?=xij?> = 0. <?
	for k,xk in ipairs(xNames) do
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(k,l)?> * partial2_gammaBar_llll[<?=from3x3to6(k,l)-1?>].<?=xij?><?
		end
	end
?>;
<? end
?>

	//B&S 11.54
	sym3 RBar_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	RBar_ll.<?=xij?> = -1./2. * partial2_gammaBar_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		+ .5 * U->gammaBar_ll.<?=sym(k,i)?> * partial_connBar_ul[<?=j-1?>].<?=xk?>
		+ .5 * U->gammaBar_ll.<?=sym(k,j)?> * partial_connBar_ul[<?=i-1?>].<?=xk?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=i-1?>].<?=sym(j,k)?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=j-1?>].<?=sym(i,k)?>
<?		for l,xl in ipairs(xNames) do
			for m,xm in ipairs(xNames) do
?>		+ connBar_ull[<?=k-1?>].<?=sym(l,i)?> * connBar_ull[<?=j-1?>].<?=sym(k,m)?>
		+ connBar_ull[<?=k-1?>].<?=sym(l,j)?> * connBar_ull[<?=i-1?>].<?=sym(k,m)?>
		+ connBar_ull[<?=k-1?>].<?=sym(i,m)?> * connBar_ull[<?=k-1?>].<?=sym(l,j)?>
<?			end
		end
	end
?>	;
<? end
?>

	sym3 DBar2_phi_ll;
<? for ij,xij in ipairs(symNames) do
?>	DBar2_phi_ll.<?=xij?> = partial2_phi_ll.<?=xij?> <?
	for k,xk in ipairs(xNames) do
?> - connBar_ull[<?=k?>].<?=xij?> * partial_phi_l.<?=xk?><?
	end
?>;
<? end
?>

	real tr_DBar2_phi = sym3_dot(gammaBar_uu, DBar2_phi_ll);

	real DBar_phi_norm = real3_weighted_norm(partial_phi_l, gammaBar_uu);

	//Baumgarte & Shapiro p.57 eqn 3.10
	//R_ll(i,j) := R_ij = RBar_ij - 2 (DBar_i DBar_j ln(psi) + gammaBar_ij gammaBar^lm DBar_l DBar_m ln(psi)) + 4((DBar_i ln(psi)) (DBar_j ln(psi)) - gammaBar_ij gammaBar^lm (DBar_l ln(psi)) (DBar_m ln(psi)))
	//Then Baumgarte & Shapiro on p.390 say RPhi_ij is the same as p.57 substituting phi for ln(psi)
	// ... but I thought phi was ln(psi)?  Then why would you need to separate R_ij = RBar_ij + RPhi_ij ?  I thought the substitution showed that R_ij was RPhi_ij?
	//phi = ln(psi), so DBar_i ln psi = partial_phi_i
	sym3 RPhi_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	RPhi_ll.<?=xij?> = RBar_ll.<?=xij?> - 2. * (DBar2_phi_ll.<?=xij?> + gammaBar_uu.<?=xij?> * tr_DBar2_phi) + 4. * (partial_phi_l.<?=xi?> * partial_phi_l.<?=xj?> - U->gammaBar_ll.<?=xij?> * DBar_phi_norm);
<? end
?>
	sym3 R_ll = sym3_add(RPhi_ll, RBar_ll);

	//traceless portion of -D^2 alpha + alpha (R_ij - 8 pi S_ij)
	sym3 tracelessPart_ll = sym3_sub(
		sym3_scale(
			sym3_add(R_ll, sym3_scale(U->S_ll, -8. * M_PI)),
			U->alpha),
		D2_alpha_ll);
	real tr_tracelessPart = sym3_dot(gamma_uu, tracelessPart_ll);
	tracelessPart_ll = sym3_sub(tracelessPart_ll, sym3_scale(gamma_ll, -1./3. * tr_tracelessPart));

#if 1
//something is asymmetric
	//B&S 11.53
	//ATilde_ij,t =
	//	exp(-4phi) (-(D_ij alpha)^TF + alpha (R_ij^TF - 8 pi S_ij^TF) )
	//	+ alpha (K ATilde_ij - 2 ATilde_il ATilde^l_j)
	//	+ beta^k ATilde_ij,k
	//	+ ATilde_ik beta^k_,j
	//	+ ATidle_kj beta^k_,i
	//	- 2/3 ATilde_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	deriv->ATilde_ll.<?=xij?> += exp_neg4phi * tracelessPart_ll.<?=xij?>
		+ U->alpha * U->K * U->ATilde_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		- 2. * U->alpha * U->ATilde_ll.<?=sym(i,k)?> * ATilde_ul.<?=xk?>.<?=xj?>
		+ partial_ATilde_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>
		+ U->ATilde_ll.<?=sym(i,k)?> * partial_beta_ul[<?=j-1?>].<?=xk?>
		+ U->ATilde_ll.<?=sym(j,k)?> * partial_beta_ul[<?=i-1?>].<?=xk?>
<?	end
?>		- 2./3. * U->ATilde_ll.<?=xij?> * tr_partial_beta;
<? end
?>
#endif

	real3 partial2_beta_ull[6];	//partial2_beta_ull[jk].i = beta^i_,jk
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_add(
			real3_scale(U->beta_u, -2.),
			real3_add(
				Up[<?=i-1?>]->beta_u,
				Um[<?=i-1?>]->beta_u)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_sub(
			real3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u),
			real3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>

	//connBar^i is the connection function / connection coefficient iteration with Hamiltonian constraint baked in (Baumgarte & Shapiro p.389, Alcubierre p.86).
	//B&S 11.55
	//partial_t connBar^i =
	//	-2 ATilde^ij alpha_,j
	//	+ 2 alpha (
	//		connBar^i_jk ATilde^kj
	//		- 2/3 gammaBar^ij K_,j
	//		- 8 pi gammaBar^ij S_j
	//		+ 6 ATilde^ij phi_,j
	//	)
	//	+ beta^j connBar^i_,j
	//	- connBar^j beta^i_,j
	//	+ 2/3 connBar^i beta^j_,j
	//	+ 1/3 gammaBar^ki beta^j_,jk
	//	+ gammaBar^kj beta^i_,jk
<?
for i,xi in ipairs(xNames) do
?>	deriv->connBar_u.<?=xi?> +=
		2./3. * U->connBar_u.<?=xi?> * tr_partial_beta
<?	for j,xj in ipairs(xNames) do
		local xij = sym(i,j)
		local jj = from3x3to6(j,j)
?>		- 2. * U->ATilde_ll.<?=xij?> * partial_alpha_l.<?=xj?>
		+ 2. * U->alpha * (
			-2./3. * gammaBar_uu.<?=xij?> * partial_K_l.<?=xj?>
			- 8. * M_PI * U->gammaBar_ll.<?=xij?> * U->S_u.<?=xj?>
			+ 6. * ATilde_uu.<?=xij?> * partial_phi_l.<?=xj?>)
		+ U->beta_u.<?=xi?> * partial_connBar_ul[<?=j-1?>].<?=xi?>
		- U->connBar_u.<?=xj?> * partial_beta_ul[<?=j-1?>].<?=xi?>
<?		for k,xk in ipairs(xNames) do
			local xik = sym(i,k)
			local jk = from3x3to6(j,k)
			local xjk = symNames[jk]
?>		+ 2. * U->alpha * connBar_ull[<?=i-1?>].<?=xjk?> * ATilde_uu.<?=xjk?>
		+ 1./3. * gammaBar_uu.<?=xik?> * partial2_beta_ull[<?=jk-1?>].<?=xj?>
		+ gammaBar_uu.<?=xjk?> * partial2_beta_ull[<?=jk-1?>].<?=xi?>
<?		end
	end
?>	;
<? end
?>
}


Barrier needed between Vertical and Horizontal filters.

This problem became apparent on AMD's CPU-based CL implementation, but would
probably also be visible on other implementations where there's less actual
compute cores than threads spawned.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, local uc*);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, local uc*);

__inline void vp8_mbfilter(signed char mask,signed char hev,local uc*);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
#endif

char vp8_char_clamp(int in){
    if (in > 127)
        return 127;
    if (in < -128)
        return -128;

    return in;
}

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

void vp8_filter(
    signed char mask,
    signed char hev,
    local uc *base
)
{

    local uc *op1 = &base[0];
    local uc *op0 = &base[1];
    local uc *oq0 = &base[2];
    local uc *oq1 = &base[3];

    char vp8_filter, Filter1, Filter2;
    char u;

    char ps1 = (char) * op1 ^ 0x80;
    char ps0 = (char) * op0 ^ 0x80;
    char qs0 = (char) * oq0 ^ 0x80;
    char qs1 = (char) * oq1 ^ 0x80;

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps1 - qs1, -128, 127);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (qs0 - ps0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter1 >>= 3;
    Filter2 >>= 3;
    u = clamp(qs0 - Filter1, -128, 127);
    *oq0 = u ^ 0x80;
    u = clamp(ps0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
    vp8_filter = Filter1;

    /* outer tap adjustments */
    vp8_filter += 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u = clamp(qs1 - vp8_filter, -128, 127);
    *oq1 = u ^ 0x80;
    u = clamp(ps1 + vp8_filter, -128, 127);
    *op1 = u ^ 0x80;
}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    int filter_level = filters[block];
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        if (cur_iter == 0 || plane == 0){
            if (cur_iter > 0){
                num_planes = 1;
            }

            if (filter_level){
                int p = pitches[plane];
                int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
                int s_off = offsets[block_offset+priority_offset];

                size_t i = get_global_id(0);

                int  hev = 0; /* high edge variance */
                signed char mask = 0;

                global signed char *limit, *flimit, *thresh;
                global loop_filter_info *lf_info;

                if (i < threads[plane]){
                    lf_info = &lfi[filter_level];
                    flimit = lf_info->flim;
                    limit = lf_info->lim;
                    thresh = lf_info->thr;

                    s_off += i;
                    s_data += plane*128+i*8;

                    s_data[0] = s_base[s_off - 4*p];
                    s_data[1] = s_base[s_off - 3*p];
                    s_data[2] = s_base[s_off - 2*p];
                    s_data[3] = s_base[s_off - 1*p];
                    s_data[4] = s_base[s_off];
                    s_data[5] = s_base[s_off + p];
                    s_data[6] = s_base[s_off + 2*p];
                    s_data[7] = s_base[s_off + 3*p];

                    mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                    hev = vp8_hevmask(thresh[i], &s_data[2]);

                    vp8_filter(mask, hev, &s_data[2]);

                    s_base[s_off - 2*p] = s_data[2];
                    s_base[s_off - p  ] = s_data[3];
                    s_base[s_off      ] = s_data[4];
                    s_base[s_off + p  ] = s_data[5];

                }
            }
        }
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if ((cur_iter ==1 || plane == 0) && filters[num_blocks*filter_type + block] > 0){
        if (cur_iter > 1){
            num_planes = 1;
        }

        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            int  hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i = get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->flim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];

                mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                hev = vp8_hevmask(thresh[i], &s_data[2]);

                vp8_filter(mask, hev, s_data+2);

                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];

            }
        }
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*ROWS_LOCATION + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = 8*num_blocks + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4*p];
                s_data[1] = s_base[s_off - 3*p];
                s_data[2] = s_base[s_off - 2*p];
                s_data[3] = s_base[s_off - 1*p];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + p];
                s_data[6] = s_base[s_off + 2*p];
                s_data[7] = s_base[s_off + 3*p];

                mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                hev = vp8_hevmask(thresh[i], &s_data[2]);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3*p] = s_data[1];
                s_base[s_off - 2*p] = s_data[2];
                s_base[s_off - 1*p] = s_data[3];
                s_base[s_off      ] = s_data[4];
                s_base[s_off + p  ] = s_data[5];
                s_base[s_off + 2*p] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim, //unused for normal filters
    int filter_type,
    int cur_iter,
    int priority_offset
){
    local unsigned char s_data[16*8*3];

    vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters,  priority_offset, s_data);
    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, priority_offset, s_data);
}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

#if 1
                prefetch(&s_base[s_off-4], 8);
                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];
#endif

                mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                hev = vp8_hevmask(thresh[i], &s_data[2]);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3] = s_data[1];
                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];
                s_base[s_off + 2] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_offset
){
    local unsigned char s_data[16*8*3];

#if 0
    event_t e;
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int block_offset = get_global_id(2)*3+plane;
        int s_off = offsets[block_offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            e = async_work_group_strided_copy(s_data+plane*128+thread*8, &s_base[s_off+p*thread-4], 8, 1, e);
        }
    }
    wait_group_events(1, &e);
#else
#if 1
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int block_offset = get_global_id(2)*3+plane;
        int s_off = offsets[block_offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+p*thread-4], 8);
        }
    }
#endif
#endif

    vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            COLS_LOCATION, priority_offset, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7, priority_offset, s_data);

    barrier(CLK_GLOBAL_MEM_FENCE);

    vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters,  priority_offset, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, priority_offset, s_data);

}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_offset
){
    local unsigned char s_data[16*8*3];

    vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            COLS_LOCATION, priority_offset, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7, priority_offset, s_data);

}

kernel void vp8_loop_filter_simple_horizontal_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_offset
)
{
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    local unsigned char s_data[16*8*3];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_offset
)
{
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    local unsigned char s_data[16*8*3];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

//Inline and non-kernel functions follow.

__inline void vp8_mbfilter(
    signed char mask,
    signed char hev,
    local uc *base
)
{
    signed char s, u;
    signed char vp8_filter;

    local uc *op2 = base;
    local uc *op1 = base + 1;
    local uc *op0 = base + 2;
    local uc *oq0 = base + 3;
    local uc *oq1 = base + 4;
    local uc *oq2 = base + 5;

    char2 filter;

    char4 ps = { *op0, *op1, *op2, 0 };
    ps ^= (char4){0x80, 0x80, 0x80, 0x80};

    char4 qs = { *oq0, *oq1, *oq2, 0 };
    qs ^= (char4){0x80, 0x80, 0x80, 0x80};

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps.s1 - qs.s1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (qs.s0 - ps.s0), -128, 127);
    vp8_filter &= mask;

    filter.s1 = vp8_filter;
    filter.s1 &= hev;
    filter.s0 = filter.s1;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter.s0 = clamp(filter.s0 + 4, -128, 127);
    filter.s1 = clamp(filter.s1 + 3, -128, 127);
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    qs.s0 = clamp(qs.s0 - filter.s0, -128, 127);
    ps.s0 = clamp(ps.s0 + filter.s1, -128, 127);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th difference across boundary */
    u = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    s = clamp(qs.s0 - u, -128, 127);
    *oq0 = s ^ 0x80;
    s = clamp(ps.s0 + u, -128, 127);
    *op0 = s ^ 0x80;

    /* roughly 2/7th difference across boundary */
    u = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    s = clamp(qs.s1 - u, -128, 127);
    *oq1 = s ^ 0x80;
    s = clamp(ps.s1 + u, -128, 127);
    *op1 = s ^ 0x80;

    /* roughly 1/7th difference across boundary */
    u = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    s = clamp(qs.s2 - u, -128, 127);
    *oq2 = s ^ 0x80;
    s = clamp(ps.s2 + u, -128, 127);
    *op2 = s ^ 0x80;
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, local uc pq[])
{
    signed char hev = 0;
    hev  |= (abs(pq[0] - pq[1]) > thresh) * -1;
    hev  |= (abs(pq[3] - pq[2]) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        local uc *pq)
{
    signed char mask = 0;
    mask |= (abs(pq[0] - pq[1]) > limit) * -1;
    mask |= (abs(pq[1] - pq[2]) > limit) * -1;
    mask |= (abs(pq[2] - pq[3]) > limit) * -1;
    mask |= (abs(pq[5] - pq[4]) > limit) * -1;
    mask |= (abs(pq[6] - pq[5]) > limit) * -1;
    mask |= (abs(pq[7] - pq[6]) > limit) * -1;
    mask |= (abs(pq[3] - pq[4]) * 2 + abs(pq[2] - pq[5]) / 2  > flimit * 2 + limit) * -1;
    mask = ~mask;
    return mask;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs(p0 - q0) * 2 + abs(p1 - q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter, Filter1, Filter2;
    signed char p1 = (signed char) * op1 ^ 0x80;
    signed char p0 = (signed char) * op0 ^ 0x80;
    signed char q0 = (signed char) * oq0 ^ 0x80;
    signed char q1 = (signed char) * oq1 ^ 0x80;
    signed char u;

    vp8_filter = clamp(p1 - q1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (q0 - p0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter1 >>= 3;
    u = clamp(q0 - Filter1, -128, 127);
    *oq0  = u ^ 0x80;

    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter2 >>= 3;
    u = clamp(p0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, local uc*);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, local uc*);

__inline void vp8_mbfilter(signed char mask,signed char hev,local uc*);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
#endif

char vp8_char_clamp(int in){
    if (in > 127)
        return 127;
    if (in < -128)
        return -128;

    return in;
}

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

void vp8_filter(
    signed char mask,
    signed char hev,
    local uc *base
)
{

    local uc *op1 = &base[0];
    local uc *op0 = &base[1];
    local uc *oq0 = &base[2];
    local uc *oq1 = &base[3];

    char vp8_filter, Filter1, Filter2;
    char u;

    char ps1 = (char) * op1 ^ 0x80;
    char ps0 = (char) * op0 ^ 0x80;
    char qs0 = (char) * oq0 ^ 0x80;
    char qs1 = (char) * oq1 ^ 0x80;

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps1 - qs1, -128, 127);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (qs0 - ps0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter1 >>= 3;
    Filter2 >>= 3;
    u = clamp(qs0 - Filter1, -128, 127);
    *oq0 = u ^ 0x80;
    u = clamp(ps0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
    vp8_filter = Filter1;

    /* outer tap adjustments */
    vp8_filter += 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u = clamp(qs1 - vp8_filter, -128, 127);
    *oq1 = u ^ 0x80;
    u = clamp(ps1 + vp8_filter, -128, 127);
    *op1 = u ^ 0x80;
}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    int filter_level = filters[block];
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        if (cur_iter == 0 || plane == 0){
            if (cur_iter > 0){
                num_planes = 1;
            }

            if (filter_level){
                int p = pitches[plane];
                int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
                int s_off = offsets[block_offset+priority_offset];

                size_t i = get_global_id(0);

                int  hev = 0; /* high edge variance */
                signed char mask = 0;

                global signed char *limit, *flimit, *thresh;
                global loop_filter_info *lf_info;

                if (i < threads[plane]){
                    lf_info = &lfi[filter_level];
                    flimit = lf_info->flim;
                    limit = lf_info->lim;
                    thresh = lf_info->thr;

                    s_off += i;
                    s_data += plane*128+i*8;

                    s_data[0] = s_base[s_off - 4*p];
                    s_data[1] = s_base[s_off - 3*p];
                    s_data[2] = s_base[s_off - 2*p];
                    s_data[3] = s_base[s_off - 1*p];
                    s_data[4] = s_base[s_off];
                    s_data[5] = s_base[s_off + p];
                    s_data[6] = s_base[s_off + 2*p];
                    s_data[7] = s_base[s_off + 3*p];

                    mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                    hev = vp8_hevmask(thresh[i], &s_data[2]);

                    vp8_filter(mask, hev, &s_data[2]);

                    s_base[s_off - 2*p] = s_data[2];
                    s_base[s_off - p  ] = s_data[3];
                    s_base[s_off      ] = s_data[4];
                    s_base[s_off + p  ] = s_data[5];

                }
            }
        }
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if ((cur_iter ==1 || plane == 0) && filters[num_blocks*filter_type + block] > 0){
        if (cur_iter > 1){
            num_planes = 1;
        }

        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            int  hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i = get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->flim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];

                mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                hev = vp8_hevmask(thresh[i], &s_data[2]);

                vp8_filter(mask, hev, s_data+2);

                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];

            }
        }
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*ROWS_LOCATION + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = 8*num_blocks + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4*p];
                s_data[1] = s_base[s_off - 3*p];
                s_data[2] = s_base[s_off - 2*p];
                s_data[3] = s_base[s_off - 1*p];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + p];
                s_data[6] = s_base[s_off + 2*p];
                s_data[7] = s_base[s_off + 3*p];

                mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                hev = vp8_hevmask(thresh[i], &s_data[2]);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3*p] = s_data[1];
                s_base[s_off - 2*p] = s_data[2];
                s_base[s_off - 1*p] = s_data[3];
                s_base[s_off      ] = s_data[4];
                s_base[s_off + p  ] = s_data[5];
                s_base[s_off + 2*p] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim, //unused for normal filters
    int filter_type,
    int cur_iter,
    int priority_offset
){
    local unsigned char s_data[16*8*3];

    vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters,  priority_offset, s_data);
    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, priority_offset, s_data);
}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int priority_offset,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

#if 1
                prefetch(&s_base[s_off-4], 8);
                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];
#endif

                mask = vp8_filter_mask(limit[i], flimit[i], s_data);

                hev = vp8_hevmask(thresh[i], &s_data[2]);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3] = s_data[1];
                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];
                s_base[s_off + 2] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_offset
){
    local unsigned char s_data[16*8*3];

#if 0
    event_t e;
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int block_offset = get_global_id(2)*3+plane;
        int s_off = offsets[block_offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            e = async_work_group_strided_copy(s_data+plane*128+thread*8, &s_base[s_off+p*thread-4], 8, 1, e);
        }
    }
    wait_group_events(1, &e);
#else
#if 1
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int block_offset = get_global_id(2)*3+plane;
        int s_off = offsets[block_offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+p*thread-4], 8);
        }
    }
#endif
#endif

    vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            COLS_LOCATION, priority_offset, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7, priority_offset, s_data);

    //barrier(CLK_GLOBAL_MEM_FENCE);

    vp8_mbloop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi,
            filters,  priority_offset, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, priority_offset, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, priority_offset, s_data);

}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_offset
){
    local unsigned char s_data[16*8*3];

    vp8_mbloop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            COLS_LOCATION, priority_offset, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6, priority_offset, s_data);
    vp8_loop_filter_vertical_edge_worker(s_base, offsets, pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7, priority_offset, s_data);

}

kernel void vp8_loop_filter_simple_horizontal_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_offset
)
{
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    local unsigned char s_data[16*8*3];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_offset
)
{
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    local unsigned char s_data[16*8*3];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

//Inline and non-kernel functions follow.

__inline void vp8_mbfilter(
    signed char mask,
    signed char hev,
    local uc *base
)
{
    signed char s, u;
    signed char vp8_filter;

    local uc *op2 = base;
    local uc *op1 = base + 1;
    local uc *op0 = base + 2;
    local uc *oq0 = base + 3;
    local uc *oq1 = base + 4;
    local uc *oq2 = base + 5;

    char2 filter;

    char4 ps = { *op0, *op1, *op2, 0 };
    ps ^= (char4){0x80, 0x80, 0x80, 0x80};

    char4 qs = { *oq0, *oq1, *oq2, 0 };
    qs ^= (char4){0x80, 0x80, 0x80, 0x80};

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps.s1 - qs.s1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (qs.s0 - ps.s0), -128, 127);
    vp8_filter &= mask;

    filter.s1 = vp8_filter;
    filter.s1 &= hev;
    filter.s0 = filter.s1;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter.s0 = clamp(filter.s0 + 4, -128, 127);
    filter.s1 = clamp(filter.s1 + 3, -128, 127);
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    qs.s0 = clamp(qs.s0 - filter.s0, -128, 127);
    ps.s0 = clamp(ps.s0 + filter.s1, -128, 127);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th difference across boundary */
    u = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    s = clamp(qs.s0 - u, -128, 127);
    *oq0 = s ^ 0x80;
    s = clamp(ps.s0 + u, -128, 127);
    *op0 = s ^ 0x80;

    /* roughly 2/7th difference across boundary */
    u = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    s = clamp(qs.s1 - u, -128, 127);
    *oq1 = s ^ 0x80;
    s = clamp(ps.s1 + u, -128, 127);
    *op1 = s ^ 0x80;

    /* roughly 1/7th difference across boundary */
    u = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    s = clamp(qs.s2 - u, -128, 127);
    *oq2 = s ^ 0x80;
    s = clamp(ps.s2 + u, -128, 127);
    *op2 = s ^ 0x80;
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, local uc pq[])
{
    signed char hev = 0;
    hev  |= (abs(pq[0] - pq[1]) > thresh) * -1;
    hev  |= (abs(pq[3] - pq[2]) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        local uc *pq)
{
    signed char mask = 0;
    mask |= (abs(pq[0] - pq[1]) > limit) * -1;
    mask |= (abs(pq[1] - pq[2]) > limit) * -1;
    mask |= (abs(pq[2] - pq[3]) > limit) * -1;
    mask |= (abs(pq[5] - pq[4]) > limit) * -1;
    mask |= (abs(pq[6] - pq[5]) > limit) * -1;
    mask |= (abs(pq[7] - pq[6]) > limit) * -1;
    mask |= (abs(pq[3] - pq[4]) * 2 + abs(pq[2] - pq[5]) / 2  > flimit * 2 + limit) * -1;
    mask = ~mask;
    return mask;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs(p0 - q0) * 2 + abs(p1 - q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter, Filter1, Filter2;
    signed char p1 = (signed char) * op1 ^ 0x80;
    signed char p0 = (signed char) * op0 ^ 0x80;
    signed char q0 = (signed char) * oq0 ^ 0x80;
    signed char q1 = (signed char) * oq1 ^ 0x80;
    signed char u;

    vp8_filter = clamp(p1 - q1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (q0 - p0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter1 >>= 3;
    u = clamp(q0 - Filter1, -128, 127);
    *oq0  = u ^ 0x80;

    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter2 >>= 3;
    u = clamp(p0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
}