found another equation error

found another equation error

/*
Baumgarte & Shapiro "Numerical Relativity: Solving Einstein's Equations on the Computer" 2010
Alcubierre "Introduction to Numerical Relativity" 2008
*/

<?
local table = require 'ext.table'
local from3x3to6_table = {
	{1, 2, 3},
	{2, 4, 5},
	{3, 5, 6},
}
local function from3x3to6(i,j)
	return from3x3to6_table[i][j]
end
local from6to3x3_table = {{1,1},{1,2},{1,3},{2,2},{2,3},{3,3}}
local function from6to3x3(i)
	return table.unpack(from6to3x3_table[i])
end

local function sym(a,b)
	assert(a >= 1 and a <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	assert(b >= 1 and b <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	if a > b then a,b = b,a end
	return xNames[a]..xNames[b]
end
?>

kernel void constrainU(
	global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* U = UBuf + index;

	/*
	det(gammaBar_ij)
	= det(gamma^-1/3 gamma_ij)
	= gamma^-1 gamma
	= 1

	det(a * 1/det(a)^(1/n) )
	det(a) * 1/det(a)^(1/n)^n
	det(a) * 1/det(a)
	1
	*/
	real det_gammaBar = sym3_det(U->gammaBar_ll);
	real _1_cbrt_det_gammaBar = 1./cbrt(det_gammaBar);
<? for ij,xij in ipairs(symNames) do
?>	U->gammaBar_ll.<?=xij?> *= _1_cbrt_det_gammaBar;
<? end
?>
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	/*
	tr(A_ij)
	= tr(K_ij - 1/3 gamma_ij K)
	= gamma^ij K_ij - 1/3 gamma^ij gamma_ij K
	= K - 1/3 3 K
	= 0

	tr(ATilde_ij) = 3 exp(-4 phi) tr(A_ij)
	= 3 exp(-4 phi) * 0
	= 0
	*/
	real tr_ATilde = sym3_dot(gammaBar_uu, U->ATilde_ll);
<? for ij,xij in ipairs(symNames) do
?>	U->ATilde_ll.<?=xij?> -= U->gammaBar_ll.<?=xij?> * tr_ATilde * (1./3.);
<? end
?>
}

kernel void calcDeriv(
	global <?=eqn.cons_t?>* derivBuf,
	const global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* deriv = derivBuf + index;
	const global <?=eqn.cons_t?>* U = UBuf + index;

	const global <?=eqn.cons_t?>* Up[dim];
	const global <?=eqn.cons_t?>* Um[dim];
	for (int i = 0; i < dim; ++i) {
		Up[i] = U + stepsize[i];
		Um[i] = U - stepsize[i];
	}

	real3 partial_alpha_l;			//alpha_,i = partial_alpha_l.i
	real3 partial_phi_l;			//phi_,i = partial_phi_l.i
	real3 partial_K_l;				//K,i = partial_K_l.i
	real3 partial_beta_ul[3];		//beta^i_,j = partial_beta_ul[j].i
	real3 partial_connBar_ul[3];	//connBar^i_,j = partial_connBar_ul[j].i
	real tr_partial_beta = 0.;		//beta^i_,i
	sym3 partial_gammaBar_lll[3];	//gammaBar_ij,k = partial_gammaBar[k].ij
	sym3 partial_ATilde_lll[3];		//ATilde_ij,k = partial_ATilde_lll[k].ij
<? for i=1,solver.dim do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = (Up[<?=i-1?>]->alpha - Um[<?=i-1?>]->alpha) / (2. * grid_dx<?=i-1?>);
	partial_phi_l.<?=xi?> = (Up[<?=i-1?>]->phi - Um[<?=i-1?>]->phi) / (2. * grid_dx<?=i-1?>);
	partial_beta_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->beta_u, Um[<?=i-1?>]->beta_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_connBar_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->connBar_u, Um[<?=i-1?>]->connBar_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_gammaBar_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->gammaBar_ll, Um[<?=i-1?>]->gammaBar_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_ATilde_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->ATilde_ll, Um[<?=i-1?>]->ATilde_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_K_l.<?=xi?> = (Up[<?=i-1?>]->K - Um[<?=i-1?>]->K) / (2. * grid_dx<?=i-1?>);
	tr_partial_beta += partial_beta_ul[<?=i-1?>].<?=xi?>;
<? end
for i=solver.dim+1,3 do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = 0;
	partial_phi_l.<?=xi?> = 0;
	partial_beta_ul[<?=i-1?>] = _real3(0,0,0);
	partial_connBar_ul[<?=i-1?>] = _real3(0,0,0);
	partial_gammaBar_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_ATilde_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_K_l.<?=xi?> = 0;
<? end
?>

	//alpha_,ij = partial2_alpha_ll.ij
	sym3 partial2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_alpha_ll.<?=xij?> = (Up[<?=i-1?>]->alpha - 2. * U->alpha + Um[<?=i-1?>]->alpha) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_alpha_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>
	//phi_,ij = partial2_phi_ll.ij
	sym3 partial2_phi_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_phi_ll.<?=xij?> = (Up[<?=i-1?>]->phi - 2. * U->phi + Um[<?=i-1?>]->phi) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_phi_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>

	real exp_4phi = exp(4. * U->phi);
	real exp_neg4phi = 1. / exp_4phi;

	//gamma_ij = exp(4 phi) gammaBar_ij
	sym3 gamma_ll = sym3_scale(U->gammaBar_ll, exp_4phi);

	//gammaBar^ij = inv gammaBar_ij
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	//gamma^ij = inv gamma_ij
	//sym3 gamma_uu = sym3_inv(gamma_ll);
	//gamma^ij = exp(-4 phi) gammaBar^ij
	sym3 gamma_uu = sym3_scale(gammaBar_uu, exp_neg4phi);

	sym3 connBar_lll[3];	//connBar_ijk = connBar_lll[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	connBar_lll[<?=i-1?>].<?=xjk?> = .5 * (partial_gammaBar_lll[<?=k-1?>].<?=sym(i,j)?> + partial_gammaBar_lll[<?=j-1?>].<?=sym(i,k)?> - partial_gammaBar_lll[<?=i-1?>].<?=xjk?>);
<?	end
end
?>
	sym3 connBar_ull[3];	//connBar^i_jk = connBar_ull[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
?>	connBar_ull[<?=i-1?>].<?=xjk?> = 0. <?
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(i,l)?> * connBar_lll[<?=l-1?>].<?=xjk?><?
		end
?>;
<?	end
end
?>

	sym3 conn_ull[3];	//conn^i_jk = conn_ull[i].jk
<? for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	conn_ull[<?=i-1?>].<?=xjk?> = connBar_ull[<?=i-1?>].<?=xjk?><?
		if i==j then
?> + 2 * partial_phi_l.<?=xNames[k]?><?
		end
		if i==k then
?> + 2 * partial_phi_l.<?=xNames[j]?><?
		end
		for l,xl in ipairs(xNames) do
?> - 2 * U->gammaBar_ll.<?=xjk?> * gammaBar_uu.<?=sym(i,l)?> * partial_phi_l.<?=xl?><?
		end
?>;
<?	end
end
?>

	//D2_alpha_ll.ij = D_i D_j alpha = partial_i partial_j alpha - conn^k_ij partial_k alpha
	sym3 D2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	D2_alpha_ll.<?=xij?> = partial2_alpha_ll.<?=xij?><?
	for k,xk in ipairs(xNames) do
?> - conn_ull[<?=k-1?>].<?=xij?> * partial_alpha_l.<?=xk?><?
	end ?>;
<? end
?>

	//Q = f(alpha) K
	real Q = calc_f(U->alpha) * U->K;

	//d/dt alpha = -alpha^2 Q = alpha,t + alpha,i beta^i
	//alpha,t = -alpha^2 Q + alpha,i beta^i
	deriv->alpha += -U->alpha * U->alpha * Q + real3_dot(partial_alpha_l, U->beta_u);

	//manuall update elsewhere?
	//deriv->beta += _real3(0,0,0);

	//B&S 11.50
	//phi,t = -1/6 alpha K + beta^i phi,i + 1/6 beta^i_,i
	deriv->phi += -U->alpha * U->K / 6. + real3_dot(U->beta_u, partial_phi_l) + tr_partial_beta / 6.;

	//B&S 11.51
	//gammaBar_ij,t = -2 alpha ATilde_ij + beta^k gammaBar_ij,k + gammaBar_ik beta^k_,j + gammaBar_kj beta^k_,i - 2/3 gammaBar_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
?>	deriv->gammaBar_ll.<?=xij?> += -2 * U->alpha * U->ATilde_ll.<?=xij?>	//-2 alpha ATilde_ij
<? 	for k,xk in ipairs(xNames) do
?>		+ partial_gammaBar_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>		//+ beta^k gammaBar_ij,k
		+ U->gammaBar_ll.<?=sym(i,k)?> * partial_beta_ul[<?=j-1?>].<?=xk?> 	//+ gammaBar_ik beta^k_,j
		+ U->gammaBar_ll.<?=sym(j,k)?> * partial_beta_ul[<?=i-1?>].<?=xk?>	//+ gammaBar_jk beta^k_,i
<? 	end
?>		- 2./3. * U->gammaBar_ll.<?=sym(i,j)?> * tr_partial_beta;		//- 2/3 gammaBar_ij beta^k_,k
<? end
?>
	mat3 ATilde_ul = sym3_sym3_mul(gammaBar_uu, U->ATilde_ll);		//ATilde^i_j = gammaBar^kl ATilde_kj
	sym3 ATilde_uu = mat3_sym3_to_sym3_mul(ATilde_ul, gammaBar_uu);	//ATilde^ij = gammaBar^ik ATilde_kl gammaBar^lj
	real tr_ATilde_sq = sym3_dot(U->ATilde_ll, ATilde_uu);			//tr_ATilde_sq := tr(ATilde^2) = ATilde_ij ATilde^ji

	real S = sym3_dot(U->S_ll, gamma_uu);

	//B&S 11.52
	//K_,t = -gamma^ij D_ij alpha + alpha (ATilde_ij ATilde^ij + K^2 / 3) + 4 pi alpha (rho + S) + beta^i K_,i
	deriv->K += -sym3_dot(gamma_uu, D2_alpha_ll)
		+ U->alpha * (tr_ATilde_sq + U->K * U->K / 3.)
		+ 4. * M_PI * U->alpha * (U->rho + S)
		+ real3_dot(U->beta_u, partial_K_l);

	sym3 partial2_gammaBar_llll[6];	//partial2_gammaBar_llll[kl].ij = gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_add(
			sym3_scale(U->gammaBar_ll, -2.),
			sym3_add(
				Up[<?=i-1?>]->gammaBar_ll,
				Um[<?=i-1?>]->gammaBar_ll)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_sub(
			sym3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll),
			sym3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>
	sym3 partial2_gammaBar_ll;	//partial2_gammaBar_ll.ij = gammaBar^kl gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
?>	partial2_gammaBar_ll.<?=xij?> = 0. <?
	for k,xk in ipairs(xNames) do
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(k,l)?> * partial2_gammaBar_llll[<?=from3x3to6(k,l)-1?>].<?=xij?><?
		end
	end
?>;
<? end
?>

	//B&S 11.54
	sym3 RBar_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	RBar_ll.<?=xij?> = -1./2. * partial2_gammaBar_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		+ .5 * U->gammaBar_ll.<?=sym(k,i)?> * partial_connBar_ul[<?=j-1?>].<?=xk?>
		+ .5 * U->gammaBar_ll.<?=sym(k,j)?> * partial_connBar_ul[<?=i-1?>].<?=xk?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=i-1?>].<?=sym(j,k)?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=j-1?>].<?=sym(i,k)?>
<?		for l,xl in ipairs(xNames) do
			for m,xm in ipairs(xNames) do
?>		+ connBar_ull[<?=k-1?>].<?=sym(l,i)?> * connBar_ull[<?=j-1?>].<?=sym(k,m)?>
		+ connBar_ull[<?=k-1?>].<?=sym(l,j)?> * connBar_ull[<?=i-1?>].<?=sym(k,m)?>
		+ connBar_ull[<?=k-1?>].<?=sym(i,m)?> * connBar_ull[<?=k-1?>].<?=sym(l,j)?>
<?			end
		end
	end
?>	;
<? end
?>

	sym3 DBar2_phi_ll;
<? for ij,xij in ipairs(symNames) do
?>	DBar2_phi_ll.<?=xij?> = partial2_phi_ll.<?=xij?> <?
	for k,xk in ipairs(xNames) do
?> - connBar_ull[<?=k?>].<?=xij?> * partial_phi_l.<?=xk?><?
	end
?>;
<? end
?>

	real tr_DBar2_phi = sym3_dot(gammaBar_uu, DBar2_phi_ll);

	real DBar_phi_norm = real3_weighted_norm(partial_phi_l, gammaBar_uu);

	//Baumgarte & Shapiro p.57 eqn 3.10
	//R_ll(i,j) := R_ij = RBar_ij - 2 (DBar_i DBar_j ln(psi) + gammaBar_ij gammaBar^lm DBar_l DBar_m ln(psi)) + 4((DBar_i ln(psi)) (DBar_j ln(psi)) - gammaBar_ij gammaBar^lm (DBar_l ln(psi)) (DBar_m ln(psi)))
	//Then Baumgarte & Shapiro on p.390 say RPhi_ij is the same as p.57 substituting phi for ln(psi)
	// ... but I thought phi was ln(psi)?  Then why would you need to separate R_ij = RBar_ij + RPhi_ij ?  I thought the substitution showed that R_ij was RPhi_ij?
	//phi = ln(psi), so DBar_i ln psi = partial_phi_i
	sym3 RPhi_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	RPhi_ll.<?=xij?> = RBar_ll.<?=xij?> - 2. * (DBar2_phi_ll.<?=xij?> + gammaBar_uu.<?=xij?> * tr_DBar2_phi) + 4. * (partial_phi_l.<?=xi?> * partial_phi_l.<?=xj?> - U->gammaBar_ll.<?=xij?> * DBar_phi_norm);
<? end
?>
	sym3 R_ll = sym3_add(RPhi_ll, RBar_ll);

	//traceless portion of -D^2 alpha + alpha (R_ij - 8 pi S_ij)
	sym3 tracelessPart_ll = sym3_sub(
		sym3_scale(
			sym3_add(R_ll, sym3_scale(U->S_ll, -8. * M_PI)),
			U->alpha),
		D2_alpha_ll);
	real tr_tracelessPart = sym3_dot(gamma_uu, tracelessPart_ll);
	tracelessPart_ll = sym3_sub(tracelessPart_ll, sym3_scale(gamma_ll, -1./3. * tr_tracelessPart));

#if 1
//something is asymmetric
	//B&S 11.53
	//ATilde_ij,t =
	//	exp(-4phi) (-(D_ij alpha)^TF + alpha (R_ij^TF - 8 pi S_ij^TF) )
	//	+ alpha (K ATilde_ij - 2 ATilde_il ATilde^l_j)
	//	+ beta^k ATilde_ij,k
	//	+ ATilde_ik beta^k_,j
	//	+ ATidle_kj beta^k_,i
	//	- 2/3 ATilde_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	deriv->ATilde_ll.<?=xij?> += exp_neg4phi * tracelessPart_ll.<?=xij?>
		+ U->alpha * U->K * U->ATilde_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		- 2. * U->alpha * U->ATilde_ll.<?=sym(i,k)?> * ATilde_ul.<?=xk?>.<?=xj?>
		+ partial_ATilde_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>
		+ U->ATilde_ll.<?=sym(i,k)?> * partial_beta_ul[<?=j-1?>].<?=xk?>
		+ U->ATilde_ll.<?=sym(j,k)?> * partial_beta_ul[<?=i-1?>].<?=xk?>
<?	end
?>		- 2./3. * U->ATilde_ll.<?=xij?> * tr_partial_beta;
<? end
?>
#endif

	real3 partial2_beta_ull[6];	//partial2_beta_ull[jk].i = beta^i_,jk
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_add(
			real3_scale(U->beta_u, -2.),
			real3_add(
				Up[<?=i-1?>]->beta_u,
				Um[<?=i-1?>]->beta_u)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_sub(
			real3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u),
			real3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>

	//connBar^i is the connection function / connection coefficient iteration with Hamiltonian constraint baked in (Baumgarte & Shapiro p.389, Alcubierre p.86).
	//B&S 11.55
	//partial_t connBar^i =
	//	-2 ATilde^ij alpha_,j
	//	+ 2 alpha (
	//		connBar^i_jk ATilde^kj
	//		- 2/3 gammaBar^ij K_,j
	//		- 8 pi gammaBar^ij S_j
	//		+ 6 ATilde^ij phi_,j
	//	)
	//	+ beta^j connBar^i_,j
	//	- connBar^j beta^i_,j
	//	+ 2/3 connBar^i beta^j_,j
	//	+ 1/3 gammaBar^ki beta^j_,jk
	//	+ gammaBar^kj beta^i_,jk
<?
for i,xi in ipairs(xNames) do
?>	deriv->connBar_u.<?=xi?> +=
		2./3. * U->connBar_u.<?=xi?> * tr_partial_beta
<?	for j,xj in ipairs(xNames) do
		local xij = sym(i,j)
		local jj = from3x3to6(j,j)
?>		- 2. * U->ATilde_ll.<?=xij?> * partial_alpha_l.<?=xj?>
		+ 2. * U->alpha * (
			-2./3. * gammaBar_uu.<?=xij?> * partial_K_l.<?=xj?>
			- 8. * M_PI * U->gammaBar_ll.<?=xij?> * U->S_u.<?=xj?>
			+ 6. * ATilde_uu.<?=xij?> * partial_phi_l.<?=xj?>)
		+ U->beta_u.<?=xi?> * partial_connBar_ul[<?=j-1?>].<?=xi?>
		- U->connBar_u.<?=xj?> * partial_beta_ul[<?=j-1?>].<?=xi?>
<?		for k,xk in ipairs(xNames) do
			local xik = sym(i,k)
			local jk = from3x3to6(j,k)
			local xjk = symNames[jk]
?>		+ 2. * U->alpha * connBar_ull[<?=i-1?>].<?=xjk?> * ATilde_uu.<?=xjk?>
		+ 1./3. * gammaBar_uu.<?=xik?> * partial2_beta_ull[<?=jk-1?>].<?=xj?>
		+ gammaBar_uu.<?=xjk?> * partial2_beta_ull[<?=jk-1?>].<?=xi?>
<?		end
	end
?>	;
<? end
?>
}

/*
Baumgarte & Shapiro "Numerical Relativity: Solving Einstein's Equations on the Computer" 2010
Alcubierre "Introduction to Numerical Relativity" 2008
*/

<?
local table = require 'ext.table'
local from3x3to6_table = {
	{1, 2, 3},
	{2, 4, 5},
	{3, 5, 6},
}
local function from3x3to6(i,j)
	return from3x3to6_table[i][j]
end
local from6to3x3_table = {{1,1},{1,2},{1,3},{2,2},{2,3},{3,3}}
local function from6to3x3(i)
	return table.unpack(from6to3x3_table[i])
end

local function sym(a,b)
	assert(a >= 1 and a <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	assert(b >= 1 and b <= 3, "tried to index sym with "..tostring(a)..", "..tostring(b))
	if a > b then a,b = b,a end
	return xNames[a]..xNames[b]
end
?>

kernel void constrainU(
	global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* U = UBuf + index;

	/*
	det(gammaBar_ij)
	= det(gamma^-1/3 gamma_ij)
	= gamma^-1 gamma
	= 1

	det(a * 1/det(a)^(1/n) )
	det(a) * 1/det(a)^(1/n)^n
	det(a) * 1/det(a)
	1
	*/
	real det_gammaBar = sym3_det(U->gammaBar_ll);
	real _1_cbrt_det_gammaBar = 1./cbrt(det_gammaBar);
<? for ij,xij in ipairs(symNames) do
?>	U->gammaBar_ll.<?=xij?> *= _1_cbrt_det_gammaBar;
<? end
?>
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	/*
	tr(A_ij)
	= tr(K_ij - 1/3 gamma_ij K)
	= gamma^ij K_ij - 1/3 gamma^ij gamma_ij K
	= K - 1/3 3 K
	= 0

	tr(ATilde_ij) = 3 exp(-4 phi) tr(A_ij)
	= 3 exp(-4 phi) * 0
	= 0
	*/
	real tr_ATilde = sym3_dot(gammaBar_uu, U->ATilde_ll);
<? for ij,xij in ipairs(symNames) do
?>	U->ATilde_ll.<?=xij?> -= U->gammaBar_ll.<?=xij?> * tr_ATilde * (1./3.);
<? end
?>
}

kernel void calcDeriv(
	global <?=eqn.cons_t?>* derivBuf,
	const global <?=eqn.cons_t?>* UBuf
) {
	SETBOUNDS(2,2);
	global <?=eqn.cons_t?>* deriv = derivBuf + index;
	const global <?=eqn.cons_t?>* U = UBuf + index;

	const global <?=eqn.cons_t?>* Up[dim];
	const global <?=eqn.cons_t?>* Um[dim];
	for (int i = 0; i < dim; ++i) {
		Up[i] = U + stepsize[i];
		Um[i] = U - stepsize[i];
	}

	real3 partial_alpha_l;			//alpha_,i = partial_alpha_l.i
	real3 partial_phi_l;			//phi_,i = partial_phi_l.i
	real3 partial_K_l;				//K,i = partial_K_l.i
	real3 partial_beta_ul[3];		//beta^i_,j = partial_beta_ul[j].i
	real3 partial_connBar_ul[3];	//connBar^i_,j = partial_connBar_ul[j].i
	real tr_partial_beta = 0.;		//beta^i_,i
	sym3 partial_gammaBar_lll[3];	//gammaBar_ij,k = partial_gammaBar[k].ij
	sym3 partial_ATilde_lll[3];		//ATilde_ij,k = partial_ATilde_lll[k].ij
<? for i=1,solver.dim do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = (Up[<?=i-1?>]->alpha - Um[<?=i-1?>]->alpha) / (2. * grid_dx<?=i-1?>);
	partial_phi_l.<?=xi?> = (Up[<?=i-1?>]->phi - Um[<?=i-1?>]->phi) / (2. * grid_dx<?=i-1?>);
	partial_beta_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->beta_u, Um[<?=i-1?>]->beta_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_connBar_ul[<?=i-1?>] = real3_scale( real3_sub(Up[<?=i-1?>]->connBar_u, Um[<?=i-1?>]->connBar_u), 1. / (2. * grid_dx<?=i-1?>));
	partial_gammaBar_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->gammaBar_ll, Um[<?=i-1?>]->gammaBar_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_ATilde_lll[<?=i-1?>] = sym3_scale( sym3_sub(Up[<?=i-1?>]->ATilde_ll, Um[<?=i-1?>]->ATilde_ll), 1. / (2. * grid_dx<?=i-1?>));
	partial_K_l.<?=xi?> = (Up[<?=i-1?>]->K - Um[<?=i-1?>]->K) / (2. * grid_dx<?=i-1?>);
	tr_partial_beta += partial_beta_ul[<?=i-1?>].<?=xi?>;
<? end
for i=solver.dim+1,3 do
	local xi = xNames[i]
?>	partial_alpha_l.<?=xi?> = 0;
	partial_phi_l.<?=xi?> = 0;
	partial_beta_ul[<?=i-1?>] = _real3(0,0,0);
	partial_connBar_ul[<?=i-1?>] = _real3(0,0,0);
	partial_gammaBar_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_ATilde_lll[<?=i-1?>] = (sym3){.s={0,0,0,0,0,0}};
	partial_K_l.<?=xi?> = 0;
<? end
?>

	//alpha_,ij = partial2_alpha_ll.ij
	sym3 partial2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_alpha_ll.<?=xij?> = (Up[<?=i-1?>]->alpha - 2. * U->alpha + Um[<?=i-1?>]->alpha) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_alpha_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].alpha
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].alpha
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>
	//phi_,ij = partial2_phi_ll.ij
	sym3 partial2_phi_ll = (sym3){.s={0,0,0,0,0,0}};
<?
for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_phi_ll.<?=xij?> = (Up[<?=i-1?>]->phi - 2. * U->phi + Um[<?=i-1?>]->phi) / (grid_dx<?=i-1?> * grid_dx<?=i-1?>);
<?	else
?>	partial2_phi_ll.<?=xij?> = (
		U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].phi
		- U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
		+ U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].phi
	) / (4 * grid_dx<?=i-1?> * grid_dx<?=j-1?>);
<? 	end
end
?>

	real exp_4phi = exp(4. * U->phi);
	real exp_neg4phi = 1. / exp_4phi;

	//gamma_ij = exp(4 phi) gammaBar_ij
	sym3 gamma_ll = sym3_scale(U->gammaBar_ll, exp_4phi);

	//gammaBar^ij = inv gammaBar_ij
	sym3 gammaBar_uu = sym3_inv(1., U->gammaBar_ll);

	//gamma^ij = inv gamma_ij
	//sym3 gamma_uu = sym3_inv(gamma_ll);
	//gamma^ij = exp(-4 phi) gammaBar^ij
	sym3 gamma_uu = sym3_scale(gammaBar_uu, exp_neg4phi);

	sym3 connBar_lll[3];	//connBar_ijk = connBar_lll[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	connBar_lll[<?=i-1?>].<?=xjk?> = .5 * (partial_gammaBar_lll[<?=k-1?>].<?=sym(i,j)?> + partial_gammaBar_lll[<?=j-1?>].<?=sym(i,k)?> - partial_gammaBar_lll[<?=i-1?>].<?=xjk?>);
<?	end
end
?>
	sym3 connBar_ull[3];	//connBar^i_jk = connBar_ull[i].jk
<?
for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
?>	connBar_ull[<?=i-1?>].<?=xjk?> = 0. <?
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(i,l)?> * connBar_lll[<?=l-1?>].<?=xjk?><?
		end
?>;
<?	end
end
?>

	sym3 conn_ull[3];	//conn^i_jk = conn_ull[i].jk
<? for i,xi in ipairs(xNames) do
	for jk,xjk in ipairs(symNames) do
		local j,k = from6to3x3(jk)
?>	conn_ull[<?=i-1?>].<?=xjk?> = connBar_ull[<?=i-1?>].<?=xjk?><?
		if i==j then
?> + 2 * partial_phi_l.<?=xNames[k]?><?
		end
		if i==k then
?> + 2 * partial_phi_l.<?=xNames[j]?><?
		end
		for l,xl in ipairs(xNames) do
?> - 2 * U->gammaBar_ll.<?=xjk?> * gammaBar_uu.<?=sym(i,l)?> * partial_phi_l.<?=xl?><?
		end
?>;
<?	end
end
?>

	//D2_alpha_ll.ij = D_i D_j alpha = partial_i partial_j alpha - conn^k_ij partial_k alpha
	sym3 D2_alpha_ll = (sym3){.s={0,0,0,0,0,0}};
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	D2_alpha_ll.<?=xij?> = partial2_alpha_ll.<?=xij?><?
	for k,xk in ipairs(xNames) do
?> - conn_ull[<?=k-1?>].<?=xij?> * partial_alpha_l.<?=xk?><?
	end ?>;
<? end
?>

	//Q = f(alpha) K
	real Q = calc_f(U->alpha) * U->K;

	//d/dt alpha = -alpha^2 Q = alpha,t + alpha,i beta^i
	//alpha,t = -alpha^2 Q + alpha,i beta^i
	deriv->alpha += -U->alpha * U->alpha * Q + real3_dot(partial_alpha_l, U->beta_u);

	//manuall update elsewhere?
	//deriv->beta += _real3(0,0,0);

	//B&S 11.50
	//phi,t = -1/6 alpha K + beta^i phi,i + 1/6 beta^i_,i
	deriv->phi += -U->alpha * U->K / 6. + real3_dot(U->beta_u, partial_phi_l) + tr_partial_beta / 6.;

	//B&S 11.51
	//gammaBar_ij,t = -2 alpha ATilde_ij + beta^k gammaBar_ij,k + gammaBar_ik beta^k_,j + gammaBar_kj beta^k_,i - 2/3 gammaBar_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
?>	deriv->gammaBar_ll.<?=xij?> += -2 * U->alpha * U->ATilde_ll.<?=xij?>	//-2 alpha ATilde_ij
<? 	for k,xk in ipairs(xNames) do
?>		+ partial_gammaBar_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>		//+ beta^k gammaBar_ij,k
		+ U->gammaBar_ll.<?=sym(i,k)?> * partial_beta_ul[<?=j-1?>].<?=xk?> 	//+ gammaBar_ik beta^k_,j
		+ U->gammaBar_ll.<?=sym(j,k)?> * partial_beta_ul[<?=i-1?>].<?=xk?>	//+ gammaBar_jk beta^k_,i
<? 	end
?>		- 2./3. * U->gammaBar_ll.<?=sym(i,j)?> * tr_partial_beta;		//- 2/3 gammaBar_ij beta^k_,k
<? end
?>
	mat3 ATilde_ul = sym3_sym3_mul(gammaBar_uu, U->ATilde_ll);		//ATilde^i_j = gammaBar^kl ATilde_kj
	sym3 ATilde_uu = mat3_sym3_to_sym3_mul(ATilde_ul, gammaBar_uu);	//ATilde^ij = gammaBar^ik ATilde_kl gammaBar^lj
	real tr_ATilde_sq = sym3_dot(U->ATilde_ll, ATilde_uu);			//tr_ATilde_sq := tr(ATilde^2) = ATilde_ij ATilde^ji

	real S = sym3_dot(U->S_ll, gamma_uu);

	//B&S 11.52
	//K_,t = -gamma^ij D_ij alpha + alpha (ATilde_ij ATilde^ij + K^2 / 3) + 4 pi alpha (rho + S) + beta^i K_,i
	deriv->K += -sym3_dot(gammaBar_uu, D2_alpha_ll)
		+ U->alpha * (tr_ATilde_sq + U->K * U->K / 3.)
		+ 4. * M_PI * U->alpha * (U->rho + S)
		+ real3_dot(U->beta_u, partial_K_l);

	sym3 partial2_gammaBar_llll[6];	//partial2_gammaBar_llll[kl].ij = gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_add(
			sym3_scale(U->gammaBar_ll, -2.),
			sym3_add(
				Up[<?=i-1?>]->gammaBar_ll,
				Um[<?=i-1?>]->gammaBar_ll)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_gammaBar_llll[<?=ij-1?>] = sym3_scale(
		sym3_sub(
			sym3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll),
			sym3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].gammaBar_ll,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].gammaBar_ll)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>
	sym3 partial2_gammaBar_ll;	//partial2_gammaBar_ll.ij = gammaBar^kl gammaBar_ij,kl
<? for ij,xij in ipairs(symNames) do
?>	partial2_gammaBar_ll.<?=xij?> = 0. <?
	for k,xk in ipairs(xNames) do
		for l,xl in ipairs(xNames) do
?> + gammaBar_uu.<?=sym(k,l)?> * partial2_gammaBar_llll[<?=from3x3to6(k,l)-1?>].<?=xij?><?
		end
	end
?>;
<? end
?>

	//B&S 11.54
	sym3 RBar_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
?>	RBar_ll.<?=xij?> = -1./2. * partial2_gammaBar_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		+ .5 * U->gammaBar_ll.<?=sym(k,i)?> * partial_connBar_ul[<?=j-1?>].<?=xk?>
		+ .5 * U->gammaBar_ll.<?=sym(k,j)?> * partial_connBar_ul[<?=i-1?>].<?=xk?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=i-1?>].<?=sym(j,k)?>
		+ .5 * U->connBar_u.<?=xk?> * connBar_lll[<?=j-1?>].<?=sym(i,k)?>
<?		for l,xl in ipairs(xNames) do
			for m,xm in ipairs(xNames) do
?>		+ connBar_ull[<?=k-1?>].<?=sym(l,i)?> * connBar_ull[<?=j-1?>].<?=sym(k,m)?>
		+ connBar_ull[<?=k-1?>].<?=sym(l,j)?> * connBar_ull[<?=i-1?>].<?=sym(k,m)?>
		+ connBar_ull[<?=k-1?>].<?=sym(i,m)?> * connBar_ull[<?=k-1?>].<?=sym(l,j)?>
<?			end
		end
	end
?>	;
<? end
?>

	sym3 DBar2_phi_ll;
<? for ij,xij in ipairs(symNames) do
?>	DBar2_phi_ll.<?=xij?> = partial2_phi_ll.<?=xij?> <?
	for k,xk in ipairs(xNames) do
?> - connBar_ull[<?=k?>].<?=xij?> * partial_phi_l.<?=xk?><?
	end
?>;
<? end
?>

	real tr_DBar2_phi = sym3_dot(gammaBar_uu, DBar2_phi_ll);

	real DBar_phi_norm = real3_weighted_norm(partial_phi_l, gammaBar_uu);

	//Baumgarte & Shapiro p.57 eqn 3.10
	//R_ll(i,j) := R_ij = RBar_ij - 2 (DBar_i DBar_j ln(psi) + gammaBar_ij gammaBar^lm DBar_l DBar_m ln(psi)) + 4((DBar_i ln(psi)) (DBar_j ln(psi)) - gammaBar_ij gammaBar^lm (DBar_l ln(psi)) (DBar_m ln(psi)))
	//Then Baumgarte & Shapiro on p.390 say RPhi_ij is the same as p.57 substituting phi for ln(psi)
	// ... but I thought phi was ln(psi)?  Then why would you need to separate R_ij = RBar_ij + RPhi_ij ?  I thought the substitution showed that R_ij was RPhi_ij?
	//phi = ln(psi), so DBar_i ln psi = partial_phi_i
	sym3 RPhi_ll;
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	RPhi_ll.<?=xij?> = RBar_ll.<?=xij?> - 2. * (DBar2_phi_ll.<?=xij?> + gammaBar_uu.<?=xij?> * tr_DBar2_phi) + 4. * (partial_phi_l.<?=xi?> * partial_phi_l.<?=xj?> - U->gammaBar_ll.<?=xij?> * DBar_phi_norm);
<? end
?>
	sym3 R_ll = sym3_add(RPhi_ll, RBar_ll);

	//traceless portion of -D^2 alpha + alpha (R_ij - 8 pi S_ij)
	sym3 tracelessPart_ll = sym3_sub(
		sym3_scale(
			sym3_add(R_ll, sym3_scale(U->S_ll, -8. * M_PI)),
			U->alpha),
		D2_alpha_ll);
	real tr_tracelessPart = sym3_dot(gamma_uu, tracelessPart_ll);
	tracelessPart_ll = sym3_sub(tracelessPart_ll, sym3_scale(gamma_ll, -1./3. * tr_tracelessPart));

#if 1
//something is asymmetric
	//B&S 11.53
	//ATilde_ij,t =
	//	exp(-4phi) (-(D_ij alpha)^TF + alpha (R_ij^TF - 8 pi S_ij^TF) )
	//	+ alpha (K ATilde_ij - 2 ATilde_il ATilde^l_j)
	//	+ beta^k ATilde_ij,k
	//	+ ATilde_ik beta^k_,j
	//	+ ATidle_kj beta^k_,i
	//	- 2/3 ATilde_ij beta^k_,k
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	local xi = xNames[i]
	local xj = xNames[j]
?>	deriv->ATilde_ll.<?=xij?> += exp_neg4phi * tracelessPart_ll.<?=xij?>
		+ U->alpha * U->K * U->ATilde_ll.<?=xij?>
<?	for k,xk in ipairs(xNames) do
?>		- 2. * U->alpha * U->ATilde_ll.<?=sym(i,k)?> * ATilde_ul.<?=xk?>.<?=xj?>
		+ partial_ATilde_lll[<?=k-1?>].<?=xij?> * U->beta_u.<?=xk?>
		+ U->ATilde_ll.<?=sym(i,k)?> * partial_beta_ul[<?=j-1?>].<?=xk?>
		+ U->ATilde_ll.<?=sym(j,k)?> * partial_beta_ul[<?=i-1?>].<?=xk?>
<?	end
?>		- 2./3. * U->ATilde_ll.<?=xij?> * tr_partial_beta;
<? end
?>
#endif

	real3 partial2_beta_ull[6];	//partial2_beta_ull[jk].i = beta^i_,jk
<? for ij,xij in ipairs(symNames) do
	local i,j = from6to3x3(ij)
	if i==j then
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_add(
			real3_scale(U->beta_u, -2.),
			real3_add(
				Up[<?=i-1?>]->beta_u,
				Um[<?=i-1?>]->beta_u)),
			1. / (grid_dx<?=i-1?> * grid_dx<?=i-1?>));
<?	else
?>	partial2_beta_ull[<?=ij-1?>] = real3_scale(
		real3_sub(
			real3_add(
				U[index + stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index - stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u),
			real3_add(
				U[index - stepsize[<?=i-1?>] + stepsize[<?=j-1?>]].beta_u,
				U[index + stepsize[<?=i-1?>] - stepsize[<?=j-1?>]].beta_u)
		), 1. / (4. * grid_dx<?=i-1?> * grid_dx<?=j-1?>));
<?	end
end
?>

	//connBar^i is the connection function / connection coefficient iteration with Hamiltonian constraint baked in (Baumgarte & Shapiro p.389, Alcubierre p.86).
	//B&S 11.55
	//partial_t connBar^i =
	//	-2 ATilde^ij alpha_,j
	//	+ 2 alpha (
	//		connBar^i_jk ATilde^kj
	//		- 2/3 gammaBar^ij K_,j
	//		- 8 pi gammaBar^ij S_j
	//		+ 6 ATilde^ij phi_,j
	//	)
	//	+ beta^j connBar^i_,j
	//	- connBar^j beta^i_,j
	//	+ 2/3 connBar^i beta^j_,j
	//	+ 1/3 gammaBar^ki beta^j_,jk
	//	+ gammaBar^kj beta^i_,jk
<?
for i,xi in ipairs(xNames) do
?>	deriv->connBar_u.<?=xi?> +=
		2./3. * U->connBar_u.<?=xi?> * tr_partial_beta
<?	for j,xj in ipairs(xNames) do
		local xij = sym(i,j)
		local jj = from3x3to6(j,j)
?>		- 2. * U->ATilde_ll.<?=xij?> * partial_alpha_l.<?=xj?>
		+ 2. * U->alpha * (
			-2./3. * gammaBar_uu.<?=xij?> * partial_K_l.<?=xj?>
			- 8. * M_PI * U->gammaBar_ll.<?=xij?> * U->S_u.<?=xj?>
			+ 6. * ATilde_uu.<?=xij?> * partial_phi_l.<?=xj?>)
		+ U->beta_u.<?=xi?> * partial_connBar_ul[<?=j-1?>].<?=xi?>
		- U->connBar_u.<?=xj?> * partial_beta_ul[<?=j-1?>].<?=xi?>
<?		for k,xk in ipairs(xNames) do
			local xik = sym(i,k)
			local jk = from3x3to6(j,k)
			local xjk = symNames[jk]
?>		+ 2. * U->alpha * connBar_ull[<?=i-1?>].<?=xjk?> * ATilde_uu.<?=xjk?>
		+ 1./3. * gammaBar_uu.<?=xik?> * partial2_beta_ull[<?=jk-1?>].<?=xj?>
		+ gammaBar_uu.<?=xjk?> * partial2_beta_ull[<?=jk-1?>].<?=xi?>
<?		end
	end
?>	;
<? end
?>
}


lib/kernel/write_image.cl: handle 1D/2D arrays

/* OpenCL built-in library: write_image()

   Copyright (c) 2013 Ville Korhonen

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

/* NOTE: this file is NOT a generic implementation; it works with vectors
   in a lot of places and requires that either device supports unaligned
   vector operations, or that memory backing the images is properly aligned.
   The maximum required alignment is 16bytes (4channels * 32bit color)

   Not all CPUs support unaligned vector operations, but the pthread / basic
   drivers allocate properly aligned memory for backing buffers; therefore
   this should work for everything supported by pthread / basic.
*/

#include "templates.h"
#include "pocl_image_rw_utils.h"

static uint4
map_channels (uint4 color, int order)
{
  switch (order)
    {
    case CL_ARGB:
      return color.wxyz;
    case CL_BGRA:
      return color.zyxw;
    case CL_RGBA:
    default:
      return color;
    }
}

/* only for CL_FLOAT, CL_SNORM_INT8, CL_UNORM_INT8,
 * CL_SNORM_INT16, CL_UNORM_INT16 channel types */
static void
write_float4_pixel (float4 color, void *data, size_t base_index, int type)
{
  if (type == CL_FLOAT)
    {
      ((float4 *)data)[base_index] = color;
      return;
    }
  if (type == CL_HALF_FLOAT)
    {
#if !defined(LLVM_OLDER_THAN_3_9) && __has_builtin(__builtin_convertvector)
      typedef float vector4float __attribute__ ((__vector_size__ (16)));
      typedef half vector4half __attribute__ ((__vector_size__ (8)));
      vector4half vh = __builtin_convertvector(color, vector4half);
      ((vector4half *)data)[base_index] = vh;
      return;
#else
      __builtin_trap ();
#endif
    }
  const float4 f127 = ((float4) (SCHAR_MAX));
  const float4 f32767 = ((float4) (SHRT_MAX));
  const float4 f255 = ((float4) (UCHAR_MAX));
  const float4 f65535 = ((float4) (USHRT_MAX));
  if (type == CL_SNORM_INT8)
    {
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float4 colorf = color * f127;
      char4 final_color = convert_char4_sat_rte (colorf);
      ((char4 *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_SNORM_INT16)
    {
      float4 colorf = color * f32767;
      short4 final_color = convert_short4_sat_rte (colorf);
      ((short4 *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT8)
    {
      /* <0, I*_MAX> to <0.0, 1.0> */
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float4 colorf = color * f255;
      uchar4 final_color = convert_uchar4_sat_rte (colorf);
      ((uchar4 *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT16)
    {
      float4 colorf = color * f65535;
      ushort4 final_color = convert_ushort4_sat_rte (colorf);
      ((ushort4 *)data)[base_index] = final_color;
      return;
    }

  return;
}

/* only for CL_FLOAT, CL_SNORM_INT8, CL_UNORM_INT8,
 * CL_SNORM_INT16, CL_UNORM_INT16 channel types */
static void
write_float_pixel (float color, void *data, size_t base_index, int type)
{
  if (type == CL_FLOAT)
    {
      ((float *)data)[base_index] = color;
      return;
    }
  const float f127 = ((float)SCHAR_MAX);
  const float f32767 = ((float)SHRT_MAX);
  const float f255 = ((float)UCHAR_MAX);
  const float f65535 = ((float)USHRT_MAX);
  if (type == CL_SNORM_INT8)
    {
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float colorf = color * f127;
      char final_color = convert_char_sat_rte (colorf);
      ((char *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_SNORM_INT16)
    {
      float colorf = color * f32767;
      short final_color = convert_short_sat_rte (colorf);
      ((short *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT8)
    {
      /* <0, I*_MAX> to <0.0, 1.0> */
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float colorf = color * f255;
      uchar final_color = convert_uchar_sat_rte (colorf);
      ((uchar *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT16)
    {
      float colorf = color * f65535;
      ushort final_color = convert_ushort_sat_rte (colorf);
      ((ushort *)data)[base_index] = final_color;
      return;
    }

  return;
}

/* for use inside filter functions
 * no channel mapping
 * no pointers to img metadata */
static void
pocl_write_pixel_fast_ui (uint4 color, size_t base_index, int order,
                          int elem_size, void *data)
{
  if (order == CL_A)
    {
      if (elem_size == 1)
        ((uchar *)data)[base_index] = convert_uchar_sat (color.w);
      else if (elem_size == 2)
        ((ushort *)data)[base_index] = convert_ushort_sat (color.w);
      else if (elem_size == 4)
        ((uint *)data)[base_index] = color.w;
      return;
    }

  if (elem_size == 1)
    {
      ((uchar4 *)data)[base_index] = convert_uchar4_sat (color);
    }
  else if (elem_size == 2)
    {
      ((ushort4 *)data)[base_index] = convert_ushort4_sat (color);
    }
  else if (elem_size == 4)
    {
      ((uint4 *)data)[base_index] = color;
    }

  return;
}

/* for use inside filter functions
 * no channel mapping
 * no pointers to img metadata */
static void
pocl_write_pixel_fast_f (float4 color, size_t base_index, int channel_type,
                         int order, void *data)
{
  if (order == CL_A)
    {
      write_float_pixel (color.w, data, base_index, channel_type);
    }
  else
    {
      write_float4_pixel (color, data, base_index, channel_type);
    }

  return;
}

/* for use inside filter functions
 * no channel mapping
 * no pointers to img metadata */
static void
pocl_write_pixel_fast_i (int4 color, size_t base_index, int order,
                         int elem_size, void *data)
{
  if (order == CL_A)
    {
      if (elem_size == 1)
        ((char *)data)[base_index] = convert_char_sat (color.w);
      else if (elem_size == 2)
        ((short *)data)[base_index] = convert_short_sat (color.w);
      else if (elem_size == 4)
        ((int *)data)[base_index] = color.w;
      return;
    }

  if (elem_size == 1)
    {
      ((char4 *)data)[base_index] = convert_char4_sat (color);
    }
  else if (elem_size == 2)
    {
      ((short4 *)data)[base_index] = convert_short4_sat (color);
    }
  else if (elem_size == 4)
    {
      ((int4 *)data)[base_index] = color;
    }
  return;
}

/* full write with channel map conversion etc
 * Writes a four element pixel to an image pixel pointed by integer coords.
 */
static void
pocl_write_pixel (uint4 color, global dev_image_t *img, int4 coord,
                  size_t array_offset_pixels, size_t row_pitch,
                  size_t slice_pitch)
{
  int width = img->_width;
  int height = img->_height;
  int depth = img->_depth;
  int order = img->_order;
  int elem_size = img->_elem_size;
  int channel_type = img->_data_type;
  void *data = img->_data;

  if ((coord.x >= width || coord.x < 0)
      || ((height != 0) && (coord.y >= height || coord.y < 0))
      || ((depth != 0) && (coord.z >= depth || coord.z < 0)))
    {
      return;
    }

  size_t base_index = array_offset_pixels + coord.x + (coord.y * row_pitch)
                      + (coord.z * slice_pitch);

  color = map_channels (color, order);

  if ((channel_type == CL_SIGNED_INT8) || (channel_type == CL_SIGNED_INT16)
      || (channel_type == CL_SIGNED_INT32))
    pocl_write_pixel_fast_i (as_int4 (color), base_index, order, elem_size,
                             data);
  else if ((channel_type == CL_UNSIGNED_INT8)
           || (channel_type == CL_UNSIGNED_INT16)
           || (channel_type == CL_UNSIGNED_INT32))
    pocl_write_pixel_fast_ui (as_uint4 (color), base_index, order, elem_size,
                              data);
  else // TODO unsupported channel types
    pocl_write_pixel_fast_f (as_float4 (color), base_index, channel_type,
                             order, data);
}

/*
write_imagei can only be used with image objects created with
image_channel_data_type set to one of the following values:
CL_SIGNED_INT8, CL_SIGNED_INT16, and CL_SIGNED_INT32.

write_imageui functions can only be used with image objects created with
image_channel_data_type set to one of the following values:
CL_UNSIGNED_INT8, CL_UNSIGNED_INT16, or CL_UNSIGNED_INT32.
*/

/*
 * write_imagef can only be used with image objects created with
 * image_channel_data_type set to one of the pre-defined packed formats,
 * or set to CL_SNORM_INT8, CL_UNORM_INT8, CL_SNORM_INT16,
 * CL_UNORM_INT16, CL_HALF_FLOAT or CL_FLOAT.
*/

#define IMPLEMENT_WRITE_IMAGE_INT_COORD(__IMGTYPE__, __POSTFIX__, __COORD__,  \
                                        __DTYPE__)                            \
  void _CL_OVERLOADABLE write_image##__POSTFIX__ (                            \
      __IMGTYPE__ image, __COORD__ coord, __DTYPE__ color)                    \
  {                                                                           \
    int4 coord4;                                                              \
    INITCOORD##__COORD__ (coord4, coord);                                     \
    global dev_image_t *i_ptr                                                 \
        = __builtin_astype (image, global dev_image_t *);                     \
    int elem_size = i_ptr->_elem_size;                                        \
    int num_channels = i_ptr->_num_channels;                                  \
    size_t elem_bytes = num_channels * elem_size;                             \
    size_t row_pitch = i_ptr->_row_pitch / elem_bytes;                        \
    size_t slice_pitch = i_ptr->_slice_pitch / elem_bytes;                    \
    pocl_write_pixel (as_uint4 (color), i_ptr, coord4, 0, row_pitch,          \
                      slice_pitch);                                           \
  }

#define IMPLEMENT_WRITE_ARRAY_INT_COORD(__IMGTYPE__, __POSTFIX__, __COORD__,  \
                                        __DTYPE__)                            \
  void _CL_OVERLOADABLE write_image##__POSTFIX__ (                            \
      __IMGTYPE__ image, __COORD__ coord, __DTYPE__ color)                    \
  {                                                                           \
    int4 coord4;                                                              \
    INITCOORD##__COORD__ (coord4, coord);                                     \
    global dev_image_t *i_ptr                                                 \
        = __builtin_astype (image, global dev_image_t *);                     \
    int asize = i_ptr->_image_array_size - 1;                                 \
    int elem_size = i_ptr->_elem_size;                                        \
    int channel_type = i_ptr->_data_type;                                     \
    int num_channels = i_ptr->_num_channels;                                  \
    size_t elem_bytes = num_channels * elem_size;                             \
    size_t slice_pitch_pixels = i_ptr->_slice_pitch / elem_bytes;             \
    size_t row_pitch = i_ptr->_row_pitch / elem_bytes;                        \
    size_t slice_pitch = i_ptr->_slice_pitch / elem_bytes;                    \
    size_t array_offset_pixels = 0;                                           \
    if (i_ptr->_height > 0)                                                   \
      {                                                                       \
        array_offset_pixels = clamp (coord4.z, 0, asize);                     \
        coord4.z = 0;                                                         \
      }                                                                       \
    else                                                                      \
      {                                                                       \
        array_offset_pixels = clamp (coord4.y, 0, asize);                     \
        coord4.y = 0;                                                         \
      }                                                                       \
    array_offset_pixels *= slice_pitch;                                       \
    pocl_write_pixel (as_uint4 (color), i_ptr, coord4, array_offset_pixels,   \
                      row_pitch, slice_pitch);                                \
  }

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_t, f, int, float4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_buffer_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_buffer_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_buffer_t, f, int, float4)

IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_WO_AQ image1d_array_t, ui, int2, uint4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_WO_AQ image1d_array_t, i, int2, int4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_WO_AQ image1d_array_t, f, int2, float4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image2d_t, ui, int2, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image2d_t, i, int2, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image2d_t, f, int2, float4)

IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_WO_AQ image2d_array_t, ui, int4, uint4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_WO_AQ image2d_array_t, i, int4, int4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_WO_AQ image2d_array_t, f, int4, float4)

#ifdef cl_khr_3d_image_writes
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image3d_t, ui, int4, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image3d_t, i, int4, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image3d_t, f, int4, float4)
#endif

#ifdef CLANG_HAS_RW_IMAGES

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_t, f, int, float4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_buffer_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_buffer_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_buffer_t, f, int, float4)

IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_RW_AQ image1d_array_t, ui, int2, uint4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_RW_AQ image1d_array_t, i, int2, int4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_RW_AQ image1d_array_t, f, int2, float4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image2d_t, ui, int2, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image2d_t, i, int2, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image2d_t, f, int2, float4)

IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_RW_AQ image2d_array_t, ui, int4, uint4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_RW_AQ image2d_array_t, i, int4, int4)
IMPLEMENT_WRITE_ARRAY_INT_COORD (IMG_RW_AQ image2d_array_t, f, int4, float4)

#ifdef cl_khr_3d_image_writes
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image3d_t, ui, int4, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image3d_t, i, int4, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image3d_t, f, int4, float4)
#endif

#endif

/* OpenCL built-in library: write_image()

   Copyright (c) 2013 Ville Korhonen

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

/* NOTE: this file is NOT a generic implementation; it works with vectors
   in a lot of places and requires that either device supports unaligned
   vector operations, or that memory backing the images is properly aligned.
   The maximum required alignment is 16bytes (4channels * 32bit color)

   Not all CPUs support unaligned vector operations, but the pthread / basic
   drivers allocate properly aligned memory for backing buffers; therefore
   this should work for everything supported by pthread / basic.
*/

#include "templates.h"
#include "pocl_image_rw_utils.h"

static uint4
map_channels (uint4 color, int order)
{
  switch (order)
    {
    case CL_ARGB:
      return color.wxyz;
    case CL_BGRA:
      return color.zyxw;
    case CL_RGBA:
    default:
      return color;
    }
}

/* only for CL_FLOAT, CL_SNORM_INT8, CL_UNORM_INT8,
 * CL_SNORM_INT16, CL_UNORM_INT16 channel types */
static void
write_float4_pixel (float4 color, void *data, size_t base_index, int type)
{
  if (type == CL_FLOAT)
    {
      ((float4 *)data)[base_index] = color;
      return;
    }
  if (type == CL_HALF_FLOAT)
    {
#if !defined(LLVM_OLDER_THAN_3_9) && __has_builtin(__builtin_convertvector)
      typedef float vector4float __attribute__ ((__vector_size__ (16)));
      typedef half vector4half __attribute__ ((__vector_size__ (8)));
      vector4half vh = __builtin_convertvector(color, vector4half);
      ((vector4half *)data)[base_index] = vh;
      return;
#else
      __builtin_trap ();
#endif
    }
  const float4 f127 = ((float4) (SCHAR_MAX));
  const float4 f32767 = ((float4) (SHRT_MAX));
  const float4 f255 = ((float4) (UCHAR_MAX));
  const float4 f65535 = ((float4) (USHRT_MAX));
  if (type == CL_SNORM_INT8)
    {
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float4 colorf = color * f127;
      char4 final_color = convert_char4_sat_rte (colorf);
      ((char4 *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_SNORM_INT16)
    {
      float4 colorf = color * f32767;
      short4 final_color = convert_short4_sat_rte (colorf);
      ((short4 *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT8)
    {
      /* <0, I*_MAX> to <0.0, 1.0> */
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float4 colorf = color * f255;
      uchar4 final_color = convert_uchar4_sat_rte (colorf);
      ((uchar4 *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT16)
    {
      float4 colorf = color * f65535;
      ushort4 final_color = convert_ushort4_sat_rte (colorf);
      ((ushort4 *)data)[base_index] = final_color;
      return;
    }

  return;
}

/* only for CL_FLOAT, CL_SNORM_INT8, CL_UNORM_INT8,
 * CL_SNORM_INT16, CL_UNORM_INT16 channel types */
static void
write_float_pixel (float color, void *data, size_t base_index, int type)
{
  if (type == CL_FLOAT)
    {
      ((float *)data)[base_index] = color;
      return;
    }
  const float f127 = ((float)SCHAR_MAX);
  const float f32767 = ((float)SHRT_MAX);
  const float f255 = ((float)UCHAR_MAX);
  const float f65535 = ((float)USHRT_MAX);
  if (type == CL_SNORM_INT8)
    {
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float colorf = color * f127;
      char final_color = convert_char_sat_rte (colorf);
      ((char *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_SNORM_INT16)
    {
      float colorf = color * f32767;
      short final_color = convert_short_sat_rte (colorf);
      ((short *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT8)
    {
      /* <0, I*_MAX> to <0.0, 1.0> */
      /*  <-1.0, 1.0> to <I*_MIN, I*_MAX> */
      float colorf = color * f255;
      uchar final_color = convert_uchar_sat_rte (colorf);
      ((uchar *)data)[base_index] = final_color;
      return;
    }
  if (type == CL_UNORM_INT16)
    {
      float colorf = color * f65535;
      ushort final_color = convert_ushort_sat_rte (colorf);
      ((ushort *)data)[base_index] = final_color;
      return;
    }

  return;
}

/* for use inside filter functions
 * no channel mapping
 * no pointers to img metadata */
static void
pocl_write_pixel_fast_ui (uint4 color, size_t base_index, int order,
                          int elem_size, void *data)
{
  if (order == CL_A)
    {
      if (elem_size == 1)
        ((uchar *)data)[base_index] = convert_uchar_sat (color.w);
      else if (elem_size == 2)
        ((ushort *)data)[base_index] = convert_ushort_sat (color.w);
      else if (elem_size == 4)
        ((uint *)data)[base_index] = color.w;
      return;
    }

  if (elem_size == 1)
    {
      ((uchar4 *)data)[base_index] = convert_uchar4_sat (color);
    }
  else if (elem_size == 2)
    {
      ((ushort4 *)data)[base_index] = convert_ushort4_sat (color);
    }
  else if (elem_size == 4)
    {
      ((uint4 *)data)[base_index] = color;
    }

  return;
}

/* for use inside filter functions
 * no channel mapping
 * no pointers to img metadata */
static void
pocl_write_pixel_fast_f (float4 color, size_t base_index, int channel_type,
                         int order, void *data)
{
  if (order == CL_A)
    {
      write_float_pixel (color.w, data, base_index, channel_type);
    }
  else
    {
      write_float4_pixel (color, data, base_index, channel_type);
    }

  return;
}

/* for use inside filter functions
 * no channel mapping
 * no pointers to img metadata */
static void
pocl_write_pixel_fast_i (int4 color, size_t base_index, int order,
                         int elem_size, void *data)
{
  if (order == CL_A)
    {
      if (elem_size == 1)
        ((char *)data)[base_index] = convert_char_sat (color.w);
      else if (elem_size == 2)
        ((short *)data)[base_index] = convert_short_sat (color.w);
      else if (elem_size == 4)
        ((int *)data)[base_index] = color.w;
      return;
    }

  if (elem_size == 1)
    {
      ((char4 *)data)[base_index] = convert_char4_sat (color);
    }
  else if (elem_size == 2)
    {
      ((short4 *)data)[base_index] = convert_short4_sat (color);
    }
  else if (elem_size == 4)
    {
      ((int4 *)data)[base_index] = color;
    }
  return;
}

/* full write with channel map conversion etc
 * Writes a four element pixel to an image pixel pointed by integer coords.
 */
static void
pocl_write_pixel (uint4 color, global dev_image_t *img, int4 coord)
{
  int width = img->_width;
  int height = img->_height;
  int depth = img->_depth;
  int order = img->_order;
  int elem_size = img->_elem_size;
  int channel_type = img->_data_type;
  void *data = img->_data;

  if ((coord.x >= width || coord.x < 0)
      || ((height != 0) && (coord.y >= height || coord.y < 0))
      || ((depth != 0) && (coord.z >= depth || coord.z < 0)))
    {
      return;
    }

  size_t base_index = coord.x + coord.y * width;
  if (depth)
    base_index += (coord.z * height * width);

  color = map_channels (color, order);

  if ((channel_type == CL_SIGNED_INT8) || (channel_type == CL_SIGNED_INT16)
      || (channel_type == CL_SIGNED_INT32))
    pocl_write_pixel_fast_i (as_int4 (color), base_index, order, elem_size,
                             data);
  else if ((channel_type == CL_UNSIGNED_INT8)
           || (channel_type == CL_UNSIGNED_INT16)
           || (channel_type == CL_UNSIGNED_INT32))
    pocl_write_pixel_fast_ui (as_uint4 (color), base_index, order, elem_size,
                              data);
  else // TODO unsupported channel types
    pocl_write_pixel_fast_f (as_float4 (color), base_index, channel_type,
                             order, data);
}

/*
write_imagei can only be used with image objects created with
image_channel_data_type set to one of the following values:
CL_SIGNED_INT8, CL_SIGNED_INT16, and CL_SIGNED_INT32.

write_imageui functions can only be used with image objects created with
image_channel_data_type set to one of the following values:
CL_UNSIGNED_INT8, CL_UNSIGNED_INT16, or CL_UNSIGNED_INT32.
*/

/*
 * write_imagef can only be used with image objects created with
 * image_channel_data_type set to one of the pre-defined packed formats,
 * or set to CL_SNORM_INT8, CL_UNORM_INT8, CL_SNORM_INT16,
 * CL_UNORM_INT16, CL_HALF_FLOAT or CL_FLOAT.
*/

#define IMPLEMENT_WRITE_IMAGE_INT_COORD(__IMGTYPE__, __POSTFIX__, __COORD__,  \
                                        __DTYPE__)                            \
  void _CL_OVERLOADABLE write_image##__POSTFIX__ (                            \
      __IMGTYPE__ image, __COORD__ coord, __DTYPE__ color)                    \
  {                                                                           \
    int4 coord4;                                                              \
    INITCOORD##__COORD__ (coord4, coord);                                     \
    global dev_image_t *i_ptr                                                 \
        = __builtin_astype (image, global dev_image_t *);                     \
    pocl_write_pixel (as_uint4 (color), i_ptr, coord4);                       \
  }

IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_WO_AQ image2d_t, ui, int2, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_WO_AQ image2d_t, ui, int4, uint4)

IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_WO_AQ image2d_t, i, int2, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_WO_AQ image2d_t, i, int4, int4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image2d_t, f, int2, float4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image3d_t, f, int4, float4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_buffer_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_array_t, ui, int2, uint4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_buffer_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_array_t, i, int2, int4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_t, f, int, float4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_buffer_t, f, int, float4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_WO_AQ image1d_array_t, f, int2, float4)

#ifdef CLANG_HAS_RW_IMAGES

IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_RW_AQ image2d_t, ui, int2, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_RW_AQ image2d_t, ui, int4, uint4)

IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_RW_AQ image2d_t, i, int2, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD ( IMG_RW_AQ image2d_t, i, int4, int4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image2d_t, f, int2, float4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image3d_t, f, int4, float4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_buffer_t, ui, int, uint4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_array_t, ui, int2, uint4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_buffer_t, i, int, int4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_array_t, i, int2, int4)

IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_t, f, int, float4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_buffer_t, f, int, float4)
IMPLEMENT_WRITE_IMAGE_INT_COORD (IMG_RW_AQ image1d_array_t, f, int2, float4)

#endif