Add sign.

Add sign.

/*
 * Copyright (C) 2011 Peter Couperus
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include "PlatformStdlib.h"

float CLANG_OVERLOADABLE degrees(float radians)
{
    return (radians * 180.0f) / M_PI_F;
}

float2 CLANG_OVERLOADABLE degrees(float2 radians)
{
    float2 result;
    int i = 0;
    for (i = 0; i < 2; i++) {
        result[i] = degrees(radians[i]);
    }
    return result;
}

float3 CLANG_OVERLOADABLE degrees(float3 radians)
{
    float3 result;
    int i = 0;
    for (i = 0; i < 3; i++) {
        result[i] = degrees(radians[i]);
    }
    return result;
}

float4 CLANG_OVERLOADABLE degrees(float4 radians)
{
    float4 result;
    int i = 0;
    for (i = 0; i < 4; i++) {
        result[i] = degrees(radians[i]);
    }
    return result;
}

float8 CLANG_OVERLOADABLE degrees(float8 radians)
{
    float8 result;
    int i = 0;
    for (i = 0; i < 8; i++) {
        result[i] = degrees(radians[i]);
    }
    return result;
}

float16 CLANG_OVERLOADABLE degrees(float16 radians)
{
    float16 result;
    int i = 0;
    for (i = 0; i < 16; i++) {
        result[i] = degrees(radians[i]);
    }
    return result;
}

// max -----------------------------------------------------------------------

float CLANG_OVERLOADABLE max(float x, float y)
{
    return (x > y ? x : y);
}

float2 CLANG_OVERLOADABLE max(float2 x, float2 y)
{
    float2 result;
    int i = 0;
    for(i = 0;i < 2;i++) {
        result[i] = max(x[i], y[i]);
    }
    return result;
}

float3 CLANG_OVERLOADABLE max(float3 x, float3 y)
{
    float3 result;
    int i = 0;
    for(i = 0;i < 3;i++) {
        result[i] = max(x[i], y[i]);
    }
    return result;
}

float4 CLANG_OVERLOADABLE max(float4 x, float4 y)
{
    float4 result;
    int i = 0;
    for(i = 0;i < 4;i++) {
        result[i] = max(x[i], y[i]);
    }
    return result;
}

float8 CLANG_OVERLOADABLE max(float8 x, float8 y)
{
    float8 result;
    int i = 0;
    for(i = 0;i < 8;i++) {
        result[i] = max(x[i], y[i]);
    }
    return result;
}

float16 CLANG_OVERLOADABLE max(float16 x, float16 y)
{
    float16 result;
    int i = 0;
    for(i = 0;i < 16;i++) {
        result[i] = max(x[i], y[i]);
    }
    return result;
}

// min -----------------------------------------------------------------------

float CLANG_OVERLOADABLE min(float x, float y)
{
    return (x < y ? x : y);
}

float2 CLANG_OVERLOADABLE min(float2 x, float2 y)
{
    float2 result;
    int i = 0;
    for(i = 0;i < 2;i++) {
        result[i] = min(x[i], y[i]);
    }
    return result;
}

float3 CLANG_OVERLOADABLE min(float3 x, float3 y)
{
    float3 result;
    int i = 0;
    for(i = 0;i < 3;i++) {
        result[i] = min(x[i], y[i]);
    }
    return result;
}

float4 CLANG_OVERLOADABLE min(float4 x, float4 y)
{
    float4 result;
    int i = 0;
    for(i = 0;i < 4;i++) {
        result[i] = min(x[i], y[i]);
    }
    return result;
}

float8 CLANG_OVERLOADABLE min(float8 x, float8 y)
{
    float8 result;
    int i = 0;
    for(i = 0;i < 8;i++) {
        result[i] = min(x[i], y[i]);
    }
    return result;
}

float16 CLANG_OVERLOADABLE min(float16 x, float16 y)
{
    float16 result;
    int i = 0;
    for(i = 0;i < 16;i++) {
        result[i] = min(x[i], y[i]);
    }
    return result;
}

// mix -----------------------------------------------------------------------

float CLANG_OVERLOADABLE mix(float x, float y, float a)
{
    return x + (y - x)*a;
}

float2 CLANG_OVERLOADABLE mix(float2 x, float2 y, float2 a)
{
    float2 result;
    int i = 0;
    for(i = 0;i < 2;i++) {
        result[i] = mix(x[i], y[i], a[i]);
    }
    return result;
}

float3 CLANG_OVERLOADABLE mix(float3 x, float3 y, float3 a)
{
    float3 result;
    int i = 0;
    for(i = 0;i < 3;i++) {
        result[i] = mix(x[i], y[i], a[i]);
    }
    return result;
}

float4 CLANG_OVERLOADABLE mix(float4 x, float4 y, float4 a)
{
    float4 result;
    int i = 0;
    for(i = 0;i < 4;i++) {
        result[i] = mix(x[i], y[i], a[i]);
    }
    return result;
}

float8 CLANG_OVERLOADABLE mix(float8 x, float8 y, float8 a)
{
    float8 result;
    int i = 0;
    for(i = 0;i < 8;i++) {
        result[i] = mix(x[i], y[i], a[i]);
    }
    return result;
}

float16 CLANG_OVERLOADABLE mix(float16 x, float16 y, float16 a)
{
    float16 result;
    int i = 0;
    for(i = 0;i < 16;i++) {
        result[i] = mix(x[i], y[i], a[i]);
    }
    return result;
}

// radians -------------------------------------------------------------------

float CLANG_OVERLOADABLE radians(float degrees)
{
    return (M_PI_F * degrees) / 180.0f;
}

float2 CLANG_OVERLOADABLE radians(float2 degrees)
{
    float2 result;
    int i = 0;
    for (i = 0; i < 2; i++) {
        result[i] = radians(degrees[i]);
    }
    return result;
}

float3 CLANG_OVERLOADABLE radians(float3 degrees)
{
    float3 result;
    int i = 0;
    for (i = 0; i < 3; i++) {
        result[i] = radians(degrees[i]);
    }
    return result;
}

float4 CLANG_OVERLOADABLE radians(float4 degrees)
{
    float4 result;
    int i = 0;
    for (i = 0; i < 4; i++) {
        result[i] = radians(degrees[i]);
    }
    return result;
}

float8 CLANG_OVERLOADABLE radians(float8 degrees)
{
    float8 result;
    int i = 0;
    for (i = 0; i < 8; i++) {
        result[i] = radians(degrees[i]);
    }
    return result;
}

float16 CLANG_OVERLOADABLE radians(float16 degrees)
{
    float16 result;
    int i = 0;
    for (i = 0; i < 16; i++) {
        result[i] = radians(degrees[i]);
    }
    return result;
}

// sign ----------------------------------------------------------------------

// TODO: nan support
float CLANG_OVERLOADABLE sign(float x)
{
    if (x == 0.0f) {
        return 0.0f;
    }
    return (x > 0.0f ? 1.0f : -1.0f);
}

float2 CLANG_OVERLOADABLE sign(float2 x)
{
    float2 result;
    int i = 0;
    for (i = 0; i < 2; i++) {
        result[i] = sign(x[i]);
    }
    return result;
}

float3 CLANG_OVERLOADABLE sign(float3 x)
{
    float3 result;
    int i = 0;
    for (i = 0; i < 3; i++) {
        result[i] = sign(x[i]);
    }
    return result;
}

float4 CLANG_OVERLOADABLE sign(float4 x)
{
    float4 result;
    int i = 0;
    for (i = 0; i < 4; i++) {
        result[i] = sign(x[i]);
    }
    return result;
}

float8 CLANG_OVERLOADABLE sign(float8 x)
{
    float8 result;
    int i = 0;
    for (i = 0; i < 8; i++) {
        result[i] = sign(x[i]);
    }
    return result;
}

float16 CLANG_OVERLOADABLE sign(float16 x)
{
    float16 result;
    int i = 0;
    for (i = 0; i < 16; i++) {
        result[i] = sign(x[i]);
    }
    return result;
}


relational: Fix signbit

relational: Fix signbit

The vector components were mistakenly using () instead of {}, which caused
all but the last vector component to be dropped on the floor.

Signed-off-by: Aaron Watry <awatry@gmail.com>
Reviewed-by: Jeroen Ketema <j.ketema@imperial.ac.uk>

git-svn-id: e7f1ab7c2a259259587475a3e7520e757882f167@211733 91177308-0d34-0410-b5e6-96231b3b80d8

#include <clc/clc.h>

#define _CLC_DEFINE_RELATIONAL_UNARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_NAME, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x){ \
	return BUILTIN_NAME(x); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( (RET_TYPE){FUNCTION(x.lo), FUNCTION(x.hi)} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( (RET_TYPE){FUNCTION(x.lo), FUNCTION(x.hi)} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( (RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2)} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( \
	(RET_TYPE){ \
		FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3) \
	} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( \
	(RET_TYPE){ \
		FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3), \
		FUNCTION(x.s4), FUNCTION(x.s5), FUNCTION(x.s6), FUNCTION(x.s7) \
	} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( \
	(RET_TYPE){ \
		FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3), \
		FUNCTION(x.s4), FUNCTION(x.s5), FUNCTION(x.s6), FUNCTION(x.s7), \
		FUNCTION(x.s8), FUNCTION(x.s9), FUNCTION(x.sa), FUNCTION(x.sb), \
		FUNCTION(x.sc), FUNCTION(x.sd), FUNCTION(x.se), FUNCTION(x.sf) \
	} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY(RET_TYPE, FUNCTION, BUILTIN_FUNCTION, ARG_TYPE) \
_CLC_DEFINE_RELATIONAL_UNARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_FUNCTION, ARG_TYPE) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE##2, FUNCTION, ARG_TYPE##2) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE##3, FUNCTION, ARG_TYPE##3) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE##4, FUNCTION, ARG_TYPE##4) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE##8, FUNCTION, ARG_TYPE##8) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE##16, FUNCTION, ARG_TYPE##16) \

_CLC_DEFINE_RELATIONAL_UNARY(int, signbit, __builtin_signbitf, float)

#ifdef cl_khr_fp64

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

// The scalar version of signbit(double) returns an int, but the vector versions
// return long.

_CLC_DEF _CLC_OVERLOAD int signbit(double x){
	return __builtin_signbit(x);
}

_CLC_DEFINE_RELATIONAL_UNARY_VEC2(long2, signbit, double2)
_CLC_DEFINE_RELATIONAL_UNARY_VEC3(long3, signbit, double3)
_CLC_DEFINE_RELATIONAL_UNARY_VEC4(long4, signbit, double4)
_CLC_DEFINE_RELATIONAL_UNARY_VEC8(long8, signbit, double8)
_CLC_DEFINE_RELATIONAL_UNARY_VEC16(long16, signbit, double16)

#endif

#undef _CLC_DEFINE_RELATIONAL_UNARY
#undef _CLC_DEFINE_RELATIONAL_UNARY_SCALAR
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC2
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC3
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC4
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC8
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC16

#include <clc/clc.h>

#define _CLC_DEFINE_RELATIONAL_UNARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_NAME, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x){ \
	return BUILTIN_NAME(x); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( (RET_TYPE){FUNCTION(x.lo), FUNCTION(x.hi)} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( (RET_TYPE){FUNCTION(x.lo), FUNCTION(x.hi)} != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)((FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2)) != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( \
	( \
		FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3) \
	) != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( \
	( \
		FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3), \
		FUNCTION(x.s4), FUNCTION(x.s5), FUNCTION(x.s6), FUNCTION(x.s7) \
	) != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE, FUNCTION, ARG_TYPE) \
_CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) { \
  return (RET_TYPE)( \
	( \
		FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3), \
		FUNCTION(x.s4), FUNCTION(x.s5), FUNCTION(x.s6), FUNCTION(x.s7), \
		FUNCTION(x.s8), FUNCTION(x.s9), FUNCTION(x.sa), FUNCTION(x.sb), \
		FUNCTION(x.sc), FUNCTION(x.sd), FUNCTION(x.se), FUNCTION(x.sf) \
	) != (RET_TYPE)0); \
} \

#define _CLC_DEFINE_RELATIONAL_UNARY(RET_TYPE, FUNCTION, BUILTIN_FUNCTION, ARG_TYPE) \
_CLC_DEFINE_RELATIONAL_UNARY_SCALAR(RET_TYPE, FUNCTION, BUILTIN_FUNCTION, ARG_TYPE) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE##2, FUNCTION, ARG_TYPE##2) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE##3, FUNCTION, ARG_TYPE##3) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE##4, FUNCTION, ARG_TYPE##4) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE##8, FUNCTION, ARG_TYPE##8) \
_CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE##16, FUNCTION, ARG_TYPE##16) \

_CLC_DEFINE_RELATIONAL_UNARY(int, signbit, __builtin_signbitf, float)

#ifdef cl_khr_fp64

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

// The scalar version of signbit(double) returns an int, but the vector versions
// return long.

_CLC_DEF _CLC_OVERLOAD int signbit(double x){
	return __builtin_signbit(x);
}

_CLC_DEFINE_RELATIONAL_UNARY_VEC2(long2, signbit, double2)
_CLC_DEFINE_RELATIONAL_UNARY_VEC3(long3, signbit, double3)
_CLC_DEFINE_RELATIONAL_UNARY_VEC4(long4, signbit, double4)
_CLC_DEFINE_RELATIONAL_UNARY_VEC8(long8, signbit, double8)
_CLC_DEFINE_RELATIONAL_UNARY_VEC16(long16, signbit, double16)

#endif

#undef _CLC_DEFINE_RELATIONAL_UNARY
#undef _CLC_DEFINE_RELATIONAL_UNARY_SCALAR
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC2
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC3
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC4
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC8
#undef _CLC_DEFINE_RELATIONAL_UNARY_VEC16


trying stuff

trying stuff

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [imageid][outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
void kernel propagate_4_by_n_outplane_smallercache( const int batchSize,
    const int pixelsPerThread,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube, local float *_pixelSums ) {
    const int globalId = get_global_id(0);

    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int n = workgroupId / gNumFilters;
    const int outPlane = workgroupId % gNumFilters;

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int numFilterPixelsPerThread = ( gFilterSizeSquared + workgroupSize - 1 ) / workgroupSize;

    //local float *_myPixelSums = _pixelSums + pixelsPerThread * localId;
    local float *_myPixelSums = _pixelSums;

    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        _myPixelSums[pixel] = 0.0f;
    }

    const int outputRow = localId / gOutputBoardSize;
    const int outputCol = localId % gOutputBoardSize;

    const int minu = gPadZeros ? max( -gHalfFilterSize, -outputRow ) : -gHalfFilterSize;
    const int maxu = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputRow  - evenPadding) : gHalfFilterSize - evenPadding;
    const int minv = gPadZeros ? max( -gHalfFilterSize, -outputCol ) : - gHalfFilterSize;
    const int maxv = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputCol - evenPadding) : gHalfFilterSize - evenPadding;

    for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
        int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int i = 0; i < numUpstreamsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gInputBoardSizeSquared ) {
                _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
            }
        }
        const int filterGlobalOffset = ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared;
        for( int i = 0; i < numFilterPixelsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gFilterSizeSquared ) {
                _filterCube[thisOffset] = filters[filterGlobalOffset + thisOffset];
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
//        int pixel = 0;
            const int virtualLocalId = localId + pixel * workgroupSize;

            float thissum = 0;
            if( virtualLocalId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u + ( gPadZeros ? 0 : gHalfFilterSize );
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v + ( gPadZeros ? 0 : gHalfFilterSize );
                        thissum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
            _myPixelSums[pixel] += thissum;
//            if( globalId == 0 ) results[0] = _upstreamBoard[pixel];
        }
    }
//    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        int pixel = 0;
        const int virtualLocalId = localId + pixel * workgroupSize;
        if( virtualLocalId < gOutputBoardSizeSquared ) {
            float sum = _myPixelSums[pixel];
            #ifdef BIASED
                sum += biases[outPlane];
            #endif
            // results are organized like [imageid][filterid][row][col]
            int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + virtualLocalId;
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
            // results[resultIndex ] = 123;
            //if( globalId == 0 ) results[0] += 0.000001f + _perPixelSums[0];
        }
//    }
}
#endif
#endif

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [imageid][outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
void kernel propagate_4_by_n_outplane_smallercache( const int batchSize,
    const int pixelsPerThread,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube, local float *_pixelSums ) {
    const int globalId = get_global_id(0);

    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int n = workgroupId / gNumFilters;
    const int outPlane = workgroupId % gNumFilters;

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int numFilterPixelsPerThread = ( gFilterSizeSquared + workgroupSize - 1 ) / workgroupSize;

    //local float *_myPixelSums = _pixelSums + pixelsPerThread * localId;
    local float *_myPixelSums = _pixelSums;

    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        _myPixelSums[pixel] = 0.0f;
    }

    const int outputRow = localId / gOutputBoardSize;
    const int outputCol = localId % gOutputBoardSize;

    const int minu = gPadZeros ? max( -gHalfFilterSize, -outputRow ) : -gHalfFilterSize;
    const int maxu = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputRow  - evenPadding) : gHalfFilterSize - evenPadding;
    const int minv = gPadZeros ? max( -gHalfFilterSize, -outputCol ) : - gHalfFilterSize;
    const int maxv = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputCol - evenPadding) : gHalfFilterSize - evenPadding;

    for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
        int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int i = 0; i < numUpstreamsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gInputBoardSizeSquared ) {
                _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
            }
        }
        const int filterGlobalOffset = ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared;
        for( int i = 0; i < numFilterPixelsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gFilterSizeSquared ) {
                _filterCube[thisOffset] = filters[filterGlobalOffset + thisOffset];
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

//        for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        int pixel = 0;
            const int virtualLocalId = localId + pixel * workgroupSize;

            float thissum = 0;
            if( virtualLocalId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u + ( gPadZeros ? 0 : gHalfFilterSize );
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v + ( gPadZeros ? 0 : gHalfFilterSize );
                        thissum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
            _myPixelSums[pixel] += thissum;
//            if( globalId == 0 ) results[0] = _upstreamBoard[pixel];
//        }
    }
//    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        int pixel = 0;
        const int virtualLocalId = localId + pixel * workgroupSize;
        if( virtualLocalId < gOutputBoardSizeSquared ) {
            float sum = _myPixelSums[pixel];
            #ifdef BIASED
                sum += biases[outPlane];
            #endif
            // results are organized like [imageid][filterid][row][col]
            int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + virtualLocalId;
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
            // results[resultIndex ] = 123;
            //if( globalId == 0 ) results[0] += 0.000001f + _perPixelSums[0];
        }
//    }
}
#endif
#endif


still haven't got flux working in 'y' direction...

//use Eqn.hasFluxFromCons to allow the calcFlux function to take advantage of this function
<? for side=0,solver.dim-1 do ?>
cons_t fluxFromCons_<?=side?>(cons_t U) {
	prim_t W = primFromCons(U);
	real vj = W.v.s<?=side?>;
	real bj = W.b.s<?=side?>;
	real bSq = real3_lenSq(W.b);
	real bDotV = real3_dot(W.b, W.v);
	real PMag = .5 * bSq;
	real PTotal = W.P + PMag;
	real HTotal = U.ETotal + PTotal;

	cons_t F;
	F.rho = U.m.s<?=side?>;
	F.m = real3_sub(real3_scale(U.m, vj), real3_scale(U.b, bj / mu0));
	F.m.s<?=side?> += PTotal;
	F.b = real3_sub(real3_scale(U.b, vj), real3_scale(W.v, bj));
	F.ETotal = HTotal * vj - bDotV * bj / mu0;
	return F;
}
<? end ?>

// https://arxiv.org/pdf/0804.0402v1.pdf
// based on Athena's version of eigenvectors of derivative of adiabatic MHD flux wrt primitives

typedef struct {
	real rho;
	real3 v;
	real hTotal;
	real3 b;
	real X, Y;
} Roe_t;

//called from calcDT
<? for side=0,solver.dim-1 do ?>
range_t calcCellMinMaxEigenvalues_<?=side?>(
	const global cons_t* U
) {
#if 0
	prim_t W = primFromCons(*U);
	real3 v = W.v;
	real3 b = W.b;

	real bSq = real3_lenSq(b);
	real invRho = 1./W.rho;

	real aSq = heatCapacityRatio * W.P * invRho;
	real CaxSq = b.s<?=side?> * b.s<?=side?> * invRho;
	real CaSq = bSq * invRho;

	real CStarSq = .5 * (CaSq + aSq);
	real sqrtCfsDiscr = sqrt(max(0., CStarSq * CStarSq - aSq * CaxSq));

	real CfSq = CStarSq + sqrtCfsDiscr;
	real CsSq = CStarSq - sqrtCfsDiscr;

	real Cf = sqrt(CfSq);
	real Cs = sqrt(max(CsSq, 0.));
	return (range_t){.min=v.s<?=side?> - Cf, .max=v.s<?=side?> + Cf};
#else
	const real gamma = heatCapacityRatio;
	const real gamma_1 = gamma - 1.;
	const real gamma_2 = gamma - 2.;

	cons_t U_ = *U;
	<? if side == 1 then ?>
	U_.m = _real3(U->m.y, -U->m.x, U->m.z);
	U_.b = _real3(U->b.y, -U->b.x, U->b.z);
	<? elseif side == 2 then ?>
	U_.m = _real3(U->m.z, U->m.y, -U->m.x);
	U_.b = _real3(U->b.z, U->b.y, -U->b.x);
	<? end ?>

	prim_t W = primFromCons(U_);

	real rho = W.rho;
	real3 v = W.v;
	real3 b = W.b;
	real hTotal = .5 * real3_lenSq(W.v) + (W.P * gamma / gamma_1 + real3_lenSq(b)) / W.rho;

	//the rest of this matches calcEigenBasis:

	real _1_rho = 1. / rho;
	real vSq = real3_lenSq(v);
	real bPerpSq = b.y*b.y + b.z*b.z;
	real bStarPerpSq = (gamma_1 - gamma_2) * bPerpSq;
	real CAxSq = b.x*b.x*_1_rho;
	real CASq = CAxSq + bPerpSq * _1_rho;
	real hHydro = hTotal - CASq;
	// hTotal = (EHydro + EMag + P)/rho
	// hHydro = hTotal - CASq, CASq = EMag/rho
	// hHydro = eHydro + P/rho = eKin + eInt + P/rho
	// hHydro - eKin = eInt + P/rho = (1./(gamma-1) + 1) P/rho = gamma/(gamma-1) P/rho
	// a^2 = (gamma-1)(hHydro - eKin) = gamma P / rho
	real aTildeSq = max((gamma_1 * (hHydro - .5 * vSq) - gamma_2), 1e-20);

	real bStarPerpSq_rho = bStarPerpSq * _1_rho;
	real CATildeSq = CAxSq + bStarPerpSq_rho;
	real CStarSq = .5 * (CATildeSq + aTildeSq);
	real CA_a_TildeSqDiff = .5 * (CATildeSq - aTildeSq);
	real sqrtDiscr = sqrt(CA_a_TildeSqDiff * CA_a_TildeSqDiff + aTildeSq * bStarPerpSq_rho);

	real CfSq = CStarSq + sqrtDiscr;
	real Cf = sqrt(CfSq);

	real CsSq = aTildeSq * CAxSq / CfSq;
	real Cs = sqrt(CsSq);

	real lambdaFastMin = v.x - Cf;
	real lambdaFastMax = v.x + Cf;

	return (range_t){
		.min = lambdaFastMin,
		.max = lambdaFastMax,
	};
#endif
}
<? end ?>

//assumes UL and UR are already rotated so the 'x' direction is our flux direction
void calcRoeValues(
	Roe_t* W,
	const cons_t* UL,
	const cons_t* UR
) {
	// should I use bx, or bxL/R, for calculating the PMag at the L and R states?
	prim_t WL = primFromCons(*UL);
	real sqrtRhoL = sqrt(UL->rho);
	real PMagL = .5 * real3_lenSq(UL->b);
	real hTotalL = (UL->ETotal + WL.P + PMagL) / UL->rho;

	prim_t WR = primFromCons(*UR);
	real sqrtRhoR = sqrt(UR->rho);
	real PMagR = .5 * real3_lenSq(UR->b);
	real hTotalR = (UR->ETotal + WR.P + PMagR) / UR->rho;

	real dby = WL.b.y - WR.b.y;
	real dbz = WL.b.z - WR.b.z;

	real invDenom = 1 / (sqrtRhoL + sqrtRhoR);

	W->rho  = sqrtRhoL * sqrtRhoR;
	W->v = real3_scale(real3_add(
		real3_scale(WL.v, sqrtRhoL),
		real3_scale(WR.v, sqrtRhoR)), invDenom);

	W->hTotal = (sqrtRhoL * hTotalL + sqrtRhoR * hTotalR) * invDenom;

	W->b.x = (sqrtRhoL * WL.b.x + sqrtRhoR * WR.b.x) * invDenom;
	// why does athena switch the weights of the by and bz components?
	W->b.y = (sqrtRhoR * WL.b.y + sqrtRhoL * WR.b.y) * invDenom;
	W->b.z = (sqrtRhoR * WL.b.z + sqrtRhoL * WR.b.z) * invDenom;

	W->X = .5 * (dby * dby + dbz * dbz) * invDenom * invDenom;
	W->Y = .5 * (UL->rho + UR->rho) / W->rho;
};

void fill(global real* ptr, int step, real a, real b, real c, real d, real e, real f, real g) {
	ptr[0*step] = a;
	ptr[1*step] = b;
	ptr[2*step] = c;
	ptr[3*step] = d;
	ptr[4*step] = e;
	ptr[5*step] = f;
	ptr[6*step] = g;
}

kernel void calcEigenBasis(
	global real* waveBuf,			//[volume][dim][numWaves]
	global eigen_t* eigenBuf,		//[volume][dim]
	<?= solver.getULRArg ?>
) {
	SETBOUNDS(2,1);
	int indexR = index;

	const real gamma = heatCapacityRatio;
	const real gamma_1 = gamma - 1.;
	const real gamma_2 = gamma - 2.;
	const real gamma_3 = gamma - 3.;

	<? for side=0,solver.dim-1 do ?>{
		const int side = <?=side?>;
		int indexL = index - stepsize[side];

		<?= solver.getULRCode ?>

		//swap the sides with x here, so all the fluxes are in the 'x' direction
		cons_t UL_ = *UL;
		cons_t UR_ = *UR;
		<? if side == 1 then ?>
		UL_.m = _real3(UL->m.y, -UL->m.x, UL->m.z);
		UR_.m = _real3(UR->m.y, -UR->m.x, UR->m.z);
		UL_.b = _real3(UL->b.y, -UL->b.x, UL->b.z);
		UR_.b = _real3(UR->b.y, -UR->b.x, UR->b.z);
		<? elseif side == 2 then ?>
		UL_.m = _real3(UL->m.z, UL->m.y, -UL->m.x);
		UR_.m = _real3(UR->m.z, UR->m.y, -UR->m.x);
		UL_.b = _real3(UL->b.z, UL->b.y, -UL->b.x);
		UR_.b = _real3(UR->b.z, UR->b.y, -UR->b.x);
		<? end ?>

		Roe_t roe;
		calcRoeValues(&roe, &UL_, &UR_);

		real rho = roe.rho;
		real3 v = roe.v;
		real hTotal = roe.hTotal;
		real3 b = roe.b;
		real X = roe.X;
		real Y = roe.Y;

		real _1_rho = 1. / rho;
		real vSq = real3_lenSq(v);
		real bDotV = real3_dot(b,v);
		real bPerpSq = b.y*b.y + b.z*b.z;
		real bStarPerpSq = (gamma_1 - gamma_2 * Y) * bPerpSq;
		real CAxSq = b.x*b.x*_1_rho;
		real CASq = CAxSq + bPerpSq * _1_rho;
		real hHydro = hTotal - CASq;
		// hTotal = (EHydro + EMag + P)/rho
		// hHydro = hTotal - CASq, CASq = EMag/rho
		// hHydro = eHydro + P/rho = eKin + eInt + P/rho
		// hHydro - eKin = eInt + P/rho = (1./(gamma-1) + 1) P/rho = gamma/(gamma-1) P/rho
		// a^2 = (gamma-1)(hHydro - eKin) = gamma P / rho
		real aTildeSq = max((gamma_1 * (hHydro - .5 * vSq) - gamma_2 * X), 1e-20);

		real bStarPerpSq_rho = bStarPerpSq * _1_rho;
		real CATildeSq = CAxSq + bStarPerpSq_rho;
		real CStarSq = .5 * (CATildeSq + aTildeSq);
		real CA_a_TildeSqDiff = .5 * (CATildeSq - aTildeSq);
		real sqrtDiscr = sqrt(CA_a_TildeSqDiff * CA_a_TildeSqDiff + aTildeSq * bStarPerpSq_rho);

		real CfSq = CStarSq + sqrtDiscr;
		real Cf = sqrt(CfSq);

		real CsSq = aTildeSq * CAxSq / CfSq;
		real Cs = sqrt(CsSq);

		real bPerpLen = sqrt(bPerpSq);
		real bStarPerpLen = sqrt(bStarPerpSq);
		real betaY, betaZ;
		if (bPerpLen == 0) {
			betaY = 1;
			betaZ = 0;
		} else {
			betaY = b.y / bPerpLen;
			betaZ = b.z / bPerpLen;
		}
		real betaStarY = betaY / sqrt(gamma_1 - gamma_2*Y);
		real betaStarZ = betaZ / sqrt(gamma_1 - gamma_2*Y);
		real betaStarSq = betaStarY*betaStarY + betaStarZ*betaStarZ;
		real vDotBeta = v.y*betaStarY + v.z*betaStarZ;

		real alphaF, alphaS;
		if (CfSq - CsSq == 0) {
			alphaF = 1;
			alphaS = 0;
		} else if (aTildeSq - CsSq <= 0) {
			alphaF = 0;
			alphaS = 1;
		} else if (CfSq - aTildeSq <= 0) {
			alphaF = 1;
			alphaS = 0;
		} else {
			alphaF = sqrt((aTildeSq - CsSq) / (CfSq - CsSq));
			alphaS = sqrt((CfSq - aTildeSq) / (CfSq - CsSq));
		}

		real sqrtRho = sqrt(rho);
		real _1_sqrtRho = 1./sqrtRho;
		real sbx = b.x >= 0 ? 1 : -1;
		real aTilde = sqrt(aTildeSq);
		real Qf = Cf*alphaF*sbx;
		real Qs = Cs*alphaS*sbx;
		real Af = aTilde*alphaF*_1_sqrtRho;
		real As = aTilde*alphaS*_1_sqrtRho;
		real Afpbb = Af*bStarPerpLen*betaStarSq;
		real Aspbb = As*bStarPerpLen*betaStarSq;

		real CAx = sqrt(CAxSq);

		int intindex = side + dim * index;
		global real* wave = waveBuf + numWaves * intindex;

		real lambdaFastMin = v.x - Cf;
		real lambdaSlowMin = v.x - Cs;
		real lambdaSlowMax = v.x + Cs;
		real lambdaFastMax = v.x + Cf;
		wave[0] = lambdaFastMin;
		wave[1] = v.x - CAx;
		wave[2] = lambdaSlowMin;
		wave[3] = v.x;
		wave[4] = lambdaSlowMax;
		wave[5] = v.x + CAx;
		wave[6] = lambdaFastMax;

		global eigen_t* eig = eigenBuf + intindex;

		// dF/dU
		<? if solver.checkFluxError then ?>
		global real* A = eig->A;
		fill(A+0,7,	0, 												1,											0,								0,								0,			0,								0								);
		fill(A+1,7,	-v.x*v.x + .5*gamma_1*vSq - gamma_2*X,			-gamma_3*v.x,								-gamma_1*v.y,					-gamma_1*v.z,					gamma_1,	-gamma_2*Y*b.y,					-gamma_2*Y*b.z					);
		fill(A+2,7,	-v.x*v.y,										v.y,										v.x,							0, 								0,			-b.x,							0								);
		fill(A+3,7,	-v.x*v.z,										v.z,										0,								v.x, 							0,			0,								-b.x							);
		fill(A+4,7,	v.x*(.5*gamma_1*vSq - hTotal) + b.x*bDotV/rho,	-gamma_1*v.x*v.x + hTotal - b.x*b.x/rho,	-gamma_1*v.x*v.y - b.x*b.y/rho,	-gamma_1*v.x*v.z - b.x*b.z/rho,	gamma*v.x,	-gamma_2*Y*b.y*v.x - b.x*v.y,	-gamma_2*Y*b.z*v.x - b.x*v.z	);
		fill(A+5,7,	(b.x*v.y - b.y*v.x)/rho,						b.y/rho,									-b.x/rho,						0, 								0,			v.x,							0								);
		fill(A+6,7,	(b.x*v.z - b.z*v.x)/rho,						b.z/rho,									0,								-b.x/rho, 						0,			0,								v.x								);
		<? end ?>

		// right eigenvectors
		real qa3 = alphaF*v.y;
		real qb3 = alphaS*v.y;
		real qc3 = Qs*betaStarY;
		real qd3 = Qf*betaStarY;
		real qa4 = alphaF*v.z;
		real qb4 = alphaS*v.z;
		real qc4 = Qs*betaStarZ;
		real qd4 = Qf*betaStarZ;
		real r52 = -(v.y*betaZ - v.z*betaY);
		real r61 = As*betaStarY;
		real r62 = -betaZ*sbx*_1_sqrtRho;
		real r63 = -Af*betaStarY;
		real r71 = As*betaStarZ;
		real r72 = betaY*sbx*_1_sqrtRho;
		real r73 = -Af*betaStarZ;
		//rows
		global real* evR = eig->evR;
		fill(evR+0,7, 	alphaF, 										0, 		alphaS, 										1, 							alphaS, 										0, 		alphaF											);
		fill(evR+1,7, 	alphaF*lambdaFastMin, 							0, 		alphaS*lambdaSlowMin, 							v.x, 	 	 	 	 	 	alphaS*lambdaSlowMax, 							0, 		alphaF*lambdaFastMax							);
		fill(evR+2,7, 	qa3 + qc3, 										-betaZ,	qb3 - qd3, 										v.y, 	 	 	 	 	 	qb3 + qd3, 										betaZ, 	qa3 - qc3										);
		fill(evR+3,7, 	qa4 + qc4, 										betaY,	qb4 - qd4, 										v.z, 	 	 	 	 	 	qb4 + qd4, 										-betaY, qa4 - qc4										);
		fill(evR+4,7, 	alphaF*(hHydro - v.x*Cf) + Qs*vDotBeta + Aspbb, r52,	alphaS*(hHydro - v.x*Cs) - Qf*vDotBeta - Afpbb, .5*vSq + gamma_2*X/gamma_1, alphaS*(hHydro + v.x*Cs) + Qf*vDotBeta - Afpbb, -r52, 	alphaF*(hHydro + v.x*Cf) - Qs*vDotBeta + Aspbb	);
		fill(evR+5,7, 	r61, 											r62, 	r63, 											0, 							r63, 											r62, 	r61												);
		fill(evR+6,7,	r71, 											r72, 	r73, 											0, 							r73, 											r72, 	r71												);

		// left eigenvectors
		real norm = .5/aTildeSq;
		real Cff = norm*alphaF*Cf;
		real Css = norm*alphaS*Cs;
		Qf = Qf * norm;
		Qs = Qs * norm;
		real AHatF = norm*Af*rho;
		real AHatS = norm*As*rho;
		real afpb = norm*Af*bStarPerpLen;
		real aspb = norm*As*bStarPerpLen;

		norm = norm * gamma_1;
		alphaF = alphaF * norm;
		alphaS = alphaS * norm;
		real QStarY = betaStarY/betaStarSq;
		real QStarZ = betaStarZ/betaStarSq;
		real vqstr = (v.y*QStarY + v.z*QStarZ);
		norm = norm * 2.;

#if 1	//computing beforehand
		real l16 = AHatS*QStarY - alphaF*b.y;
		real l17 = AHatS*QStarZ - alphaF*b.z;
		real l21 = .5*(v.y*betaZ - v.z*betaY);
		real l23 = .5*betaZ;
		real l24 = .5*betaY;
		real l26 = -.5*sqrtRho*betaZ*sbx;
		real l27 = .5*sqrtRho*betaY*sbx;
		real l36 = -AHatF*QStarY - alphaS*b.y;
		real l37 = -AHatF*QStarZ - alphaS*b.z;
		//rows
		global real* evL = eig->evL;
		fill(evL+0,7,	alphaF*(vSq-hHydro) + Cff*(Cf+v.x) - Qs*vqstr - aspb, -alphaF*v.x - Cff, -alphaF*v.y + Qs*QStarY, -alphaF*v.z + Qs*QStarZ, alphaF, l16, l17);
		fill(evL+1,7,	l21, 0, l23, l24, 0, l26, l27);
		fill(evL+2,7,	alphaS*(vSq-hHydro) + Css*(Cs+v.x) + Qf*vqstr + afpb, -alphaS*v.x - Css, -alphaS*v.y - Qf*QStarY, -alphaS*v.z - Qf*QStarZ, alphaS, l36, l37);
		fill(evL+3,7,	1. - norm*(.5*vSq - gamma_2*X/gamma_1) , norm*v.x, norm*v.y, norm*v.z, -norm, norm*b.y, norm*b.z);
		fill(evL+4,7,	alphaS*(vSq-hHydro) + Css*(Cs-v.x) - Qf*vqstr + afpb, -alphaS*v.x + Css, -alphaS*v.y + Qf*QStarY, -alphaS*v.z + Qf*QStarZ, alphaS, l36, l37);
		fill(evL+5,7,	-l21, 0, -l23, -l24, 0, l26, l27);
		fill(evL+6,7,	alphaF*(vSq-hHydro) + Cff*(Cf-v.x) + Qs*vqstr - aspb, -alphaF*v.x + Cff, -alphaF*v.y - Qs*QStarY, -alphaF*v.z - Qs*QStarZ, alphaF, l16, l17);
#else	//self-referencing evL
		//rows
		global real* evL = eig->evL;
		fill(evL+0,7,	alphaF*(vSq-hHydro) + Cff*(Cf+v.x) - Qs*vqstr - aspb, -alphaF*v.x - Cff, -alphaF*v.y + Qs*QStarY, -alphaF*v.z + Qs*QStarZ, alphaF, AHatS*QStarY - alphaF*b.y, AHatS*QStarZ - alphaF*b.z);
		fill(evL+1,7,	.5*(v.y*betaZ - v.z*betaY), 0, -.5*betaZ, .5*betaY, 0, -.5*sqrtRho*betaZ*sbx, .5*sqrtRho*betaY*sbx);
		fill(evL+2,7,	alphaS*(vSq-hHydro) + Css*(Cs+v.x) + Qf*vqstr + afpb, -alphaS*v.x - Css, -alphaS*v.y - Qf*QStarY, -alphaS*v.z - Qf*QStarZ, alphaS, -AHatF*QStarY - alphaS*b.y, -AHatF*QStarZ - alphaS*b.z);
		fill(evL+3,7,	1. - norm*(.5*vSq - gamma_2*X/gamma_1) , norm*v.x, norm*v.y, norm*v.z, -norm, norm*b.y, norm*b.z);
		fill(evL+4,7,	alphaS*(vSq-hHydro) + Css*(Cs-v.x) - Qf*vqstr + afpb, -alphaS*v.x + Css, -alphaS*v.y + Qf*QStarY, -alphaS*v.z + Qf*QStarZ, alphaS, evL[2+7*5], evL[2+7*6]);
		fill(evL+5,7,	-evL[1+7*0], 0, -evL[1+7*2], -evL[1+7*3], 0, evL[1+7*5], evL[1+7*6]);
		fill(evL+6,7,	alphaF*(vSq-hHydro) + Cff*(Cf-v.x) + Qs*vqstr - aspb, -alphaF*v.x + Cff, -alphaF*v.y - Qs*QStarY, -alphaF*v.z - Qs*QStarZ, alphaF, evL[0+7*5], evL[0+7*6]);
#endif
	}<? end ?>
}

<?
-- swap x and side, and get rid of bx
local function _7to8code(side)
	return ({
		-- rho mx my mz ETotal (bx) by bz
		[0] = [[ real x[7] = { x_[0], x_[1], x_[2], x_[3], x_[4], x_[6], x_[7] }; ]],
		-- rho my -mx mz ETotal (by) -bx bz
		[1] = [[ real x[7] = { x_[0], x_[2], -x_[1], x_[3], x_[4], -x_[5], x_[7] }; ]],
		-- rho mz my -mx ETotal (bz) by -bx
		[2] = [[ real x[7] = { x_[0], x_[3], x_[2], -x_[1], x_[4], x_[6], -x_[5] }; ]],
	})[side]
end

-- swap x and side and insert 0 for bx
local function _8to7code(side)
	return ({
		[0] = [[ y[7] = y[6]; y[6] = y[5]; y[5] = 0; ]],
		[1] = [[ real tmp = y[1]; y[1] = -y[2]; y[2] = tmp; y[5] = -y[5]; y[7] = y[6]; y[6] = 0; ]],
		[2] = [[ real tmp = y[1]; y[1] = -y[3]; y[3] = tmp; y[7] = 0; tmp = y[5]; y[5] = -y[6]; y[6] = tmp; ]],
	})[side]
end

for _,addr0 in ipairs{'', 'global'} do
	for _,addr1 in ipairs{'', 'global'} do
		for _,addr2 in ipairs{'', 'global'} do
			for side=0,2 do ?>
void eigen_leftTransform_<?=side?>_<?=addr0?>_<?=addr1?>_<?=addr2?>(
	<?=addr0?> real* y,
	<?=addr1?> const eigen_t* eig,
	<?=addr2?> const real* x_
) {
	<?=_7to8code(side)?>

	<?=addr1?> const real* A = eig->evL;
	for (int i = 0; i < 7; ++i) {
		real sum = 0;
		for (int j = 0; j < 7; ++j) {
			sum += A[i+7*j] * x[j];
		}
		y[i] = sum;
	}
}

void eigen_rightTransform_<?=side?>_<?=addr0?>_<?=addr1?>_<?=addr2?>(
	<?=addr0?> real* y,
	<?=addr1?> const eigen_t* eig,
	<?=addr2?> const real* x
) {
	<?=addr1?> const real* A = eig->evR;
	for (int i = 0; i < 7; ++i) {
		real sum = 0;
		for (int j = 0; j < 7; ++j) {
			sum += A[i+7*j] * x[j];
		}
		y[i] = sum;
	}

	<?=_8to7code(side)?>
}

<? 				if solver.checkFluxError then ?>
void eigen_fluxTransform_<?=side?>_<?=addr0?>_<?=addr1?>_<?=addr2?>(
	<?=addr0?> real* y,
	<?=addr1?> const eigen_t* eig,
	<?=addr2?> const real* x_
) {
	<?=_7to8code(side)?>

	<?=addr1?> const real* A = eig->A;
	for (int i = 0; i < 7; ++i) {
		real sum = 0;
		for (int j = 0; j < 7; ++j) {
			sum += A[i+7*j] * x[j];
		}
		y[i] = sum;
	}

	<?=_8to7code(side)?>
}
<?				end
			end
		end
	end
end ?>

//use Eqn.hasFluxFromCons to allow the calcFlux function to take advantage of this function
<? for side=0,solver.dim-1 do ?>
cons_t fluxFromCons_<?=side?>(cons_t U) {
	prim_t W = primFromCons(U);
	real vj = W.v.s<?=side?>;
	real bj = W.b.s<?=side?>;
	real bSq = real3_lenSq(W.b);
	real bDotV = real3_dot(W.b, W.v);
	real PMag = .5 * bSq;
	real PTotal = W.P + PMag;
	real HTotal = U.ETotal + PTotal;

	cons_t F;
	F.rho = U.m.s<?=side?>;
	F.m = real3_sub(real3_scale(U.m, vj), real3_scale(U.b, bj / mu0));
	F.m.s<?=side?> += PTotal;
	F.b = real3_sub(real3_scale(U.b, vj), real3_scale(W.v, bj));
	F.ETotal = HTotal * vj - bDotV * bj / mu0;
	return F;
}
<? end ?>

// https://arxiv.org/pdf/0804.0402v1.pdf
// based on Athena's version of eigenvectors of derivative of adiabatic MHD flux wrt primitives

typedef struct {
	real rho;
	real3 v;
	real hTotal;
	real3 b;
	real X, Y;
} Roe_t;

//called from calcDT
<? for side=0,solver.dim-1 do ?>
range_t calcCellMinMaxEigenvalues_<?=side?>(
	const global cons_t* U
) {
#if 0
	prim_t W = primFromCons(*U);
	real3 v = W.v;
	real3 b = W.b;

	real bSq = real3_lenSq(b);
	real invRho = 1./W.rho;

	real aSq = heatCapacityRatio * W.P * invRho;
	real CaxSq = b.s<?=side?> * b.s<?=side?> * invRho;
	real CaSq = bSq * invRho;

	real CStarSq = .5 * (CaSq + aSq);
	real sqrtCfsDiscr = sqrt(max(0., CStarSq * CStarSq - aSq * CaxSq));

	real CfSq = CStarSq + sqrtCfsDiscr;
	real CsSq = CStarSq - sqrtCfsDiscr;

	real Cf = sqrt(CfSq);
	real Cs = sqrt(max(CsSq, 0.));
	return (range_t){.min=v.s<?=side?> - Cf, .max=v.s<?=side?> + Cf};
#else
	const real gamma = heatCapacityRatio;
	const real gamma_1 = gamma - 1.;
	const real gamma_2 = gamma - 2.;

	prim_t W = primFromCons(*U);

	real rho = W.rho;
	real3 v = W.v;
	real hTotal = .5 * real3_lenSq(W.v) + (W.P * gamma / gamma_1 + real3_lenSq(W.b)) / W.rho;
	real3 b = W.b;

	//the rest of this matches calcEigenBasis:

	real _1_rho = 1. / rho;
	real vSq = real3_lenSq(v);
	real bPerpSq = b.y*b.y + b.z*b.z;
	real bStarPerpSq = (gamma_1 - gamma_2) * bPerpSq;
	real CAxSq = b.x*b.x*_1_rho;
	real CASq = CAxSq + bPerpSq * _1_rho;
	real hHydro = hTotal - CASq;
	// hTotal = (EHydro + EMag + P)/rho
	// hHydro = hTotal - CASq, CASq = EMag/rho
	// hHydro = eHydro + P/rho = eKin + eInt + P/rho
	// hHydro - eKin = eInt + P/rho = (1./(gamma-1) + 1) P/rho = gamma/(gamma-1) P/rho
	// a^2 = (gamma-1)(hHydro - eKin) = gamma P / rho
	real aTildeSq = max((gamma_1 * (hHydro - .5 * vSq) - gamma_2), 1e-20);

	real bStarPerpSq_rho = bStarPerpSq * _1_rho;
	real CATildeSq = CAxSq + bStarPerpSq_rho;
	real CStarSq = .5 * (CATildeSq + aTildeSq);
	real CA_a_TildeSqDiff = .5 * (CATildeSq - aTildeSq);
	real sqrtDiscr = sqrt(CA_a_TildeSqDiff * CA_a_TildeSqDiff + aTildeSq * bStarPerpSq_rho);

	real CfSq = CStarSq + sqrtDiscr;
	real Cf = sqrt(CfSq);

	real CsSq = aTildeSq * CAxSq / CfSq;
	real Cs = sqrt(CsSq);

	real lambdaFastMin = v.x - Cf;
	real lambdaFastMax = v.x + Cf;

	return (range_t){
		.min = lambdaFastMin,
		.max = lambdaFastMax,
	};
#endif
}
<? end ?>

void calcRoeValues(
	Roe_t* W,
	const cons_t* UL,
	const cons_t* UR
) {
	// should I use bx, or bxL/R, for calculating the PMag at the L and R states?
	prim_t WL = primFromCons(*UL);
	real sqrtRhoL = sqrt(UL->rho);
	real PMagL = .5 * real3_lenSq(UL->b);
	real hTotalL = (UL->ETotal + WL.P + PMagL) / UL->rho;

	prim_t WR = primFromCons(*UR);
	real sqrtRhoR = sqrt(UR->rho);
	real PMagR = .5 * real3_lenSq(UR->b);
	real hTotalR = (UR->ETotal + WR.P + PMagR) / UR->rho;

	real dby = WL.b.y - WR.b.y;
	real dbz = WL.b.z - WR.b.z;

	real invDenom = 1 / (sqrtRhoL + sqrtRhoR);

	W->rho  = sqrtRhoL * sqrtRhoR;
	W->v = real3_scale(real3_add(
		real3_scale(WL.v, sqrtRhoL),
		real3_scale(WR.v, sqrtRhoR)), invDenom);

	W->hTotal = (sqrtRhoL * hTotalL + sqrtRhoR * hTotalR) * invDenom;

	W->b.x = (sqrtRhoL * WL.b.x + sqrtRhoR * WR.b.x) * invDenom;
	// why does athena switch the weights of the by and bz components?
	W->b.y = (sqrtRhoR * WL.b.y + sqrtRhoL * WR.b.y) * invDenom;
	W->b.z = (sqrtRhoR * WL.b.z + sqrtRhoL * WR.b.z) * invDenom;

	W->X = .5 * (dby * dby + dbz * dbz) * invDenom * invDenom;
	W->Y = .5 * (UL->rho + UR->rho) / W->rho;
};

void fill(global real* ptr, int step, real a, real b, real c, real d, real e, real f, real g) {
	ptr[0*step] = a;
	ptr[1*step] = b;
	ptr[2*step] = c;
	ptr[3*step] = d;
	ptr[4*step] = e;
	ptr[5*step] = f;
	ptr[6*step] = g;
}

kernel void calcEigenBasis(
	global real* waveBuf,			//[volume][dim][numWaves]
	global eigen_t* eigenBuf,		//[volume][dim]
	<?= solver.getULRArg ?>
) {
	SETBOUNDS(2,1);
	int indexR = index;

	const real gamma = heatCapacityRatio;
	const real gamma_1 = gamma - 1.;
	const real gamma_2 = gamma - 2.;
	const real gamma_3 = gamma - 3.;

	<? for side=0,solver.dim-1 do ?>{
		const int side = <?=side?>;
		int indexL = index - stepsize[side];

		<?= solver.getULRCode ?>

		//swap the sides with x here, so all the fluxes are in the 'x' direction
		cons_t UL_ = *UL;
		cons_t UR_ = *UR;
		real tmp;
		<? if side == 1 then ?>
		UL_.v = _real3(UL.v.y, -UL.v.x, UL.v.z);
		UR_.v = _real3(UR.v.y, -UR.v.x, UR.v.z);
		UL_.b = _real3(UL.b.y, -UL.b.x, UL.b.z);
		UR_.b = _real3(UR.b.y, -UR.b.x, UR.b.z);
		<? elseif side == 2 then ?>
		UL_.v = _real3(UL.v.z, UL.v.y, -UL.v.x);
		UR_.v = _real3(UR.v.z, UR.v.y, -UR.v.x);
		UL_.b = _real3(UL.b.z, UL.b.y, -UL.b.x);
		UR_.b = _real3(UR.b.z, UR.b.y, -UR.b.x);
		<? end ?>

		Roe_t roe;
		calcRoeValues(&roe, &UL_, &UR_);

		real rho = roe.rho;
		real3 v = roe.v;
		real hTotal = roe.hTotal;
		real3 b = roe.b;
		real X = roe.X;
		real Y = roe.Y;

		real _1_rho = 1. / rho;
		real vSq = real3_lenSq(v);
		real bDotV = real3_dot(b,v);
		real bPerpSq = b.y*b.y + b.z*b.z;
		real bStarPerpSq = (gamma_1 - gamma_2 * Y) * bPerpSq;
		real CAxSq = b.x*b.x*_1_rho;
		real CASq = CAxSq + bPerpSq * _1_rho;
		real hHydro = hTotal - CASq;
		// hTotal = (EHydro + EMag + P)/rho
		// hHydro = hTotal - CASq, CASq = EMag/rho
		// hHydro = eHydro + P/rho = eKin + eInt + P/rho
		// hHydro - eKin = eInt + P/rho = (1./(gamma-1) + 1) P/rho = gamma/(gamma-1) P/rho
		// a^2 = (gamma-1)(hHydro - eKin) = gamma P / rho
		real aTildeSq = max((gamma_1 * (hHydro - .5 * vSq) - gamma_2 * X), 1e-20);

		real bStarPerpSq_rho = bStarPerpSq * _1_rho;
		real CATildeSq = CAxSq + bStarPerpSq_rho;
		real CStarSq = .5 * (CATildeSq + aTildeSq);
		real CA_a_TildeSqDiff = .5 * (CATildeSq - aTildeSq);
		real sqrtDiscr = sqrt(CA_a_TildeSqDiff * CA_a_TildeSqDiff + aTildeSq * bStarPerpSq_rho);

		real CfSq = CStarSq + sqrtDiscr;
		real Cf = sqrt(CfSq);

		real CsSq = aTildeSq * CAxSq / CfSq;
		real Cs = sqrt(CsSq);

		real bPerpLen = sqrt(bPerpSq);
		real bStarPerpLen = sqrt(bStarPerpSq);
		real betaY, betaZ;
		if (bPerpLen == 0) {
			betaY = 1;
			betaZ = 0;
		} else {
			betaY = b.y / bPerpLen;
			betaZ = b.z / bPerpLen;
		}
		real betaStarY = betaY / sqrt(gamma_1 - gamma_2*Y);
		real betaStarZ = betaZ / sqrt(gamma_1 - gamma_2*Y);
		real betaStarSq = betaStarY*betaStarY + betaStarZ*betaStarZ;
		real vDotBeta = v.y*betaStarY + v.z*betaStarZ;

		real alphaF, alphaS;
		if (CfSq - CsSq == 0) {
			alphaF = 1;
			alphaS = 0;
		} else if (aTildeSq - CsSq <= 0) {
			alphaF = 0;
			alphaS = 1;
		} else if (CfSq - aTildeSq <= 0) {
			alphaF = 1;
			alphaS = 0;
		} else {
			alphaF = sqrt((aTildeSq - CsSq) / (CfSq - CsSq));
			alphaS = sqrt((CfSq - aTildeSq) / (CfSq - CsSq));
		}

		real sqrtRho = sqrt(rho);
		real _1_sqrtRho = 1./sqrtRho;
		real sbx = b.x >= 0 ? 1 : -1;
		real aTilde = sqrt(aTildeSq);
		real Qf = Cf*alphaF*sbx;
		real Qs = Cs*alphaS*sbx;
		real Af = aTilde*alphaF*_1_sqrtRho;
		real As = aTilde*alphaS*_1_sqrtRho;
		real Afpbb = Af*bStarPerpLen*betaStarSq;
		real Aspbb = As*bStarPerpLen*betaStarSq;

		real CAx = sqrt(CAxSq);

		int intindex = side + dim * index;
		global real* wave = waveBuf + numWaves * intindex;

		real lambdaFastMin = v.x - Cf;
		real lambdaSlowMin = v.x - Cs;
		real lambdaSlowMax = v.x + Cs;
		real lambdaFastMax = v.x + Cf;
		wave[0] = lambdaFastMin;
		wave[1] = v.x - CAx;
		wave[2] = lambdaSlowMin;
		wave[3] = v.x;
		wave[4] = lambdaSlowMax;
		wave[5] = v.x + CAx;
		wave[6] = lambdaFastMax;

		global eigen_t* eig = eigenBuf + intindex;

		// dF/dU
		<? if solver.checkFluxError then ?>
		global real* A = eig->A;
		fill(A+0,7,	0, 												1,											0,								0,								0,			0,								0								);
		fill(A+1,7,	-v.x*v.x + .5*gamma_1*vSq - gamma_2*X,			-gamma_3*v.x,								-gamma_1*v.y,					-gamma_1*v.z,					gamma_1,	-gamma_2*Y*b.y,					-gamma_2*Y*b.z					);
		fill(A+2,7,	-v.x*v.y,										v.y,										v.x,							0, 								0,			-b.x,							0								);
		fill(A+3,7,	-v.x*v.z,										v.z,										0,								v.x, 							0,			0,								-b.x							);
		fill(A+4,7,	v.x*(.5*gamma_1*vSq - hTotal) + b.x*bDotV/rho,	-gamma_1*v.x*v.x + hTotal - b.x*b.x/rho,	-gamma_1*v.x*v.y - b.x*b.y/rho,	-gamma_1*v.x*v.z - b.x*b.z/rho,	gamma*v.x,	-gamma_2*Y*b.y*v.x - b.x*v.y,	-gamma_2*Y*b.z*v.x - b.x*v.z	);
		fill(A+5,7,	(b.x*v.y - b.y*v.x)/rho,						b.y/rho,									-b.x/rho,						0, 								0,			v.x,							0								);
		fill(A+6,7,	(b.x*v.z - b.z*v.x)/rho,						b.z/rho,									0,								-b.x/rho, 						0,			0,								v.x								);
		<? end ?>

		// right eigenvectors
		real qa3 = alphaF*v.y;
		real qb3 = alphaS*v.y;
		real qc3 = Qs*betaStarY;
		real qd3 = Qf*betaStarY;
		real qa4 = alphaF*v.z;
		real qb4 = alphaS*v.z;
		real qc4 = Qs*betaStarZ;
		real qd4 = Qf*betaStarZ;
		real r52 = -(v.y*betaZ - v.z*betaY);
		real r61 = As*betaStarY;
		real r62 = -betaZ*sbx*_1_sqrtRho;
		real r63 = -Af*betaStarY;
		real r71 = As*betaStarZ;
		real r72 = betaY*sbx*_1_sqrtRho;
		real r73 = -Af*betaStarZ;
		//rows
		global real* evR = eig->evR;
		fill(evR+0,7, 	alphaF, 										0, 		alphaS, 										1, 							alphaS, 										0, 		alphaF											);
		fill(evR+1,7, 	alphaF*lambdaFastMin, 							0, 		alphaS*lambdaSlowMin, 							v.x, 	 	 	 	 	 	alphaS*lambdaSlowMax, 							0, 		alphaF*lambdaFastMax							);
		fill(evR+2,7, 	qa3 + qc3, 										-betaZ,	qb3 - qd3, 										v.y, 	 	 	 	 	 	qb3 + qd3, 										betaZ, 	qa3 - qc3										);
		fill(evR+3,7, 	qa4 + qc4, 										betaY,	qb4 - qd4, 										v.z, 	 	 	 	 	 	qb4 + qd4, 										-betaY, qa4 - qc4										);
		fill(evR+4,7, 	alphaF*(hHydro - v.x*Cf) + Qs*vDotBeta + Aspbb, r52,	alphaS*(hHydro - v.x*Cs) - Qf*vDotBeta - Afpbb, .5*vSq + gamma_2*X/gamma_1, alphaS*(hHydro + v.x*Cs) + Qf*vDotBeta - Afpbb, -r52, 	alphaF*(hHydro + v.x*Cf) - Qs*vDotBeta + Aspbb	);
		fill(evR+5,7, 	r61, 											r62, 	r63, 											0, 							r63, 											r62, 	r61												);
		fill(evR+6,7,	r71, 											r72, 	r73, 											0, 							r73, 											r72, 	r71												);

		// left eigenvectors
		real norm = .5/aTildeSq;
		real Cff = norm*alphaF*Cf;
		real Css = norm*alphaS*Cs;
		Qf = Qf * norm;
		Qs = Qs * norm;
		real AHatF = norm*Af*rho;
		real AHatS = norm*As*rho;
		real afpb = norm*Af*bStarPerpLen;
		real aspb = norm*As*bStarPerpLen;

		norm = norm * gamma_1;
		alphaF = alphaF * norm;
		alphaS = alphaS * norm;
		real QStarY = betaStarY/betaStarSq;
		real QStarZ = betaStarZ/betaStarSq;
		real vqstr = (v.y*QStarY + v.z*QStarZ);
		norm = norm * 2.;

#if 1	//computing beforehand
		real l16 = AHatS*QStarY - alphaF*b.y;
		real l17 = AHatS*QStarZ - alphaF*b.z;
		real l21 = .5*(v.y*betaZ - v.z*betaY);
		real l23 = .5*betaZ;
		real l24 = .5*betaY;
		real l26 = -.5*sqrtRho*betaZ*sbx;
		real l27 = .5*sqrtRho*betaY*sbx;
		real l36 = -AHatF*QStarY - alphaS*b.y;
		real l37 = -AHatF*QStarZ - alphaS*b.z;
		//rows
		global real* evL = eig->evL;
		fill(evL+0,7,	alphaF*(vSq-hHydro) + Cff*(Cf+v.x) - Qs*vqstr - aspb, -alphaF*v.x - Cff, -alphaF*v.y + Qs*QStarY, -alphaF*v.z + Qs*QStarZ, alphaF, l16, l17);
		fill(evL+1,7,	l21, 0, l23, l24, 0, l26, l27);
		fill(evL+2,7,	alphaS*(vSq-hHydro) + Css*(Cs+v.x) + Qf*vqstr + afpb, -alphaS*v.x - Css, -alphaS*v.y - Qf*QStarY, -alphaS*v.z - Qf*QStarZ, alphaS, l36, l37);
		fill(evL+3,7,	1. - norm*(.5*vSq - gamma_2*X/gamma_1) , norm*v.x, norm*v.y, norm*v.z, -norm, norm*b.y, norm*b.z);
		fill(evL+4,7,	alphaS*(vSq-hHydro) + Css*(Cs-v.x) - Qf*vqstr + afpb, -alphaS*v.x + Css, -alphaS*v.y + Qf*QStarY, -alphaS*v.z + Qf*QStarZ, alphaS, l36, l37);
		fill(evL+5,7,	-l21, 0, -l23, -l24, 0, l26, l27);
		fill(evL+6,7,	alphaF*(vSq-hHydro) + Cff*(Cf-v.x) + Qs*vqstr - aspb, -alphaF*v.x + Cff, -alphaF*v.y - Qs*QStarY, -alphaF*v.z - Qs*QStarZ, alphaF, l16, l17);
#else	//self-referencing evL
		//rows
		global real* evL = eig->evL;
		fill(evL+0,7,	alphaF*(vSq-hHydro) + Cff*(Cf+v.x) - Qs*vqstr - aspb, -alphaF*v.x - Cff, -alphaF*v.y + Qs*QStarY, -alphaF*v.z + Qs*QStarZ, alphaF, AHatS*QStarY - alphaF*b.y, AHatS*QStarZ - alphaF*b.z);
		fill(evL+1,7,	.5*(v.y*betaZ - v.z*betaY), 0, -.5*betaZ, .5*betaY, 0, -.5*sqrtRho*betaZ*sbx, .5*sqrtRho*betaY*sbx);
		fill(evL+2,7,	alphaS*(vSq-hHydro) + Css*(Cs+v.x) + Qf*vqstr + afpb, -alphaS*v.x - Css, -alphaS*v.y - Qf*QStarY, -alphaS*v.z - Qf*QStarZ, alphaS, -AHatF*QStarY - alphaS*b.y, -AHatF*QStarZ - alphaS*b.z);
		fill(evL+3,7,	1. - norm*(.5*vSq - gamma_2*X/gamma_1) , norm*v.x, norm*v.y, norm*v.z, -norm, norm*b.y, norm*b.z);
		fill(evL+4,7,	alphaS*(vSq-hHydro) + Css*(Cs-v.x) - Qf*vqstr + afpb, -alphaS*v.x + Css, -alphaS*v.y + Qf*QStarY, -alphaS*v.z + Qf*QStarZ, alphaS, evL[2+7*5], evL[2+7*6]);
		fill(evL+5,7,	-evL[1+7*0], 0, -evL[1+7*2], -evL[1+7*3], 0, evL[1+7*5], evL[1+7*6]);
		fill(evL+6,7,	alphaF*(vSq-hHydro) + Cff*(Cf-v.x) + Qs*vqstr - aspb, -alphaF*v.x + Cff, -alphaF*v.y - Qs*QStarY, -alphaF*v.z - Qs*QStarZ, alphaF, evL[0+7*5], evL[0+7*6]);
#endif
	}<? end ?>
}

<?
for _,addr0 in ipairs{'', 'global'} do
	for _,addr1 in ipairs{'', 'global'} do
		for _,addr2 in ipairs{'', 'global'} do
			for side=0,2 do ?>
void eigen_leftTransform_<?=side?>_<?=addr0?>_<?=addr1?>_<?=addr2?>(
	<?=addr0?> real* y,
	<?=addr1?> const eigen_t* eig,
	<?=addr2?> const real* x_
) {
	//swap x and side, and get rid of bx
	real x[7] = {
	<? if side == 0 then ?>
		// rho mx my mz ETotal _ by bz
		x_[0], x_[1], x_[2], x_[3], x_[4], x_[6], x_[7]
	<? elseif side == 1 then ?>
		// rho my -mx mz ETotal _ -bx bz
		x_[0], x_[2], -x_[1], x_[3], x_[4], -x_[5], x_[7]
	<? elseif side == 2 then ?>
		// rho mz my -mx ETotal _ by -bx
		x_[0], x_[3], x_[2], -x_[1], x_[4], x_[6], -x_[5]
	<? end ?>
	};

	<?=addr1?> const real* A = eig->evL;
	for (int i = 0; i < 7; ++i) {
		real sum = 0;
		for (int j = 0; j < 7; ++j) {
			sum += A[i+7*j] * x[j];
		}
		y[i] = sum;
	}
}

void eigen_rightTransform_<?=side?>_<?=addr0?>_<?=addr1?>_<?=addr2?>(
	<?=addr0?> real* y,
	<?=addr1?> const eigen_t* eig,
	<?=addr2?> const real* x
) {
	<?=addr1?> const real* A = eig->evR;
	for (int i = 0; i < 7; ++i) {
		real sum = 0;
		for (int j = 0; j < 7; ++j) {
			sum += A[i+7*j] * x[j];
		}
		y[i] = sum;
	}

	//swap x and side and insert 0 for bx
	<? if side == 0 then ?>
	y[7] = y[6]; y[6] = y[5]; y[5] = 0;
	<? elseif side == 1 then ?>
	real tmp = y[1]; y[1] = -y[2]; y[2] = tmp;
	y[5] = -y[5]; y[7] = y[6]; y[6] = 0;
	<? elseif side == 2 then ?>
	real tmp = y[1]; y[1] = -y[3]; y[3] = tmp;
	y[7] = 0;
	tmp = y[5]; y[5] = -y[6]; y[6] = y[5];
	<? end ?>
}

<? 				if solver.checkFluxError then ?>
void eigen_fluxTransform_<?=side?>_<?=addr0?>_<?=addr1?>_<?=addr2?>(
	<?=addr0?> real* y,
	<?=addr1?> const eigen_t* eig,
	<?=addr2?> const real* x_
) {
	//swap x and side, and get rid of bx
	real x[7] = {
	<? if side == 0 then ?>
		// rho mx my mz ETotal _ by bz
		x_[0], x_[1], x_[2], x_[3], x_[4], x_[6], x_[7]
	<? elseif side == 1 then ?>
		// rho my -mx mz ETotal _ -bx bz
		x_[0], x_[2], -x_[1], x_[3], x_[4], -x_[5], x_[7]
	<? elseif side == 2 then ?>
		// rho mz my -mx ETotal _ by -bx
		x_[0], x_[3], x_[2], -x_[1], x_[4], x_[6], -x_[5]
	<? end ?>
	};

	<?=addr1?> const real* A = eig->A;
	for (int i = 0; i < 7; ++i) {
		real sum = 0;
		for (int j = 0; j < 7; ++j) {
			sum += A[i+7*j] * x[j];
		}
		y[i] = sum;
	}

	//swap x and side and insert 0 for bx
	<? if side == 0 then ?>
	y[7] = y[6]; y[6] = y[5]; y[5] = 0;
	<? elseif side == 1 then ?>
	real tmp = y[1]; y[1] = -y[2]; y[2] = tmp;
	y[5] = -y[5]; y[7] = y[6]; y[6] = 0;
	<? elseif side == 2 then ?>
	real tmp = y[1]; y[1] = -y[3]; y[3] = tmp;
	y[7] = 0;
	tmp = y[5]; y[5] = -y[6]; y[6] = y[5];
	<? end ?>
}
<?				end
			end
		end
	end
end ?>


Comment + simplify

Comment + simplify

/******************************************************************************
 * function: wah_bitmap                                                       *
 * input:    32-bit integer                                                   *
 * output:   wah compressed bitmap index, offsets for its values              *
 ******************************************************************************/

#define FENCE_TYPE CLK_LOCAL_MEM_FENCE
#define WORK_GROUP_SIZE 1024

// Processing steps
void sort_rids_by_value(global uint* input, global  uint* rids,
                        size_t idx, size_t total);
void produce_chunck_id_literals(global uint* rids, global uint* chids,
                                global uint* lits, size_t idx, size_t total);
size_t merged_lit_by_val_chids(global uint* input, global uint* chids,
                               global uint* lits, size_t idx, size_t total);
void produce_fills(global uint* input, global uint* chids,
                   size_t idx, size_t total);
size_t fuse_fill_literals(global uint* chids, global uint* lits,
                          global uint* index, size_t idx, size_t total);
size_t compute_colum_length(global uint* input, global uint* chids,
                            global uint* offsets, size_t idx, size_t total);

// helper functions
void parallel_selection_sort(global uint* key, global uint* data,
                             size_t idx, size_t total);
size_t reduce_by_key_OR(global uint* keys_high, global uint* keys_low,
                        global uint* data, size_t idx, size_t total);
size_t reduce_by_key_SUM(global uint* keys, local uint* data,
                         size_t idx, size_t total);

// main kernel
kernel void kernel_wah_index(global uint* config,
                             global uint* input,
                             global uint* index,
                             global uint* offsets,
                             global uint* rids,
                             global uint* chids,
                             global uint* lits) {
  // just 1 dimension here: 0
  size_t total = get_global_size(0);
  size_t idx = get_global_id(0);
  if (total > WORK_GROUP_SIZE) {
    offsets[idx] = total;
    index[idx] = WORK_GROUP_SIZE;
    return;
  }
  // CONSIDERATIONS:
  // * some scaling state of the are functions required
  //  - sorting
  //  - scan
  //  - stream compaction
  //  - reduce by key
  // * when do we need barriers?
  //  - at the end of functions, before return?
  //  - can be just assign return values to global stuff
  //  - ...

  // read config data
  // uint input_size       = config[0];
  // uint index_size       = config[1]; should be 2* input size
  // uint processed_ values = config[2];

  // Currently 1 work item per value
  total = config[0];

  //__gloabl uint[index_size] rids;
  sort_rids_by_value(input, rids, idx, total);
  barrier(FENCE_TYPE);
  //index[idx] = rids[idx];
  //offsets[idx] = 0;
  //config[4] = total;
  produce_chunck_id_literals(rids, chids, lits, idx, total);
  barrier(FENCE_TYPE);
  //input[idx] = rids[idx];
  //index[idx] = chids[idx];
  //offsets[idx] = lits[idx];
  //config[4] = total;
  size_t k = merged_lit_by_val_chids(input, chids, lits, idx, total);
  barrier(FENCE_TYPE);
  //index[idx] = chids[idx];
  //offsets[idx] = lits[idx];
  //config[3] = (uint) k;
  //config[4] = (uint) k;
  //config[5] = (uint) k;
  produce_fills(input, chids, idx, k);
  barrier(FENCE_TYPE);
  //config[4] = k;
  //index[idx] = chids[idx];
  //offsets[idx] = lits[idx];
  uint idxlen = fuse_fill_literals(chids, lits, index, idx, k);
  barrier(FENCE_TYPE);
  config[1] = idxlen;
  uint keycnt = compute_colum_length(input, chids, offsets, idx, k);
  barrier(FENCE_TYPE);
  config[0] = keycnt;
}

void sort_rids_by_value(global uint* input,global uint* rids,
                        size_t idx, size_t total) {
  rids[idx] = idx;
  barrier(FENCE_TYPE); // is this needed here?
  // sort by input value
  parallel_selection_sort(input, rids, idx, total);
}

void produce_chunck_id_literals(global uint* rids, global uint* chids,
                                global uint* lits, size_t idx, size_t total) {
  (void) total;
  lits[idx] = 1u << (rids[idx] % 31u);
  lits[idx] |= 1u << 31;
  chids[idx] = (uint) rids[idx] / 31;
}

size_t merged_lit_by_val_chids(global uint* input, global uint* chids,
                               global uint* lits, size_t idx, size_t total) {
  // avoid merge of chids and input into keys,
  // simply pass them both
  size_t k = reduce_by_key_OR(chids, input, lits, idx, total);
  return k;
}

void produce_fills(global uint* input, global uint* chids,
                   size_t idx, size_t total) {
  uint tmp = chids[idx];
  if (idx != 0 && (input[idx] == input[idx - 1] &&
                   chids[idx] == chids[idx - 1])) {
    tmp = chids[idx] - chids[idx - 1] - 1;
  }
  // This branch leads to loss of fills at the beginning of
  // a bitmap index.
  /* else {
    if (chids[idx] != 0) {
      chids[idx] = chids[idx] - 1;
    }
  }*/
  barrier(FENCE_TYPE);
  chids[idx] = tmp;
}

size_t fuse_fill_literals(global uint* chids, global uint* lits,
                          global uint* index, size_t idx, size_t total) {
  local uint markers [WORK_GROUP_SIZE * 2];
  local uint position[WORK_GROUP_SIZE * 2];
  volatile local int len;
  //uint a =  2 * idx;
  //uint b = (2 * idx) + 1;
  len = 0;
  index[ 2 * idx     ] = chids[idx];
  index[(2 * idx) + 1] = lits[idx];
  barrier(FENCE_TYPE);
  // stream compaction
  markers[ 2 * idx     ] = index[ 2 * idx     ] != 0; // ? 1 : 0;
  markers[(2 * idx) + 1] = index[(2 * idx) + 1] != 0; // ? 1 : 0;
  barrier(FENCE_TYPE);
  // should be a parallel scan
  if (idx == 0) {
    position[0] = 0;
    for (uint i = 1; i < 2 * total; ++i) {
      position[i] = position[i - 1] + markers[i - 1];
    }
  }
  // end parallel scan
  uint tmp_a = index[2 * idx];
  uint tmp_b = index[(2 * idx) + 1];
  barrier(FENCE_TYPE);
  if (idx < total) {
    if (markers[2 * idx] == 1) {
      index[position[2 * idx]] = tmp_a;
      atomic_add(&len, 1);
    }
    if (markers[(2 * idx) + 1] == 1) {
      index[position[(2 * idx) + 1]] = tmp_b;
      atomic_add(&len, 1);
    }
  }
  barrier(FENCE_TYPE); // <-- should there be a barrier here?
  return len;
}

size_t compute_colum_length(global uint* input, global uint* chids,
                            global uint* offsets, size_t idx, size_t k) {
  local uint tmp[WORK_GROUP_SIZE];
  tmp[idx] = (1 + (chids[idx] == 0 ? 0 : 1));
  uint keycnt = reduce_by_key_SUM(input, tmp, idx, k);
  // inclusive scan to create offsets, should be parallel
  barrier(FENCE_TYPE);
  if (idx == 0) {
    offsets[0] = 0;
    for (uint i = 1; i < keycnt; ++i) {
      offsets[i] = offsets[i - 1] + tmp[i - 1];
    }
  }
  return keycnt;
}

// Helper functions

// we have 64 bit keys consisting of (high << 32 | low), but want to
// avoid the extra copy, so ...
size_t reduce_by_key_OR(global uint* keys_high, global uint* keys_low,
                        global uint* data, size_t idx, size_t total) {
  local uint heads[WORK_GROUP_SIZE];
  volatile local int k;
  heads[idx] = (idx == 0) ||
               (keys_high[idx] != keys_high[idx - 1]) ||
               (keys_low [idx] != keys_low [idx - 1]);
  barrier(FENCE_TYPE);
  if (heads[idx] != 0) {
    uint val = data[idx];
    uint curr = idx + 1;
    while (heads[curr] == 0 && curr < total) {
      val |= data[curr]; // OR operation
      curr += 1;
    }
    atomic_add(&k, 1);
    data[idx] = val;
  }
  barrier(FENCE_TYPE);
  if (idx == 0) {
    uint pos = 0;
    for (uint i = 0; i < total; ++i) {
      if (heads[i] != 0) {
        keys_high[pos] = keys_high[i];
        keys_low[pos] = keys_low[i];
        data[pos] = data[i];
        pos += 1;
      }
    }
  }
  return (uint) k;
}

size_t reduce_by_key_SUM(global uint* keys, local uint* data,
                         size_t idx, size_t total) {
  local uint heads[WORK_GROUP_SIZE];
  heads[idx] = idx == 0 || (keys[idx] != keys[idx - 1]);
  volatile local int k;
  if (heads[idx] != 0 && idx < total) {
    uint val = data[idx];
    uint curr = idx + 1;
    while (heads[curr] == 0 && curr < total) {
      val += data[curr]; // SUM operation
      curr += 1;
    }
    atomic_add(&k, 1);
    data[idx] = val;
  }
  barrier(FENCE_TYPE);
  if (idx == 0) {
    uint pos = 0;
    for (uint i = 0; i < total; ++i) {
      if (heads[i] != 0) {
        keys[pos] = keys[i];
        data[pos] = data[i];
        pos += 1;
      }
    }
  }
  return (uint) k;
}

/**
 * Sort found on: http://www.bealto.com/gpu-sorting_intro.html
 * License:set syntax=:
 *  This code is released under the following license (BSD-style).
 *  --
 *
 *  Copyright (c) 2011, Eric Bainville
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are met:
 *      * Redistributions of source code must retain the above copyright
 *        notice, this list of conditions and the following disclaimer.
 *      * Redistributions in binary form must reproduce the above copyright
 *        notice, this list of conditions and the following disclaimer in the
 *        documentation and/or other materials provided with the distribution.
 *      * Neither the name of Eric Bainville nor the
 *        names of its contributors may be used to endorse or promote products
 *        derived from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY ERIC BAINVILLE ''AS IS'' AND ANY
 *  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 *  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 *  DISCLAIMED. IN NO EVENT SHALL ERIC BAINVILLE BE LIABLE FOR ANY
 *  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 *  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 *  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 *  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 **/

void parallel_selection_sort(global uint* key, global uint* data,
                             size_t idx, size_t total) {
  uint key_value = key[idx];
  uint data_value = data[idx];
  // Compute position of in[i] in output
  int pos = 0;
  for (size_t j = 0; j < total; ++j) {
    uint curr = key[j]; // broadcasted
    bool smaller = (curr < key_value) || (curr == key_value && j < idx);
    pos += (smaller) ? 1 : 0;
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  key[pos] = key_value;
  data[pos] = data_value;
}

/******************************************************************************
 * function: wah_bitmap                                                       *
 * input:    32-bit integer                                                   *
 * output:   wah compressed bitmap index, offsets for its values              *
 ******************************************************************************/

#define FENCE_TYPE CLK_LOCAL_MEM_FENCE
#define WORK_GROUP_SIZE 1024

// Processing steps
void sort_rids_by_value(global uint* input, global  uint* rids,
                        size_t idx, size_t total);
void produce_chunck_id_literals(global uint* rids, global uint* chids,
                                global uint* lits, size_t idx, size_t total);
size_t merged_lit_by_val_chids(global uint* input, global uint* chids,
                               global uint* lits, size_t idx, size_t total);
void produce_fills(global uint* input, global uint* chids,
                   size_t idx, size_t total);
size_t fuse_fill_literals(global uint* chids, global uint* lits,
                          global uint* index, size_t idx, size_t total);
size_t compute_colum_length(global uint* input, global uint* chids,
                            global uint* offsets, size_t idx, size_t total);

// helper functions
void parallel_selection_sort(global uint* key, global uint* data,
                             size_t idx, size_t total);
size_t reduce_by_key_OR(global uint* keys_high, global uint* keys_low,
                        global uint* data, size_t idx, size_t total);
size_t reduce_by_key_SUM(global uint* keys, local uint* data,
                         size_t idx, size_t total);

// main kernel
kernel void kernel_wah_index(global uint* config,
                             global uint* input,
                             global uint* index,
                             global uint* offsets,
                             global uint* rids,
                             global uint* chids,
                             global uint* lits) {
  // just 1 dimension here: 0
  size_t total = get_global_size(0);
  size_t idx = get_global_id(0);
  if (total > WORK_GROUP_SIZE) {
    offsets[idx] = total;
    index[idx] = WORK_GROUP_SIZE;
    return;
  }
  // CONSIDERATIONS:
  // * some scaling state of the are functions required
  //  - sorting
  //  - scan
  //  - stream compaction
  //  - reduce by key
  // * when do we need barriers?
  //  - at the end of functions, before return?
  //  - can be just assign return values to global stuff
  //  - ...

  // read config data
  // uint input_size       = config[0];
  // uint index_size       = config[1]; should be 2* input size
  // uint processed_ values = config[2];

  // Currently 1 work item per value
  total = config[0];

  //__gloabl uint[index_size] rids;
  sort_rids_by_value(input, rids, idx, total);
  barrier(FENCE_TYPE);
  //index[idx] = rids[idx];
  //offsets[idx] = 0;
  //config[4] = total;
  produce_chunck_id_literals(rids, chids, lits, idx, total);
  barrier(FENCE_TYPE);
  //input[idx] = rids[idx];
  //index[idx] = chids[idx];
  //offsets[idx] = lits[idx];
  //config[4] = total;
  size_t k = merged_lit_by_val_chids(input, chids, lits, idx, total);
  barrier(FENCE_TYPE);
  //index[idx] = chids[idx];
  //offsets[idx] = lits[idx];
  //config[3] = (uint) k;
  //config[4] = (uint) k;
  //config[5] = (uint) k;
  produce_fills(input, chids, idx, k);
  barrier(FENCE_TYPE);
  //config[4] = k;
  //index[idx] = chids[idx];
  //offsets[idx] = lits[idx];
  uint idxlen = fuse_fill_literals(chids, lits, index, idx, k);
  barrier(FENCE_TYPE);
  config[1] = idxlen;
  uint keycnt = compute_colum_length(input, chids, offsets, idx, k);
  barrier(FENCE_TYPE);
  config[0] = keycnt;
}

void sort_rids_by_value(global uint* input,global uint* rids,
                        size_t idx, size_t total) {
  rids[idx] = idx;
  barrier(FENCE_TYPE); // is this needed here?
  // sort by input value
  parallel_selection_sort(input, rids, idx, total);
}

void produce_chunck_id_literals(global uint* rids, global uint* chids,
                                global uint* lits, size_t idx, size_t total) {
  (void) total;
  lits[idx] = 1u << (rids[idx] % 31u);
  lits[idx] |= 1u << 31;
  chids[idx] = (uint) rids[idx] / 31;
}

size_t merged_lit_by_val_chids(global uint* input, global uint* chids,
                               global uint* lits, size_t idx, size_t total) {
  // avoid merge of chids and input into keys,
  // simply pass them both
  size_t k = reduce_by_key_OR(chids, input, lits, idx, total);
  return k;
}

void produce_fills(global uint* input, global uint* chids,
                   size_t idx, size_t total) {
  uint tmp = chids[idx];
  if (idx != 0 && (input[idx] == input[idx - 1] &&
                   chids[idx] == chids[idx - 1])) {
    tmp = chids[idx] - chids[idx - 1] - 1;
  } /*else {
    if (chids[idx] != 0) {
      chids[idx] = chids[idx] - 1;
    }
  }*/
  barrier(FENCE_TYPE);
  chids[idx] = tmp;
}

size_t fuse_fill_literals(global uint* chids, global uint* lits,
                          global uint* index, size_t idx, size_t total) {
  local uint markers [WORK_GROUP_SIZE * 2];
  local uint position[WORK_GROUP_SIZE * 2];
  volatile local int len;
  //uint a =  2 * idx;
  //uint b = (2 * idx) + 1;
  len = 0;
  if (idx < total) {
    index[ 2 * idx     ] = chids[idx];
    index[(2 * idx) + 1] = lits[idx];
    barrier(FENCE_TYPE);
    // stream compaction
    markers[ 2 * idx     ] = index[ 2 * idx     ] != 0; // ? 1 : 0;
    markers[(2 * idx) + 1] = index[(2 * idx) + 1] != 0; // ? 1 : 0;
    barrier(FENCE_TYPE);
    // should be a parallel scan
    if (idx == 0) {
      position[0] = 0;
      for (uint i = 1; i < 2 * total; ++i) {
        position[i] = position[i - 1] + markers[i - 1];
      }
    }
    uint tmp_a = index[2 * idx];
    uint tmp_b = index[(2 * idx) + 1];
    barrier(FENCE_TYPE);
    if (markers[2 * idx] == 1) {
      index[position[2 * idx]] = tmp_a;
      atomic_add(&len, 1);
    }
    if (markers[(2 * idx) + 1] == 1) {
      index[position[(2 * idx) + 1]] = tmp_b;
      atomic_add(&len, 1);
    }
  }
  barrier(FENCE_TYPE); // <-- should there be a barrier here?
  return len;
}

size_t compute_colum_length(global uint* input, global uint* chids,
                            global uint* offsets, size_t idx, size_t k) {
  local uint tmp[WORK_GROUP_SIZE];
  tmp[idx] = (1 + (chids[idx] == 0 ? 0 : 1));
  uint keycnt = reduce_by_key_SUM(input, tmp, idx, k);
  // inclusive scan to create offsets, should be parallel
  barrier(FENCE_TYPE);
  if (idx == 0) {
    offsets[0] = 0;
    for (uint i = 1; i < keycnt; ++i) {
      offsets[i] = offsets[i - 1] + tmp[i - 1];
    }
  }
  return keycnt;
}

// Helper functions

// we have 64 bit keys consisting of (high << 32 | low), but want to
// avoid the extra copy, so ...
size_t reduce_by_key_OR(global uint* keys_high, global uint* keys_low,
                        global uint* data, size_t idx, size_t total) {
  local uint heads[WORK_GROUP_SIZE];
  volatile local int k;
  heads[idx] = (idx == 0) ||
               (keys_high[idx] != keys_high[idx - 1]) ||
               (keys_low [idx] != keys_low [idx - 1]);
  barrier(FENCE_TYPE);
  if (heads[idx] != 0) {
    uint val = data[idx];
    uint curr = idx + 1;
    while (heads[curr] == 0 && curr < total) {
      val |= data[curr]; // OR operation
      curr += 1;
    }
    atomic_add(&k, 1);
    data[idx] = val;
  }
  barrier(FENCE_TYPE);
  if (idx == 0) {
    uint pos = 0;
    for (uint i = 0; i < total; ++i) {
      if (heads[i] != 0) {
        keys_high[pos] = keys_high[i];
        keys_low[pos] = keys_low[i];
        data[pos] = data[i];
        pos += 1;
      }
    }
  }
  return (uint) k;
}

size_t reduce_by_key_SUM(global uint* keys, local uint* data,
                         size_t idx, size_t total) {
  local uint heads[WORK_GROUP_SIZE];
  heads[idx] = idx == 0 || (keys[idx] != keys[idx - 1]);
  volatile local int k;
  if (heads[idx] != 0 && idx < total) {
    uint val = data[idx];
    uint curr = idx + 1;
    while (heads[curr] == 0 && curr < total) {
      val += data[curr]; // SUM operation
      curr += 1;
    }
    atomic_add(&k, 1);
    data[idx] = val;
  }
  barrier(FENCE_TYPE);
  if (idx == 0) {
    uint pos = 0;
    for (uint i = 0; i < total; ++i) {
      if (heads[i] != 0) {
        keys[pos] = keys[i];
        data[pos] = data[i];
        pos += 1;
      }
    }
  }
  return (uint) k;
}

/**
 * Sort found on: http://www.bealto.com/gpu-sorting_intro.html
 * License:set syntax=:
 *  This code is released under the following license (BSD-style).
 *  --
 *
 *  Copyright (c) 2011, Eric Bainville
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are met:
 *      * Redistributions of source code must retain the above copyright
 *        notice, this list of conditions and the following disclaimer.
 *      * Redistributions in binary form must reproduce the above copyright
 *        notice, this list of conditions and the following disclaimer in the
 *        documentation and/or other materials provided with the distribution.
 *      * Neither the name of Eric Bainville nor the
 *        names of its contributors may be used to endorse or promote products
 *        derived from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY ERIC BAINVILLE ''AS IS'' AND ANY
 *  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 *  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 *  DISCLAIMED. IN NO EVENT SHALL ERIC BAINVILLE BE LIABLE FOR ANY
 *  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 *  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 *  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 *  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 **/

void parallel_selection_sort(global uint* key, global uint* data,
                             size_t idx, size_t total) {
  uint key_value = key[idx];
  uint data_value = data[idx];
  // Compute position of in[i] in output
  int pos = 0;
  for (size_t j = 0; j < total; ++j) {
    uint curr = key[j]; // broadcasted
    bool smaller = (curr < key_value) || (curr == key_value && j < idx);
    pos += (smaller) ? 1 : 0;
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  key[pos] = key_value;
  data[pos] = data_value;
}


fixed bug in boundary mirror enforcement

#pragma OPENCL EXTENSION cl_khr_3d_image_writes : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVFInit(__read_only image3d_t volume, __write_only image3d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    float f100 = read_imagef(volume, sampler, pos + (int4)(1,0,0,0)).x;
    float f_100 = read_imagef(volume, sampler, pos - (int4)(1,0,0,0)).x;
    float f010 = read_imagef(volume, sampler, pos + (int4)(0,1,0,0)).x;
    float f0_10 = read_imagef(volume, sampler, pos - (int4)(0,1,0,0)).x;
    float f001 = read_imagef(volume, sampler, pos + (int4)(0,0,1,0)).x;
    float f00_1 = read_imagef(volume, sampler, pos - (int4)(0,0,1,0)).x;

    float4 gradient = {
        0.5f*(f100-f_100),
        0.5f*(f010-f0_10),
        0.5f*(f001-f00_1),
        0};

    gradient.w = gradient.x*gradient.x + gradient.y*gradient.y + gradient.z*gradient.z;

    write_imagef(vector_field, pos, gradient);
}

__kernel void GVFIteration(__read_only image3d_t init_vector_field, __read_only image3d_t read_vector_field, __write_only image3d_t write_vector_field, __private float mu) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    // Enforce mirror boundary conditions
    int4 size = {get_global_size(0), get_global_size(1), get_global_size(2), 0};
    pos = select(pos, (int4)(2,2,2,0), pos == (int4)(0,0,0,0));
    pos = select(pos, size-3, pos == size-1);

    /*
    if(pos.x == 0)
        pos.x = 2;
    if(pos.y == 0)
        pos.y = 2;
    if(pos.z == 0)
        pos.z = 2;
    if(pos.x == get_global_size(0)-1)
        pos.x = get_global_size(0)-3;
    if(pos.y == get_global_size(1)-1)
        pos.y = get_global_size(1)-3;
    if(pos.z == get_global_size(2)-1)
        pos.z = get_global_size(2)-3;

    int4 diff = (pos2 != pos);
    if(diff.x+diff.y+diff.z+diff.w > 0)
        printf("iiiik!\n");
    */

    float4 vector = read_imagef(read_vector_field, sampler, pos);
    float4 fx1 = read_imagef(read_vector_field, sampler, pos + (int4)(1,0,0,0));
    float4 fx_1 = read_imagef(read_vector_field, sampler, pos - (int4)(1,0,0,0));
    float4 fy1 = read_imagef(read_vector_field, sampler, pos + (int4)(0,1,0,0));
    float4 fy_1 = read_imagef(read_vector_field, sampler, pos - (int4)(0,1,0,0));
    float4 fz1 = read_imagef(read_vector_field, sampler, pos + (int4)(0,0,1,0));
    float4 fz_1 = read_imagef(read_vector_field, sampler, pos - (int4)(0,0,1,0));
    float4 init_vector = read_imagef(init_vector_field, sampler, pos);

    // Update the vector field: Calculate Laplacian using a 3D central difference scheme
    float4 laplacian = -6*vector + fx1 + fx_1 + fy1 + fy_1 + fz1 + fz_1;

    vector += mu * laplacian - (vector - init_vector)*init_vector.w;

    write_imagef(write_vector_field, pos, vector);
}

__kernel void GVFResult(__write_only image3d_t result, __read_only image3d_t vectorField) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.w = 0;
    //if(1 < length(vector))
    //    printf("%f %f %f\n", vector.x, vector.y, vector.z);
    write_imagef(result, pos, length(vector));
}

#pragma OPENCL EXTENSION cl_khr_3d_image_writes : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVFInit(__read_only image3d_t volume, __write_only image3d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    float f100 = read_imagef(volume, sampler, pos + (int4)(1,0,0,0)).x;
    float f_100 = read_imagef(volume, sampler, pos - (int4)(1,0,0,0)).x;
    float f010 = read_imagef(volume, sampler, pos + (int4)(0,1,0,0)).x;
    float f0_10 = read_imagef(volume, sampler, pos - (int4)(0,1,0,0)).x;
    float f001 = read_imagef(volume, sampler, pos + (int4)(0,0,1,0)).x;
    float f00_1 = read_imagef(volume, sampler, pos - (int4)(0,0,1,0)).x;

    float4 gradient = {
        0.5f*(f100-f_100),
        0.5f*(f010-f0_10),
        0.5f*(f001-f00_1),
        0};

    gradient.w = gradient.x*gradient.x + gradient.y*gradient.y + gradient.z*gradient.z;

    write_imagef(vector_field, pos, gradient);
}

__kernel void GVFIteration(__read_only image3d_t init_vector_field, __read_only image3d_t read_vector_field, __write_only image3d_t write_vector_field, __private float mu) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    // Enforce mirror boundary conditions
    int4 size = (get_global_size(0), get_global_size(1), get_global_size(2), 0);
    pos = select(pos, (int4)(2,2,2,0), pos == (int4)(0,0,0,0));
    pos = select(pos, size-3, pos == size);

    /*
    if(pos.x == 0)
        pos.x = 2;
    if(pos.y == 0)
        pos.y = 2;
    if(pos.z == 0)
        pos.z = 2;
    if(pos.x == get_global_size(0))
        pos.x = get_global_size(0)-3;
    if(pos.y == get_global_size(1))
        pos.y = get_global_size(1)-3;
    if(pos.z == get_global_size(2))
        pos.z = get_global_size(2)-3;

    int4 diff = (pos2 != pos);
    if(diff.x+diff.y+diff.z+diff.w > 0)
        printf("iiiik!\n");
    */

    float4 vector = read_imagef(read_vector_field, sampler, pos);
    float4 fx1 = read_imagef(read_vector_field, sampler, pos + (int4)(1,0,0,0));
    float4 fx_1 = read_imagef(read_vector_field, sampler, pos - (int4)(1,0,0,0));
    float4 fy1 = read_imagef(read_vector_field, sampler, pos + (int4)(0,1,0,0));
    float4 fy_1 = read_imagef(read_vector_field, sampler, pos - (int4)(0,1,0,0));
    float4 fz1 = read_imagef(read_vector_field, sampler, pos + (int4)(0,0,1,0));
    float4 fz_1 = read_imagef(read_vector_field, sampler, pos - (int4)(0,0,1,0));
    float4 init_vector = read_imagef(init_vector_field, sampler, pos);

    // Update the vector field: Calculate Laplacian using a 3D central difference scheme
    float4 laplacian = -6*vector + fx1 + fx_1 + fy1 + fy_1 + fz1 + fz_1;

    vector += mu * laplacian - (vector - init_vector)*init_vector.w;

    write_imagef(write_vector_field, pos, vector);
}

__kernel void GVFResult(__write_only image3d_t result, __read_only image3d_t vectorField) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.w = 0;
    write_imagef(result, pos, length(vector));
}