Optimize the performance of cascade OpenCL kernel.

1. Use built-in mad() instead of += and *.
2. For stump stages, if weight.z == 0, avoid unnecessary calculation
because some features only have 2 rectangles..

It could improve
OCL_Cascade_Image_MinSize_CascadeClassifier.CascadeClassifier/*
about 10% mean values.

Signed-off-by: Yan Wang <8ca54580088a7ffe1dd243d1c16c6d1f04463d55@linux.intel.com>

///////////////////////////// OpenCL kernels for face detection //////////////////////////////
////////////////////////////// see the opencv/doc/license.txt ///////////////////////////////

//
// the code has been derived from the OpenCL Haar cascade kernel by
//
//    Niko Li, newlife20080214@gmail.com
//    Wang Weiyan, wangweiyanster@gmail.com
//    Jia Haipeng, jiahaipeng95@gmail.com
//    Nathan, liujun@multicorewareinc.com
//    Peng Xiao, pengxiao@outlook.com
//    Erping Pang, erping@multicorewareinc.com
//

#ifdef HAAR
typedef struct __attribute__((aligned(4))) OptHaarFeature
{
    int4 ofs[3] __attribute__((aligned (4)));
    float4 weight __attribute__((aligned (4)));
}
OptHaarFeature;
#endif

#ifdef LBP
typedef struct __attribute__((aligned(4))) OptLBPFeature
{
    int16 ofs __attribute__((aligned (4)));
}
OptLBPFeature;
#endif

typedef struct __attribute__((aligned(4))) Stump
{
    float4 st __attribute__((aligned (4)));
}
Stump;

typedef struct __attribute__((aligned(4))) Node
{
    int4 n __attribute__((aligned (4)));
}
Node;

typedef struct __attribute__((aligned (4))) Stage
{
    int first __attribute__((aligned (4)));
    int ntrees __attribute__((aligned (4)));
    float threshold __attribute__((aligned (4)));
}
Stage;

typedef struct __attribute__((aligned (4))) ScaleData
{
    float scale __attribute__((aligned (4)));
    int szi_width __attribute__((aligned (4)));
    int szi_height __attribute__((aligned (4)));
    int layer_ofs __attribute__((aligned (4)));
    int ystep __attribute__((aligned (4)));
}
ScaleData;

#ifndef SUM_BUF_SIZE
#define SUM_BUF_SIZE 0
#endif

#ifndef NODE_COUNT
#define NODE_COUNT 1
#endif

#ifdef HAAR
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X,LOCAL_SIZE_Y,1)))
void runHaarClassifier(
    int nscales, __global const ScaleData* scaleData,
    __global const int* sum,
    int _sumstep, int sumoffset,
    __global const OptHaarFeature* optfeatures,
    __global const Stage* stages,
    __global const Node* nodes,
    __global const float* leaves0,

    volatile __global int* facepos,
    int4 normrect, int sqofs, int2 windowsize)
{
    int lx = get_local_id(0);
    int ly = get_local_id(1);
    int groupIdx = get_group_id(0);
    int i, ngroups = get_global_size(0)/LOCAL_SIZE_X;
    int scaleIdx, tileIdx, stageIdx;
    int sumstep = (int)(_sumstep/sizeof(int));
    int4 nofs0 = (int4)(mad24(normrect.y, sumstep, normrect.x),
                        mad24(normrect.y, sumstep, normrect.x + normrect.z),
                        mad24(normrect.y + normrect.w, sumstep, normrect.x),
                        mad24(normrect.y + normrect.w, sumstep, normrect.x + normrect.z));
    int normarea = normrect.z * normrect.w;
    float invarea = 1.f/normarea;
    int lidx = ly*LOCAL_SIZE_X + lx;

    #if SUM_BUF_SIZE > 0
    int4 nofs = (int4)(mad24(normrect.y, SUM_BUF_STEP, normrect.x),
                       mad24(normrect.y, SUM_BUF_STEP, normrect.x + normrect.z),
                       mad24(normrect.y + normrect.w, SUM_BUF_STEP, normrect.x),
                       mad24(normrect.y + normrect.w, SUM_BUF_STEP, normrect.x + normrect.z));
    #else
    int4 nofs = nofs0;
    #endif
    #define LOCAL_SIZE (LOCAL_SIZE_X*LOCAL_SIZE_Y)
    __local int lstore[SUM_BUF_SIZE + LOCAL_SIZE*5/2+1];
    #if SUM_BUF_SIZE > 0
    __local int* ibuf = lstore;
    __local int* lcount = ibuf + SUM_BUF_SIZE;
    #else
    __local int* lcount = lstore;
    #endif
    __local float* lnf = (__local float*)(lcount + 1);
    __local float* lpartsum = lnf + LOCAL_SIZE;
    __local short* lbuf = (__local short*)(lpartsum + LOCAL_SIZE);

    for( scaleIdx = nscales-1; scaleIdx >= 0; scaleIdx-- )
    {
        __global const ScaleData* s = scaleData + scaleIdx;
        int ystep = s->ystep;
        int2 worksize = (int2)(max(s->szi_width - windowsize.x, 0), max(s->szi_height - windowsize.y, 0));
        int2 ntiles = (int2)((worksize.x + LOCAL_SIZE_X-1)/LOCAL_SIZE_X,
                             (worksize.y + LOCAL_SIZE_Y-1)/LOCAL_SIZE_Y);
        int totalTiles = ntiles.x*ntiles.y;

        for( tileIdx = groupIdx; tileIdx < totalTiles; tileIdx += ngroups )
        {
            int ix0 = (tileIdx % ntiles.x)*LOCAL_SIZE_X;
            int iy0 = (tileIdx / ntiles.x)*LOCAL_SIZE_Y;
            int ix = lx, iy = ly;
            __global const int* psum0 = sum + mad24(iy0, sumstep, ix0) + s->layer_ofs;
            __global const int* psum1 = psum0 + mad24(iy, sumstep, ix);

            if( ix0 >= worksize.x || iy0 >= worksize.y )
                continue;
            #if SUM_BUF_SIZE > 0
            for( i = lidx*4; i < SUM_BUF_SIZE; i += LOCAL_SIZE_X*LOCAL_SIZE_Y*4 )
            {
                int dy = i/SUM_BUF_STEP, dx = i - dy*SUM_BUF_STEP;
                vstore4(vload4(0, psum0 + mad24(dy, sumstep, dx)), 0, ibuf+i);
            }
            #endif

            if( lidx == 0 )
                lcount[0] = 0;
            barrier(CLK_LOCAL_MEM_FENCE);

            if( ix0 + ix < worksize.x && iy0 + iy < worksize.y )
            {
                #if NODE_COUNT==1
                __global const Stump* stump = (__global const Stump*)nodes;
                #else
                __global const Node* node = nodes;
                __global const float* leaves = leaves0;
                #endif
                #if SUM_BUF_SIZE > 0
                __local const int* psum = ibuf + mad24(iy, SUM_BUF_STEP, ix);
                #else
                __global const int* psum = psum1;
                #endif

                __global const float* psqsum = (__global const float*)(psum1 + sqofs);
                float sval = (psum[nofs.x] - psum[nofs.y] - psum[nofs.z] + psum[nofs.w])*invarea;
                float sqval = (psqsum[nofs0.x] - psqsum[nofs0.y] - psqsum[nofs0.z] + psqsum[nofs0.w])*invarea;
                float nf = (float)normarea * sqrt(max(sqval - sval * sval, 0.f));
                nf = nf > 0 ? nf : 1.f;

                for( stageIdx = 0; stageIdx < SPLIT_STAGE; stageIdx++ )
                {
                    int ntrees = stages[stageIdx].ntrees;
                    float s = 0.f;
                    #if NODE_COUNT==1
                    for( i = 0; i < ntrees; i++ )
                    {
                        float4 st = stump[i].st;
                        __global const OptHaarFeature* f = optfeatures + as_int(st.x);
                        float4 weight = f->weight;

                        int4 ofs = f->ofs[0];
                        sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                        ofs = f->ofs[1];
                        sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.y, sval);
                        if( weight.z > 0 )
                        {
                            ofs = f->ofs[2];
                            sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.z, sval);
                        }

                        s += (sval < st.y*nf) ? st.z : st.w;
                    }
                    stump += ntrees;
                    #else
                    for( i = 0; i < ntrees; i++, node += NODE_COUNT, leaves += NODE_COUNT+1 )
                    {
                        int idx = 0;
                        do
                        {
                            int4 n = node[idx].n;
                            __global const OptHaarFeature* f = optfeatures + n.x;
                            float4 weight = f->weight;

                            int4 ofs = f->ofs[0];

                            sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                            ofs = f->ofs[1];
                            sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.y, sval);
                            if( weight.z > 0 )
                            {
                                ofs = f->ofs[2];
                                sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.z, sval);
                            }

                            idx = (sval < as_float(n.y)*nf) ? n.z : n.w;
                        }
                        while(idx > 0);
                        s += leaves[-idx];
                    }
                    #endif

                    if( s < stages[stageIdx].threshold )
                        break;
                }

                if( stageIdx == SPLIT_STAGE && (ystep == 1 || ((ix | iy) & 1) == 0) )
                {
                    int count = atomic_inc(lcount);
                    lbuf[count] = (int)(ix | (iy << 8));
                    lnf[count] = nf;
                }
            }

            for( stageIdx = SPLIT_STAGE; stageIdx < N_STAGES; stageIdx++ )
            {
                barrier(CLK_LOCAL_MEM_FENCE);
                int nrects = lcount[0];

                if( nrects == 0 )
                    break;
                barrier(CLK_LOCAL_MEM_FENCE);
                if( lidx == 0 )
                    lcount[0] = 0;

                {
                    #if NODE_COUNT == 1
                    __global const Stump* stump = (__global const Stump*)nodes + stages[stageIdx].first;
                    #else
                    __global const Node* node = nodes + stages[stageIdx].first*NODE_COUNT;
                    __global const float* leaves = leaves0 + stages[stageIdx].first*(NODE_COUNT+1);
                    #endif
                    int nparts = LOCAL_SIZE / nrects;
                    int ntrees = stages[stageIdx].ntrees;
                    int ntrees_p = (ntrees + nparts - 1)/nparts;
                    int nr = lidx / nparts;
                    int partidx = -1, idxval = 0;
                    float partsum = 0.f, nf = 0.f;

                    if( nr < nrects )
                    {
                        partidx = lidx % nparts;
                        idxval = lbuf[nr];
                        nf = lnf[nr];

                        {
                        int ntrees0 = ntrees_p*partidx;
                        int ntrees1 = min(ntrees0 + ntrees_p, ntrees);
                        int ix1 = idxval & 255, iy1 = idxval >> 8;
                        #if SUM_BUF_SIZE > 0
                        __local const int* psum = ibuf + mad24(iy1, SUM_BUF_STEP, ix1);
                        #else
                        __global const int* psum = psum0 + mad24(iy1, sumstep, ix1);
                        #endif

                        #if NODE_COUNT == 1
                        for( i = ntrees0; i < ntrees1; i++ )
                        {
                            float4 st = stump[i].st;
                            __global const OptHaarFeature* f = optfeatures + as_int(st.x);
                            float4 weight = f->weight;

                            int4 ofs = f->ofs[0];
                            float sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                            ofs = f->ofs[1];
                            sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.y, sval);
                            //if( weight.z > 0 )
                            if( fabs(weight.z) > 0 )
                            {
                                ofs = f->ofs[2];
                                sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.z, sval);
                            }

                            partsum += (sval < st.y*nf) ? st.z : st.w;
                        }
                        #else
                        for( i = ntrees0; i < ntrees1; i++ )
                        {
                            int idx = 0;
                            do
                            {
                                int4 n = node[i*2 + idx].n;
                                __global const OptHaarFeature* f = optfeatures + n.x;
                                float4 weight = f->weight;
                                int4 ofs = f->ofs[0];

                                float sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                                ofs = f->ofs[1];
                                sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.y, sval);
                                if( weight.z > 0 )
                                {
                                    ofs = f->ofs[2];
                                    sval = mad((psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w]), weight.z, sval);
                                }

                                idx = (sval < as_float(n.y)*nf) ? n.z : n.w;
                            }
                            while(idx > 0);
                            partsum += leaves[i*3-idx];
                        }
                        #endif
                        }
                    }
                    lpartsum[lidx] = partsum;
                    barrier(CLK_LOCAL_MEM_FENCE);

                    if( partidx == 0 )
                    {
                        float s = lpartsum[nr*nparts];
                        for( i = 1; i < nparts; i++ )
                            s += lpartsum[i + nr*nparts];
                        if( s >= stages[stageIdx].threshold )
                        {
                            int count = atomic_inc(lcount);
                            lbuf[count] = idxval;
                            lnf[count] = nf;
                        }
                    }
                }
            }

            barrier(CLK_LOCAL_MEM_FENCE);
            if( stageIdx == N_STAGES )
            {
                int nrects = lcount[0];
                if( lidx < nrects )
                {
                    int nfaces = atomic_inc(facepos);
                    if( nfaces < MAX_FACES )
                    {
                        volatile __global int* face = facepos + 1 + nfaces*3;
                        int val = lbuf[lidx];
                        face[0] = scaleIdx;
                        face[1] = ix0 + (val & 255);
                        face[2] = iy0 + (val >> 8);
                    }
                }
            }
        }
    }
}
#endif

#ifdef LBP
#undef CALC_SUM_OFS_
#define CALC_SUM_OFS_(p0, p1, p2, p3, ptr) \
    ((ptr)[p0] - (ptr)[p1] - (ptr)[p2] + (ptr)[p3])

__kernel void runLBPClassifierStumpSimple(
    int nscales, __global const ScaleData* scaleData,
    __global const int* sum,
    int _sumstep, int sumoffset,
    __global const OptLBPFeature* optfeatures,
    __global const Stage* stages,
    __global const Stump* stumps,
    __global const int* bitsets,
    int bitsetSize,

    volatile __global int* facepos,
    int2 windowsize)
{
    int lx = get_local_id(0);
    int ly = get_local_id(1);
    int local_size_x = get_local_size(0);
    int local_size_y = get_local_size(1);
    int groupIdx = get_group_id(1)*get_num_groups(0) + get_group_id(0);
    int ngroups = get_num_groups(0)*get_num_groups(1);
    int scaleIdx, tileIdx, stageIdx;
    int sumstep = (int)(_sumstep/sizeof(int));

    for( scaleIdx = nscales-1; scaleIdx >= 0; scaleIdx-- )
    {
        __global const ScaleData* s = scaleData + scaleIdx;
        int ystep = s->ystep;
        int2 worksize = (int2)(max(s->szi_width - windowsize.x, 0), max(s->szi_height - windowsize.y, 0));
        int2 ntiles = (int2)((worksize.x/ystep + local_size_x-1)/local_size_x,
                             (worksize.y/ystep + local_size_y-1)/local_size_y);
        int totalTiles = ntiles.x*ntiles.y;

        for( tileIdx = groupIdx; tileIdx < totalTiles; tileIdx += ngroups )
        {
            int iy = mad24((tileIdx / ntiles.x), local_size_y, ly) * ystep;
            int ix = mad24((tileIdx % ntiles.x), local_size_x, lx) * ystep;

            if( ix < worksize.x && iy < worksize.y )
            {
                __global const int* p = sum + mad24(iy, sumstep, ix) + s->layer_ofs;
                __global const Stump* stump = stumps;
                __global const int* bitset = bitsets;

                for( stageIdx = 0; stageIdx < N_STAGES; stageIdx++ )
                {
                    int i, ntrees = stages[stageIdx].ntrees;
                    float s = 0.f;
                    for( i = 0; i < ntrees; i++, stump++, bitset += bitsetSize )
                    {
                        float4 st = stump->st;
                        __global const OptLBPFeature* f = optfeatures + as_int(st.x);
                        int16 ofs = f->ofs;

                        int cval = CALC_SUM_OFS_( ofs.s5, ofs.s6, ofs.s9, ofs.sa, p );

                        int mask, idx = (CALC_SUM_OFS_( ofs.s0, ofs.s1, ofs.s4, ofs.s5, p ) >= cval ? 4 : 0); // 0
                        idx |= (CALC_SUM_OFS_( ofs.s1, ofs.s2, ofs.s5, ofs.s6, p ) >= cval ? 2 : 0); // 1
                        idx |= (CALC_SUM_OFS_( ofs.s2, ofs.s3, ofs.s6, ofs.s7, p ) >= cval ? 1 : 0); // 2

                        mask = (CALC_SUM_OFS_( ofs.s6, ofs.s7, ofs.sa, ofs.sb, p ) >= cval ? 16 : 0); // 5
                        mask |= (CALC_SUM_OFS_( ofs.sa, ofs.sb, ofs.se, ofs.sf, p ) >= cval ? 8 : 0);  // 8
                        mask |= (CALC_SUM_OFS_( ofs.s9, ofs.sa, ofs.sd, ofs.se, p ) >= cval ? 4 : 0);  // 7
                        mask |= (CALC_SUM_OFS_( ofs.s8, ofs.s9, ofs.sc, ofs.sd, p ) >= cval ? 2 : 0);  // 6
                        mask |= (CALC_SUM_OFS_( ofs.s4, ofs.s5, ofs.s8, ofs.s9, p ) >= cval ? 1 : 0);  // 7

                        s += (bitset[idx] & (1 << mask)) ? st.z : st.w;
                    }

                    if( s < stages[stageIdx].threshold )
                        break;
                }

                if( stageIdx == N_STAGES )
                {
                    int nfaces = atomic_inc(facepos);
                    if( nfaces < MAX_FACES )
                    {
                        volatile __global int* face = facepos + 1 + nfaces*3;
                        face[0] = scaleIdx;
                        face[1] = ix;
                        face[2] = iy;
                    }
                }
            }
        }
    }
}

__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X,LOCAL_SIZE_Y,1)))
void runLBPClassifierStump(
    int nscales, __global const ScaleData* scaleData,
    __global const int* sum,
    int _sumstep, int sumoffset,
    __global const OptLBPFeature* optfeatures,
    __global const Stage* stages,
    __global const Stump* stumps,
    __global const int* bitsets,
    int bitsetSize,

    volatile __global int* facepos,
    int2 windowsize)
{
    int lx = get_local_id(0);
    int ly = get_local_id(1);
    int groupIdx = get_group_id(0);
    int i, ngroups = get_global_size(0)/LOCAL_SIZE_X;
    int scaleIdx, tileIdx, stageIdx;
    int sumstep = (int)(_sumstep/sizeof(int));
    int lidx = ly*LOCAL_SIZE_X + lx;

    #define LOCAL_SIZE (LOCAL_SIZE_X*LOCAL_SIZE_Y)
    __local int lstore[SUM_BUF_SIZE + LOCAL_SIZE*3/2+1];
    #if SUM_BUF_SIZE > 0
    __local int* ibuf = lstore;
    __local int* lcount = ibuf + SUM_BUF_SIZE;
    #else
    __local int* lcount = lstore;
    #endif
    __local float* lpartsum = (__local float*)(lcount + 1);
    __local short* lbuf = (__local short*)(lpartsum + LOCAL_SIZE);

    for( scaleIdx = nscales-1; scaleIdx >= 0; scaleIdx-- )
    {
        __global const ScaleData* s = scaleData + scaleIdx;
        int ystep = s->ystep;
        int2 worksize = (int2)(max(s->szi_width - windowsize.x, 0), max(s->szi_height - windowsize.y, 0));
        int2 ntiles = (int2)((worksize.x + LOCAL_SIZE_X-1)/LOCAL_SIZE_X,
                             (worksize.y + LOCAL_SIZE_Y-1)/LOCAL_SIZE_Y);
        int totalTiles = ntiles.x*ntiles.y;

        for( tileIdx = groupIdx; tileIdx < totalTiles; tileIdx += ngroups )
        {
            int ix0 = (tileIdx % ntiles.x)*LOCAL_SIZE_X;
            int iy0 = (tileIdx / ntiles.x)*LOCAL_SIZE_Y;
            int ix = lx, iy = ly;
            __global const int* psum0 = sum + mad24(iy0, sumstep, ix0) + s->layer_ofs;

            if( ix0 >= worksize.x || iy0 >= worksize.y )
                continue;
            #if SUM_BUF_SIZE > 0
            for( i = lidx*4; i < SUM_BUF_SIZE; i += LOCAL_SIZE_X*LOCAL_SIZE_Y*4 )
            {
                int dy = i/SUM_BUF_STEP, dx = i - dy*SUM_BUF_STEP;
                vstore4(vload4(0, psum0 + mad24(dy, sumstep, dx)), 0, ibuf+i);
            }
            barrier(CLK_LOCAL_MEM_FENCE);
            #endif

            if( lidx == 0 )
                lcount[0] = 0;
            barrier(CLK_LOCAL_MEM_FENCE);

            if( ix0 + ix < worksize.x && iy0 + iy < worksize.y )
            {
                __global const Stump* stump = stumps;
                __global const int* bitset = bitsets;
                #if SUM_BUF_SIZE > 0
                __local const int* p = ibuf + mad24(iy, SUM_BUF_STEP, ix);
                #else
                __global const int* p = psum0 + mad24(iy, sumstep, ix);
                #endif

                for( stageIdx = 0; stageIdx < SPLIT_STAGE; stageIdx++ )
                {
                    int ntrees = stages[stageIdx].ntrees;
                    float s = 0.f;
                    for( i = 0; i < ntrees; i++, stump++, bitset += bitsetSize )
                    {
                        float4 st = stump->st;
                        __global const OptLBPFeature* f = optfeatures + as_int(st.x);
                        int16 ofs = f->ofs;

                        int cval = CALC_SUM_OFS_( ofs.s5, ofs.s6, ofs.s9, ofs.sa, p );

                        int mask, idx = (CALC_SUM_OFS_( ofs.s0, ofs.s1, ofs.s4, ofs.s5, p ) >= cval ? 4 : 0); // 0
                        idx |= (CALC_SUM_OFS_( ofs.s1, ofs.s2, ofs.s5, ofs.s6, p ) >= cval ? 2 : 0); // 1
                        idx |= (CALC_SUM_OFS_( ofs.s2, ofs.s3, ofs.s6, ofs.s7, p ) >= cval ? 1 : 0); // 2

                        mask = (CALC_SUM_OFS_( ofs.s6, ofs.s7, ofs.sa, ofs.sb, p ) >= cval ? 16 : 0); // 5
                        mask |= (CALC_SUM_OFS_( ofs.sa, ofs.sb, ofs.se, ofs.sf, p ) >= cval ? 8 : 0);  // 8
                        mask |= (CALC_SUM_OFS_( ofs.s9, ofs.sa, ofs.sd, ofs.se, p ) >= cval ? 4 : 0);  // 7
                        mask |= (CALC_SUM_OFS_( ofs.s8, ofs.s9, ofs.sc, ofs.sd, p ) >= cval ? 2 : 0);  // 6
                        mask |= (CALC_SUM_OFS_( ofs.s4, ofs.s5, ofs.s8, ofs.s9, p ) >= cval ? 1 : 0);  // 7

                        s += (bitset[idx] & (1 << mask)) ? st.z : st.w;
                    }

                    if( s < stages[stageIdx].threshold )
                        break;
                }

                if( stageIdx == SPLIT_STAGE && (ystep == 1 || ((ix | iy) & 1) == 0) )
                {
                    int count = atomic_inc(lcount);
                    lbuf[count] = (int)(ix | (iy << 8));
                }
            }

            for( stageIdx = SPLIT_STAGE; stageIdx < N_STAGES; stageIdx++ )
            {
                int nrects = lcount[0];

                barrier(CLK_LOCAL_MEM_FENCE);
                if( nrects == 0 )
                    break;
                if( lidx == 0 )
                    lcount[0] = 0;

                {
                    __global const Stump* stump = stumps + stages[stageIdx].first;
                    __global const int* bitset = bitsets + stages[stageIdx].first*bitsetSize;
                    int nparts = LOCAL_SIZE / nrects;
                    int ntrees = stages[stageIdx].ntrees;
                    int ntrees_p = (ntrees + nparts - 1)/nparts;
                    int nr = lidx / nparts;
                    int partidx = -1, idxval = 0;
                    float partsum = 0.f, nf = 0.f;

                    if( nr < nrects )
                    {
                        partidx = lidx % nparts;
                        idxval = lbuf[nr];

                        {
                            int ntrees0 = ntrees_p*partidx;
                            int ntrees1 = min(ntrees0 + ntrees_p, ntrees);
                            int ix1 = idxval & 255, iy1 = idxval >> 8;
                            #if SUM_BUF_SIZE > 0
                            __local const int* p = ibuf + mad24(iy1, SUM_BUF_STEP, ix1);
                            #else
                            __global const int* p = psum0 + mad24(iy1, sumstep, ix1);
                            #endif

                            for( i = ntrees0; i < ntrees1; i++ )
                            {
                                float4 st = stump[i].st;
                                __global const OptLBPFeature* f = optfeatures + as_int(st.x);
                                int16 ofs = f->ofs;

                                #define CALC_SUM_OFS_(p0, p1, p2, p3, ptr) \
                                    ((ptr)[p0] - (ptr)[p1] - (ptr)[p2] + (ptr)[p3])

                                int cval = CALC_SUM_OFS_( ofs.s5, ofs.s6, ofs.s9, ofs.sa, p );

                                int mask, idx = (CALC_SUM_OFS_( ofs.s0, ofs.s1, ofs.s4, ofs.s5, p ) >= cval ? 4 : 0); // 0
                                idx |= (CALC_SUM_OFS_( ofs.s1, ofs.s2, ofs.s5, ofs.s6, p ) >= cval ? 2 : 0); // 1
                                idx |= (CALC_SUM_OFS_( ofs.s2, ofs.s3, ofs.s6, ofs.s7, p ) >= cval ? 1 : 0); // 2

                                mask = (CALC_SUM_OFS_( ofs.s6, ofs.s7, ofs.sa, ofs.sb, p ) >= cval ? 16 : 0); // 5
                                mask |= (CALC_SUM_OFS_( ofs.sa, ofs.sb, ofs.se, ofs.sf, p ) >= cval ? 8 : 0);  // 8
                                mask |= (CALC_SUM_OFS_( ofs.s9, ofs.sa, ofs.sd, ofs.se, p ) >= cval ? 4 : 0);  // 7
                                mask |= (CALC_SUM_OFS_( ofs.s8, ofs.s9, ofs.sc, ofs.sd, p ) >= cval ? 2 : 0);  // 6
                                mask |= (CALC_SUM_OFS_( ofs.s4, ofs.s5, ofs.s8, ofs.s9, p ) >= cval ? 1 : 0);  // 7

                                partsum += (bitset[i*bitsetSize + idx] & (1 << mask)) ? st.z : st.w;
                            }
                        }
                    }
                    lpartsum[lidx] = partsum;
                    barrier(CLK_LOCAL_MEM_FENCE);

                    if( partidx == 0 )
                    {
                        float s = lpartsum[nr*nparts];
                        for( i = 1; i < nparts; i++ )
                            s += lpartsum[i + nr*nparts];
                        if( s >= stages[stageIdx].threshold )
                        {
                            int count = atomic_inc(lcount);
                            lbuf[count] = idxval;
                        }
                    }
                }
            }

            barrier(CLK_LOCAL_MEM_FENCE);
            if( stageIdx == N_STAGES )
            {
                int nrects = lcount[0];
                if( lidx < nrects )
                {
                    int nfaces = atomic_inc(facepos);
                    if( nfaces < MAX_FACES )
                    {
                        volatile __global int* face = facepos + 1 + nfaces*3;
                        int val = lbuf[lidx];
                        face[0] = scaleIdx;
                        face[1] = ix0 + (val & 255);
                        face[2] = iy0 + (val >> 8);
                    }
                }
            }
        }
    }
}
#endif

///////////////////////////// OpenCL kernels for face detection //////////////////////////////
////////////////////////////// see the opencv/doc/license.txt ///////////////////////////////

//
// the code has been derived from the OpenCL Haar cascade kernel by
//
//    Niko Li, newlife20080214@gmail.com
//    Wang Weiyan, wangweiyanster@gmail.com
//    Jia Haipeng, jiahaipeng95@gmail.com
//    Nathan, liujun@multicorewareinc.com
//    Peng Xiao, pengxiao@outlook.com
//    Erping Pang, erping@multicorewareinc.com
//

#ifdef HAAR
typedef struct __attribute__((aligned(4))) OptHaarFeature
{
    int4 ofs[3] __attribute__((aligned (4)));
    float4 weight __attribute__((aligned (4)));
}
OptHaarFeature;
#endif

#ifdef LBP
typedef struct __attribute__((aligned(4))) OptLBPFeature
{
    int16 ofs __attribute__((aligned (4)));
}
OptLBPFeature;
#endif

typedef struct __attribute__((aligned(4))) Stump
{
    float4 st __attribute__((aligned (4)));
}
Stump;

typedef struct __attribute__((aligned(4))) Node
{
    int4 n __attribute__((aligned (4)));
}
Node;

typedef struct __attribute__((aligned (4))) Stage
{
    int first __attribute__((aligned (4)));
    int ntrees __attribute__((aligned (4)));
    float threshold __attribute__((aligned (4)));
}
Stage;

typedef struct __attribute__((aligned (4))) ScaleData
{
    float scale __attribute__((aligned (4)));
    int szi_width __attribute__((aligned (4)));
    int szi_height __attribute__((aligned (4)));
    int layer_ofs __attribute__((aligned (4)));
    int ystep __attribute__((aligned (4)));
}
ScaleData;

#ifndef SUM_BUF_SIZE
#define SUM_BUF_SIZE 0
#endif

#ifndef NODE_COUNT
#define NODE_COUNT 1
#endif

#ifdef HAAR
__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X,LOCAL_SIZE_Y,1)))
void runHaarClassifier(
    int nscales, __global const ScaleData* scaleData,
    __global const int* sum,
    int _sumstep, int sumoffset,
    __global const OptHaarFeature* optfeatures,
    __global const Stage* stages,
    __global const Node* nodes,
    __global const float* leaves0,

    volatile __global int* facepos,
    int4 normrect, int sqofs, int2 windowsize)
{
    int lx = get_local_id(0);
    int ly = get_local_id(1);
    int groupIdx = get_group_id(0);
    int i, ngroups = get_global_size(0)/LOCAL_SIZE_X;
    int scaleIdx, tileIdx, stageIdx;
    int sumstep = (int)(_sumstep/sizeof(int));
    int4 nofs0 = (int4)(mad24(normrect.y, sumstep, normrect.x),
                        mad24(normrect.y, sumstep, normrect.x + normrect.z),
                        mad24(normrect.y + normrect.w, sumstep, normrect.x),
                        mad24(normrect.y + normrect.w, sumstep, normrect.x + normrect.z));
    int normarea = normrect.z * normrect.w;
    float invarea = 1.f/normarea;
    int lidx = ly*LOCAL_SIZE_X + lx;

    #if SUM_BUF_SIZE > 0
    int4 nofs = (int4)(mad24(normrect.y, SUM_BUF_STEP, normrect.x),
                       mad24(normrect.y, SUM_BUF_STEP, normrect.x + normrect.z),
                       mad24(normrect.y + normrect.w, SUM_BUF_STEP, normrect.x),
                       mad24(normrect.y + normrect.w, SUM_BUF_STEP, normrect.x + normrect.z));
    #else
    int4 nofs = nofs0;
    #endif
    #define LOCAL_SIZE (LOCAL_SIZE_X*LOCAL_SIZE_Y)
    __local int lstore[SUM_BUF_SIZE + LOCAL_SIZE*5/2+1];
    #if SUM_BUF_SIZE > 0
    __local int* ibuf = lstore;
    __local int* lcount = ibuf + SUM_BUF_SIZE;
    #else
    __local int* lcount = lstore;
    #endif
    __local float* lnf = (__local float*)(lcount + 1);
    __local float* lpartsum = lnf + LOCAL_SIZE;
    __local short* lbuf = (__local short*)(lpartsum + LOCAL_SIZE);

    for( scaleIdx = nscales-1; scaleIdx >= 0; scaleIdx-- )
    {
        __global const ScaleData* s = scaleData + scaleIdx;
        int ystep = s->ystep;
        int2 worksize = (int2)(max(s->szi_width - windowsize.x, 0), max(s->szi_height - windowsize.y, 0));
        int2 ntiles = (int2)((worksize.x + LOCAL_SIZE_X-1)/LOCAL_SIZE_X,
                             (worksize.y + LOCAL_SIZE_Y-1)/LOCAL_SIZE_Y);
        int totalTiles = ntiles.x*ntiles.y;

        for( tileIdx = groupIdx; tileIdx < totalTiles; tileIdx += ngroups )
        {
            int ix0 = (tileIdx % ntiles.x)*LOCAL_SIZE_X;
            int iy0 = (tileIdx / ntiles.x)*LOCAL_SIZE_Y;
            int ix = lx, iy = ly;
            __global const int* psum0 = sum + mad24(iy0, sumstep, ix0) + s->layer_ofs;
            __global const int* psum1 = psum0 + mad24(iy, sumstep, ix);

            if( ix0 >= worksize.x || iy0 >= worksize.y )
                continue;
            #if SUM_BUF_SIZE > 0
            for( i = lidx*4; i < SUM_BUF_SIZE; i += LOCAL_SIZE_X*LOCAL_SIZE_Y*4 )
            {
                int dy = i/SUM_BUF_STEP, dx = i - dy*SUM_BUF_STEP;
                vstore4(vload4(0, psum0 + mad24(dy, sumstep, dx)), 0, ibuf+i);
            }
            #endif

            if( lidx == 0 )
                lcount[0] = 0;
            barrier(CLK_LOCAL_MEM_FENCE);

            if( ix0 + ix < worksize.x && iy0 + iy < worksize.y )
            {
                #if NODE_COUNT==1
                __global const Stump* stump = (__global const Stump*)nodes;
                #else
                __global const Node* node = nodes;
                __global const float* leaves = leaves0;
                #endif
                #if SUM_BUF_SIZE > 0
                __local const int* psum = ibuf + mad24(iy, SUM_BUF_STEP, ix);
                #else
                __global const int* psum = psum1;
                #endif

                __global const float* psqsum = (__global const float*)(psum1 + sqofs);
                float sval = (psum[nofs.x] - psum[nofs.y] - psum[nofs.z] + psum[nofs.w])*invarea;
                float sqval = (psqsum[nofs0.x] - psqsum[nofs0.y] - psqsum[nofs0.z] + psqsum[nofs0.w])*invarea;
                float nf = (float)normarea * sqrt(max(sqval - sval * sval, 0.f));
                nf = nf > 0 ? nf : 1.f;

                for( stageIdx = 0; stageIdx < SPLIT_STAGE; stageIdx++ )
                {
                    int ntrees = stages[stageIdx].ntrees;
                    float s = 0.f;
                    #if NODE_COUNT==1
                    for( i = 0; i < ntrees; i++ )
                    {
                        float4 st = stump[i].st;
                        __global const OptHaarFeature* f = optfeatures + as_int(st.x);
                        float4 weight = f->weight;

                        int4 ofs = f->ofs[0];
                        sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                        ofs = f->ofs[1];
                        sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.y;
                        if( weight.z > 0 )
                        {
                            ofs = f->ofs[2];
                            sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.z;
                        }

                        s += (sval < st.y*nf) ? st.z : st.w;
                    }
                    stump += ntrees;
                    #else
                    for( i = 0; i < ntrees; i++, node += NODE_COUNT, leaves += NODE_COUNT+1 )
                    {
                        int idx = 0;
                        do
                        {
                            int4 n = node[idx].n;
                            __global const OptHaarFeature* f = optfeatures + n.x;
                            float4 weight = f->weight;

                            int4 ofs = f->ofs[0];

                            sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                            ofs = f->ofs[1];
                            sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.y;
                            if( weight.z > 0 )
                            {
                                ofs = f->ofs[2];
                                sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.z;
                            }

                            idx = (sval < as_float(n.y)*nf) ? n.z : n.w;
                        }
                        while(idx > 0);
                        s += leaves[-idx];
                    }
                    #endif

                    if( s < stages[stageIdx].threshold )
                        break;
                }

                if( stageIdx == SPLIT_STAGE && (ystep == 1 || ((ix | iy) & 1) == 0) )
                {
                    int count = atomic_inc(lcount);
                    lbuf[count] = (int)(ix | (iy << 8));
                    lnf[count] = nf;
                }
            }

            for( stageIdx = SPLIT_STAGE; stageIdx < N_STAGES; stageIdx++ )
            {
                barrier(CLK_LOCAL_MEM_FENCE);
                int nrects = lcount[0];

                if( nrects == 0 )
                    break;
                barrier(CLK_LOCAL_MEM_FENCE);
                if( lidx == 0 )
                    lcount[0] = 0;

                {
                    #if NODE_COUNT == 1
                    __global const Stump* stump = (__global const Stump*)nodes + stages[stageIdx].first;
                    #else
                    __global const Node* node = nodes + stages[stageIdx].first*NODE_COUNT;
                    __global const float* leaves = leaves0 + stages[stageIdx].first*(NODE_COUNT+1);
                    #endif
                    int nparts = LOCAL_SIZE / nrects;
                    int ntrees = stages[stageIdx].ntrees;
                    int ntrees_p = (ntrees + nparts - 1)/nparts;
                    int nr = lidx / nparts;
                    int partidx = -1, idxval = 0;
                    float partsum = 0.f, nf = 0.f;

                    if( nr < nrects )
                    {
                        partidx = lidx % nparts;
                        idxval = lbuf[nr];
                        nf = lnf[nr];

                        {
                        int ntrees0 = ntrees_p*partidx;
                        int ntrees1 = min(ntrees0 + ntrees_p, ntrees);
                        int ix1 = idxval & 255, iy1 = idxval >> 8;
                        #if SUM_BUF_SIZE > 0
                        __local const int* psum = ibuf + mad24(iy1, SUM_BUF_STEP, ix1);
                        #else
                        __global const int* psum = psum0 + mad24(iy1, sumstep, ix1);
                        #endif

                        #if NODE_COUNT == 1
                        for( i = ntrees0; i < ntrees1; i++ )
                        {
                            float4 st = stump[i].st;
                            __global const OptHaarFeature* f = optfeatures + as_int(st.x);
                            float4 weight = f->weight;

                            int4 ofs = f->ofs[0];
                            float sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                            ofs = f->ofs[1];
                            sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.y;
                            //if( weight.z > 0 )
                            {
                                ofs = f->ofs[2];
                                sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.z;
                            }

                            partsum += (sval < st.y*nf) ? st.z : st.w;
                        }
                        #else
                        for( i = ntrees0; i < ntrees1; i++ )
                        {
                            int idx = 0;
                            do
                            {
                                int4 n = node[i*2 + idx].n;
                                __global const OptHaarFeature* f = optfeatures + n.x;
                                float4 weight = f->weight;
                                int4 ofs = f->ofs[0];

                                float sval = (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.x;
                                ofs = f->ofs[1];
                                sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.y;
                                if( weight.z > 0 )
                                {
                                    ofs = f->ofs[2];
                                    sval += (psum[ofs.x] - psum[ofs.y] - psum[ofs.z] + psum[ofs.w])*weight.z;
                                }

                                idx = (sval < as_float(n.y)*nf) ? n.z : n.w;
                            }
                            while(idx > 0);
                            partsum += leaves[i*3-idx];
                        }
                        #endif
                        }
                    }
                    lpartsum[lidx] = partsum;
                    barrier(CLK_LOCAL_MEM_FENCE);

                    if( partidx == 0 )
                    {
                        float s = lpartsum[nr*nparts];
                        for( i = 1; i < nparts; i++ )
                            s += lpartsum[i + nr*nparts];
                        if( s >= stages[stageIdx].threshold )
                        {
                            int count = atomic_inc(lcount);
                            lbuf[count] = idxval;
                            lnf[count] = nf;
                        }
                    }
                }
            }

            barrier(CLK_LOCAL_MEM_FENCE);
            if( stageIdx == N_STAGES )
            {
                int nrects = lcount[0];
                if( lidx < nrects )
                {
                    int nfaces = atomic_inc(facepos);
                    if( nfaces < MAX_FACES )
                    {
                        volatile __global int* face = facepos + 1 + nfaces*3;
                        int val = lbuf[lidx];
                        face[0] = scaleIdx;
                        face[1] = ix0 + (val & 255);
                        face[2] = iy0 + (val >> 8);
                    }
                }
            }
        }
    }
}
#endif

#ifdef LBP
#undef CALC_SUM_OFS_
#define CALC_SUM_OFS_(p0, p1, p2, p3, ptr) \
    ((ptr)[p0] - (ptr)[p1] - (ptr)[p2] + (ptr)[p3])

__kernel void runLBPClassifierStumpSimple(
    int nscales, __global const ScaleData* scaleData,
    __global const int* sum,
    int _sumstep, int sumoffset,
    __global const OptLBPFeature* optfeatures,
    __global const Stage* stages,
    __global const Stump* stumps,
    __global const int* bitsets,
    int bitsetSize,

    volatile __global int* facepos,
    int2 windowsize)
{
    int lx = get_local_id(0);
    int ly = get_local_id(1);
    int local_size_x = get_local_size(0);
    int local_size_y = get_local_size(1);
    int groupIdx = get_group_id(1)*get_num_groups(0) + get_group_id(0);
    int ngroups = get_num_groups(0)*get_num_groups(1);
    int scaleIdx, tileIdx, stageIdx;
    int sumstep = (int)(_sumstep/sizeof(int));

    for( scaleIdx = nscales-1; scaleIdx >= 0; scaleIdx-- )
    {
        __global const ScaleData* s = scaleData + scaleIdx;
        int ystep = s->ystep;
        int2 worksize = (int2)(max(s->szi_width - windowsize.x, 0), max(s->szi_height - windowsize.y, 0));
        int2 ntiles = (int2)((worksize.x/ystep + local_size_x-1)/local_size_x,
                             (worksize.y/ystep + local_size_y-1)/local_size_y);
        int totalTiles = ntiles.x*ntiles.y;

        for( tileIdx = groupIdx; tileIdx < totalTiles; tileIdx += ngroups )
        {
            int iy = mad24((tileIdx / ntiles.x), local_size_y, ly) * ystep;
            int ix = mad24((tileIdx % ntiles.x), local_size_x, lx) * ystep;

            if( ix < worksize.x && iy < worksize.y )
            {
                __global const int* p = sum + mad24(iy, sumstep, ix) + s->layer_ofs;
                __global const Stump* stump = stumps;
                __global const int* bitset = bitsets;

                for( stageIdx = 0; stageIdx < N_STAGES; stageIdx++ )
                {
                    int i, ntrees = stages[stageIdx].ntrees;
                    float s = 0.f;
                    for( i = 0; i < ntrees; i++, stump++, bitset += bitsetSize )
                    {
                        float4 st = stump->st;
                        __global const OptLBPFeature* f = optfeatures + as_int(st.x);
                        int16 ofs = f->ofs;

                        int cval = CALC_SUM_OFS_( ofs.s5, ofs.s6, ofs.s9, ofs.sa, p );

                        int mask, idx = (CALC_SUM_OFS_( ofs.s0, ofs.s1, ofs.s4, ofs.s5, p ) >= cval ? 4 : 0); // 0
                        idx |= (CALC_SUM_OFS_( ofs.s1, ofs.s2, ofs.s5, ofs.s6, p ) >= cval ? 2 : 0); // 1
                        idx |= (CALC_SUM_OFS_( ofs.s2, ofs.s3, ofs.s6, ofs.s7, p ) >= cval ? 1 : 0); // 2

                        mask = (CALC_SUM_OFS_( ofs.s6, ofs.s7, ofs.sa, ofs.sb, p ) >= cval ? 16 : 0); // 5
                        mask |= (CALC_SUM_OFS_( ofs.sa, ofs.sb, ofs.se, ofs.sf, p ) >= cval ? 8 : 0);  // 8
                        mask |= (CALC_SUM_OFS_( ofs.s9, ofs.sa, ofs.sd, ofs.se, p ) >= cval ? 4 : 0);  // 7
                        mask |= (CALC_SUM_OFS_( ofs.s8, ofs.s9, ofs.sc, ofs.sd, p ) >= cval ? 2 : 0);  // 6
                        mask |= (CALC_SUM_OFS_( ofs.s4, ofs.s5, ofs.s8, ofs.s9, p ) >= cval ? 1 : 0);  // 7

                        s += (bitset[idx] & (1 << mask)) ? st.z : st.w;
                    }

                    if( s < stages[stageIdx].threshold )
                        break;
                }

                if( stageIdx == N_STAGES )
                {
                    int nfaces = atomic_inc(facepos);
                    if( nfaces < MAX_FACES )
                    {
                        volatile __global int* face = facepos + 1 + nfaces*3;
                        face[0] = scaleIdx;
                        face[1] = ix;
                        face[2] = iy;
                    }
                }
            }
        }
    }
}

__kernel __attribute__((reqd_work_group_size(LOCAL_SIZE_X,LOCAL_SIZE_Y,1)))
void runLBPClassifierStump(
    int nscales, __global const ScaleData* scaleData,
    __global const int* sum,
    int _sumstep, int sumoffset,
    __global const OptLBPFeature* optfeatures,
    __global const Stage* stages,
    __global const Stump* stumps,
    __global const int* bitsets,
    int bitsetSize,

    volatile __global int* facepos,
    int2 windowsize)
{
    int lx = get_local_id(0);
    int ly = get_local_id(1);
    int groupIdx = get_group_id(0);
    int i, ngroups = get_global_size(0)/LOCAL_SIZE_X;
    int scaleIdx, tileIdx, stageIdx;
    int sumstep = (int)(_sumstep/sizeof(int));
    int lidx = ly*LOCAL_SIZE_X + lx;

    #define LOCAL_SIZE (LOCAL_SIZE_X*LOCAL_SIZE_Y)
    __local int lstore[SUM_BUF_SIZE + LOCAL_SIZE*3/2+1];
    #if SUM_BUF_SIZE > 0
    __local int* ibuf = lstore;
    __local int* lcount = ibuf + SUM_BUF_SIZE;
    #else
    __local int* lcount = lstore;
    #endif
    __local float* lpartsum = (__local float*)(lcount + 1);
    __local short* lbuf = (__local short*)(lpartsum + LOCAL_SIZE);

    for( scaleIdx = nscales-1; scaleIdx >= 0; scaleIdx-- )
    {
        __global const ScaleData* s = scaleData + scaleIdx;
        int ystep = s->ystep;
        int2 worksize = (int2)(max(s->szi_width - windowsize.x, 0), max(s->szi_height - windowsize.y, 0));
        int2 ntiles = (int2)((worksize.x + LOCAL_SIZE_X-1)/LOCAL_SIZE_X,
                             (worksize.y + LOCAL_SIZE_Y-1)/LOCAL_SIZE_Y);
        int totalTiles = ntiles.x*ntiles.y;

        for( tileIdx = groupIdx; tileIdx < totalTiles; tileIdx += ngroups )
        {
            int ix0 = (tileIdx % ntiles.x)*LOCAL_SIZE_X;
            int iy0 = (tileIdx / ntiles.x)*LOCAL_SIZE_Y;
            int ix = lx, iy = ly;
            __global const int* psum0 = sum + mad24(iy0, sumstep, ix0) + s->layer_ofs;

            if( ix0 >= worksize.x || iy0 >= worksize.y )
                continue;
            #if SUM_BUF_SIZE > 0
            for( i = lidx*4; i < SUM_BUF_SIZE; i += LOCAL_SIZE_X*LOCAL_SIZE_Y*4 )
            {
                int dy = i/SUM_BUF_STEP, dx = i - dy*SUM_BUF_STEP;
                vstore4(vload4(0, psum0 + mad24(dy, sumstep, dx)), 0, ibuf+i);
            }
            barrier(CLK_LOCAL_MEM_FENCE);
            #endif

            if( lidx == 0 )
                lcount[0] = 0;
            barrier(CLK_LOCAL_MEM_FENCE);

            if( ix0 + ix < worksize.x && iy0 + iy < worksize.y )
            {
                __global const Stump* stump = stumps;
                __global const int* bitset = bitsets;
                #if SUM_BUF_SIZE > 0
                __local const int* p = ibuf + mad24(iy, SUM_BUF_STEP, ix);
                #else
                __global const int* p = psum0 + mad24(iy, sumstep, ix);
                #endif

                for( stageIdx = 0; stageIdx < SPLIT_STAGE; stageIdx++ )
                {
                    int ntrees = stages[stageIdx].ntrees;
                    float s = 0.f;
                    for( i = 0; i < ntrees; i++, stump++, bitset += bitsetSize )
                    {
                        float4 st = stump->st;
                        __global const OptLBPFeature* f = optfeatures + as_int(st.x);
                        int16 ofs = f->ofs;

                        int cval = CALC_SUM_OFS_( ofs.s5, ofs.s6, ofs.s9, ofs.sa, p );

                        int mask, idx = (CALC_SUM_OFS_( ofs.s0, ofs.s1, ofs.s4, ofs.s5, p ) >= cval ? 4 : 0); // 0
                        idx |= (CALC_SUM_OFS_( ofs.s1, ofs.s2, ofs.s5, ofs.s6, p ) >= cval ? 2 : 0); // 1
                        idx |= (CALC_SUM_OFS_( ofs.s2, ofs.s3, ofs.s6, ofs.s7, p ) >= cval ? 1 : 0); // 2

                        mask = (CALC_SUM_OFS_( ofs.s6, ofs.s7, ofs.sa, ofs.sb, p ) >= cval ? 16 : 0); // 5
                        mask |= (CALC_SUM_OFS_( ofs.sa, ofs.sb, ofs.se, ofs.sf, p ) >= cval ? 8 : 0);  // 8
                        mask |= (CALC_SUM_OFS_( ofs.s9, ofs.sa, ofs.sd, ofs.se, p ) >= cval ? 4 : 0);  // 7
                        mask |= (CALC_SUM_OFS_( ofs.s8, ofs.s9, ofs.sc, ofs.sd, p ) >= cval ? 2 : 0);  // 6
                        mask |= (CALC_SUM_OFS_( ofs.s4, ofs.s5, ofs.s8, ofs.s9, p ) >= cval ? 1 : 0);  // 7

                        s += (bitset[idx] & (1 << mask)) ? st.z : st.w;
                    }

                    if( s < stages[stageIdx].threshold )
                        break;
                }

                if( stageIdx == SPLIT_STAGE && (ystep == 1 || ((ix | iy) & 1) == 0) )
                {
                    int count = atomic_inc(lcount);
                    lbuf[count] = (int)(ix | (iy << 8));
                }
            }

            for( stageIdx = SPLIT_STAGE; stageIdx < N_STAGES; stageIdx++ )
            {
                int nrects = lcount[0];

                barrier(CLK_LOCAL_MEM_FENCE);
                if( nrects == 0 )
                    break;
                if( lidx == 0 )
                    lcount[0] = 0;

                {
                    __global const Stump* stump = stumps + stages[stageIdx].first;
                    __global const int* bitset = bitsets + stages[stageIdx].first*bitsetSize;
                    int nparts = LOCAL_SIZE / nrects;
                    int ntrees = stages[stageIdx].ntrees;
                    int ntrees_p = (ntrees + nparts - 1)/nparts;
                    int nr = lidx / nparts;
                    int partidx = -1, idxval = 0;
                    float partsum = 0.f, nf = 0.f;

                    if( nr < nrects )
                    {
                        partidx = lidx % nparts;
                        idxval = lbuf[nr];

                        {
                            int ntrees0 = ntrees_p*partidx;
                            int ntrees1 = min(ntrees0 + ntrees_p, ntrees);
                            int ix1 = idxval & 255, iy1 = idxval >> 8;
                            #if SUM_BUF_SIZE > 0
                            __local const int* p = ibuf + mad24(iy1, SUM_BUF_STEP, ix1);
                            #else
                            __global const int* p = psum0 + mad24(iy1, sumstep, ix1);
                            #endif

                            for( i = ntrees0; i < ntrees1; i++ )
                            {
                                float4 st = stump[i].st;
                                __global const OptLBPFeature* f = optfeatures + as_int(st.x);
                                int16 ofs = f->ofs;

                                #define CALC_SUM_OFS_(p0, p1, p2, p3, ptr) \
                                    ((ptr)[p0] - (ptr)[p1] - (ptr)[p2] + (ptr)[p3])

                                int cval = CALC_SUM_OFS_( ofs.s5, ofs.s6, ofs.s9, ofs.sa, p );

                                int mask, idx = (CALC_SUM_OFS_( ofs.s0, ofs.s1, ofs.s4, ofs.s5, p ) >= cval ? 4 : 0); // 0
                                idx |= (CALC_SUM_OFS_( ofs.s1, ofs.s2, ofs.s5, ofs.s6, p ) >= cval ? 2 : 0); // 1
                                idx |= (CALC_SUM_OFS_( ofs.s2, ofs.s3, ofs.s6, ofs.s7, p ) >= cval ? 1 : 0); // 2

                                mask = (CALC_SUM_OFS_( ofs.s6, ofs.s7, ofs.sa, ofs.sb, p ) >= cval ? 16 : 0); // 5
                                mask |= (CALC_SUM_OFS_( ofs.sa, ofs.sb, ofs.se, ofs.sf, p ) >= cval ? 8 : 0);  // 8
                                mask |= (CALC_SUM_OFS_( ofs.s9, ofs.sa, ofs.sd, ofs.se, p ) >= cval ? 4 : 0);  // 7
                                mask |= (CALC_SUM_OFS_( ofs.s8, ofs.s9, ofs.sc, ofs.sd, p ) >= cval ? 2 : 0);  // 6
                                mask |= (CALC_SUM_OFS_( ofs.s4, ofs.s5, ofs.s8, ofs.s9, p ) >= cval ? 1 : 0);  // 7

                                partsum += (bitset[i*bitsetSize + idx] & (1 << mask)) ? st.z : st.w;
                            }
                        }
                    }
                    lpartsum[lidx] = partsum;
                    barrier(CLK_LOCAL_MEM_FENCE);

                    if( partidx == 0 )
                    {
                        float s = lpartsum[nr*nparts];
                        for( i = 1; i < nparts; i++ )
                            s += lpartsum[i + nr*nparts];
                        if( s >= stages[stageIdx].threshold )
                        {
                            int count = atomic_inc(lcount);
                            lbuf[count] = idxval;
                        }
                    }
                }
            }

            barrier(CLK_LOCAL_MEM_FENCE);
            if( stageIdx == N_STAGES )
            {
                int nrects = lcount[0];
                if( lidx < nrects )
                {
                    int nfaces = atomic_inc(facepos);
                    if( nfaces < MAX_FACES )
                    {
                        volatile __global int* face = facepos + 1 + nfaces*3;
                        int val = lbuf[lidx];
                        face[0] = scaleIdx;
                        face[1] = ix0 + (val & 255);
                        face[2] = iy0 + (val >> 8);
                    }
                }
            }
        }
    }
}
#endif


stupid up enter... undoing that last one

// Roe solver:

<?
-- for cartesian, use the normal in cartesian coordinates
-- however that 'x' is still being used for who knows what
--  ... and could be used for grid-metric-based norms
-- but you can't use 'cartesianFromCoord' because we're using the cartesian basis already
-- for coordinate or coordinate-orthonormal, use the coordinate-aligned normal
local gridIsCartesian = require 'coord.cartesian'.is(solver.coord)
local nonAxisAlignedNormals = solver.coord.vectorComponent == 'cartesian' and not gridIsCartesian
?>

//call this, initializing nL to the normalBasisForSide<?=side?>
// this calculates the rest based on the x coordinate
// TODO put this somewhere above where display functions can use it
void calcNormalBasisForGrid(
	real3x3* nL,
	real3x3* nU,
	real *nLen,
	real3 x
) {
	// if we are non-cartesian grid with cartesian components then the normal of the non-cartesian grid needs to be in cartesian components as well ...
<? if nonAxisAlignedNormals then ?>
	nL.x = coord_cartesianFromCoord(nL.x, xInt);
	nL.y = coord_cartesianFromCoord(nL.y, xInt);
	nL.z = coord_cartesianFromCoord(nL.z, xInt);
<? end ?>
<? if gridIsCartesian then -- or if the metric is identity ... cylinder surface ?>
	*nU = *nL;
	*nLen = 1;
<? else ?>
	//if we have a non-cartesian grid + cartesian component normal ... or if we have a non cartesian component normal ... then raising is different than the lowered version
	*nU = (real3x3){
		.x = coord_raise(nL->x, xInt),
		.y = coord_raise(nL->y, xInt),
		.z = coord_raise(nL->z, xInt),
	};
	*nLen = sqrt(real3_dot(nL->x, nU->x));
<? end ?>
}

//TODO entropy fix ... for the Euler equations at least
kernel void calcFlux(
	constant <?=solver.solver_t?>* solver,
	global <?=eqn.cons_t?>* fluxBuf,
	const global <?=solver.getULRArg?>,
	realparam dt
) {
	typedef <?=eqn.cons_t?> cons_t;
	typedef <?=eqn.eigen_t?> eigen_t;
	typedef <?=eqn.waves_t?> waves_t;

	SETBOUNDS(numGhost,numGhost-1);
	real3 xR = cell_x(i);
	int indexR = index;
	<? for side=0,solver.dim-1 do ?>{
		const int side = <?=side?>;
		int indexL = index - solver->stepsize.s<?=side?>;

		real dx = solver->grid_dx.s<?=side?>;

		real3 xL = xR;
		xL.s<?=side?> -= dx;

		real3 xInt = xR;
		xInt.s<?=side?> -= .5 * dx;

		//this is used for the flux limiter
		//should it be using the coordinate dx or the grid dx?
		//real dt_dx = dt / cell_dx<?=side?>(xInt);
		real dt_dx = dt / dx;

		real3 xIntL = xInt;
		xIntL.s<?=side?> -= dx;

		real3 xIntR = xInt;
		xIntR.s<?=side?> += dx;

<?=solver:getULRCode():gsub('\t', '\t\t')?>

		// here is where parallel propagator comes into play
		cons_t pUL = cons_parallelPropagate<?=side?>(*UL, xL, .5 * dx);
		cons_t pUR = cons_parallelPropagate<?=side?>(*UR, xR, -.5 * dx);

		int indexInt = side + dim * index;

		real3x3 nL = normalBasisForSide<?=side?>;
		real3x3 nU;
		real nLen;
		calcNormalBasisForGrid(&nL, &nU, &nLen, xInt);

		eigen_t eig = eigen_forInterface(solver, pUL, pUR, xInt, nL, nU, nLen);

<? if nonAxisAlignedNormals then ?>
<?=eqn:eigenWaveCodePrefixForNormal('nL', 'nU', 'nLen', 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? else ?>
<?=eqn:eigenWaveCodePrefixForSide(side, 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? end ?>

		waves_t fluxEig;
<? if not eqn.roeUseFluxFromCons then
?>		cons_t UAvg;
		for (int j = 0; j < numIntStates; ++j) {
			UAvg.ptr[j] = .5 * (pUL.ptr[j] + pUR.ptr[j]);
		}

<? 	if nonAxisAlignedNormals then ?>
		fluxEig = eigen_leftTransformForNormal(solver, eig, UAvg, xInt, nL, nU, nLen);
<?	else ?>
		fluxEig = eigen_leftTransformForSide<?=side?>(solver, eig, UAvg, xInt);
<?	end
end
?>
		cons_t deltaU;
<? if solver.fluxLimiter > 1 then
?>		int indexR2 = indexR + solver->stepsize.s<?=side?>;
		int indexL2 = indexL - solver->stepsize.s<?=side?>;
		<?=solver:getULRCode{indexL = 'indexL2', indexR = 'indexL', suffix='_L'}?>
		<?=solver:getULRCode{indexL = 'indexR', indexR = 'indexR2', suffix='_R'}?>

		// here is where parallel propagator comes into play
		cons_t pUL_L = cons_parallelPropagate<?=side?>(*UL_L, xIntL, 1.5 * dx);		//xIntL2?
		cons_t pUL_R = cons_parallelPropagate<?=side?>(*UL_R, xIntL, .5 * dx);
		cons_t pUR_L = cons_parallelPropagate<?=side?>(*UR_L, xIntR, -.5 * dx);
		cons_t pUR_R = cons_parallelPropagate<?=side?>(*UR_R, xIntR, -1.5 * dx);	// xIntR2?

		cons_t deltaUL, deltaUR;
<? end
?>
		for (int j = 0; j < numStates; ++j) {
			deltaU.ptr[j] = pUR.ptr[j] - pUL.ptr[j];
<? if solver.fluxLimiter > 1 then
?>			deltaUL.ptr[j] = pUR_L.ptr[j] - pUL_L.ptr[j];
			deltaUR.ptr[j] = pUR_R.ptr[j] - pUL_R.ptr[j];
<? end
?>		}

<? if nonAxisAlignedNormals then ?>
		waves_t deltaUEig = eigen_leftTransformForNormal(solver, eig, deltaU, xInt, nL, nU, nLen);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForNormal(solver, eigL, deltaUL, xIntL, nL, nU, nLen);
		waves_t deltaUEigR = eigen_leftTransformForNormal(solver, eigR, deltaUR, xIntR, nL, nU, nLen);
<? 	end ?>
<? else ?>
		waves_t deltaUEig = eigen_leftTransformForSide<?=side?>(solver, eig, deltaU, xInt);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForSide<?=side?>(solver, eigL, deltaUL, xIntL);
		waves_t deltaUEigR = eigen_leftTransformForSide<?=side?>(solver, eigR, deltaUR, xIntR);
<? 	end ?>
<? end ?>

		<? for j=0,eqn.numWaves-1 do ?>{
			const int j = <?=j?>;
			real lambda = <?=eqn:eigenWaveCode(side, 'eig', 'xInt', j)?>;

<? if not eqn.roeUseFluxFromCons then
?>			fluxEig.ptr[j] *= lambda;
<? else
?>			fluxEig.ptr[j] = 0.;
<? end
?>			real sgnLambda = lambda >= 0 ? 1 : -1;

<? if solver.fluxLimiter > 1 then
?>			real rEig;
			if (deltaUEig.ptr[j] == 0) {
				rEig = 0;
			} else {
				if (lambda >= 0) {
					rEig = deltaUEigL.ptr[j] / deltaUEig.ptr[j];
				} else {
					rEig = deltaUEigR.ptr[j] / deltaUEig.ptr[j];
				}
			}
			real phi = fluxLimiter(rEig);
<? end
?>
			fluxEig.ptr[j] -= .5 * lambda * deltaUEig.ptr[j] * (sgnLambda
<? if solver.fluxLimiter > 1 then
?>				+ phi * (lambda * dt_dx - sgnLambda)
<? end
?>			);
		}<? end ?>

		global cons_t* flux = fluxBuf + indexInt;

<? if nonAxisAlignedNormals then ?>
		*flux = eigen_rightTransformForNormal(solver, eig, fluxEig, xInt, nL, nU, nLen);
<? else ?>
		*flux = eigen_rightTransformForSide<?=side?>(solver, eig, fluxEig, xInt);
<? end ?>

<? if eqn.roeUseFluxFromCons then ?>
<? 	if nonAxisAlignedNormals then ?>
		cons_t FL = fluxFromConsForNormal(solver, *UL, xL, nL, nU, nLen);
		cons_t FR = fluxFromConsForNormal(solver, *UR, xR, nL, nU, nLen);
<?	else ?>
		cons_t FL = fluxFromConsForSide<?=side?>(solver, *UL, xL);
		cons_t FR = fluxFromConsForSide<?=side?>(solver, *UR, xR);
<?	end ?>

		cons_t pFL = cons_parallelPropagate<?=side?>(FL, xL, .5 * dx);
		cons_t pFR = cons_parallelPropagate<?=side?>(FR, xR, -.5 * dx);

		for (int j = 0; j < numIntStates; ++j) {
			flux->ptr[j] += .5 * (pFL.ptr[j] + pFR.ptr[j]);
		}
<? end
?>	}<? end ?>
}

// Roe solver:

<?
-- for cartesian, use the normal in cartesian coordinates
-- however that 'x' is still being used for who knows what
--  ... and could be used for grid-metric-based norms
-- but you can't use 'cartesianFromCoord' because we're using the cartesian basis already
-- for coordinate or coordinate-orthonormal, use the coordinate-aligned normal
local gridIsCartesian = require 'coord.cartesian'.is(solver.coord)
local nonAxisAlignedNormals = solver.coord.vectorComponent == 'cartesian' and not gridIsCartesian
?>

//call this, initializing nL to the normalBasisForSide<?=side?>
// this calculates the rest based on the x coordinate
// TODO put this somewhere above where display functions can use it
void calcNormalBasisForGrid(
	real3x3* nL,
	real3x3* nU,
	real *nLen,
	real3 x
) {
	// if we are non-cartesian grid with cartesian components then the normal of the non-cartesian grid needs to be in cartesian components as well ...
<? if nonAxisAlignedNormals then ?>
	nL.x = coord_cartesianFromCoord(nL.x, xInt);
	nL.y = coord_cartesianFromCoord(nL.y, xInt);
	nL.z = coord_cartesianFromCoord(nL.z, xInt);
<? end ?>
<? if gridIsCartesian then -- or if the metric is identity ... cylinder surface ?>
	*nU = *nL;
	*nLen = 1;
<? else ?>
	//if we have a non-cartesian grid + cartesian component normal ... or if we have a non cartesian component normal ... then raising is different than the lowered version
	*nU = (real3x3){
		.x = coord_raise(nL->x, xInt),
		.y = coord_raise(nL->y, xInt),
		.z = coord_raise(nL->z, xInt),
	};
	*nLen = sqrt(real3_dot(nL->x, nU->x));
<? end ?>
}

//TODO entropy fix ... for the Euler equations at least
kernel void calcFlux(
	constant <?=solver.solver_t?>* solver,
	global <?=eqn.cons_t?>* fluxBuf,
	const global <?=solver.getULRArg?>,
	realparam dt
) {
	typedef <?=eqn.cons_t?> cons_t;
	typedef <?=eqn.eigen_t?> eigen_t;
	typedef <?=eqn.waves_t?> waves_t;

	SETBOUNDS(numGhost,numGhost-1);
	real3 xR = cell_x(i);
	int indexR = index;
	<? for side=0,solver.dim-1 do ?>{
		const int side = <?=side?>;
		int indexL = index - solver->stepsize.s<?=side?>;

		real dx = solver->grid_dx.s<?=side?>;

		real3 xL = xR;
		xL.s<?=side?> -= dx;

		real3 xInt = xR;
		xInt.s<?=side?> -= .5 * dx;

		//this is used for the flux limiter
		//should it be using the coordinate dx or the grid dx?
		//real dt_dx = dt / cell_dx<?=side?>(xInt);
		real dt_dx = dt / dx;

		real3 xIntL = xInt;
		xIntL.s<?=side?> -= dx;

		real3 xIntR = xInt;
		xIntR.s<?=side?> += dx;

<?=solver:getULRCode():gsub('\t', '\t\t')?>

		// here is where parallel propagator comes into play
		cons_t pUL = cons_parallelPropagate<?=side?>(*UL, xL, .5 * dx);
		cons_t pUR = cons_parallelPropagate<?=side?>(*UR, xR, -.5 * dx);

		int indexInt = side + dim * index;

		real3x3 nL = normalBasisForSide<?=side?>;
		real3x3 nU;
		real nLen;
		calcNormalBasisForGrid(&nL, &nU, &nLen, xInt);

		eigen_t eig = eigen_forInterface(solver, pUL, pUR, xInt, nL, nU, nLen);

<? if nonAxisAlignedNormals then ?>
<?=eqn:eigenWaveCodePrefixForNormal('nL', 'nU', 'nLen', 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? else ?>
<?=eqn:eigenWaveCodePrefixForSide(side, 'eig', 'xInt'):gsub('\t', '\t\t')?>
<? end ?>

		waves_t fluxEig;
<? if not eqn.roeUseFluxFromCons then
?>		cons_t UAvg;
		for (int j = 0; j < numIntStates; ++j) {
			UAvg.ptr[j] = .5 * (pUL.ptr[j] + pUR.ptr[j]);
		}

<? 	if nonAxisAlignedNormals then ?>
#error here
		fluxEig = eigen_leftTransformForNormal(solver, eig, UAvg, xInt, nL, nU, nLen);
<?	else ?>
		fluxEig = eigen_leftTransformForSide<?=side?>(solver, eig, UAvg, xInt);
<?	end
end
?>
		cons_t deltaU;
<? if solver.fluxLimiter > 1 then
?>		int indexR2 = indexR + solver->stepsize.s<?=side?>;
		int indexL2 = indexL - solver->stepsize.s<?=side?>;
		<?=solver:getULRCode{indexL = 'indexL2', indexR = 'indexL', suffix='_L'}?>
		<?=solver:getULRCode{indexL = 'indexR', indexR = 'indexR2', suffix='_R'}?>

		// here is where parallel propagator comes into play
		cons_t pUL_L = cons_parallelPropagate<?=side?>(*UL_L, xIntL, 1.5 * dx);		//xIntL2?
		cons_t pUL_R = cons_parallelPropagate<?=side?>(*UL_R, xIntL, .5 * dx);
		cons_t pUR_L = cons_parallelPropagate<?=side?>(*UR_L, xIntR, -.5 * dx);
		cons_t pUR_R = cons_parallelPropagate<?=side?>(*UR_R, xIntR, -1.5 * dx);	// xIntR2?

		cons_t deltaUL, deltaUR;
<? end
?>
		for (int j = 0; j < numStates; ++j) {
			deltaU.ptr[j] = pUR.ptr[j] - pUL.ptr[j];
<? if solver.fluxLimiter > 1 then
?>			deltaUL.ptr[j] = pUR_L.ptr[j] - pUL_L.ptr[j];
			deltaUR.ptr[j] = pUR_R.ptr[j] - pUL_R.ptr[j];
<? end
?>		}

<? if nonAxisAlignedNormals then ?>
		waves_t deltaUEig = eigen_leftTransformForNormal(solver, eig, deltaU, xInt, nL, nU, nLen);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForNormal(solver, eigL, deltaUL, xIntL, nL, nU, nLen);
		waves_t deltaUEigR = eigen_leftTransformForNormal(solver, eigR, deltaUR, xIntR, nL, nU, nLen);
<? 	end ?>
<? else ?>
		waves_t deltaUEig = eigen_leftTransformForSide<?=side?>(solver, eig, deltaU, xInt);
<? 	if solver.fluxLimiter > 1 then ?>
		eigen_t eigL = eigen_forInterface(solver, pUL_L, pUR_L, xInt, nL, nU, nLen);
		eigen_t eigR = eigen_forInterface(solver, pUL_R, pUR_R, xInt, nL, nU, nLen);
		waves_t deltaUEigL = eigen_leftTransformForSide<?=side?>(solver, eigL, deltaUL, xIntL);
		waves_t deltaUEigR = eigen_leftTransformForSide<?=side?>(solver, eigR, deltaUR, xIntR);
<? 	end ?>
<? end ?>

		<? for j=0,eqn.numWaves-1 do ?>{
			const int j = <?=j?>;
			real lambda = <?=eqn:eigenWaveCode(side, 'eig', 'xInt', j)?>;

<? if not eqn.roeUseFluxFromCons then
?>			fluxEig.ptr[j] *= lambda;
<? else
?>			fluxEig.ptr[j] = 0.;
<? end
?>			real sgnLambda = lambda >= 0 ? 1 : -1;

<? if solver.fluxLimiter > 1 then
?>			real rEig;
			if (deltaUEig.ptr[j] == 0) {
				rEig = 0;
			} else {
				if (lambda >= 0) {
					rEig = deltaUEigL.ptr[j] / deltaUEig.ptr[j];
				} else {
					rEig = deltaUEigR.ptr[j] / deltaUEig.ptr[j];
				}
			}
			real phi = fluxLimiter(rEig);
<? end
?>
			fluxEig.ptr[j] -= .5 * lambda * deltaUEig.ptr[j] * (sgnLambda
<? if solver.fluxLimiter > 1 then
?>				+ phi * (lambda * dt_dx - sgnLambda)
<? end
?>			);
		}<? end ?>

		global cons_t* flux = fluxBuf + indexInt;

<? if nonAxisAlignedNormals then ?>
		*flux = eigen_rightTransformForNormal(solver, eig, fluxEig, xInt, nL, nU, nLen);
<? else ?>
		*flux = eigen_rightTransformForSide<?=side?>(solver, eig, fluxEig, xInt);
<? end ?>

<? if eqn.roeUseFluxFromCons then ?>
<? 	if nonAxisAlignedNormals then ?>
		cons_t FL = fluxFromConsForNormal(solver, *UL, xL, nL, nU, nLen);
		cons_t FR = fluxFromConsForNormal(solver, *UR, xR, nL, nU, nLen);
<?	else ?>
		cons_t FL = fluxFromConsForSide<?=side?>(solver, *UL, xL);
		cons_t FR = fluxFromConsForSide<?=side?>(solver, *UR, xR);
<?	end ?>

		cons_t pFL = cons_parallelPropagate<?=side?>(FL, xL, .5 * dx);
		cons_t pFR = cons_parallelPropagate<?=side?>(FR, xR, -.5 * dx);

		for (int j = 0; j < numIntStates; ++j) {
			flux->ptr[j] += .5 * (pFL.ptr[j] + pFR.ptr[j]);
		}
<? end
?>	}<? end ?>
}


Fix test_prime (#134)

Fix test_prime (#134)

__kernel void test_prime(const int maybePrime, __global bool* out)
{
	float tmp = maybePrime;
	tmp = sqrt(tmp);
	int root = ceil(tmp);
	for(int i = 2; i <= root; i++)
	{
		if(maybePrime % i == 0)
		{
			*out = false;
			return;
		}
	}
	*out = true;
}

__kernel void test_prime(const int maybePrime, __global bool* out)
{
	float tmp = maybePrime;
	tmp = sqrt(tmp);
	int root = ceil(tmp);
	for(int i = 2; i <= root; i++)
	{
		if(root % i == 0)
		{
			*out = false;
			return;
		}
	}
	*out = true;
}


Add debug output to printf implementation (disabled by default)

/* OpenCL built-in library: printf()

   Copyright (c) 2013 Erik Schnetter <eschnetter@perimeterinstitute.ca>
                      Perimeter Institute for Theoretical Physics

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

// Make the C99 printf visible again
#undef printf

#include <stdarg.h>

// We implement the OpenCL printf by calling the C99 printf. This is
// not very efficient, but is easy to implement.
int printf(const char* restrict fmt, ...);
int snprintf(char* restrict str, size_t size, const char* restrict fmt, ...);

// For debugging
// Use as: DEBUG_PRINTF((fmt, args...)) -- note double parentheses!
// #define DEBUG_PRINTF(args) printf(args)
#define DEBUG_PRINTF(args) ((void)0)

// Conversion flags
typedef struct {
  bool left;
  bool plus;
  bool space;
  bool alt;
  bool zero;
} flags_t;

// Helper routines to output integers

#define INT_CONV_char  "hh"
#define INT_CONV_short "h"
#define INT_CONV_int   ""
#define INT_CONV_long  "ll"     // C99 printf uses "ll" for int64_t

#define DEFINE_PRINT_INTS(WIDTH)                                        \
  void _cl_print_ints_##WIDTH(flags_t flags, int field_width, int precision, \
                              char conv, const void* vals, int n)       \
  {                                                                     \
    DEBUG_PRINTF(("[printf:ints:n=%df]\n", n));                         \
    char outfmt[1000];                                                  \
    snprintf(outfmt, sizeof outfmt,                                     \
             "%%%s%s%s%s%s%.0d%s%.0d" INT_CONV_##WIDTH "%c",            \
             flags.left ? "-" : "",                                     \
             flags.plus ? "+" : "",                                     \
             flags.space ? " " : "",                                    \
             flags.alt ? "#" : "",                                      \
             flags.zero ? "0" : "",                                     \
             field_width,                                               \
             precision != -1 ? "." : "",                                \
             precision != -1 ? precision : 0,                           \
             conv);                                                     \
    DEBUG_PRINTF(("[printf:ints:outfmt=%s]\n", outfmt));                \
    for (int d=0; d<n; ++d) {                                           \
      DEBUG_PRINTF(("[printf:ints:d=%d]\n", d));                        \
      if (d != 0) printf(",");                                          \
      printf(outfmt, ((const WIDTH*)vals)[d]);                          \
    }                                                                   \
    DEBUG_PRINTF(("[printf:ints:done]\n"));                             \
  }

DEFINE_PRINT_INTS(char)
DEFINE_PRINT_INTS(short)
DEFINE_PRINT_INTS(int)
#ifdef cles_khr_int64
DEFINE_PRINT_INTS(long)
#endif

#undef DEFINE_PRINT_INTS

// Helper routines to output floats

// Note: To simplify implementation, we print double values with %lf,
// although %f would suffice as well
#define FLOAT_CONV_half   "h"
#define FLOAT_CONV_float  ""
#define FLOAT_CONV_double "l"

#define DEFINE_PRINT_FLOATS(WIDTH)                                      \
  void _cl_print_floats_##WIDTH(flags_t flags, int field_width, int precision, \
                                char conv, const void* vals, int n)     \
  {                                                                     \
    char outfmt[1000];                                                  \
    snprintf(outfmt, sizeof outfmt,                                     \
             "%%%s%s%s%s%s%.0d%s%.0d" FLOAT_CONV_##WIDTH "%c",          \
             flags.left ? "-" : "",                                     \
             flags.plus ? "+" : "",                                     \
             flags.space ? " " : "",                                    \
             flags.alt ? "#" : "",                                      \
             flags.zero ? "0" : "",                                     \
             field_width,                                               \
             precision != -1 ? "." : "",                                \
             precision != -1 ? precision : 0,                           \
             conv);                                                     \
    for (int d=0; d<n; ++d) {                                           \
      if (d != 0) printf(",");                                          \
      printf(outfmt, ((const WIDTH*)vals)[d]);                          \
    }                                                                   \
  }

#ifdef cl_khr_fp16
DEFINE_PRINT_FLOATS(half)
#endif
DEFINE_PRINT_FLOATS(float)
#ifdef cl_khr_fp64
DEFINE_PRINT_FLOATS(double)
#endif

#undef DEFINE_PRINT_FLOATS

// The OpenCL printf routine.

// The implementation is straightforward:
// - walk through the format string
// - when a variable should be output, parse flags, field width,
//   precision, vector specifier, length, and conversion specifier
// - call a helper routine to perform the actual output
// - the helper routine is based on calling C99 printf, and constructs
//   a format string via snprintf
// - if there is an error during parsing, a "goto error" aborts the
//   routine, returning -1

int _cl_printf(const char* restrict format, ...)
{
  DEBUG_PRINTF(("[printf:format=%s]\n", format));
  va_list ap;
  va_start(ap, format);

  char ch = *format;
  while (ch) {
    if (ch == '%') {
      ch = *++format;

      if (ch == '%') {
        DEBUG_PRINTF(("[printf:%%]\n"));
        printf("%%");           // literal %
        ch = *++format;
      } else {
        DEBUG_PRINTF(("[printf:arg]\n"));
        // Flags
        flags_t flags;
        flags.left = false;
        flags.plus = false;
        flags.space = false;
        flags.alt = false;
        flags.zero = false;
        for (;;) {
          switch (ch) {
          case '-': if (flags.left) goto error; flags.left = true; break;
          case '+': if (flags.plus) goto error; flags.plus = true; break;
          case ' ': if (flags.space) goto error; flags.space = true; break;
          case '#': if (flags.alt) goto error; flags.alt = true; break;
          case '0': if (flags.zero) goto error; flags.zero = true; break;
          default: goto flags_done;
          }
          ch = *++format;
        }
      flags_done:;
        DEBUG_PRINTF(("[printf:flags:left=%d,plus=%d,space=%d,alt=%d,zero=%d]\n",
                      flags.left, flags.plus, flags.space, flags.alt, flags.zero));

        // Field width
        int field_width = 0;
        while (ch >= '0' && ch <= '9') {
          if (ch == '0' && field_width == 0) goto error;
          if (field_width > (INT_MAX - 9) / 10) goto error;
          field_width = 10 * field_width + (ch - '0');
          ch = *++format;
        }
        DEBUG_PRINTF(("[printf:width=%d]\n", field_width));

        // Precision
        int precision;
        if (ch == '.') {
          ch = *++format;
          precision = 0;
          while (ch >= '0' && ch <= '9') {
            if (precision > (INT_MAX - 9) / 10) goto error;
            precision = 10 * precision + (ch - '0');
            ch = *++format;
          }
        } else {
          precision = -1;
        }
        DEBUG_PRINTF(("[printf:precision=%d]\n", precision));

        // Vector specifier
        int vector_length = 0;
        if (ch == 'v') {
          ch = *++format;
          while (ch >= '0' && ch <= '9') {
            if (ch == '0' && vector_length == 0) goto error;
            if (vector_length > (INT_MAX - 9) / 10) goto error;
            vector_length = 10 * vector_length + (ch - '0');
            ch = *++format;
          }
          if (! (vector_length == 2 ||
                 vector_length == 3 ||
                 vector_length == 4 ||
                 vector_length == 8 ||
                 vector_length == 16)) goto error;

        }
        DEBUG_PRINTF(("[printf:vector_length=%d]\n", vector_length));

        // Length modifier
        int length = 0;           // default
        if (ch == 'h') {
          ch = *++format;
          if (ch == 'h') {
            ch = *++format;
            length = 1;           // "hh" -> char
          } else if (ch == 'l') {
            ch = *++format;
            length = 4;           // "hl" -> int
          } else {
            length = 2;           // "h" -> short
          }
        } else if (ch == 'l') {
          ch = *++format;
          length = 8;             // "l" -> long
        }
        if (vector_length > 0 && length == 0) goto error;
        if (vector_length == 0 && length == 4) goto error;
        if (vector_length == 0) vector_length = 1;
        DEBUG_PRINTF(("[printf:length=%d]\n", length));

        // Conversion specifier
        switch (ch) {

          // Output integers
        case 'd':
        case 'i':
        case 'o':
        case 'u':
        case 'x':
        case 'X':

#define CALL_PRINT_INTS(WIDTH, PROMOTED_WIDTH)                          \
          {                                                             \
            WIDTH##16 val;                                              \
            switch (vector_length) {                                    \
            default: __builtin_unreachable();                           \
            case 1: val.s0 = va_arg(ap, PROMOTED_WIDTH); break;         \
            case 2: val.s01 = va_arg(ap, WIDTH##2); break;              \
            case 3: val.s012 = va_arg(ap, WIDTH##3); break;             \
            case 4: val.s0123 = va_arg(ap, WIDTH##4); break;            \
            case 8: val.lo = va_arg(ap, WIDTH##8); break;               \
            case 16: val = va_arg(ap, WIDTH##16); break;                \
            }                                                           \
            _cl_print_ints_##WIDTH(flags, field_width, precision,       \
                                   ch, &val, vector_length);            \
          }

          DEBUG_PRINTF(("[printf:int:conversion=%c]\n", ch));
          switch (length) {
          default: __builtin_unreachable();
          case 1: CALL_PRINT_INTS(char, int); break;
          case 2: CALL_PRINT_INTS(short, int); break;
          case 0:
          case 4: CALL_PRINT_INTS(int, int); break;
#ifdef cles_khr_int64
          case 8: CALL_PRINT_INTS(long, long); break;
#endif
          }

#undef CALL_PRINT_INTS

          break;

          // Output floats
        case 'f':
        case 'F':
        case 'e':
        case 'E':
        case 'g':
        case 'G':
        case 'a':
        case 'A':

#define CALL_PRINT_FLOATS(WIDTH, PROMOTED_WIDTH)                        \
          {                                                             \
            WIDTH##16 val;                                              \
            switch (vector_length) {                                    \
            default: __builtin_unreachable();                           \
            case 1: val.s0 = va_arg(ap, PROMOTED_WIDTH); break;         \
            case 2: val.s01 = va_arg(ap, WIDTH##2); break;              \
            case 3: val.s012 = va_arg(ap, WIDTH##3); break;             \
            case 4: val.s0123 = va_arg(ap, WIDTH##4); break;            \
            case 8: val.lo = va_arg(ap, WIDTH##8); break;               \
            case 16: val = va_arg(ap, WIDTH##16); break;                \
            }                                                           \
            _cl_print_floats_##WIDTH(flags, field_width, precision,     \
                                     ch, &val, vector_length);          \
          }

          DEBUG_PRINTF(("[printf:float:conversion=%c]\n", ch));
          switch (length) {
          default: __builtin_unreachable();
#ifdef cles_khr_fp16
          case 2: CALL_PRINT_FLOATS(half, double); break;
#endif
          case 0:
            // Note: width 0 cleverly falls through to float if double
            // is not supported
#ifdef cles_khr_fp64
          case 8: CALL_PRINT_FLOATS(double, double); break;
#endif
          case 4: CALL_PRINT_FLOATS(float, double); break;
          }

#undef CALL_PRINT_FLOATS

          break;

        case 'c': goto error;   // not yet implemented
        case 's': goto error;   // not yet implemented
        case 'p': goto error;   // not yet implemented

        default: goto error;
        }
        ch = *++format;

      } // not a literal %

    } else {
      DEBUG_PRINTF(("[printf:char]\n"));
      printf("%c", ch);
      ch = *++format;
    }
  }

  va_end(ap);
  DEBUG_PRINTF(("[printf:done]\n"));
  return 0;

 error:;
  va_end(ap);
  DEBUG_PRINTF(("[printf:error]\n"));
  return -1;
}

/* OpenCL built-in library: printf()

   Copyright (c) 2013 Erik Schnetter <eschnetter@perimeterinstitute.ca>
                      Perimeter Institute for Theoretical Physics

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

// Make the C99 printf visible again
#undef printf

#include <stdarg.h>

// We implement the OpenCL printf by calling the C99 printf. This is
// not very efficient, but is easy to implement.
int printf(const char* restrict fmt, ...);
int snprintf(char* restrict str, size_t size, const char* restrict fmt, ...);

// Conversion flags
typedef struct {
  bool left;
  bool plus;
  bool space;
  bool alt;
  bool zero;
} flags_t;

// Helper routines to output integers

#define INT_CONV_char  "hh"
#define INT_CONV_short "h"
#define INT_CONV_int   ""
#define INT_CONV_long  "ll"     // C99 printf uses "ll" for int64_t

#define DEFINE_PRINT_INTS(WIDTH)                                        \
  void _cl_print_ints_##WIDTH(flags_t flags, int field_width, int precision, \
                              char conv, const void* vals, int n)       \
  {                                                                     \
    char outfmt[1000];                                                  \
    snprintf(outfmt, sizeof outfmt,                                     \
             "%%%s%s%s%s%s%.0d%s%.0d" INT_CONV_##WIDTH "%c",            \
             flags.left ? "-" : "",                                     \
             flags.plus ? "+" : "",                                     \
             flags.space ? " " : "",                                    \
             flags.alt ? "#" : "",                                      \
             flags.zero ? "0" : "",                                     \
             field_width,                                               \
             precision != -1 ? "." : "",                                \
             precision != -1 ? precision : 0,                           \
             conv);                                                     \
    for (int d=0; d<n; ++d) {                                           \
      if (d != 0) printf(",");                                          \
      printf(outfmt, ((const WIDTH*)vals)[d]);                          \
    }                                                                   \
  }

DEFINE_PRINT_INTS(char)
DEFINE_PRINT_INTS(short)
DEFINE_PRINT_INTS(int)
#ifdef cles_khr_int64
DEFINE_PRINT_INTS(long)
#endif

#undef DEFINE_PRINT_INTS

// Helper routines to output floats

// Note: To simplify implementation, we print double values with %lf,
// although %f would suffice as well
#define FLOAT_CONV_half   "h"
#define FLOAT_CONV_float  ""
#define FLOAT_CONV_double "l"

#define DEFINE_PRINT_FLOATS(WIDTH)                                      \
  void _cl_print_floats_##WIDTH(flags_t flags, int field_width, int precision, \
                                char conv, const void* vals, int n)     \
  {                                                                     \
    char outfmt[1000];                                                  \
    snprintf(outfmt, sizeof outfmt,                                     \
             "%%%s%s%s%s%s%.0d%s%.0d" FLOAT_CONV_##WIDTH "%c",          \
             flags.left ? "-" : "",                                     \
             flags.plus ? "+" : "",                                     \
             flags.space ? " " : "",                                    \
             flags.alt ? "#" : "",                                      \
             flags.zero ? "0" : "",                                     \
             field_width,                                               \
             precision != -1 ? "." : "",                                \
             precision != -1 ? precision : 0,                           \
             conv);                                                     \
    for (int d=0; d<n; ++d) {                                           \
      if (d != 0) printf(",");                                          \
      printf(outfmt, ((const WIDTH*)vals)[d]);                          \
    }                                                                   \
  }

#ifdef cl_khr_fp16
DEFINE_PRINT_FLOATS(half)
#endif
DEFINE_PRINT_FLOATS(float)
#ifdef cl_khr_fp64
DEFINE_PRINT_FLOATS(double)
#endif

#undef DEFINE_PRINT_FLOATS

// The OpenCL printf routine.

// The implementation is straightforward:
// - walk through the format string
// - when a variable should be output, parse flags, field width,
//   precision, vector specifier, length, and conversion specifier
// - call a helper routine to perform the actual output
// - the helper routine is based on calling C99 printf, and constructs
//   a format string via snprintf
// - if there is an error during parsing, a "goto error" aborts the
//   routine, returning -1

int _cl_printf(const char* restrict format, ...)
{
  va_list ap;
  va_start(ap, format);

  char ch = *format;
  while (ch) {
    if (ch == '%') {
      ch = *++format;

      if (ch == '%') {
        printf("%%");           // literal %
        ch = *++format;
      } else {

        // Flags
        flags_t flags;
        flags.left = false;
        flags.plus = false;
        flags.space = false;
        flags.alt = false;
        flags.zero = false;
        for (;;) {
          switch (ch) {
          case '-': if (flags.left) goto error; flags.left = true; break;
          case '+': if (flags.plus) goto error; flags.plus = true; break;
          case ' ': if (flags.space) goto error; flags.space = true; break;
          case '#': if (flags.alt) goto error; flags.alt = true; break;
          case '0': if (flags.zero) goto error; flags.zero = true; break;
          default: goto flags_done;
          }
          ch = *++format;
        }
      flags_done:;

        // Field width
        int field_width = 0;
        while (ch >= '0' && ch <= '9') {
          if (ch == '0' && field_width == 0) goto error;
          if (field_width > (INT_MAX - 9) / 10) goto error;
          field_width = 10 * field_width + (ch - '0');
          ch = *++format;
        }

        // Precision
        int precision;
        if (ch == '.') {
          ch = *++format;
          precision = 0;
          while (ch >= '0' && ch <= '9') {
            if (precision > (INT_MAX - 9) / 10) goto error;
            precision = 10 * precision + (ch - '0');
            ch = *++format;
          }
        } else {
          precision = -1;
        }

        // Vector specifier
        int vector_length = 0;
        if (ch == 'v') {
          ch = *++format;
          while (ch >= '0' && ch <= '9') {
            if (ch == '0' && vector_length == 0) goto error;
            if (vector_length > (INT_MAX - 9) / 10) goto error;
            vector_length = 10 * vector_length + (ch - '0');
            ch = *++format;
          }
          if (! (vector_length == 2 ||
                 vector_length == 3 ||
                 vector_length == 4 ||
                 vector_length == 8 ||
                 vector_length == 16)) goto error;

        }

        // Length modifier
        int length = 0;           // default
        if (ch == 'h') {
          ch = *++format;
          if (ch == 'h') {
            ch = *++format;
            length = 1;           // "hh" -> char
          } else if (ch == 'l') {
            ch = *++format;
            length = 4;           // "hl" -> int
          } else {
            length = 2;           // "h" -> short
          }
        } else if (ch == 'l') {
          ch = *++format;
          length = 8;             // "l" -> long
        }
        if (vector_length > 0 && length == 0) goto error;
        if (vector_length == 0 && length == 4) goto error;
        if (vector_length == 0) vector_length = 1;

        // Conversion specifier
        switch (ch) {

          // Output integers
        case 'd':
        case 'i':
        case 'o':
        case 'u':
        case 'x':
        case 'X':

#define CALL_PRINT_INTS(WIDTH, PROMOTED_WIDTH)                          \
          {                                                             \
            WIDTH##16 val;                                              \
            switch (vector_length) {                                    \
            default: __builtin_unreachable();                           \
            case 1: val.s0 = va_arg(ap, PROMOTED_WIDTH); break;         \
            case 2: val.s01 = va_arg(ap, WIDTH##2); break;              \
            case 3: val.s012 = va_arg(ap, WIDTH##3); break;             \
            case 4: val.s0123 = va_arg(ap, WIDTH##4); break;            \
            case 8: val.lo = va_arg(ap, WIDTH##8); break;               \
            case 16: val = va_arg(ap, WIDTH##16); break;                \
            }                                                           \
            _cl_print_ints_##WIDTH(flags, field_width, precision,       \
                                   ch, &val, vector_length);            \
          }

          switch (length) {
          default: __builtin_unreachable();
          case 1: CALL_PRINT_INTS(char, int); break;
          case 2: CALL_PRINT_INTS(short, int); break;
          case 0:
          case 4: CALL_PRINT_INTS(int, int); break;
#ifdef cles_khr_int64
          case 8: CALL_PRINT_INTS(long, long); break;
#endif
          }

#undef CALL_PRINT_INTS

          break;

          // Output floats
        case 'f':
        case 'F':
        case 'e':
        case 'E':
        case 'g':
        case 'G':
        case 'a':
        case 'A':

#define CALL_PRINT_FLOATS(WIDTH, PROMOTED_WIDTH)                        \
          {                                                             \
            WIDTH##16 val;                                              \
            switch (vector_length) {                                    \
            default: __builtin_unreachable();                           \
            case 1: val.s0 = va_arg(ap, PROMOTED_WIDTH); break;         \
            case 2: val.s01 = va_arg(ap, WIDTH##2); break;              \
            case 3: val.s012 = va_arg(ap, WIDTH##3); break;             \
            case 4: val.s0123 = va_arg(ap, WIDTH##4); break;            \
            case 8: val.lo = va_arg(ap, WIDTH##8); break;               \
            case 16: val = va_arg(ap, WIDTH##16); break;                \
            }                                                           \
            _cl_print_floats_##WIDTH(flags, field_width, precision,     \
                                     ch, &val, vector_length);          \
          }

          switch (length) {
          default: __builtin_unreachable();
#ifdef cles_khr_fp16
          case 2: CALL_PRINT_FLOATS(half, double); break;
#endif
          case 0:
            // Note: width 0 cleverly falls through to float if double
            // is not supported
#ifdef cles_khr_fp64
          case 8: CALL_PRINT_FLOATS(double, double); break;
#endif
          case 4: CALL_PRINT_FLOATS(float, double); break;
          }

#undef CALL_PRINT_FLOATS

          break;

        case 'c': goto error;   // not yet implemented
        case 's': goto error;   // not yet implemented
        case 'p': goto error;   // not yet implemented

        default: goto error;
        }
        ch = *++format;

      } // not a literal %

    } else {
      printf("%c", ch);
      ch = *++format;
    }
  }

  va_end(ap);
  return 0;

 error:;
  va_end(ap);
  return -1;
}