double backprop speed :-)

double backprop speed :-)

// Copyright Hugh Perkins 2014 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) tanh(output)
    #define ACTIVATION_DERIV(output) 1 - output * output
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) output> 0 ? output : 0
    #define ACTIVATION_DERIV(output) output > 0 ? 1 : 0
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) output
    #define ACTIVATION_DERIV(output) 1
#endif

void kernel convolve_ints( global const int *p_boardSize, global const int *p_filterSize,
      global const int *image, global const int *filter, global int *result ) {
    int id = get_global_id(0);
    int boardSize = p_boardSize[0];
    int filterSize = p_filterSize[0];
    int boardOffset = id / (boardSize * boardSize ) * (boardSize * boardSize );
    int localid = id % (boardSize * boardSize );
    int row = localid / boardSize;
    int col = localid % boardSize;
    int halfFilterSize = filterSize >> 1;
    int sum = 0;
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int m = minm;
    while( m <= maxm ) {
        int x = ( row + m );
        int xboard = boardOffset + x * boardSize;
        int filterrowoffset = (m+halfFilterSize) * filterSize + halfFilterSize;
        int n = minn;
        while( n <= maxn ) {
            int y = col + n;
            sum += image[ xboard + y] * filter[ filterrowoffset + n ];
            n++;
        }
        m++;
    }
    result[id] = sum;
}

void kernel convolve_floats( global const int *p_boardSize, global const int *p_filterSize,
      global const float *image, global const float *filter, global float *result ) {
    int id = get_global_id(0);
    int boardSize = p_boardSize[0];
    int filterSize = p_filterSize[0];
    int boardOffset = id / (boardSize * boardSize ) * (boardSize * boardSize );
    int localid = id % (boardSize * boardSize );
    int row = localid / boardSize;
    int col = localid % boardSize;
    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int m = minm;
    while( m <= maxm ) {
        int x = ( row + m );
        int xboard = boardOffset + x * boardSize;
        int filterrowoffset = (m+halfFilterSize) * filterSize + halfFilterSize;
        int n = minn;
        while( n <= maxn ) {
            int y = col + n;
            sum += image[ xboard + y] * filter[ filterrowoffset + n ];
            n++;
        }
        m++;
    }
    result[id] = sum;
}

void kernel convolve_imagecubes_int( global const int *p_numInputPlanes, global const int *p_numFilters,
      global const int *p_boardSize, global const int *p_filterSize,
      global const int *images, global const int *filters, global int *results ) {
    int globalId = get_global_id(0);

    int numInputPlanes = p_numInputPlanes[0];
    int numFilters = p_numFilters[0];
    int boardSize = p_boardSize[0];
    int filterSize = p_filterSize[0];
    int boardSizeSquared = boardSize * boardSize;

    int outputBoard2Id = globalId / boardSizeSquared;
    int filterId = outputBoard2Id % numFilters;
    int inputBoard3Id = outputBoard2Id / numFilters;

    int filterOffset = filterId * filterSize * filterSize;
    int inputBoard3Offset = inputBoard3Id * numInputPlanes * boardSizeSquared;

    // intraboard coords
    int localid = globalId % boardSizeSquared;
    int row = localid / boardSize;
    int col = localid % boardSize;

    int halfFilterSize = filterSize >> 1;
    int sum = 0;
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int plane = 0;
    while( plane < numInputPlanes ) {
        int inputBoardOffset = inputBoard3Offset + plane * boardSizeSquared;
        int filterPlaneOffset = filterOffset + plane * filterSize * filterSize;
        int m = minm;
        while( m <= maxm ) {
            int y = row + m;
            int inputboardrowoffset = inputBoardOffset + y * boardSize;
            int filterrowoffset = filterPlaneOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int x = col + n;
                sum += images[ inputboardrowoffset + x] * filters[ filterrowoffset + n ];
                n++;
            }
            m++;
        }
        plane++;
    }
    results[globalId] = sum;
}

// receive images as a stack of images
// globalid = n * numfilters * boardsize * boardsize + filter * boardsize * boardsize + imagerow * boardsize + imagecol
//                                 globalid              globalid
//  inputboard3 1 inputboard2 1----filter 1             -> outputboard2 1   outputboard3 1
//                inputboard2 2_/\_filter 2             -> outputboard2 2
//  inputboard3 2 inputboard2 3    filter 1             -> outputboard2 3   outputboard3 2
//                inputboard2 4    filter 2             -> outputboard2 4
//
// each outputboard is only written once, by a combination of:
// - one inputboard3
// - one filter
// each inputboard3 is mapped to each filter once, each time writing to one outputboard
//
// images is:
//       numimages * numinputplanes * boardsizesquared
// filters is:
//       numfilters * numinputplanes * filtersizesquared
// outputs is:
//       numimages * numfilters * outputboardsizesquared

// images are organized like [imageId][plane][row][col]
// filters are organized like [filterid][plane][filterrow][filtercol]
// results are organized like [imageid][filterid][row][col]
void kernel convolve_imagecubes_float(
      const int numInputPlanes, const int numFilters,
      const int boardSize, const int filterSize,
      global const float *images, global const float *filters, global float *results ) {
    int globalId = get_global_id(0);

    int boardSizeSquared = boardSize * boardSize;

    int outputBoard2Id = globalId / boardSizeSquared;
    int filterId = outputBoard2Id % numFilters;
    int inputBoard3Id = outputBoard2Id / numFilters;

    int filterOffset = filterId * filterSize * filterSize;
    int inputBoard3Offset = inputBoard3Id * numInputPlanes * boardSizeSquared;

    // intraboard coords
    int localid = globalId % boardSizeSquared;
    int row = localid / boardSize;
    int col = localid % boardSize;

    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    // m should vary from -halfFilterSize through 0 to halfFilterSize
    // n too...
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int inputPlane = 0;
    while( inputPlane < numInputPlanes ) {
        int inputBoardOffset = inputBoard3Offset + inputPlane * boardSizeSquared;
        int m = minm;
        while( m <= maxm ) {
            int y = row + m;
            int inputboardrowoffset = inputBoardOffset + y * boardSize;
            int filterrowoffset = filterOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int x = col + n;
                sum += images[ inputboardrowoffset + x] * filters[ filterrowoffset + n ];
                n++;
            }
            m++;
        }
        inputPlane++;
    }

    results[globalId] = sum;
}

void kernel convolve_imagecubes_float_nopadzeros(
      const int numInputPlanes, const int numFilters,
      const int inputBoardSize, const int filterSize,
      global const float *images, global const float *filters, global float *results ) {
    int globalId = get_global_id(0);

    int inputBoardSizeSquared = inputBoardSize * inputBoardSize;
    int outputBoardSize = inputBoardSize - filterSize + 1;
    int outputBoardSizeSquared = outputBoardSize * outputBoardSize;

    int outputBoard2Id = globalId / outputBoardSizeSquared;
    int filterId = outputBoard2Id % numFilters;
    int inputBoard3Id = outputBoard2Id / numFilters;

    int filterOffset = filterId * filterSize * filterSize;
    int inputBoard3Offset = inputBoard3Id * numInputPlanes * inputBoardSizeSquared;

    // intraboard coords
    int localid = globalId % outputBoardSizeSquared;
    int outputRow = localid / outputBoardSize;
    int outputCol = localid % outputBoardSize;

    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    int minm = -halfFilterSize;
    int maxm = halfFilterSize;
    int minn = -halfFilterSize;
    int maxn = halfFilterSize;
    int inputPlane = 0;
    while( inputPlane < numInputPlanes ) {
        int inputBoardOffset = inputBoard3Offset + inputPlane * inputBoardSizeSquared;
        int m = minm;
        while( m <= maxm ) {
            int inputRow = outputRow + m + halfFilterSize;
            int inputboardrowoffset = inputBoardOffset + inputRow * inputBoardSize;
            int filterrowoffset = filterOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int inputCol = outputCol + n + halfFilterSize;
                sum += images[ inputboardrowoffset + inputCol] * filters[ filterrowoffset + n ];
                n++;
            }
            m++;
        }
        inputPlane++;
    }
    results[globalId] = sum;
}

// images are organized like [imageId][plane][row][col]
// filters are organized like [filterid][inplane][filterrow][filtercol]
// results are organized like [imageid][filterid][row][col]
// global id is organized like results, ie: [imageid][filterid][row][col]
#ifdef ACTIVATION_FUNCTION // protect against not defined
void kernel convolve_imagecubes_float2( const int numExamples,
      const int numInputPlanes, const int numFilters,
      const int inputBoardSize, const int filterSize, const int padZeros,
      global const float *images, global const float *filters,
#ifdef BIASED
global const float*biases,
#endif
    global float *results ) {
    int globalId = get_global_id(0);

    const int evenPadding = filterSize % 2 == 0 ? 1 : 0;

    int inputBoardSizeSquared = inputBoardSize * inputBoardSize;
    int outputBoardSize = padZeros ? inputBoardSize + evenPadding : inputBoardSize - filterSize + 1;
    int outputBoardSizeSquared = outputBoardSize * outputBoardSize;
    int filterSizeSquared = filterSize * filterSize;

    int outputBoard2Id = globalId / outputBoardSizeSquared;
    int exampleId = outputBoard2Id / numFilters;
    int filterId = outputBoard2Id % numFilters;

    if( exampleId >= numExamples ) {
        return;
    }

    int inputCubeOffset = exampleId * numInputPlanes * inputBoardSizeSquared;
    int filterCubeOffset = filterId * numInputPlanes * filterSizeSquared;

    // intraboard coords
    int localid = globalId % outputBoardSizeSquared;
    int outputRow = localid / outputBoardSize;
    int outputCol = localid % outputBoardSize;

    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    // for odd, boardsize and filtersize 3, padZeros = 0:
    // output is a single square
    // m and n should vary between -1,0,1
    // for even, boardsize and filtersize 2, padzeros = 0
    // output is a single square, which we can position at topleft or bottomrigth
    // lets position it in bottomright
    // then m and n should vary as -1,0
    //
    // for even, boardsize and filtersize 2, padzeros = 1
    // output is 2 by 2
    // well... if it is even:
    // - if we are not padding zeros, then we simply move our filter around the board somehow
    // - if we are padding zeros, then we conceptually pad the bottom and right edge of the board with zeros by 1
    // filtersize remains the same
    //      m will vary as -1,0,1
    //       outputrow is fixed by globalid
    //       inputrow should be unchanged...
    // padzeros = 0:
    //  x x .  . . .
    //  x x .  . x x
    //  . . .  . x x
    // when filtersize even:
    //    new boardsize = oldboardsize - filtersize + 1
    // when filtersize odd:
    //    x x x .
    //    x x x .
    //    x x x .
    //    . . . .
    //    new boardsize = oldboardsize - filtersize + 1
    // padzeros = 1:
    // x x
    // x x . .   x x .    . . .     . . .
    //   . . .   x x .    . x x     . . .
    //   . . .   . . .    . x x     . . x x
    // outrow=0 outrow=1  outrow=2      x x
    // outcol=0 outcol=1  outcol=2    outrow=3
    //                                outcol=3
    // when filtersize is even, and padzeros, boardsize grows by 1 each time...
    //    boardsize = oldboardsize + 1
    // when filtersize is odd
    //  x x x
    //  x x x .   x x x    . . .
    //  x x x .   x x x    . x x x
    //    . . .   x x x    . x x x
    //                       x x x
    //  boardsize = oldboardsize
    int minm = padZeros ? max( -halfFilterSize, -outputRow ) : -halfFilterSize;
    int maxm = padZeros ? min( halfFilterSize - evenPadding, outputBoardSize - 1 - outputRow  - evenPadding) : halfFilterSize - evenPadding;
    int minn = padZeros ? max( -halfFilterSize, -outputCol ) : - halfFilterSize;
    int maxn = padZeros ? min( halfFilterSize - evenPadding, outputBoardSize - 1 - outputCol - evenPadding) : halfFilterSize - evenPadding;
    int inputPlane = 0;
//    float probe = 0;
    while( inputPlane < numInputPlanes ) {
        int inputBoardOffset = inputCubeOffset + inputPlane * inputBoardSizeSquared;
        int filterBoardOffset = filterCubeOffset + inputPlane * filterSizeSquared;
        int m = minm;
        while( m <= maxm ) {
            int inputRow = outputRow + m + ( padZeros ? 0 : halfFilterSize );
            int inputboardrowoffset = inputBoardOffset + inputRow * inputBoardSize;
            int filterrowoffset = filterBoardOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int inputCol = outputCol + n + ( padZeros ? 0 : halfFilterSize );
                sum += images[ inputboardrowoffset + inputCol] * filters[ filterrowoffset + n ];
//                probe += 10000 * pown(100, inputPlane) *( inputboardrowoffset + inputCol );
            //    probe += pown(100, inputPlane) *( images[inputboardrowoffset + inputCol] );
                //probe += pown(100, inputPlane) *( filterrowoffset + n );
             //   probe += pown(1000, inputPlane) *( floor(filters[ filterrowoffset + n ]*100)/100 );

//                sum = filters[filterrowoffset + n];
                //sum = filterrowoffset;
                n++;
            }
            m++;
        }
//        probe += pown(100, inputPlane ) * filterBoardOffset;
        inputPlane++;
    }
//     probe = exampleId * 100 + filterCubeOffset;

#ifdef BIASED
    sum += biases[filterId];
#endif
    results[globalId] = ACTIVATION_FUNCTION(sum);
//    results[0] = 1234.0;
//     results[1024+globalId] = maxn;
//     results[1] = maxMm;
//     results[2] = minm;
}
#endif

// images are organized like [imageId][plane][row][col]    128*32*19*19=1,500,000
// filters are organized like [filterid][inplane][filterrow][filtercol] 32*32*5*5=25600 = 100k bytes, or 3.2KB per filter
// results are organized like [imageid][filterid][row][col]   128*32*19*19=1,500,000 = 6MB, or 46KB per image,
//
//                  if w updates are per image,then 25600*128 = 3.3 million
// globalid is for: [outPlane][upstreamPlane][filterRow][filterCol]
// eg 32 * 32 * 5 * 5 = 25600 ...
// then we are aggregating over [outRow][outCol][n]
//      eg 19 * 19 * 128 = 46208
// derivtype: 0=relu 1=tanh
#ifdef ACTIVATION_DERIV // protect against if activation_function not defined
void kernel backprop_floats( const float learningRateMultiplier,
        const int batchSize, const int upstreamNumPlanes, const int numPlanes,
         const int upstreamBoardSize, const int filterSize, const int outBoardSize, const int padZeros,
         global const float *images, global const float *results, global const float *errors, global float *weightChanges ) {
    int globalId = get_global_id(0);

    int filterSizeSquared = filterSize * filterSize;

    int IntraFilterOffset = globalId % filterSizeSquared;
    int filterRow = IntraFilterOffset / filterSize;
    int filterCol = IntraFilterOffset % filterSize;

    int filter2Id = globalId / filterSizeSquared;
    int outPlane = filter2Id / upstreamNumPlanes;
    int upstreamPlane = filter2Id % upstreamNumPlanes;

    const int halfFilterSize = filterSize >> 1;
    const int margin = padZeros ? halfFilterSize : 0;
    float thiswchange = 0;
    // weights:     [outPlane][upstreamPlane][filterRow][filterCol]
    //       aggregate over:  [outRow][outCol][n]
    int n = 0;
    while( n < batchSize ) {
        int outRow = 0;
        while( outRow < outBoardSize ) {
            int upstreamRow = outRow - margin + filterRow;
            int outCol = 0;
            while( outCol < outBoardSize ) {
                int upstreamCol = outCol - margin + filterCol;
                int resultIndex = ( ( n * numPlanes
                          + outPlane ) * outBoardSize
                          + outRow ) * outBoardSize
                          + outCol;
                float error = errors[resultIndex];
                float actualOutput = results[resultIndex];
                float activationDerivative = ACTIVATION_DERIV( actualOutput);
                int upstreamDataIndex = ( ( n * upstreamNumPlanes
                                 + upstreamPlane ) * upstreamBoardSize
                                 + upstreamRow ) * upstreamBoardSize
                                 + upstreamCol;
                float upstreamResult = images[upstreamDataIndex];
                float thisimagethiswchange = upstreamResult * activationDerivative *
                    error;
                thiswchange += thisimagethiswchange;
                outCol++;
            }
            outRow++;
        }
        n++;
    }
    // weights:     [outPlane][upstreamPlane][filterRow][filterCol]
    //       aggregate over:  [outRow][outCol][n]
    weightChanges[ globalId ] = - learningRateMultiplier * thiswchange;
}
#endif

// handle lower layer...
// errors for upstream look like [n][inPlane][inRow][inCol]
// need to aggregate over: [outPlane][outRow][outCol] (?)
// need to backprop errors along each possible weight
// each upstream feeds to:
//    - each of our filters (so numPlanes filters)
//    - each of our outpoint points (so boardSize * boardSize)
// for our own backprop, we updated weights for:
//      [outPlane][inPlane][filterRow][filtercol]
//    aggregating over: [n][outRow][outCol]
// errors are provider per [n][inPlane][inRow][inCol]
// globalid is structured as: [n][upstreamPlane][upstreamRow][upstreamCol]
void kernel calcErrorsForUpstream(
        const int upstreamNumPlanes, const int upstreamBoardSize, const int filterSize,
        const int outNumPlanes, const int outBoardSize,
        const int padZeros,
        global const float *weights, global const float *errors, global float *errorsForUpstream ) {
    int globalId = get_global_id(0);
    const int halfFilterSize = filterSize >> 1;
    const int margin = padZeros ? halfFilterSize : 0;

    const int upstreamBoardSizeSquared = upstreamBoardSize * upstreamBoardSize;
    const int upstreamBoard2dId = globalId / upstreamBoardSizeSquared;

    const int intraBoardOffset = globalId % upstreamBoardSizeSquared;
    const int upstreamRow = intraBoardOffset / upstreamBoardSize;
    const int upstreamCol = intraBoardOffset % upstreamBoardSize;

    const int upstreamPlane = upstreamBoard2dId % upstreamNumPlanes;
    const int n = upstreamBoard2dId / upstreamNumPlanes;

    float sumWeightTimesOutError = 0;
    // aggregate over [outPlane][outRow][outCol]
    for( int outPlane = 0; outPlane < outNumPlanes; outPlane++ ) {
        for( int outRow = 0; outRow < outBoardSize; outRow++ ) {
            // need to derive filterRow and filterCol, given outRow and outCol
            int filterRow = upstreamRow + margin - outRow;
            for( int outCol = 0; outCol < outBoardSize; outCol++ ) {
               // need to derive filterRow and filterCol, given outRow and outCol
                int filterCol = upstreamCol + margin - outCol;
                int resultIndex = ( ( n * outNumPlanes
                          + outPlane ) * outBoardSize
                          + outRow ) * outBoardSize
                          + outCol;
                float thisError = errors[resultIndex];
                int thisWeightIndex = ( ( outPlane * upstreamNumPlanes
                                    + upstreamPlane ) * filterSize
                                    + filterRow ) * filterSize
                                    + filterCol;
                float thisWeight = weights[thisWeightIndex];
                float thisWeightTimesError = thisWeight * thisError;
                sumWeightTimesOutError += thisWeightTimesError;
            }
        }
    }
    errorsForUpstream[globalId] = sumWeightTimesOutError;
}

// Copyright Hugh Perkins 2014 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) tanh(output)
    #define ACTIVATION_DERIV(output) 1 - output * output
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) output> 0 ? output : 0
    #define ACTIVATION_DERIV(output) output > 0 ? 1 : 0
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) output
    #define ACTIVATION_DERIV(output) 1
#endif

void kernel convolve_ints( global const int *p_boardSize, global const int *p_filterSize,
      global const int *image, global const int *filter, global int *result ) {
    int id = get_global_id(0);
    int boardSize = p_boardSize[0];
    int filterSize = p_filterSize[0];
    int boardOffset = id / (boardSize * boardSize ) * (boardSize * boardSize );
    int localid = id % (boardSize * boardSize );
    int row = localid / boardSize;
    int col = localid % boardSize;
    int halfFilterSize = filterSize >> 1;
    int sum = 0;
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int m = minm;
    while( m <= maxm ) {
        int x = ( row + m );
        int xboard = boardOffset + x * boardSize;
        int filterrowoffset = (m+halfFilterSize) * filterSize + halfFilterSize;
        int n = minn;
        while( n <= maxn ) {
            int y = col + n;
            sum += image[ xboard + y] * filter[ filterrowoffset + n ];
            n++;
        }
        m++;
    }
    result[id] = sum;
}

void kernel convolve_floats( global const int *p_boardSize, global const int *p_filterSize,
      global const float *image, global const float *filter, global float *result ) {
    int id = get_global_id(0);
    int boardSize = p_boardSize[0];
    int filterSize = p_filterSize[0];
    int boardOffset = id / (boardSize * boardSize ) * (boardSize * boardSize );
    int localid = id % (boardSize * boardSize );
    int row = localid / boardSize;
    int col = localid % boardSize;
    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int m = minm;
    while( m <= maxm ) {
        int x = ( row + m );
        int xboard = boardOffset + x * boardSize;
        int filterrowoffset = (m+halfFilterSize) * filterSize + halfFilterSize;
        int n = minn;
        while( n <= maxn ) {
            int y = col + n;
            sum += image[ xboard + y] * filter[ filterrowoffset + n ];
            n++;
        }
        m++;
    }
    result[id] = sum;
}

void kernel convolve_imagecubes_int( global const int *p_numInputPlanes, global const int *p_numFilters,
      global const int *p_boardSize, global const int *p_filterSize,
      global const int *images, global const int *filters, global int *results ) {
    int globalId = get_global_id(0);

    int numInputPlanes = p_numInputPlanes[0];
    int numFilters = p_numFilters[0];
    int boardSize = p_boardSize[0];
    int filterSize = p_filterSize[0];
    int boardSizeSquared = boardSize * boardSize;

    int outputBoard2Id = globalId / boardSizeSquared;
    int filterId = outputBoard2Id % numFilters;
    int inputBoard3Id = outputBoard2Id / numFilters;

    int filterOffset = filterId * filterSize * filterSize;
    int inputBoard3Offset = inputBoard3Id * numInputPlanes * boardSizeSquared;

    // intraboard coords
    int localid = globalId % boardSizeSquared;
    int row = localid / boardSize;
    int col = localid % boardSize;

    int halfFilterSize = filterSize >> 1;
    int sum = 0;
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int plane = 0;
    while( plane < numInputPlanes ) {
        int inputBoardOffset = inputBoard3Offset + plane * boardSizeSquared;
        int filterPlaneOffset = filterOffset + plane * filterSize * filterSize;
        int m = minm;
        while( m <= maxm ) {
            int y = row + m;
            int inputboardrowoffset = inputBoardOffset + y * boardSize;
            int filterrowoffset = filterPlaneOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int x = col + n;
                sum += images[ inputboardrowoffset + x] * filters[ filterrowoffset + n ];
                n++;
            }
            m++;
        }
        plane++;
    }
    results[globalId] = sum;
}

// receive images as a stack of images
// globalid = n * numfilters * boardsize * boardsize + filter * boardsize * boardsize + imagerow * boardsize + imagecol
//                                 globalid              globalid
//  inputboard3 1 inputboard2 1----filter 1             -> outputboard2 1   outputboard3 1
//                inputboard2 2_/\_filter 2             -> outputboard2 2
//  inputboard3 2 inputboard2 3    filter 1             -> outputboard2 3   outputboard3 2
//                inputboard2 4    filter 2             -> outputboard2 4
//
// each outputboard is only written once, by a combination of:
// - one inputboard3
// - one filter
// each inputboard3 is mapped to each filter once, each time writing to one outputboard
//
// images is:
//       numimages * numinputplanes * boardsizesquared
// filters is:
//       numfilters * numinputplanes * filtersizesquared
// outputs is:
//       numimages * numfilters * outputboardsizesquared

// images are organized like [imageId][plane][row][col]
// filters are organized like [filterid][plane][filterrow][filtercol]
// results are organized like [imageid][filterid][row][col]
void kernel convolve_imagecubes_float(
      const int numInputPlanes, const int numFilters,
      const int boardSize, const int filterSize,
      global const float *images, global const float *filters, global float *results ) {
    int globalId = get_global_id(0);

    int boardSizeSquared = boardSize * boardSize;

    int outputBoard2Id = globalId / boardSizeSquared;
    int filterId = outputBoard2Id % numFilters;
    int inputBoard3Id = outputBoard2Id / numFilters;

    int filterOffset = filterId * filterSize * filterSize;
    int inputBoard3Offset = inputBoard3Id * numInputPlanes * boardSizeSquared;

    // intraboard coords
    int localid = globalId % boardSizeSquared;
    int row = localid / boardSize;
    int col = localid % boardSize;

    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    // m should vary from -halfFilterSize through 0 to halfFilterSize
    // n too...
    int minm = max( -halfFilterSize, -row );
    int maxm = min( halfFilterSize, boardSize - 1 - row );
    int minn = max( -halfFilterSize, -col );
    int maxn = min( halfFilterSize, boardSize - 1 - col );
    int inputPlane = 0;
    while( inputPlane < numInputPlanes ) {
        int inputBoardOffset = inputBoard3Offset + inputPlane * boardSizeSquared;
        int m = minm;
        while( m <= maxm ) {
            int y = row + m;
            int inputboardrowoffset = inputBoardOffset + y * boardSize;
            int filterrowoffset = filterOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int x = col + n;
                sum += images[ inputboardrowoffset + x] * filters[ filterrowoffset + n ];
                n++;
            }
            m++;
        }
        inputPlane++;
    }

    results[globalId] = sum;
}

void kernel convolve_imagecubes_float_nopadzeros(
      const int numInputPlanes, const int numFilters,
      const int inputBoardSize, const int filterSize,
      global const float *images, global const float *filters, global float *results ) {
    int globalId = get_global_id(0);

    int inputBoardSizeSquared = inputBoardSize * inputBoardSize;
    int outputBoardSize = inputBoardSize - filterSize + 1;
    int outputBoardSizeSquared = outputBoardSize * outputBoardSize;

    int outputBoard2Id = globalId / outputBoardSizeSquared;
    int filterId = outputBoard2Id % numFilters;
    int inputBoard3Id = outputBoard2Id / numFilters;

    int filterOffset = filterId * filterSize * filterSize;
    int inputBoard3Offset = inputBoard3Id * numInputPlanes * inputBoardSizeSquared;

    // intraboard coords
    int localid = globalId % outputBoardSizeSquared;
    int outputRow = localid / outputBoardSize;
    int outputCol = localid % outputBoardSize;

    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    int minm = -halfFilterSize;
    int maxm = halfFilterSize;
    int minn = -halfFilterSize;
    int maxn = halfFilterSize;
    int inputPlane = 0;
    while( inputPlane < numInputPlanes ) {
        int inputBoardOffset = inputBoard3Offset + inputPlane * inputBoardSizeSquared;
        int m = minm;
        while( m <= maxm ) {
            int inputRow = outputRow + m + halfFilterSize;
            int inputboardrowoffset = inputBoardOffset + inputRow * inputBoardSize;
            int filterrowoffset = filterOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int inputCol = outputCol + n + halfFilterSize;
                sum += images[ inputboardrowoffset + inputCol] * filters[ filterrowoffset + n ];
                n++;
            }
            m++;
        }
        inputPlane++;
    }
    results[globalId] = sum;
}

// images are organized like [imageId][plane][row][col]
// filters are organized like [filterid][inplane][filterrow][filtercol]
// results are organized like [imageid][filterid][row][col]
// global id is organized like results, ie: [imageid][filterid][row][col]
#ifdef ACTIVATION_FUNCTION // protect against not defined
void kernel convolve_imagecubes_float2( const int numExamples,
      const int numInputPlanes, const int numFilters,
      const int inputBoardSize, const int filterSize, const int padZeros,
      global const float *images, global const float *filters,
#ifdef BIASED
global const float*biases,
#endif
    global float *results ) {
    int globalId = get_global_id(0);

    const int evenPadding = filterSize % 2 == 0 ? 1 : 0;

    int inputBoardSizeSquared = inputBoardSize * inputBoardSize;
    int outputBoardSize = padZeros ? inputBoardSize + evenPadding : inputBoardSize - filterSize + 1;
    int outputBoardSizeSquared = outputBoardSize * outputBoardSize;
    int filterSizeSquared = filterSize * filterSize;

    int outputBoard2Id = globalId / outputBoardSizeSquared;
    int exampleId = outputBoard2Id / numFilters;
    int filterId = outputBoard2Id % numFilters;

    if( exampleId >= numExamples ) {
        return;
    }

    int inputCubeOffset = exampleId * numInputPlanes * inputBoardSizeSquared;
    int filterCubeOffset = filterId * numInputPlanes * filterSizeSquared;

    // intraboard coords
    int localid = globalId % outputBoardSizeSquared;
    int outputRow = localid / outputBoardSize;
    int outputCol = localid % outputBoardSize;

    int halfFilterSize = filterSize >> 1;
    float sum = 0;
    // for odd, boardsize and filtersize 3, padZeros = 0:
    // output is a single square
    // m and n should vary between -1,0,1
    // for even, boardsize and filtersize 2, padzeros = 0
    // output is a single square, which we can position at topleft or bottomrigth
    // lets position it in bottomright
    // then m and n should vary as -1,0
    //
    // for even, boardsize and filtersize 2, padzeros = 1
    // output is 2 by 2
    // well... if it is even:
    // - if we are not padding zeros, then we simply move our filter around the board somehow
    // - if we are padding zeros, then we conceptually pad the bottom and right edge of the board with zeros by 1
    // filtersize remains the same
    //      m will vary as -1,0,1
    //       outputrow is fixed by globalid
    //       inputrow should be unchanged...
    // padzeros = 0:
    //  x x .  . . .
    //  x x .  . x x
    //  . . .  . x x
    // when filtersize even:
    //    new boardsize = oldboardsize - filtersize + 1
    // when filtersize odd:
    //    x x x .
    //    x x x .
    //    x x x .
    //    . . . .
    //    new boardsize = oldboardsize - filtersize + 1
    // padzeros = 1:
    // x x
    // x x . .   x x .    . . .     . . .
    //   . . .   x x .    . x x     . . .
    //   . . .   . . .    . x x     . . x x
    // outrow=0 outrow=1  outrow=2      x x
    // outcol=0 outcol=1  outcol=2    outrow=3
    //                                outcol=3
    // when filtersize is even, and padzeros, boardsize grows by 1 each time...
    //    boardsize = oldboardsize + 1
    // when filtersize is odd
    //  x x x
    //  x x x .   x x x    . . .
    //  x x x .   x x x    . x x x
    //    . . .   x x x    . x x x
    //                       x x x
    //  boardsize = oldboardsize
    int minm = padZeros ? max( -halfFilterSize, -outputRow ) : -halfFilterSize;
    int maxm = padZeros ? min( halfFilterSize - evenPadding, outputBoardSize - 1 - outputRow  - evenPadding) : halfFilterSize - evenPadding;
    int minn = padZeros ? max( -halfFilterSize, -outputCol ) : - halfFilterSize;
    int maxn = padZeros ? min( halfFilterSize - evenPadding, outputBoardSize - 1 - outputCol - evenPadding) : halfFilterSize - evenPadding;
    int inputPlane = 0;
//    float probe = 0;
    while( inputPlane < numInputPlanes ) {
        int inputBoardOffset = inputCubeOffset + inputPlane * inputBoardSizeSquared;
        int filterBoardOffset = filterCubeOffset + inputPlane * filterSizeSquared;
        int m = minm;
        while( m <= maxm ) {
            int inputRow = outputRow + m + ( padZeros ? 0 : halfFilterSize );
            int inputboardrowoffset = inputBoardOffset + inputRow * inputBoardSize;
            int filterrowoffset = filterBoardOffset + (m+halfFilterSize) * filterSize + halfFilterSize;
            int n = minn;
            while( n <= maxn ) {
                int inputCol = outputCol + n + ( padZeros ? 0 : halfFilterSize );
                sum += images[ inputboardrowoffset + inputCol] * filters[ filterrowoffset + n ];
//                probe += 10000 * pown(100, inputPlane) *( inputboardrowoffset + inputCol );
            //    probe += pown(100, inputPlane) *( images[inputboardrowoffset + inputCol] );
                //probe += pown(100, inputPlane) *( filterrowoffset + n );
             //   probe += pown(1000, inputPlane) *( floor(filters[ filterrowoffset + n ]*100)/100 );

//                sum = filters[filterrowoffset + n];
                //sum = filterrowoffset;
                n++;
            }
            m++;
        }
//        probe += pown(100, inputPlane ) * filterBoardOffset;
        inputPlane++;
    }
//     probe = exampleId * 100 + filterCubeOffset;

#ifdef BIASED
    sum += biases[filterId];
#endif
    results[globalId] = ACTIVATION_FUNCTION(sum);
//    results[0] = 1234.0;
//     results[1024+globalId] = maxn;
//     results[1] = maxMm;
//     results[2] = minm;
}
#endif

// images are organized like [imageId][plane][row][col]    128*32*19*19=1,500,000
// filters are organized like [filterid][inplane][filterrow][filtercol] 32*32*5*5=25600 = 100k bytes, or 3.2KB per filter
// results are organized like [imageid][filterid][row][col]   128*32*19*19=1,500,000 = 6MB, or 46KB per image,
//
//                  if w updates are per image,then 25600*128 = 3.3 million
// globalid is for: [outPlane][upstreamPlane][filterRow][filterCol]
// eg 32 * 32 * 5 * 5 = 25600 ...
// then we are aggregating over [outRow][outCol][n]
//      eg 19 * 19 * 128 = 46208
// derivtype: 0=relu 1=tanh
#ifdef ACTIVATION_DERIV // protect against if activation_function not defined
void kernel backprop_floats( const float learningRateMultiplier,
        const int batchSize, const int upstreamNumPlanes, const int numPlanes,
         const int upstreamBoardSize, const int filterSize, const int outBoardSize, const int padZeros,
         global const float *images, global const float *results, global const float *errors, global float *weightChanges ) {
    int globalId = get_global_id(0);

    int filterSizeSquared = filterSize * filterSize;

    int IntraFilterOffset = globalId % filterSizeSquared;
    int filterRow = IntraFilterOffset / filterSize;
    int filterCol = IntraFilterOffset % filterSize;

    int filter2Id = globalId / filterSizeSquared;
    int outPlane = filter2Id / upstreamNumPlanes;
    int upstreamPlane = filter2Id % upstreamNumPlanes;

    const int halfFilterSize = filterSize >> 1;
    const int margin = padZeros ? halfFilterSize : 0;
    float thiswchange = 0;
    // weights:     [outPlane][upstreamPlane][filterRow][filterCol]
    //       aggregate over:  [outRow][outCol][n]
    int outRow = 0;
    while( outRow < outBoardSize ) {
        int upstreamRow = outRow - margin + filterRow;
        int outCol = 0;
        while( outCol < outBoardSize ) {
            int upstreamCol = outCol - margin + filterCol;
            int n = 0;
            while( n < batchSize ) {
                int resultIndex = ( ( n * numPlanes
                          + outPlane ) * outBoardSize
                          + outRow ) * outBoardSize
                          + outCol;
                float error = errors[resultIndex];
                float actualOutput = results[resultIndex];
                float activationDerivative = ACTIVATION_DERIV( actualOutput);
                int upstreamDataIndex = ( ( n * upstreamNumPlanes
                                 + upstreamPlane ) * upstreamBoardSize
                                 + upstreamRow ) * upstreamBoardSize
                                 + upstreamCol;
                float upstreamResult = images[upstreamDataIndex];
                float thisimagethiswchange = upstreamResult * activationDerivative *
                    error;
                thiswchange += thisimagethiswchange;
                n++;
            }
            outCol++;
        }
        outRow++;
    }
    // weights:     [outPlane][upstreamPlane][filterRow][filterCol]
    //       aggregate over:  [outRow][outCol][n]
    weightChanges[ globalId ] = - learningRateMultiplier * thiswchange;
}
#endif

// handle lower layer...
// errors for upstream look like [n][inPlane][inRow][inCol]
// need to aggregate over: [outPlane][outRow][outCol] (?)
// need to backprop errors along each possible weight
// each upstream feeds to:
//    - each of our filters (so numPlanes filters)
//    - each of our outpoint points (so boardSize * boardSize)
// for our own backprop, we updated weights for:
//      [outPlane][inPlane][filterRow][filtercol]
//    aggregating over: [n][outRow][outCol]
// errors are provider per [n][inPlane][inRow][inCol]
// globalid is structured as: [n][upstreamPlane][upstreamRow][upstreamCol]
void kernel calcErrorsForUpstream(
        const int upstreamNumPlanes, const int upstreamBoardSize, const int filterSize,
        const int outNumPlanes, const int outBoardSize,
        const int padZeros,
        global const float *weights, global const float *errors, global float *errorsForUpstream ) {
    int globalId = get_global_id(0);
    const int halfFilterSize = filterSize >> 1;
    const int margin = padZeros ? halfFilterSize : 0;

    const int upstreamBoardSizeSquared = upstreamBoardSize * upstreamBoardSize;
    const int upstreamBoard2dId = globalId / upstreamBoardSizeSquared;

    const int intraBoardOffset = globalId % upstreamBoardSizeSquared;
    const int upstreamRow = intraBoardOffset / upstreamBoardSize;
    const int upstreamCol = intraBoardOffset % upstreamBoardSize;

    const int upstreamPlane = upstreamBoard2dId % upstreamNumPlanes;
    const int n = upstreamBoard2dId / upstreamNumPlanes;

    float sumWeightTimesOutError = 0;
    // aggregate over [outPlane][outRow][outCol]
    for( int outPlane = 0; outPlane < outNumPlanes; outPlane++ ) {
        for( int outRow = 0; outRow < outBoardSize; outRow++ ) {
            // need to derive filterRow and filterCol, given outRow and outCol
            int filterRow = upstreamRow + margin - outRow;
            for( int outCol = 0; outCol < outBoardSize; outCol++ ) {
               // need to derive filterRow and filterCol, given outRow and outCol
                int filterCol = upstreamCol + margin - outCol;
                int resultIndex = ( ( n * outNumPlanes
                          + outPlane ) * outBoardSize
                          + outRow ) * outBoardSize
                          + outCol;
                float thisError = errors[resultIndex];
                int thisWeightIndex = ( ( outPlane * upstreamNumPlanes
                                    + upstreamPlane ) * filterSize
                                    + filterRow ) * filterSize
                                    + filterCol;
                float thisWeight = weights[thisWeightIndex];
                float thisWeightTimesError = thisWeight * thisError;
                sumWeightTimesOutError += thisWeightTimesError;
            }
        }
    }
    errorsForUpstream[globalId] = sumWeightTimesOutError;
}


Code cleanup

Code cleanup

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		uchar
#define _INT_MASK_LOCAL			uchar

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
       UNUSED
#endif
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
#endif
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);
				int pool_size = (hend - hstart) * (wend - wstart);
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		uchar
#define _INT_MASK_LOCAL			uchar

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
       UNUSED
#endif
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
#endif
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);
				int pool_size = (hend - hstart) * (wend - wstart);
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							int src_x = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0 + i;
							int src_y = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1 + j;
							mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}


Removed waste

Removed waste

kernel void label_with_id(global int *data, int w, int h) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int loc = w * y + x;
  if (data[loc] == 1) {
    data[loc] = loc + 2;
  }
}

kernel void neighbour_propagate(global int *data, int w, int h,
                                global char *changed) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int oldlabel = data[w * y + x];
  int curlabel = oldlabel;
  int otherlabel = 0;

  if (curlabel == 0) {
    return;
  }

  if (y + 1 < h) {
    otherlabel = data[w * (y + 1) + (x)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }
  if (y - 1 >= 0) {
    otherlabel = data[w * (y - 1) + (x)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }
  if (x + 1 < w) {
    otherlabel = data[w * (y) + (x + 1)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }
  if (x - 1 >= 0) {
    otherlabel = data[w * (y) + (x - 1)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }

  if (curlabel < oldlabel) {
    *changed = 1;
    data[w * y + x] = curlabel;
  }
}

kernel void plus_propagate(global int *data, int w, int h,
                           global char *changed) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int oldlabel = data[w * y + x];
  int curlabel = oldlabel;
  int otherlabel = 0;
  int diff;

  if (curlabel == 0) {
    return;
  }

  diff = 1;
  while (true) {
    if (y + diff < 0 || y + diff >= h) {
      break;
    }
    otherlabel = data[w * (y + diff) + (x)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }
  diff = 1;
  while (true) {
    if (y - diff < 0 || y - diff >= h) {
      break;
    }
    otherlabel = data[w * (y - diff) + (x)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }
  diff = 1;
  while (true) {
    if (x + diff < 0 || x + diff >= w) {
      break;
    }
    otherlabel = data[w * y + (x + diff)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }
  diff = 1;
  while (true) {
    if (x - diff < 0 || x - diff >= w) {
      break;
    }
    otherlabel = data[w * y + (x - diff)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }

  if (curlabel < oldlabel) {
    *changed = 1;
    data[w * y + x] = curlabel;
  }
}

int find_set(global int *data, int loc) {
  // All loc of found elements should be in range.
  // Also assuming there are no cycles in the links.
  while (loc != data[loc] - 2) {
    loc = data[loc] - 2;
  }

  // +2 is the correct LABEL of root at LOCATION loc
  return loc + 2;
}

kernel void union_find(global int *data, int w, int h, global char *changed) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int oldlabel = data[w * y + x];
  int lowest = oldlabel;

  if (oldlabel == 0) {
    return;
  }

  bool ok_N = y - 1 >= 0 && data[w * (y - 1) + (x)];
  bool ok_E = x + 1 < w && data[w * (y) + (x + 1)];
  bool ok_S = y + 1 < h && data[w * (y + 1) + (x)];
  bool ok_W = x - 1 >= 0 && data[w * (y) + (x - 1)];
  int root_N;
  int root_E;
  int root_S;
  int root_W;

  if (ok_N) {
    root_N = find_set(data, w * (y - 1) + (x));
    if (root_N < lowest) {
      lowest = root_N;
    }
  }
  if (ok_E) {
    root_E = find_set(data, w * (y) + (x + 1));
    if (root_E < lowest) {
      lowest = root_E;
    }
  }
  if (ok_S) {
    root_S = find_set(data, w * (y + 1) + (x));
    if (root_S < lowest) {
      lowest = root_S;
    }
  }
  if (ok_W) {
    root_W = find_set(data, w * (y) + (x - 1));
    if (root_W < lowest) {
      lowest = root_W;
    }
  }

  if (lowest < oldlabel) {
    *changed = 1;
    data[w * y + x] = lowest;
    if (ok_N && root_N > lowest) {
      data[root_N - 2] = lowest;
    }
    if (ok_E && root_E > lowest) {
      data[root_E - 2] = lowest;
    }
    if (ok_S && root_S > lowest) {
      data[root_S - 2] = lowest;
    }
    if (ok_W && root_W > lowest) {
      data[root_W - 2] = lowest;
    }
  }
}

kernel void lineedit_right(global int *data, int w, int h,
                           global char *changed) {
  int x = 0;
  int y = get_global_id(0);
  int lowest = 1 << 30;

  while (x < w) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    ++x;
  }
}

kernel void lineedit_left(global int *data, int w, int h,
                          global char *changed) {
  int x = w - 1;
  int y = get_global_id(0);
  int lowest = 1 << 30;

  while (x >= 0) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    --x;
  }
}

kernel void lineedit_up(global int *data, int w, int h, global char *changed) {
  int x = get_global_id(0);
  int y = 0;
  int lowest = 1 << 30;

  while (y < h) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    ++y;
  }
}

kernel void lineedit_down(global int *data, int w, int h,
                          global char *changed) {
  int x = get_global_id(0);
  int y = h - 1;
  int lowest = 1 << 30;

  while (y >= 0) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    --y;
  }
}

kernel void lines_up(global int *data, int w, int h, global char *changed) {
  int x = get_global_id(0);
  int y = 0;
  char localchanged = 0;

  while (y < h) {
    if (!data[w * y + x]) {
      ++y;
      continue;
    }
    int i = y;
    int min = data[w * y + x];
    int tmpmin = min;

    while (i < h && (tmpmin = data[w * i + x])) {
      if (tmpmin != min) {
        localchanged = 1; // Not all equal, i.e. will have change
      }
      if (tmpmin < min) {
        min = tmpmin;
      }
      ++i;
    }

    while (y != i) {
      data[w * y + x] = min;
      ++y;
    }
  }

  if (localchanged) {
    *changed = localchanged;
  }
}

kernel void lines_right(global int *data, int w, int h, global char *changed) {
  int x = 0;
  int y = get_global_id(0);
  char localchanged = 0;

  while (x < w) {
    if (!data[w * y + x]) {
      ++x;
      continue;
    }
    int i = x;
    int min = data[w * y + x];
    int tmpmin = min;

    while (i < w && (tmpmin = data[w * y + i])) {
      if (tmpmin != min) {
        localchanged = 1; // Not all equal, i.e. will have change
      }
      if (tmpmin < min) {
        min = tmpmin;
      }
      ++i;
    }

    while (x != i) {
      data[w * y + x] = min;
      ++x;
    }
  }

  if (localchanged) {
    *changed = localchanged;
  }
}

kernel void id_accessor(global int *data, int w) {
  unsigned int x = get_global_id(0);
  unsigned int y = get_global_id(1);

  int loc = w * y + x;
  data[loc] = data[loc];
}

// Can't allocate if not known at compile-time anyway
#define lw 32
#define lh 8

kernel void solve_locally_nprop(global int *data, int w, int h) {
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int x = get_global_id(0);
  int y = get_global_id(1);

  char valid = 1;
  local int buffer[lw * lh];
  local char changed;

  if (y >= h || x >= w) {
    valid = 0;
  }
  buffer[lw * ly + lx] = valid ? data[w * y + x] : 0;
  changed = 1;

  while (changed) {
    barrier(CLK_LOCAL_MEM_FENCE);
    changed = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (valid) {
      int min = 1 << 30;
      int tmp;
      if (lx > 0) {
        tmp = buffer[lw * (ly) + (lx - 1)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (lx < lw - 1) {
        tmp = buffer[lw * (ly) + (lx + 1)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (ly > 0) {
        tmp = buffer[lw * (ly - 1) + (lx)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (ly < lh - 1) {
        tmp = buffer[lw * (ly + 1) + (lx)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (min < buffer[lw * ly + lx]) {
        changed = 1;
        buffer[lw * ly + lx] = min;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
  }

  if (valid) {
    data[w * y + x] = buffer[lw * ly + lx];
  }
}

kernel void solve_locally_plus(global int *data, int w, int h) {
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int x = get_global_id(0);
  int y = get_global_id(1);

  char valid = 1;
  int diff;
  local int buffer[lw * lh];
  local char changed;

  if (y >= h || x >= w) {
    valid = 0;
  }
  buffer[lw * ly + lx] = valid ? data[w * y + x] : 0;
  changed = 1;

  while (changed) {
    barrier(CLK_LOCAL_MEM_FENCE);
    changed = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (valid) {
      int min = 1 << 30;
      int tmp;

      diff = 1;
      while (lx - diff >= 0) {
        tmp = buffer[lw * (ly) + (lx - diff)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }

      diff = 1;
      while (lx + diff < lw) {
        tmp = buffer[lw * (ly) + (lx + diff)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }

      diff = 1;
      while (ly - diff >= 0) {
        tmp = buffer[lw * (ly - diff) + (lx)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }

      diff = 1;
      while (ly + diff < lh) {
        tmp = buffer[lw * (ly + diff) + (lx)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }
      if (min < buffer[lw * ly + lx]) {
        changed = 1;
        buffer[lw * ly + lx] = min;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
  }

  if (valid) {
    data[w * y + x] = buffer[lw * ly + lx];
  }
}

kernel void plus_once_locally(global int *data, int w, int h) {
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int x = get_global_id(0);
  int y = get_global_id(1);

  int diff;
  local int buffer[lw * lh];

  if (y >= h || x >= w) {
    buffer[lw * ly + lx] = 0;
    return;
  }
  buffer[lw * ly + lx] = data[w * y + x];
  if (buffer[lw * ly + lx] == 0) {
    return;
  }

  barrier(CLK_LOCAL_MEM_FENCE);

  int min = 1 << 30;
  int tmp;

  diff = 1;
  while (lx - diff >= 0) {
    tmp = buffer[lw * (ly) + (lx - diff)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }

  diff = 1;
  while (lx + diff < lw) {
    tmp = buffer[lw * (ly) + (lx + diff)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }

  diff = 1;
  while (ly - diff >= 0) {
    tmp = buffer[lw * (ly - diff) + (lx)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }

  diff = 1;
  while (ly + diff < lh) {
    tmp = buffer[lw * (ly + diff) + (lx)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }
  if (min < buffer[lw * ly + lx]) {
    buffer[lw * ly + lx] = min;
    data[w * y + x] = min;
  }
}

// A too large value here silently fails, even if it doesn't exceed
// LOCAL_MEM_MAX. TODO
#define BUFFS 256

#define NORTH (w * (y - 1) + (x))
#define EAST (w * (y) + (x + 1))
#define SOUTH (w * (y + 1) + (x))
#define WEST (w * (y) + (x - 1))
#define CENTER (w * y + x)

#define OK_NORTH (y > 0)
#define OK_EAST (x < w - 1)
#define OK_SOUTH (y < h - 1)
#define OK_WEST (x > 0)
#define VALID (x < w && y < h)

kernel void recursively_win(global int *data, int w, int h,
                            global char *changed) {
  int x, y;
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int tmp, thistmp;
  char eligible;

  local int lowest[1];
  local int stack_x[BUFFS];
  local int stack_y[BUFFS];
  local int stack_ptr[1];
  int own_pointer;

  while (1) {
    x = get_global_id(0);
    y = get_global_id(1);
    eligible = 0;

    if (lx == 0 && ly == 0) {
      *lowest = 1 << 30;
    }

    // ELIGIBILITY PHASE
    // Only pick something as the label to handle if
    // values around it can be reduced to it
    barrier(CLK_LOCAL_MEM_FENCE);

    if (VALID && (thistmp = data[CENTER])) {
      if (OK_NORTH && data[NORTH] > thistmp) {
        eligible = 1;
      }
      if (OK_EAST && data[EAST] > thistmp) {
        eligible = 1;
      }
      if (OK_SOUTH && data[SOUTH] > thistmp) {
        eligible = 1;
      }
      if (OK_WEST && data[WEST] > thistmp) {
        eligible = 1;
      }

      if (eligible) {
        atomic_min(lowest, thistmp);
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
    if (*lowest == 1 << 30) {
      // Found no label inside this region that can be improved upon
      return;
    } else if (lx == 0 && ly == 0) {
      // Found an improvement, so a change will be had
      *changed = 1;
    }

    // FIRST PUSH PHASE
    if (VALID && thistmp == *lowest) {

      if (OK_NORTH) {
        if (data[NORTH] > thistmp) {
          data[NORTH] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x;
            stack_y[own_pointer] = y - 1;
          }
        }
      }
      if (OK_EAST) {
        if (data[EAST] > thistmp) {
          data[EAST] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x + 1;
            stack_y[own_pointer] = y;
          }
        }
      }
      if (OK_SOUTH) {
        if (data[SOUTH] > thistmp) {
          data[SOUTH] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x;
            stack_y[own_pointer] = y + 1;
          }
        }
      }
      if (OK_WEST) {
        if (data[WEST] > thistmp) {
          data[WEST] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x - 1;
            stack_y[own_pointer] = y;
          }
        }
      }
    }

    thistmp = *lowest; // For unlucky threads to participate

    if (lx == 0 && ly == 0 && thistmp == 0) {
      *changed = 0;
      return;
    }
    while (1) {
      // Test-if-there's-work-at-all phase
      barrier(CLK_LOCAL_MEM_FENCE);
      if (*stack_ptr == 0) {
        break;
      }

      // POP PHASE
      barrier(CLK_LOCAL_MEM_FENCE);
      own_pointer = atomic_dec(stack_ptr) - 1;
      if (own_pointer < 0) {
        atomic_inc(stack_ptr);
      } else {
        x = stack_x[own_pointer];
        y = stack_y[own_pointer];
      }

      // PUSH PHASE
      barrier(CLK_LOCAL_MEM_FENCE);

      if (own_pointer >= 0) {
        // Not trash in x, y

        if (OK_NORTH) {
          if (data[NORTH] > thistmp) {
            data[NORTH] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x;
              stack_y[own_pointer] = y - 1;
            }
          }
        }
        if (OK_EAST) {
          if (data[EAST] > thistmp) {
            data[EAST] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x + 1;
              stack_y[own_pointer] = y;
            }
          }
        }
        if (OK_SOUTH) {
          if (data[SOUTH] > thistmp) {
            data[SOUTH] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x;
              stack_y[own_pointer] = y + 1;
            }
          }
        }
        if (OK_WEST) {
          if (data[WEST] > thistmp) {
            data[WEST] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x - 1;
              stack_y[own_pointer] = y;
            }
          }
        }

      }

    }
  }
}

kernel void label_with_id(global int *data, int w, int h) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int loc = w * y + x;
  if (data[loc] == 1) {
    data[loc] = loc + 2;
  }
}

kernel void neighbour_propagate(global int *data, int w, int h,
                                global char *changed) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int oldlabel = data[w * y + x];
  int curlabel = oldlabel;
  int otherlabel = 0;

  if (curlabel == 0) {
    return;
  }

  if (y + 1 < h) {
    otherlabel = data[w * (y + 1) + (x)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }
  if (y - 1 >= 0) {
    otherlabel = data[w * (y - 1) + (x)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }
  if (x + 1 < w) {
    otherlabel = data[w * (y) + (x + 1)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }
  if (x - 1 >= 0) {
    otherlabel = data[w * (y) + (x - 1)];
    if (otherlabel && otherlabel < curlabel) {
      curlabel = otherlabel;
    }
  }

  if (curlabel < oldlabel) {
    *changed = 1;
    data[w * y + x] = curlabel;
  }
}

kernel void plus_propagate(global int *data, int w, int h,
                           global char *changed) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int oldlabel = data[w * y + x];
  int curlabel = oldlabel;
  int otherlabel = 0;
  int diff;

  if (curlabel == 0) {
    return;
  }

  diff = 1;
  while (true) {
    if (y + diff < 0 || y + diff >= h) {
      break;
    }
    otherlabel = data[w * (y + diff) + (x)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }
  diff = 1;
  while (true) {
    if (y - diff < 0 || y - diff >= h) {
      break;
    }
    otherlabel = data[w * (y - diff) + (x)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }
  diff = 1;
  while (true) {
    if (x + diff < 0 || x + diff >= w) {
      break;
    }
    otherlabel = data[w * y + (x + diff)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }
  diff = 1;
  while (true) {
    if (x - diff < 0 || x - diff >= w) {
      break;
    }
    otherlabel = data[w * y + (x - diff)];
    if (otherlabel == 0) {
      break;
    }
    if (otherlabel < curlabel) {
      curlabel = otherlabel;
    }
    ++diff;
  }

  if (curlabel < oldlabel) {
    *changed = 1;
    data[w * y + x] = curlabel;
  }
}

int find_set(global int *data, int loc) {
  // All loc of found elements should be in range.
  // Also assuming there are no cycles in the links.
  while (loc != data[loc] - 2) {
    loc = data[loc] - 2;
  }

  // +2 is the correct LABEL of root at LOCATION loc
  return loc + 2;
}

kernel void union_find(global int *data, int w, int h, global char *changed) {
  int x = get_global_id(0);
  int y = get_global_id(1);
  if (x >= w || y >= h) {
    return;
  }

  int oldlabel = data[w * y + x];
  int lowest = oldlabel;

  if (oldlabel == 0) {
    return;
  }

  bool ok_N = y - 1 >= 0 && data[w * (y - 1) + (x)];
  bool ok_E = x + 1 < w && data[w * (y) + (x + 1)];
  bool ok_S = y + 1 < h && data[w * (y + 1) + (x)];
  bool ok_W = x - 1 >= 0 && data[w * (y) + (x - 1)];
  int root_N;
  int root_E;
  int root_S;
  int root_W;

  if (ok_N) {
    root_N = find_set(data, w * (y - 1) + (x));
    if (root_N < lowest) {
      lowest = root_N;
    }
  }
  if (ok_E) {
    root_E = find_set(data, w * (y) + (x + 1));
    if (root_E < lowest) {
      lowest = root_E;
    }
  }
  if (ok_S) {
    root_S = find_set(data, w * (y + 1) + (x));
    if (root_S < lowest) {
      lowest = root_S;
    }
  }
  if (ok_W) {
    root_W = find_set(data, w * (y) + (x - 1));
    if (root_W < lowest) {
      lowest = root_W;
    }
  }

  if (lowest < oldlabel) {
    *changed = 1;
    data[w * y + x] = lowest;
    if (ok_N && root_N > lowest) {
      data[root_N - 2] = lowest;
    }
    if (ok_E && root_E > lowest) {
      data[root_E - 2] = lowest;
    }
    if (ok_S && root_S > lowest) {
      data[root_S - 2] = lowest;
    }
    if (ok_W && root_W > lowest) {
      data[root_W - 2] = lowest;
    }
  }
}

kernel void lineedit_right(global int *data, int w, int h,
                           global char *changed) {
  int x = 0;
  int y = get_global_id(0);
  int lowest = 1 << 30;

  while (x < w) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    ++x;
  }
}

kernel void lineedit_left(global int *data, int w, int h,
                          global char *changed) {
  int x = w - 1;
  int y = get_global_id(0);
  int lowest = 1 << 30;

  while (x >= 0) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    --x;
  }
}

kernel void lineedit_up(global int *data, int w, int h, global char *changed) {
  int x = get_global_id(0);
  int y = 0;
  int lowest = 1 << 30;

  while (y < h) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    ++y;
  }
}

kernel void lineedit_down(global int *data, int w, int h,
                          global char *changed) {
  int x = get_global_id(0);
  int y = h - 1;
  int lowest = 1 << 30;

  while (y >= 0) {
    int curlabel = data[w * y + x];

    if (curlabel == 0) {
      lowest = 1 << 30;
    } else {
      if (curlabel < lowest) {
        lowest = curlabel;
      } else if (curlabel > lowest) {
        data[w * y + x] = lowest;
        *changed = 1;
      }
    }

    --y;
  }
}

kernel void lines_up(global int *data, int w, int h, global char *changed) {
  int x = get_global_id(0);
  int y = 0;
  char localchanged = 0;

  while (y < h) {
    if (!data[w * y + x]) {
      ++y;
      continue;
    }
    int i = y;
    int min = data[w * y + x];
    int tmpmin = min;

    while (i < h && (tmpmin = data[w * i + x])) {
      if (tmpmin != min) {
        localchanged = 1; // Not all equal, i.e. will have change
      }
      if (tmpmin < min) {
        min = tmpmin;
      }
      ++i;
    }

    while (y != i) {
      data[w * y + x] = min;
      ++y;
    }
  }

  if (localchanged) {
    *changed = localchanged;
  }
}

kernel void lines_right(global int *data, int w, int h, global char *changed) {
  int x = 0;
  int y = get_global_id(0);
  char localchanged = 0;

  while (x < w) {
    if (!data[w * y + x]) {
      ++x;
      continue;
    }
    int i = x;
    int min = data[w * y + x];
    int tmpmin = min;

    while (i < w && (tmpmin = data[w * y + i])) {
      if (tmpmin != min) {
        localchanged = 1; // Not all equal, i.e. will have change
      }
      if (tmpmin < min) {
        min = tmpmin;
      }
      ++i;
    }

    while (x != i) {
      data[w * y + x] = min;
      ++x;
    }
  }

  if (localchanged) {
    *changed = localchanged;
  }
}

kernel void id_accessor(global int *data, int w) {
  unsigned int x = get_global_id(0);
  unsigned int y = get_global_id(1);

  int loc = w * y + x;
  data[loc] = data[loc];
}

// Can't allocate if not known at compile-time anyway
#define lw 32
#define lh 8

kernel void solve_locally_nprop(global int *data, int w, int h) {
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int x = get_global_id(0);
  int y = get_global_id(1);

  char valid = 1;
  local int buffer[lw * lh];
  local char changed;

  if (y >= h || x >= w) {
    valid = 0;
  }
  buffer[lw * ly + lx] = valid ? data[w * y + x] : 0;
  changed = 1;

  while (changed) {
    barrier(CLK_LOCAL_MEM_FENCE);
    changed = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (valid) {
      int min = 1 << 30;
      int tmp;
      if (lx > 0) {
        tmp = buffer[lw * (ly) + (lx - 1)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (lx < lw - 1) {
        tmp = buffer[lw * (ly) + (lx + 1)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (ly > 0) {
        tmp = buffer[lw * (ly - 1) + (lx)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (ly < lh - 1) {
        tmp = buffer[lw * (ly + 1) + (lx)];
        if (tmp && tmp < min) {
          min = tmp;
        }
      }
      if (min < buffer[lw * ly + lx]) {
        changed = 1;
        buffer[lw * ly + lx] = min;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
  }

  if (valid) {
    data[w * y + x] = buffer[lw * ly + lx];
  }
}

kernel void solve_locally_plus(global int *data, int w, int h) {
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int x = get_global_id(0);
  int y = get_global_id(1);

  char valid = 1;
  int diff;
  local int buffer[lw * lh];
  local char changed;

  if (y >= h || x >= w) {
    valid = 0;
  }
  buffer[lw * ly + lx] = valid ? data[w * y + x] : 0;
  changed = 1;

  while (changed) {
    barrier(CLK_LOCAL_MEM_FENCE);
    changed = 0;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (valid) {
      int min = 1 << 30;
      int tmp;

      diff = 1;
      while (lx - diff >= 0) {
        tmp = buffer[lw * (ly) + (lx - diff)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }

      diff = 1;
      while (lx + diff < lw) {
        tmp = buffer[lw * (ly) + (lx + diff)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }

      diff = 1;
      while (ly - diff >= 0) {
        tmp = buffer[lw * (ly - diff) + (lx)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }

      diff = 1;
      while (ly + diff < lh) {
        tmp = buffer[lw * (ly + diff) + (lx)];
        if (!tmp) {
          break;
        }
        if (tmp < min) {
          min = tmp;
        }
        ++diff;
      }
      if (min < buffer[lw * ly + lx]) {
        changed = 1;
        buffer[lw * ly + lx] = min;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
  }

  if (valid) {
    data[w * y + x] = buffer[lw * ly + lx];
  }
}

kernel void plus_once_locally(global int *data, int w, int h) {
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int x = get_global_id(0);
  int y = get_global_id(1);

  int diff;
  local int buffer[lw * lh];

  if (y >= h || x >= w) {
    buffer[lw * ly + lx] = 0;
    return;
  }
  buffer[lw * ly + lx] = data[w * y + x];
  if (buffer[lw * ly + lx] == 0) {
    return;
  }

  barrier(CLK_LOCAL_MEM_FENCE);

  int min = 1 << 30;
  int tmp;

  diff = 1;
  while (lx - diff >= 0) {
    tmp = buffer[lw * (ly) + (lx - diff)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }

  diff = 1;
  while (lx + diff < lw) {
    tmp = buffer[lw * (ly) + (lx + diff)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }

  diff = 1;
  while (ly - diff >= 0) {
    tmp = buffer[lw * (ly - diff) + (lx)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }

  diff = 1;
  while (ly + diff < lh) {
    tmp = buffer[lw * (ly + diff) + (lx)];
    if (!tmp) {
      break;
    }
    if (tmp < min) {
      min = tmp;
    }
    ++diff;
  }
  if (min < buffer[lw * ly + lx]) {
    buffer[lw * ly + lx] = min;
    data[w * y + x] = min;
  }
}

// A too large value here silently fails, even if it doesn't exceed
// LOCAL_MEM_MAX. TODO
#define BUFFS 256

#define NORTH (w * (y - 1) + (x))
#define EAST (w * (y) + (x + 1))
#define SOUTH (w * (y + 1) + (x))
#define WEST (w * (y) + (x - 1))
#define CENTER (w * y + x)

#define OK_NORTH (y > 0 && data[NORTH])
#define OK_EAST (x < w - 1 && data[EAST])
#define OK_SOUTH (y < h - 1 && data[SOUTH])
#define OK_WEST (x > 0 && data[WEST])
#define VALID (x < w && y < h)

kernel void recursively_win(global int *data, int w, int h,
                            global char *changed) {
  int x, y;
  int lx = get_local_id(0);
  int ly = get_local_id(1);
  int tmp, thistmp;
  char eligible;

  local int lowest[1];
  local int stack_x[BUFFS];
  local int stack_y[BUFFS];
  local int stack_ptr[1];
  int own_pointer;

  while (1) {
    x = get_global_id(0);
    y = get_global_id(1);
    eligible = 0;

    if (lx == 0 && ly == 0) {
      *lowest = 1 << 30;
    }

    // ELIGIBILITY PHASE
    // Only pick something as the label to handle if
    // values around it can be reduced to it
    barrier(CLK_LOCAL_MEM_FENCE);

    if (VALID && (thistmp = data[CENTER])) {
      if (OK_NORTH && data[NORTH] > thistmp) {
        eligible = 1;
      }
      if (OK_EAST && data[EAST] > thistmp) {
        eligible = 1;
      }
      if (OK_SOUTH && data[SOUTH] > thistmp) {
        eligible = 1;
      }
      if (OK_WEST && data[WEST] > thistmp) {
        eligible = 1;
      }

      if (eligible) {
        atomic_min(lowest, thistmp);
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
    if (*lowest == 1 << 30) {
      // Found no label inside this region that can be improved upon
      return;
    } else if (lx == 0 && ly == 0) {
      // Found an improvement, so a change will be had
      *changed = 1;
    }

    // FIRST PUSH PHASE
    if (VALID && thistmp == *lowest) {

      if (OK_NORTH) {
        if (data[NORTH] > thistmp) {
          data[NORTH] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x;
            stack_y[own_pointer] = y - 1;
          }
        }
      }
      if (OK_EAST) {
        if (data[EAST] > thistmp) {
          data[EAST] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x + 1;
            stack_y[own_pointer] = y;
          }
        }
      }
      if (OK_SOUTH) {
        if (data[SOUTH] > thistmp) {
          data[SOUTH] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x;
            stack_y[own_pointer] = y + 1;
          }
        }
      }
      if (OK_WEST) {
        if (data[WEST] > thistmp) {
          data[WEST] = thistmp;
          own_pointer = atomic_inc(stack_ptr);
          if (own_pointer >= BUFFS) {
            atomic_dec(stack_ptr);
          } else {
            stack_x[own_pointer] = x - 1;
            stack_y[own_pointer] = y;
          }
        }
      }
    }

    thistmp = *lowest; // For unlucky threads to participate

    if (lx == 0 && ly == 0 && thistmp == 0) {
      *changed = 0;
      return;
    }
    while (1) {
      // Test-if-there's-work-at-all phase
      barrier(CLK_LOCAL_MEM_FENCE);
      if (*stack_ptr == 0) {
        break;
      }

      // POP PHASE
      barrier(CLK_LOCAL_MEM_FENCE);
      own_pointer = atomic_dec(stack_ptr) - 1;
      if (own_pointer < 0) {
        atomic_inc(stack_ptr);
      } else {
        x = stack_x[own_pointer];
        y = stack_y[own_pointer];
      }

      // PUSH PHASE
      barrier(CLK_LOCAL_MEM_FENCE);

      if (own_pointer >= 0) {
        // Not trash in x, y

        if (OK_NORTH) {
          if (data[NORTH] > thistmp) {
            data[NORTH] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x;
              stack_y[own_pointer] = y - 1;
            }
          }
        }
        if (OK_EAST) {
          if (data[EAST] > thistmp) {
            data[EAST] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x + 1;
              stack_y[own_pointer] = y;
            }
          }
        }
        if (OK_SOUTH) {
          if (data[SOUTH] > thistmp) {
            data[SOUTH] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x;
              stack_y[own_pointer] = y + 1;
            }
          }
        }
        if (OK_WEST) {
          if (data[WEST] > thistmp) {
            data[WEST] = thistmp;
            own_pointer = atomic_inc(stack_ptr);
            if (own_pointer >= BUFFS) {
              atomic_dec(stack_ptr);
            } else {
              stack_x[own_pointer] = x - 1;
              stack_y[own_pointer] = y;
            }
          }
        }

      }

    }
  }
}