this version runs unoptimized on the GPU and gives good results on the first 9 cases and 0.005 difference on the tenth

// -*- c++ -*-
// UjoImro, 2013
// Experimental code for the CARP Project
// Copyright (c) RealEyes, 2013
// This is a c-implementation of the PCA->MLP response map calculation

typedef struct {
    int rows;
    int cols;
    int step;
    int start;
} clMat; // struct clMatFloat

typedef struct {
    int size;
    int step;
    int start;
} clVector; // struct clVector

typedef struct {
    int m_patchSize;
    clMat /* uint8_t */ alignedImage;
    clMat /* int32_t */ shape;
//        clMat /* float */ m_wIn;
    clMat /* float */ m_wOut;
//        clMat /* float */ m_U;
    clMat /* float */ wIn;
    clMat /* float */ bIn;
} calcinput; // struct

typedef struct {
    int x;
    int y;
} Point2i; // struct Point2i

typedef struct {
    // temporaries
    clVector /* clMat[] */ patches;
    clVector /* clMat[] */ xOuts;
    clVector /* clMat[] */ es;
} calctemp; // struct

typedef struct {
    // results
    clMat /*float*/ responseMap;
} calcoutput;

typedef struct {
    calcinput  input;
    calctemp   tmp;
    calcoutput output;
} calcpackage;

__constant const int MAX_INT = ~(1 << (8*sizeof(int) - 1));
// __constant const int false = (1!=1);
__constant const int gangsize = 32;

clMat GetMatFromVector( __global void * self, clVector /*clMat*/ vec, int idx )
{
//    assert(self);
//    assert(idx>=0);
//    assert(idx<vec.size);

    return ((__global clMat*)self)[ vec.start + vec.step * idx ];
} // GetMatFromVector

void SetMatToVector( __global void * self, clVector /*clMat*/ vec, int idx, clMat val )
{
//    assert(self);
//    assert(idx>=0);
//    assert(idx<vec.size);

    ((__global clMat*)self)[ vec.start + vec.step * idx ] = val;
    return;
} // GetMatFromVector

void
copyToFloat( __global void * self, clMat /*float*/input, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
                ((__global float*)self)[ q * input.step + w + input.start ];

    return;
} // copyTo

void
transposeFloat( __global void * self, clMat /*float*/input, clMat /*float*/ output )
{
    int q,w;
    // assert(output.rows == input.cols);
    // assert(output.cols == input.rows);
    for (q=0; q<input.rows; q++)
        for (w=0; w<input.cols; w++)
            ((__global float*)self)[ w * output.step + q + output.start ]
        	= ((__global float*)self)[ q * input.step + w + input.start ];

    return;
} // transposeFloat

void
transposeFloatGang( __global void * self, clMat /*float*/input, int localid, clMat /*float*/ output )
{
    // assert(output.rows == input.cols);
    // assert(output.cols == input.rows);

    int worksize = input.cols * input.rows;
    int work = 0;

    for ( work=0; work < (worksize/gangsize) + 1; work++) {
        int index = work * gangsize + localid;
        if (index>=worksize) continue;
        int q = index / input.cols;
        int w = index % input.cols;

        ((__global float*)self)[ w * output.step + q + output.start ]
            = ((__global float*)self)[ q * input.step + w + input.start ];
    }
} // transposeFloatGang

float
meanChar( __global void * self, clMat /*uint8_t*/  input )
{
    int q,w;
    float sum=0;
    float c = 0; // kahan summation

    for ( q=0; q<input.rows; q++ )
      for ( w=0; w<input.cols; w++ )
      {
        float y = ((__global uchar*)self)[ q * input.step + w + input.start ] - c;
        float t = sum + y;
        c = (t - sum) - y;
        sum = t;
      }

    return sum / ( input.rows * input.cols );
} // meanFloat

uchar
minChar( __global void * self, clMat /*uint8_t*/  input )
{
    int q,w;
    uchar minvalue = 255;

    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            minvalue = min( minvalue, ((__global uchar*)self)[ q * input.step + w + input.start ] );

    return minvalue;
} // minFloat

uchar
maxChar( __global void * self, clMat /*uint8_t*/  input )
{
    int q,w;
    uchar maxvalue = 0;

    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            maxvalue = max( maxvalue, ((__global uchar*)self)[ q * input.step + w + input.start ] );

    return maxvalue;
} // maxFloat

clMat /*uint8_t*/
GetBlockChar( __global void * self, clMat /*uint8_t*/  smat, int row_from, int row_to, int col_from, int col_to )
{
    // assert(row_from>=0);
    // assert(col_from>=0);
    // assert(row_from<row_to);
    // assert(col_from<col_to);
    // assert(row_to<=smat.rows);
    // assert(col_to<=smat.cols);

    clMat /*uchar*/  result;
    result.rows  = row_to - row_from;
    result.cols  = col_to - col_from;
    result.step  = smat.step;
    result.start = smat.start + row_from * smat.step + col_from;

    return result;
}

clMat
GetBlockFloat( __global void * self, clMat /*float*/smat, int row_from, int row_to, int col_from, int col_to )
{
    // assert(row_from>=0);
    // assert(col_from>=0);
    // assert(row_from<row_to);
    // assert(col_from<col_to);
    // assert(row_to<=smat.rows);
    // assert(col_to<=smat.cols);

    clMat /*float*/result;
    result.rows  = row_to - row_from;
    result.cols  = col_to - col_from;
    result.step  = smat.step;
    result.start = smat.start + row_from * smat.step + col_from;

    return result;
}

void
convertFromCharToFloat( __global void * self, clMat /*uint8_t*/  from, float quotient, float shift, clMat /*float*/ to )
{
    // assert(from.rows == to.rows);
    // assert(from.cols == to.cols);

    int q, w;
    for ( q=0; q<from.rows; q++ )
        for ( w=0; w<from.cols; w++ )
            ((__global float*)self)[ q * to.step + w + to.start ] =
        	quotient * ((__global uchar*)self)[ q * from.step + w + from.start ] + shift;
    return;
}

clMat
reshapeFloat( __global void * self, clMat /*float*/smat, int new_rows )
{

    // assert(smat.cols == smat.step);
    // assert( (smat.cols * smat.rows) % new_rows == 0 );

    clMat /*float*/result;
    result.rows = new_rows;
    result.cols = smat.cols * smat.rows / new_rows;
    result.step = result.cols;
    result.start = smat.start;

    return result;
} // reshapeFloat

// returns alpha*A*B + beta * C
void
gemmFloatDirDirDir( __global void * self, clMat /*float*/A, clMat /*float*/B, float alpha, clMat /*float*/C, float beta, clMat /*float*/ result )
{
    // assert(A.rows == C.rows);
    // assert(A.cols == B.rows);
    // assert(B.cols == C.cols);
    // assert(C.rows == result.rows);
    // assert(C.cols == result.cols);

    int q, w, e;
    float sum=0.;
    float c;

    if ( fabs(beta) > 0.000001 ) {
        for ( q=0; q<C.rows; q++ )
            for ( w=0; w<C.cols; w++ )
            {
                sum = 0;
                for ( e=0; e<A.cols; e++ )
                {
                    float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                    float t = sum + y;
                    c = (t - sum) - y;
                    sum = t;
                }

                ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum  + beta * ((__global float*)self)[ q * C.step + w + C.start ];
            }
    }
    else // NOT fabs(beta) > 0.000001
    {
        for ( q=0; q<C.rows; q++ )
            for ( w=0; w<C.cols; w++ )
            {
                sum = 0;
                for ( e=0; e<A.cols; e++ )
                {
                    float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                    float t = sum + y;
                    c = (t - sum) - y;
                    sum = t;
                }

                ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum;
            }
    }

    return;
} // gemmFloatDirDirDir

void
gemmFloatDirDirDirGang( __global void * self, clMat /*float*/A, clMat /*float*/B, float alpha, clMat /*float*/C, float beta, int localid, clMat /*float*/ result )
{
    // assert(A.rows == C.rows);
    // assert(A.cols == B.rows);
    // assert(B.cols == C.cols);
    // assert(C.rows == result.rows);
    // assert(C.cols == result.cols);

    int q, w, e;
    float sum=0.;
    float c;
    if ( fabs(beta) > 0.000001 ) {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum  + beta * ((__global float*)self)[ q * C.step + w + C.start ];
        }
    }
    else // NOT fabs(beta) > 0.000001
    {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum;
        }
    }

    return;
} // gemmFloatDirDirDirGang

void
gemmFloatDirTransDirGang( __global void * self, clMat /*float*/A, clMat /*float*/B, float alpha, clMat /*float*/C, float beta, int localid, clMat /*float*/ result )
{
    // assert(A.rows == C.rows);
    // assert(A.cols == B.cols);
    // assert(B.rows == C.cols);
    // assert(C.rows == result.rows);
    // assert(C.cols == result.cols);

    int q, w, e;
    float sum=0.;
    float c;
    if ( fabs(beta) > 0.000001 ) {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ w * B.step + e + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum  + beta * ((__global float*)self)[ q * C.step + w + C.start ];
        }
    }
    else // NOT fabs(beta) > 0.000001
    {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ w * B.step + e + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum;
        }
    }

    return;
} // gemmFloatDirTransDirGang

void
expFloat( __global void * self, clMat /*float*/input, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
                exp(((__global float*)self)[ q * input.step + w + input.start ]);

    return;
}

void
addFloat( __global void * self, clMat /*float*/input, float val, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
        	val + ((__global float*)self)[ q * input.step + w + input.start ];

    return;
}

void
divideFloat( __global void * self, float val, clMat /*float*/input, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
        	val / ((__global float*)self)[ q * input.step + w + input.start ];

    return;
}

void
subtractFloat( __global void * self, clMat /*float*/input, float val, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
        	((__global float*)self)[ q * input.step + w + input.start ] - val;

    return;
}

float
GetValueFloat( __global void * self, clMat /*float*/smat, int row, int col )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);
    // assert(smat.step % smat.cols == 0);

    return ((__global float*)self)[ row * smat.step + col + smat.start ];
    // return -1231.;
}

uchar
GetValueChar( __global void * self, clMat /* uint8_t*/ smat, int row, int col )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);

    return ((__global uchar*)self)[ row * smat.step + col + smat.start ];
    // return -1231.;
}

void
SetValueFloat( __global void * self, clMat /*float*/ smat, int row, int col, float value )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);
    // assert(smat.step % smat.cols == 0);

    ((__global float*)self)[ row * smat.step + col + smat.start ] = value;
    return;
}

void
SetValueChar( __global void * self, clMat /*float*/ smat, int row, int col, uchar value )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);
    // assert(smat.step % smat.cols == 0);

    ((__global uchar*)self)[ row * smat.step + col + smat.start ] = value;
    return;
}

float dotProductDirDir( __global void * self, clMat /*float*/A, clMat /*float*/B )
{
  // assert( A.cols == 1 );
  // assert( B.cols == 1 );
  // assert( A.rows == B.rows );

  float result = 0.;
  float c = 0.;

  int q;
  for (q=0; q<A.rows; q++) {
      float y = ((__global float*)self)[ q * A.step + A.start ] * ((__global float*)self)[ q * B.step + B.start ] - c;
      float t = result + y;
      c = (t - result) - y;
      result = t ;
  }

  return result;
}

float dotProductTransDir( __global void * self, clMat /*float*/A, clMat /*float*/B )
{
  // assert( A.rows == 1 );
  // assert( B.cols == 1 );
  // assert( A.cols == B.rows );

  float result = 0.;
  float c = 0.;

  int q;
  for (q=0; q<A.cols; q++) {
      float y = ((__global float*)self)[ q + A.start ] * ((__global float*)self)[ q * B.step + B.start ] - c;
      float t = result + y;
      c = (t - result) - y;
      result = t ;
  }

  return result;
}

void normalizeSample( __global void * self, clMat /*uint8_t*/  image, clMat /*float*/ * result )
{
  // assert(result->cols == image.cols);
  // assert(result->rows == image.rows);

    float sampleMean = meanChar(self, image);
    float sampleMin  = minChar(self, image);
    float sampleMax  = maxChar(self, image);

     sampleMax -= sampleMean;
     sampleMin -= sampleMean;

     sampleMax = fmax( fabs(sampleMin), fabs(sampleMax));

     if (sampleMax == 0.0) sampleMax = 1.0;

     convertFromCharToFloat( self, image, 1.0/sampleMax, -(1.0/sampleMax)*sampleMean, *result );

     *result = reshapeFloat( self, *result, image.rows * image.cols );

  return;
} // normalizeSample

void
generateResponseMap(
    __global void * self,
    const clMat /*uint8_t*/  image,
    const Point2i center,
    int mapSize,
    int m_patchSize,
    // clMat /*float*/m_wIn,
    clMat /*float*/m_wOut,
    // clMat /*float*/m_U,
    clMat wIn,
    clMat bIn,

    // temporary variables
    clVector /*cMat float*/ patches,
    clVector /*cMat float*/ xOuts,
    clVector /*cMat float*/ es,
    // result
    clMat /*float*/ result
    )
{

  // assert( result.rows == 2 * mapSize + 1 );
  // assert( result.cols == 2 * mapSize + 1 );

  //  assert(wIn.rows == wIn_A.rows);
  //  assert(wIn.cols == m_U.rows);

    clMat /*float*/wOut_tmp = GetBlockFloat( self, m_wOut, 0, m_wOut.rows, 0, m_wOut.cols - 1 );

    {
        int localid = 0;
        for ( localid=0; localid<gangsize; localid++ ) {

            int work = 0;
            int worksize = (2 * mapSize + 1) * (2 * mapSize + 1);

            float bOut = GetValueFloat( self, m_wOut, 0, m_wOut.cols - 1 );

            for ( work=0; work < (worksize / gangsize) + 1; work++ )
            {
              int index = work * gangsize + localid;
              if (index>=worksize) continue;
              int ncy = index / (2 * mapSize + 1);
              int cy  = center.y - mapSize + ncy;
              int ncx = index % (2 * mapSize + 1);
              int cx  = center.x - mapSize + ncx;

              clMat /*uint8_t*/   imagePatch = GetBlockChar( self, image, cy - m_patchSize, cy + m_patchSize + 1, cx - m_patchSize, cx + m_patchSize + 1 );

              clMat patch = GetMatFromVector( self, patches, localid );

              normalizeSample( self, imagePatch, &patch );

              clMat xOut = GetMatFromVector( self, xOuts, localid );

              gemmFloatDirDirDir( self, wIn, patch, -1.0, bIn, -1.0, xOut );

              clMat e = GetMatFromVector( self, es, localid );

              expFloat( self, xOut, e );

              addFloat( self, e, 1.0, xOut );
              divideFloat( self, 2.0, xOut, e);
              addFloat( self, e, -1.0, xOut);

              SetValueFloat( self, result, ncy, ncx, 1./( 1. + exp(- dotProductTransDir( self, wOut_tmp, xOut) - bOut ) ) );
            } // for gangsize
        } // for localid
    } // localid block

//  // assert(false);

  // freeMatFloat( self, allocator, &wIn);
  // end of classic impl
    return;
} // generateResponseMap

int
cvRound( float value )
{
    return (int)(value + (value >= 0 ? 0.5 : -0.5));
} // cvRound

__kernel void
calculateMaps(
    // global variables
    __global void * self,
    __constant int memory_segments[], // representing the start of the gang's memory segment
    int m_visibleLandmarks_size,
    int m_mapSize,

    __constant calcpackage packages[],

    // local buffer
    __local void * buffer
    )
{
    if (get_global_id(0)>0) return;

    int q;
    for (q=0; q < m_visibleLandmarks_size; q++ )
    {
        int idx = q;

        Point2i center;

        float shape_x;
        float shape_y;
        shape_x = GetValueFloat( self + memory_segments[q], packages[q].input.shape, 2*idx, 0 );
        shape_y = GetValueFloat( self + memory_segments[q], packages[q].input.shape, 2*idx+1, 0 );

        center.x = cvRound(shape_x);
        center.y = cvRound(shape_y);

        generateResponseMap(
            self + memory_segments[idx],
            packages[idx].input.alignedImage,
            center,
            m_mapSize,
            packages[idx].input.m_patchSize,
            packages[idx].input.m_wOut,
            packages[idx].input.wIn,
            packages[idx].input.bIn,

            // temporary
            packages[idx].tmp.patches,
            packages[idx].tmp.xOuts,
            packages[idx].tmp.es,
            // result
            packages[idx].output.responseMap );

    } // for q in visiblelandmarks_size

    return;
} // calculateMaps

// LuM end of file

// -*- c++ -*-
// UjoImro, 2013
// Experimental code for the CARP Project
// Copyright (c) RealEyes, 2013
// This is a c-implementation of the PCA->MLP response map calculation

typedef struct {
    int rows;
    int cols;
    int step;
    int start;
} clMat; // struct clMatFloat

typedef struct {
    int size;
    int step;
    int start;
} clVector; // struct clVector

typedef struct {
    int m_patchSize;
    clMat /* uint8_t */ alignedImage;
    clMat /* int32_t */ shape;
//        clMat /* float */ m_wIn;
    clMat /* float */ m_wOut;
//        clMat /* float */ m_U;
    clMat /* float */ wIn;
    clMat /* float */ bIn;
} calcinput; // struct

typedef struct {
    int x;
    int y;
} Point2i; // struct Point2i

typedef struct {
    // temporaries
    clVector /* clMat[] */ patches;
    clVector /* clMat[] */ xOuts;
    clVector /* clMat[] */ es;
} calctemp; // struct

typedef struct {
    // results
    clMat /*float*/ responseMap;
} calcoutput;

typedef struct {
    calcinput  input;
    calctemp   tmp;
    calcoutput output;
} calcpackage;

__constant const int MAX_INT = ~(1 << (8*sizeof(int) - 1));
// __constant const int false = (1!=1);
__constant const int gangsize = 32;

clMat GetMatFromVector( __global void * self, clVector /*clMat*/ vec, int idx )
{
//    assert(self);
//    assert(idx>=0);
//    assert(idx<vec.size);

    return ((__global clMat*)self)[ vec.start + vec.step * idx ];
} // GetMatFromVector

void SetMatToVector( __global void * self, clVector /*clMat*/ vec, int idx, clMat val )
{
//    assert(self);
//    assert(idx>=0);
//    assert(idx<vec.size);

    ((__global clMat*)self)[ vec.start + vec.step * idx ] = val;
    return;
} // GetMatFromVector

void
copyToFloat( __global void * self, clMat /*float*/input, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
                ((__global float*)self)[ q * input.step + w + input.start ];

    return;
} // copyTo

void
transposeFloat( __global void * self, clMat /*float*/input, clMat /*float*/ output )
{
    int q,w;
    // assert(output.rows == input.cols);
    // assert(output.cols == input.rows);
    for (q=0; q<input.rows; q++)
        for (w=0; w<input.cols; w++)
            ((__global float*)self)[ w * output.step + q + output.start ]
        	= ((__global float*)self)[ q * input.step + w + input.start ];

    return;
} // transposeFloat

void
transposeFloatGang( __global void * self, clMat /*float*/input, int localid, clMat /*float*/ output )
{
    // assert(output.rows == input.cols);
    // assert(output.cols == input.rows);

    int worksize = input.cols * input.rows;
    int work = 0;

    for ( work=0; work < (worksize/gangsize) + 1; work++) {
        int index = work * gangsize + localid;
        if (index>=worksize) continue;
        int q = index / input.cols;
        int w = index % input.cols;

        ((__global float*)self)[ w * output.step + q + output.start ]
            = ((__global float*)self)[ q * input.step + w + input.start ];
    }
} // transposeFloatGang

float
meanChar( __global void * self, clMat /*uint8_t*/  input )
{
    int q,w;
    float sum=0;
    float c = 0; // kahan summation

    for ( q=0; q<input.rows; q++ )
      for ( w=0; w<input.cols; w++ )
      {
        float y = ((__global uchar*)self)[ q * input.step + w + input.start ] - c;
        float t = sum + y;
        c = (t - sum) - y;
        sum = t;
      }

    return sum / ( input.rows * input.cols );
} // meanFloat

uchar
minChar( __global void * self, clMat /*uint8_t*/  input )
{
    int q,w;
    uchar minvalue = 255;

    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            minvalue = min( minvalue, ((__global uchar*)self)[ q * input.step + w + input.start ] );

    return minvalue;
} // minFloat

uchar
maxChar( __global void * self, clMat /*uint8_t*/  input )
{
    int q,w;
    uchar maxvalue = 0;

    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            maxvalue = max( maxvalue, ((__global uchar*)self)[ q * input.step + w + input.start ] );

    return maxvalue;
} // maxFloat

clMat /*uint8_t*/
GetBlockChar( __global void * self, clMat /*uint8_t*/  smat, int row_from, int row_to, int col_from, int col_to )
{
    // assert(row_from>=0);
    // assert(col_from>=0);
    // assert(row_from<row_to);
    // assert(col_from<col_to);
    // assert(row_to<=smat.rows);
    // assert(col_to<=smat.cols);

    clMat /*uchar*/  result;
    result.rows  = row_to - row_from;
    result.cols  = col_to - col_from;
    result.step  = smat.step;
    result.start = smat.start + row_from * smat.step + col_from;

    return result;
}

clMat
GetBlockFloat( __global void * self, clMat /*float*/smat, int row_from, int row_to, int col_from, int col_to )
{
    // assert(row_from>=0);
    // assert(col_from>=0);
    // assert(row_from<row_to);
    // assert(col_from<col_to);
    // assert(row_to<=smat.rows);
    // assert(col_to<=smat.cols);

    clMat /*float*/result;
    result.rows  = row_to - row_from;
    result.cols  = col_to - col_from;
    result.step  = smat.step;
    result.start = smat.start + row_from * smat.step + col_from;

    return result;
}

void
convertFromCharToFloat( __global void * self, clMat /*uint8_t*/  from, float quotient, float shift, clMat /*float*/ to )
{
    // assert(from.rows == to.rows);
    // assert(from.cols == to.cols);

    int q, w;
    for ( q=0; q<from.rows; q++ )
        for ( w=0; w<from.cols; w++ )
            ((__global float*)self)[ q * to.step + w + to.start ] =
        	quotient * ((__global uchar*)self)[ q * from.step + w + from.start ] + shift;
    return;
}

clMat
reshapeFloat( __global void * self, clMat /*float*/smat, int new_rows )
{

    // assert(smat.cols == smat.step);
    // assert( (smat.cols * smat.rows) % new_rows == 0 );

    clMat /*float*/result;
    result.rows = new_rows;
    result.cols = smat.cols * smat.rows / new_rows;
    result.step = result.cols;
    result.start = smat.start;

    return result;
} // reshapeFloat

// returns alpha*A*B + beta * C
void
gemmFloatDirDirDir( __global void * self, clMat /*float*/A, clMat /*float*/B, float alpha, clMat /*float*/C, float beta, clMat /*float*/ result )
{
    // assert(A.rows == C.rows);
    // assert(A.cols == B.rows);
    // assert(B.cols == C.cols);
    // assert(C.rows == result.rows);
    // assert(C.cols == result.cols);

    int q, w, e;
    float sum=0.;
    float c;

    if ( fabs(beta) > 0.000001 ) {
        for ( q=0; q<C.rows; q++ )
            for ( w=0; w<C.cols; w++ )
            {
                sum = 0;
                for ( e=0; e<A.cols; e++ )
                {
                    float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                    float t = sum + y;
                    c = (t - sum) - y;
                    sum = t;
                }

                ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum  + beta * ((__global float*)self)[ q * C.step + w + C.start ];
            }
    }
    else // NOT fabs(beta) > 0.000001
    {
        for ( q=0; q<C.rows; q++ )
            for ( w=0; w<C.cols; w++ )
            {
                sum = 0;
                for ( e=0; e<A.cols; e++ )
                {
                    float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                    float t = sum + y;
                    c = (t - sum) - y;
                    sum = t;
                }

                ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum;
            }
    }

    return;
} // gemmFloatDirDirDir

void
gemmFloatDirDirDirGang( __global void * self, clMat /*float*/A, clMat /*float*/B, float alpha, clMat /*float*/C, float beta, int localid, clMat /*float*/ result )
{
    // assert(A.rows == C.rows);
    // assert(A.cols == B.rows);
    // assert(B.cols == C.cols);
    // assert(C.rows == result.rows);
    // assert(C.cols == result.cols);

    int q, w, e;
    float sum=0.;
    float c;
    if ( fabs(beta) > 0.000001 ) {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum  + beta * ((__global float*)self)[ q * C.step + w + C.start ];
        }
    }
    else // NOT fabs(beta) > 0.000001
    {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ e * B.step + w + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum;
        }
    }

    return;
} // gemmFloatDirDirDirGang

void
gemmFloatDirTransDirGang( __global void * self, clMat /*float*/A, clMat /*float*/B, float alpha, clMat /*float*/C, float beta, int localid, clMat /*float*/ result )
{
    // assert(A.rows == C.rows);
    // assert(A.cols == B.cols);
    // assert(B.rows == C.cols);
    // assert(C.rows == result.rows);
    // assert(C.cols == result.cols);

    int q, w, e;
    float sum=0.;
    float c;
    if ( fabs(beta) > 0.000001 ) {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ w * B.step + e + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum  + beta * ((__global float*)self)[ q * C.step + w + C.start ];
        }
    }
    else // NOT fabs(beta) > 0.000001
    {
        int worksize = C.rows * C.cols;
        int work = 0;

        for ( work=0; work < (worksize/gangsize) + 1; work++) {
            int index = work * gangsize + localid;
            if (index>=worksize) continue;
            int q = index / C.cols;
            int w = index % C.cols;

            sum = 0;
            for ( e=0; e<A.cols; e++ )
            {
                float y = ((__global float*)self)[ q * A.step + e + A.start ] * ((__global float*)self)[ w * B.step + e + B.start ] - c;
                float t = sum + y;
                c = (t - sum) - y;
                sum = t;
            }

            ((__global float*)self)[ q * result.step + w + result.start ] = alpha * sum;
        }
    }

    return;
} // gemmFloatDirTransDirGang

void
expFloat( __global void * self, clMat /*float*/input, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
                exp(((__global float*)self)[ q * input.step + w + input.start ]);

    return;
}

void
addFloat( __global void * self, clMat /*float*/input, float val, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
        	val + ((__global float*)self)[ q * input.step + w + input.start ];

    return;
}

void
divideFloat( __global void * self, float val, clMat /*float*/input, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
        	val / ((__global float*)self)[ q * input.step + w + input.start ];

    return;
}

void
subtractFloat( __global void * self, clMat /*float*/input, float val, clMat /*float*/ output )
{
    // assert(input.rows == output.rows);
    // assert(input.cols == output.cols);

    int q, w;
    for ( q=0; q<input.rows; q++ )
        for ( w=0; w<input.cols; w++ )
            ((__global float*)self)[ q * output.step + w + output.start ] =
        	((__global float*)self)[ q * input.step + w + input.start ] - val;

    return;
}

float
GetValueFloat( __global void * self, clMat /*float*/smat, int row, int col )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);
    // assert(smat.step % smat.cols == 0);

    return ((__global float*)self)[ row * smat.step + col + smat.start ];
    // return -1231.;
}

uchar
GetValueChar( __global void * self, clMat /* uint8_t*/ smat, int row, int col )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);

    return ((__global uchar*)self)[ row * smat.step + col + smat.start ];
    // return -1231.;
}

void
SetValueFloat( __global void * self, clMat /*float*/ smat, int row, int col, float value )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);
    // assert(smat.step % smat.cols == 0);

    ((__global float*)self)[ row * smat.step + col + smat.start ] = value;
    return;
}

void
SetValueChar( __global void * self, clMat /*float*/ smat, int row, int col, uchar value )
{
    // assert(self);
    // assert(row >= 0);
    // assert(col >= 0);
    // assert(row < smat.rows);
    // assert(col < smat.cols);
    // assert(smat.step >= smat.cols);
    // assert(smat.step % smat.cols == 0);

    ((__global uchar*)self)[ row * smat.step + col + smat.start ] = value;
    return;
}

float dotProductDirDir( __global void * self, clMat /*float*/A, clMat /*float*/B )
{
  // assert( A.cols == 1 );
  // assert( B.cols == 1 );
  // assert( A.rows == B.rows );

  float result = 0.;
  float c = 0.;

  int q;
  for (q=0; q<A.rows; q++) {
      float y = ((__global float*)self)[ q * A.step + A.start ] * ((__global float*)self)[ q * B.step + B.start ] - c;
      float t = result + y;
      c = (t - result) - y;
      result = t ;
  }

  return result;
}

float dotProductTransDir( __global void * self, clMat /*float*/A, clMat /*float*/B )
{
  // assert( A.rows == 1 );
  // assert( B.cols == 1 );
  // assert( A.cols == B.rows );

  float result = 0.;
  float c = 0.;

  int q;
  for (q=0; q<A.cols; q++) {
      float y = ((__global float*)self)[ q + A.start ] * ((__global float*)self)[ q * B.step + B.start ] - c;
      float t = result + y;
      c = (t - result) - y;
      result = t ;
  }

  return result;
}

void normalizeSample( __global void * self, clMat /*uint8_t*/  image, clMat /*float*/ * result )
{
  // assert(result->cols == image.cols);
  // assert(result->rows == image.rows);

    float sampleMean = meanChar(self, image);
    float sampleMin  = minChar(self, image);
    float sampleMax  = maxChar(self, image);

     sampleMax -= sampleMean;
     sampleMin -= sampleMean;

     sampleMax = fmax( fabs(sampleMin), fabs(sampleMax));

     if (sampleMax == 0.0) sampleMax = 1.0;

     convertFromCharToFloat( self, image, 1.0/sampleMax, -(1.0/sampleMax)*sampleMean, *result );

     *result = reshapeFloat( self, *result, image.rows * image.cols );

  return;
} // normalizeSample

void
generateResponseMap(
    __global void * self,
    const clMat /*uint8_t*/  image,
    const Point2i center,
    int mapSize,
    int m_patchSize,
    // clMat /*float*/m_wIn,
    clMat /*float*/m_wOut,
    // clMat /*float*/m_U,
    clMat wIn,
    clMat bIn,

    // temporary variables
    clVector /*cMat float*/ patches,
    clVector /*cMat float*/ xOuts,
    clVector /*cMat float*/ es,
    // result
    clMat /*float*/ result
    )
{

  // assert( result.rows == 2 * mapSize + 1 );
  // assert( result.cols == 2 * mapSize + 1 );

  //  assert(wIn.rows == wIn_A.rows);
  //  assert(wIn.cols == m_U.rows);

    clMat /*float*/wOut_tmp = GetBlockFloat( self, m_wOut, 0, m_wOut.rows, 0, m_wOut.cols - 1 );

    {
        int localid = 0;
        for ( localid=0; localid<gangsize; localid++ ) {

            int work = 0;
            int worksize = (2 * mapSize + 1) * (2 * mapSize + 1);

            float bOut = GetValueFloat( self, m_wOut, 0, m_wOut.cols - 1 );

            for ( work=0; work < (worksize / gangsize) + 1; work++ )
            {
              int index = work * gangsize + localid;
              if (index>=worksize) continue;
              int ncy = index / (2 * mapSize + 1);
              int cy  = center.y - mapSize + ncy;
              int ncx = index % (2 * mapSize + 1);
              int cx  = center.x - mapSize + ncx;

              clMat /*uint8_t*/   imagePatch = GetBlockChar( self, image, cy - m_patchSize, cy + m_patchSize + 1, cx - m_patchSize, cx + m_patchSize + 1 );

              clMat patch = GetMatFromVector( self, patches, localid );

              normalizeSample( self, imagePatch, &patch );

              clMat xOut = GetMatFromVector( self, xOuts, localid );

              gemmFloatDirDirDir( self, wIn, patch, -1.0, bIn, -1.0, xOut );

              clMat e = GetMatFromVector( self, es, localid );

              expFloat( self, xOut, e );

              addFloat( self, e, 1.0, xOut );
              divideFloat( self, 2.0, xOut, e);
              addFloat( self, e, -1.0, xOut);

              SetValueFloat( self, result, ncy, ncx, 1./( 1. + exp(- dotProductTransDir( self, wOut_tmp, xOut) - bOut ) ) );
            } // for gangsize
        } // for localid
    } // localid block

//  // assert(false);

  // freeMatFloat( self, allocator, &wIn);
  // end of classic impl
    return;
} // generateResponseMap

int
cvRound( float value )
{
    return (int)(value + (value >= 0 ? 0.5 : -0.5));
} // cvRound

__kernel void
calculateMaps(
    // global variables
    __global void * self,
    __constant int memory_segments[], // representing the start of the gang's memory segment
    int m_visibleLandmarks_size,
    int m_mapSize,

    __constant calcpackage packages[],

    // local buffer
    __local void * buffer
    )
{
    if (get_global_id(0)>0) return;

    int q;
    for (q=0; q<2 /*m_visibleLandmarks_size*/; q++ )
    {
        int idx = q;

        Point2i center;

        float shape_x;
        float shape_y;
        shape_x = GetValueFloat( self + memory_segments[q], packages[q].input.shape, 2*idx, 0 );
        shape_y = GetValueFloat( self + memory_segments[q], packages[q].input.shape, 2*idx+1, 0 );

        center.x = cvRound(shape_x);
        center.y = cvRound(shape_y);

        generateResponseMap(
            self + memory_segments[idx],
            packages[idx].input.alignedImage,
            center,
            m_mapSize,
            packages[idx].input.m_patchSize,
            packages[idx].input.m_wOut,
            packages[idx].input.wIn,
            packages[idx].input.bIn,

            // temporary
            packages[idx].tmp.patches,
            packages[idx].tmp.xOuts,
            packages[idx].tmp.es,
            // result
            packages[idx].output.responseMap );

        return;

    } // for q in visiblelandmarks_size

    return;
} // calculateMaps

// LuM end of file


simplify local buffer structure

simplify local buffer structure

#include "ocl_platform.cl"

// number of banks in local memory
// LDS_BANKS

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns

 **/

// two vertical neighbours: pointer diff:
#define TOTAL_BUFFER_SIZE     1024          // WIN_SIZE_X * WIN_SIZE_Y
#define TOTAL_BUFFER_SIZE_X2  2048
#define TOTAL_BUFFER_SIZE_X3  3072

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

// read pixel from local buffer
int4 readPixel( LOCAL int*  restrict  src) {
	return (int4)(*src, *(src+TOTAL_BUFFER_SIZE),  *(src+TOTAL_BUFFER_SIZE_X2),  *(src+TOTAL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(int4 pix, LOCAL int*  restrict  dest) {
	*dest = pix.x;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.y;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.z;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.w;
}

void writeColumnToOutput(LOCAL int* restrict currentScratch, __write_only image2d_t odata, int firstY, int height, int halfHeight){
	// write points to destination
	int2 posOut = {getGlobalId(0), firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even
	    if (posOut.y >= halfHeight)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		currentScratch += WIN_SIZE_X ;
		posOut.y+= halfHeight + 1;

		// odd
		if (posOut.y >= height)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		currentScratch += WIN_SIZE_X;
		posOut.y -= halfHeight;
	}
}

// offset when transforming columns
inline int getScratchColumnOffset(){
   return (getLocalId(0) >> 1) + (getLocalId(0)&1) * (WIN_SIZE_X>>1);
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	if (getGlobalId(0) >= width)
	    return;

	const int halfWinSizeX = WIN_SIZE_X >> 1;

	int boundaryShift = 0;
	if (getGlobalId(0) == 1)				// left odd boundary column
		boundaryShift =  -1;
	else if (getGlobalId(0) == 2)           // left even boundary column
		boundaryShift = -2;
	else if (getGlobalId(0) == width - 2)   // right even boundary column
	    boundaryShift =  1;

    const unsigned int halfHeight = height >> 1;
	LOCAL int scratch[TOTAL_BUFFER_SIZE << 2];
	float yDelta = 1.0/height;
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)

	// read -1 point
	const float2 posIn = (float2)(getGlobalId(0), firstY - 1) /  (float2)(width, height);
	int4 minusOne = read_imagei(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	int4 current = read_imagei(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= ( read_imagei(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) >> 1;

	for (int i = 0; i < steps; ++i) {

		// 1. read from source image, transform columns, and store in local scratch
		LOCAL int* currentScratch = scratch + getScratchColumnOffset();
		for (int j = 0; j < WIN_SIZE_Y>>1; ++j) {

			///////////////////////////////////////////////////////////////////////////////////////////
			// fetch next two pixels, then transform and write current (odd) and last (even)

			// read current plus one (odd) point
			posIn.y += yDelta;
			if (posIn.y >= 1)
				break;
			int4 currentPlusOne = read_imagei(idata, sampler, posIn);

			// read current plus two (even) point
			posIn.y += yDelta;
			int4 currentPlusTwo = read_imagei(idata, sampler, posIn);

			// transform current plus one (odd) point
			currentPlusOne -= (current + currentPlusTwo) >> 1;

			// transform current (even) point
			current += (minusOne + currentPlusOne + (int4)(2,2,2,2)) >> 2;

			//write current (even)
			writePixel(current, currentScratch);

			//write odd
			currentScratch += WIN_SIZE_X;
			writePixel(currentPlusOne, currentScratch);

			//advance scratch pointer
			currentScratch += WIN_SIZE_X;

			//update registers
			minusOne = currentPlusOne;
			current = currentPlusTwo;
		}

		//4. transform rows

		//5. write local buffer to destination image
		writeColumnToOutput(scratch + getScratchColumnOffset(), odata, firstY, height, halfHeight);

		// move to next step
		firstY += WIN_SIZE_Y;
	}
}

////////////////////////////////////////////
/*
/// Returns index ranging from 0 to num work items, such that first half
/// of the work items get even indices and others get odd indices. Each work item
/// gets a different index.
/// Example: (for WIN_SiZE_X == 8)   getLocalId(0):   0  1  2  3  4  5  6  7
///									 parityIdx:		  0  2  4  6  1  3  5  7

/// @return parity-separated index of work item
inline int parityIdx() {
	return (getLocalId(0) << 1) - (WIN_SIZE_X - 1) * (getLocalId(0) / (WIN_SIZE_X >> 1));
}

// two vertical neighbours: pointer diff:
#define VERTICAL_STRIDE_EVEN 66			// 2*BOUNDARY_X + (WIN_SIZE_X / 2)

#define VERTICAL_STRIDE_ODD  65          //  BOUNDARY_X  + (WIN_SIZE_X / 2)

// size of one of two buffers (odd or even)
#define BUFFER_SIZE_EVEN     528          // VERTICAL_STRIDE_EVEN * WIN_SIZE_Y
#define BUFFER_SIZE_ODD     528          // VERTICAL_STRIDE_ODD * WIN_SIZE_Y

/// padding between even and odd buffers: used to reduce bank conflicts
#define PADDING			32           // LDS_BANKS - ((BUFFER_SIZE + LDS_BANKS / 2) % LDS_BANKS);

// offset of the odd columns buffer from the beginning of data buffer
#define ODD_OFFSET 560              // BUFFER_SIZE + PADDING

// size of buffer for both even and odd columns
#define TOTAL_BUFFER_SIZE  1088     // 2 * BUFFER_SIZE + PADDING;
#define TOTAL_BUFFER_SIZE_X2  2176
#define TOTAL_BUFFER_SIZE_X3  3264
*/
////////////////////////////////////////////

#include "ocl_platform.cl"

// number of banks in local memory
// LDS_BANKS

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

//  scratch buffer (in local memory of GPU) where block of input image is stored.
//  All operations expect WIN_SIZE_X threads.

 /**

Layout for scratch buffer

Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
All even columns are stored first, then all odd columns.

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns

 **/

 #define BOUNDARY_X_LEFT 1
 #define BOUNDARY_X_MIDDLE 2

// two vertical neighbours: pointer diff:
#define VERTICAL_STRIDE 131			// BOUNDARY_X_LEFT + BOUNDARY_X_MIDDLE + WIN_SIZE_X

#define TOTAL_BUFFER_SIZE     1048          // VERTICAL_STRIDE * WIN_SIZE_Y
#define TOTAL_BUFFER_SIZE_X2  2096
#define TOTAL_BUFFER_SIZE_X3  3144

///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

// read pixel from local buffer
int4 readPixel( LOCAL int*  restrict  src) {
	return (int4)(*src, *(src+TOTAL_BUFFER_SIZE),  *(src+TOTAL_BUFFER_SIZE_X2),  *(src+TOTAL_BUFFER_SIZE_X3)) ;
}

//write pixel to column
inline void writePixel(int4 pix, LOCAL int*  restrict  dest) {
	*dest = pix.x;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.y;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.z;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.w;
}

// write pixel to both designated column and mirrored boundary column
inline void writePixelAndBoundary(int4 pix, LOCAL int*  restrict  dest, int boundaryShift) {
	writePixel(pix, dest);
	writePixel(pix, dest+boundaryShift);
}

void writeColumnToOutput(LOCAL int* restrict currentScratch, __write_only image2d_t odata, int firstY, int height, int halfHeight){
	// write points to destination
	int2 posOut = {getGlobalId(0), firstY>>1};
	for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	    // even
	    if (posOut.y >= halfHeight)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE ;
		posOut.y+= halfHeight + 1;

		// odd
		if (posOut.y >= height)
			break;

		write_imagei(odata, posOut,readPixel(currentScratch));

		currentScratch += VERTICAL_STRIDE;
		posOut.y -= halfHeight;
	}
}

// offset when transforming columns
inline int getScratchColumnOffset(){
   return BOUNDARY_X_LEFT + (getLocalId(0) >> 1) + ( (getLocalId(0)&1) * (BOUNDARY_X_MIDDLE + (WIN_SIZE_X>>1))  );
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	if (getGlobalId(0) >= width)
	    return;

	const int halfWinSizeX = WIN_SIZE_X >> 1;

    //check if this column needs to handle a boundary column
	int boundaryShift = 0;
	if (getGlobalId(0) == 1)				// left odd boundary column
		boundaryShift =  -1;
	else if (getGlobalId(0) == 2)           // left even boundary column
		boundaryShift = -2;
	else if (getGlobalId(0) == width - 2)   // right even boundary column
	    boundaryShift =  1;

    const unsigned int halfHeight = height >> 1;
	LOCAL int scratch[TOTAL_BUFFER_SIZE << 2];
	float yDelta = 1.0/height;
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	//0. Initialize: fetch first pixel (and 2 top boundary pixels)

	// read -1 point
	const float2 posIn = (float2)(getGlobalId(0), firstY - 1) /  (float2)(width, height);
	int4 minusOne = read_imagei(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	int4 current = read_imagei(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= ( read_imagei(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) >> 1;

	{
		for (int i = 0; i < steps; ++i) {

		   // 1. read from source image, transform columns, and store in local scratch
		   LOCAL int* currentScratch = scratch + getScratchColumnOffset();
		   for (int j = 0; j < WIN_SIZE_Y>>1; ++j) {

				///////////////////////////////////////////////////////////////////////////////////////////
				// fetch next two pixels, then transform and write current (odd) and last (even)

				// read current plus one (odd) point
				posIn.y += yDelta;
				if (posIn.y >= 1)
				   break;
				int4 currentPlusOne = read_imagei(idata, sampler, posIn);

				// read current plus two (even) point
				posIn.y += yDelta;
				int4 currentPlusTwo = read_imagei(idata, sampler, posIn);

				// transform current plus one (odd) point
				currentPlusOne -= (current + currentPlusTwo) >> 1;

				// transform current (even) point
				current += (minusOne + currentPlusOne + (int4)(2,2,2,2)) >> 2;

				//write current (even)
				writePixel(current, currentScratch);

				//write odd
				currentScratch += VERTICAL_STRIDE;
				writePixel(currentPlusOne, currentScratch);

				//advance scratch pointer
				currentScratch += VERTICAL_STRIDE;

				//update registers
				minusOne = currentPlusOne;
				current = currentPlusTwo;
			}

			//4. transform rows

			// transform odd (two rows at a time)
			currentScratch = scratch + getScratchColumnOffset() + (1+ getLocalId(0) % halfWinSizeX ) * VERTICAL_STRIDE;
			for (int j = 1; j < WIN_SIZE_Y; j+=2) {

				currentScratch += VERTICAL_STRIDE;
			}

			// transform even (two rows at a time
			for (int j = 0; j < WIN_SIZE_Y; j+=2) {

			}

			//3. write local buffer to destination image
			writeColumnToOutput(scratch + getScratchColumnOffset(), odata, firstY, height, halfHeight);

			// move to next step
			firstY += WIN_SIZE_Y;
		}
	}
}

////////////////////////////////////////////
/*
/// Returns index ranging from 0 to num work items, such that first half
/// of the work items get even indices and others get odd indices. Each work item
/// gets a different index.
/// Example: (for WIN_SiZE_X == 8)   getLocalId(0):   0  1  2  3  4  5  6  7
///									 parityIdx:		  0  2  4  6  1  3  5  7

/// @return parity-separated index of work item
inline int parityIdx() {
	return (getLocalId(0) << 1) - (WIN_SIZE_X - 1) * (getLocalId(0) / (WIN_SIZE_X >> 1));
}

// two vertical neighbours: pointer diff:
#define VERTICAL_STRIDE_EVEN 66			// 2*BOUNDARY_X + (WIN_SIZE_X / 2)

#define VERTICAL_STRIDE_ODD  65          //  BOUNDARY_X  + (WIN_SIZE_X / 2)

// size of one of two buffers (odd or even)
#define BUFFER_SIZE_EVEN     528          // VERTICAL_STRIDE_EVEN * WIN_SIZE_Y
#define BUFFER_SIZE_ODD     528          // VERTICAL_STRIDE_ODD * WIN_SIZE_Y

/// padding between even and odd buffers: used to reduce bank conflicts
#define PADDING			32           // LDS_BANKS - ((BUFFER_SIZE + LDS_BANKS / 2) % LDS_BANKS);

// offset of the odd columns buffer from the beginning of data buffer
#define ODD_OFFSET 560              // BUFFER_SIZE + PADDING

// size of buffer for both even and odd columns
#define TOTAL_BUFFER_SIZE  1088     // 2 * BUFFER_SIZE + PADDING;
#define TOTAL_BUFFER_SIZE_X2  2176
#define TOTAL_BUFFER_SIZE_X3  3264
*/
////////////////////////////////////////////


Remove some old comments

Remove some old comments

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, size_t subblock, size_t hash_lane,
       size_t bw, size_t bh, size_t dx, size_t dy, size_t offset)
{
    size_t index[4];
    for (size_t i = 0; i < 4; i++) {
        size_t bpos = (hash_lane + i * offset) % 4;
        size_t x = (subblock * dy + i * dx) * bw + bpos % bw;
        size_t y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, size_t thread)
{
    size_t subblock = (thread >> 2) & 0x7;
    size_t hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif

__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            size_t addr_index_x = addr_index % 16;
            size_t addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
#endif
            }

            ++mem_curr;
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
#endif
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

/*
 * BLAMKA ROUND:
 *  G is sequential (atomic)
 *  4 G's in parallel (2 passes)
 *
 * FILL BLOCK:
 *  1. some moving and xoring...
 *  2. 8 x BLAMKA ROUND in parallel
 *  3. 8 x BLAMKA ROUND in parallel (on diff. distributed data)
 *  4. some moving and xoring again...
 *
 * Conclusion:
 *  BLAMKA ROUND == 4 threads
 *  FILL BLOCK == 32 threads
 */

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, size_t subblock, size_t hash_lane,
       size_t bw, size_t bh, size_t dx, size_t dy, size_t offset)
{
    size_t index[4];
    for (size_t i = 0; i < 4; i++) {
        size_t bpos = (hash_lane + i * offset) % 4;
        size_t x = (subblock * dy + i * dx) * bw + bpos % bw;
        size_t y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, size_t thread)
{
    /* |  x   x   x  |   x   x   | */
    /* |  subblock   | hash_lane | */
    size_t subblock = (thread >> 2) & 0x7;
    size_t hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    size_t thread)
{
    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        next_block->lo[pos] ^= prev_block->lo[pos];
        next_block->hi[pos] ^= prev_block->hi[pos];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    size_t thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (size_t i = 1; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

/* 1 fill_segment call == throughput of 1 fill_block call */
/* 1 fill_block call == 64 threads */
/* CONCLUSION: we need 64 threads per fill_segment (i. e. per lane) */

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif
/*
 * how to map thread ids to position/sub-tasks?
 *
 * GLOBAL ID == (job, lane, thread)
 * GROUP SIZE == (1, lanes, 32)
 */
__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    size_t thread = get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
        size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (size_t index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            size_t addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            size_t addr_index_x = addr_index % 16;
            size_t addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                    size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (size_t i = 0; i < QWORDS_PER_THREAD; i++) {
                size_t pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
#endif
            }

            ++mem_curr;
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
#endif
        mem_curr = mem_lane;
    }
}