stubborn inlining of everything

stubborn inlining of everything

int blake2b( uchar *out, uchar *in );

// The kernel that grinds nonces until it finds a hash below the target
__kernel void nonceGrind(__global uchar *headerIn, __global uchar *hashOut, __global uchar *targ, __global uchar *nonceOut) {
	private uchar header[256];
	private uchar headerHash[32];
	private uchar target[32];
	headerHash[0] = 255;

	int i;
#pragma unroll
	for (i = 0; i < 32; i++) {
		target[i] = targ[i];
		header[i] = headerIn[i];
	}
#pragma unroll
	for (i = 32; i < 80; i++) {
		header[i] = headerIn[i];
	}

	// Set nonce
	private int id = get_global_id(0);
	// Support global work sizes of up to 256^4 - 1
	header[32] = id / (256 * 256 * 256);
	header[33] = id / (256 * 256);
	header[34] = id / 256;
	header[35] = id % 256;

	// Hash the header
	blake2b(headerHash, header);

	// Compare header to target
	int z = 0;
	while (target[z] == headerHash[z]) {
		z++;
	}
	if (headerHash[z] < target[z]) {
		// Transfer the output to global space.
		for (i = 0; i < 8; i++) {
			nonceOut[i] = header[i + 32];
		}
		for (i = 0; i < 32; i++) {
			hashOut[i] = headerHash[i];
		}
		return;
	}
}

// Implementations of clmemset and memcopy
void *clmemset( __private void *s, __private int c, __private size_t n) {
	uchar *p = s;
	while(n--) {
		*p++ = (uchar)c;
	}
	return s;
}

void clmemcpy( __private void *dest, __private const void *src, __private size_t num) {
	int i = 0 ;
	char *dest8 = (char*)dest;
	char *src8 = (char*)src;
#pragma unroll
	for (int i = 0; i < num; i++) {
		dest8[i] = src8[i];
	}
}

#if defined(_MSC_VER)
#define ALIGN(x) __declspec(align(x))
#else
#define ALIGN(x) __attribute__((aligned(x)))
#endif

  typedef struct __blake2b_state
  {
	ulong h[8];
	ulong t[2];
	ulong f[2];
	uchar  buf[256];
	uchar  last_node;
  } blake2b_state;

static inline ulong load64( __private const void *src )
{
  return *( ulong * )( src );
}

static inline void store64( __private void *dst, __private ulong w )
{
  *( ulong * )( dst ) = w;
}

static inline ulong rotr64( __private const ulong w, __private const unsigned c )
{
  return ( w >> c ) | ( w << ( 64 - c ) );
}

// blake2b-ref.c
__constant ulong blake2b_IV[8] =
{
	0x6a09e667f3bcc908, 0xbb67ae8584caa73b,
	0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1,
	0x510e527fade682d1, 0x9b05688c2b3e6c1f,
	0x1f83d9abfb41bd6b, 0x5be0cd19137e2179
};

__constant uchar blake2b_sigma[12][16] =
{
	{	0,	1,	2,	3,	4,	5,	6,	7,	8,	9, 10, 11, 12, 13, 14, 15 } ,
	{ 14, 10,	4,	8,	9, 15, 13,	6,	1, 12,	0,	2, 11,	7,	5,	3 } ,
	{ 11,	8, 12,	0,	5,	2, 15, 13, 10, 14,	3,	6,	7,	1,	9,	4 } ,
	{	7,	9,	3,	1, 13, 12, 11, 14,	2,	6,	5, 10,	4,	0, 15,	8 } ,
	{	9,	0,	5,	7,	2,	4, 10, 15, 14,	1, 11, 12,	6,	8,	3, 13 } ,
	{	2, 12,	6, 10,	0, 11,	8,	3,	4, 13,	7,	5, 15, 14,	1,	9 } ,
	{ 12,	5,	1, 15, 14, 13,	4, 10,	0,	7,	6,	3,	9,	2,	8, 11 } ,
	{ 13, 11,	7, 14, 12,	1,	3,	9,	5,	0, 15,	4,	8,	6,	2, 10 } ,
	{	6, 15, 14,	9, 11,	3,	0,	8, 12,	2, 13,	7,	1,	4, 10,	5 } ,
	{ 10,	2,	8,	4,	7,	6,	1,	5, 15, 11,	9, 14,	3, 12, 13 , 0 } ,
	{	0,	1,	2,	3,	4,	5,	6,	7,	8,	9, 10, 11, 12, 13, 14, 15 } ,
	{ 14, 10,	4,	8,	9, 15, 13,	6,	1, 12,	0,	2, 11,	7,	5,	3 }
};

static int blake2b_compress( __private blake2b_state *S, __private const uchar block[128] )
{
	return 0;
}

int blake2b( __private uchar *out, __private uchar *in )
{
	// Initialize a state.
	private blake2b_state S[1];
	clmemset( S, 0, sizeof( blake2b_state ) );
	for( int i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i];
	S->h[0] ^= 0x0000000001010020UL;

	S->t[0] += 80;
	S->f[0] = ~((ulong)0);
	clmemset( in + 80, 0, 2 * 128 - 80 ); // Padding

	ulong m[16];
	ulong v[16];
	int i;

	for( i = 0; i < 16; ++i )
		m[i] = load64( in + i * sizeof( m[i] ) );

	for( i = 0; i < 8; ++i )
		v[i] = S->h[i];

	v[ 8] = blake2b_IV[0];
	v[ 9] = blake2b_IV[1];
	v[10] = blake2b_IV[2];
	v[11] = blake2b_IV[3];
	v[12] = S->t[0] ^ blake2b_IV[4];
	v[13] = S->t[1] ^ blake2b_IV[5];
	v[14] = S->f[0] ^ blake2b_IV[6];
	v[15] = S->f[1] ^ blake2b_IV[7];
#define G(r,i,a,b,c,d) \
	do { \
		a = a + b + m[blake2b_sigma[r][2*i+0]]; \
		d = rotr64(d ^ a, 32); \
		c = c + d; \
		b = rotr64(b ^ c, 24); \
		a = a + b + m[blake2b_sigma[r][2*i+1]]; \
		d = rotr64(d ^ a, 16); \
		c = c + d; \
		b = rotr64(b ^ c, 63); \
	} while(0)
#define ROUND(r)	\
	do { \
		G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \
		G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \
		G(r,2,v[ 2],v[ 6],v[10],v[14]); \
		G(r,3,v[ 3],v[ 7],v[11],v[15]); \
		G(r,4,v[ 0],v[ 5],v[10],v[15]); \
		G(r,5,v[ 1],v[ 6],v[11],v[12]); \
		G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \
		G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \
	} while(0)
	ROUND( 0 );
	ROUND( 1 );
	ROUND( 2 );
	ROUND( 3 );
	ROUND( 4 );
	ROUND( 5 );
	ROUND( 6 );
	ROUND( 7 );
	ROUND( 8 );
	ROUND( 9 );
	ROUND( 10 );
	ROUND( 11 );

	for( i = 0; i < 8; ++i )
		S->h[i] = S->h[i] ^ v[i] ^ v[i + 8];

#undef G
#undef ROUND

	uchar buffer[64];
	for( int i = 0; i < 8; ++i ) // Output full hash to temp buffer
		store64( buffer + sizeof( S->h[i] ) * i, S->h[i] );

	clmemcpy( out, buffer, 32 );

	return 0;
}

int blake2b( uchar *out, uchar *in );

// The kernel that grinds nonces until it finds a hash below the target
__kernel void nonceGrind(__global uchar *headerIn, __global uchar *hashOut, __global uchar *targ, __global uchar *nonceOut) {
	private uchar header[80];
	private uchar headerHash[32];
	private uchar target[32];
	headerHash[0] = 255;

	int i;
#pragma unroll
	for (i = 0; i < 32; i++) {
		target[i] = targ[i];
		header[i] = headerIn[i];
	}
#pragma unroll
	for (i = 32; i < 80; i++) {
		header[i] = headerIn[i];
	}

	// Set nonce
	private int id = get_global_id(0);
	// Support global work sizes of up to 256^4 - 1
	header[32] = id / (256 * 256 * 256);
	header[33] = id / (256 * 256);
	header[34] = id / 256;
	header[35] = id % 256;

	// Hash the header
	blake2b(headerHash, header);

	// Compare header to target
	int z = 0;
	while (target[z] == headerHash[z]) {
		z++;
	}
	if (headerHash[z] < target[z]) {
		// Transfer the output to global space.
		for (i = 0; i < 8; i++) {
			nonceOut[i] = header[i + 32];
		}
		for (i = 0; i < 32; i++) {
			hashOut[i] = headerHash[i];
		}
		return;
	}
}

// Implementations of clmemset and memcopy
void *clmemset( __private void *s, __private int c, __private size_t n) {
	uchar *p = s;
	while(n--) {
		*p++ = (uchar)c;
	}
	return s;
}

void clmemcpy( __private void *dest, __private const void *src, __private size_t num) {
	int i = 0 ;
	char *dest8 = (char*)dest;
	char *src8 = (char*)src;
#pragma unroll
	for (int i = 0; i < num; i++) {
		dest8[i] = src8[i];
	}
}

#if defined(_MSC_VER)
#define ALIGN(x) __declspec(align(x))
#else
#define ALIGN(x) __attribute__((aligned(x)))
#endif

  typedef struct __blake2b_state
  {
	ulong h[8];
	ulong t[2];
	ulong f[2];
	uchar  buf[256];
	size_t   buflen;
	uchar  last_node;
  } blake2b_state;

  // Streaming API
  int blake2b_update( __private blake2b_state *S, __private const uchar *in, __private ulong inlen );
  int blake2b_final( __private blake2b_state *S, __private uchar *out );

static inline ulong load64( __private const void *src )
{
  return *( ulong * )( src );
}

static inline void store64( __private void *dst, __private ulong w )
{
  *( ulong * )( dst ) = w;
}

static inline ulong rotr64( __private const ulong w, __private const unsigned c )
{
  return ( w >> c ) | ( w << ( 64 - c ) );
}

// blake2b-ref.c
__constant ulong blake2b_IV[8] =
{
	0x6a09e667f3bcc908, 0xbb67ae8584caa73b,
	0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1,
	0x510e527fade682d1, 0x9b05688c2b3e6c1f,
	0x1f83d9abfb41bd6b, 0x5be0cd19137e2179
};

__constant uchar blake2b_sigma[12][16] =
{
	{	0,	1,	2,	3,	4,	5,	6,	7,	8,	9, 10, 11, 12, 13, 14, 15 } ,
	{ 14, 10,	4,	8,	9, 15, 13,	6,	1, 12,	0,	2, 11,	7,	5,	3 } ,
	{ 11,	8, 12,	0,	5,	2, 15, 13, 10, 14,	3,	6,	7,	1,	9,	4 } ,
	{	7,	9,	3,	1, 13, 12, 11, 14,	2,	6,	5, 10,	4,	0, 15,	8 } ,
	{	9,	0,	5,	7,	2,	4, 10, 15, 14,	1, 11, 12,	6,	8,	3, 13 } ,
	{	2, 12,	6, 10,	0, 11,	8,	3,	4, 13,	7,	5, 15, 14,	1,	9 } ,
	{ 12,	5,	1, 15, 14, 13,	4, 10,	0,	7,	6,	3,	9,	2,	8, 11 } ,
	{ 13, 11,	7, 14, 12,	1,	3,	9,	5,	0, 15,	4,	8,	6,	2, 10 } ,
	{	6, 15, 14,	9, 11,	3,	0,	8, 12,	2, 13,	7,	1,	4, 10,	5 } ,
	{ 10,	2,	8,	4,	7,	6,	1,	5, 15, 11,	9, 14,	3, 12, 13 , 0 } ,
	{	0,	1,	2,	3,	4,	5,	6,	7,	8,	9, 10, 11, 12, 13, 14, 15 } ,
	{ 14, 10,	4,	8,	9, 15, 13,	6,	1, 12,	0,	2, 11,	7,	5,	3 }
};

static int blake2b_compress( __private blake2b_state *S, __private const uchar block[128] )
{
	ulong m[16];
	ulong v[16];
	int i;

	for( i = 0; i < 16; ++i )
		m[i] = load64( block + i * sizeof( m[i] ) );

	for( i = 0; i < 8; ++i )
		v[i] = S->h[i];

	v[ 8] = blake2b_IV[0];
	v[ 9] = blake2b_IV[1];
	v[10] = blake2b_IV[2];
	v[11] = blake2b_IV[3];
	v[12] = S->t[0] ^ blake2b_IV[4];
	v[13] = S->t[1] ^ blake2b_IV[5];
	v[14] = S->f[0] ^ blake2b_IV[6];
	v[15] = S->f[1] ^ blake2b_IV[7];
#define G(r,i,a,b,c,d) \
	do { \
		a = a + b + m[blake2b_sigma[r][2*i+0]]; \
		d = rotr64(d ^ a, 32); \
		c = c + d; \
		b = rotr64(b ^ c, 24); \
		a = a + b + m[blake2b_sigma[r][2*i+1]]; \
		d = rotr64(d ^ a, 16); \
		c = c + d; \
		b = rotr64(b ^ c, 63); \
	} while(0)
#define ROUND(r)	\
	do { \
		G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \
		G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \
		G(r,2,v[ 2],v[ 6],v[10],v[14]); \
		G(r,3,v[ 3],v[ 7],v[11],v[15]); \
		G(r,4,v[ 0],v[ 5],v[10],v[15]); \
		G(r,5,v[ 1],v[ 6],v[11],v[12]); \
		G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \
		G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \
	} while(0)
	ROUND( 0 );
	ROUND( 1 );
	ROUND( 2 );
	ROUND( 3 );
	ROUND( 4 );
	ROUND( 5 );
	ROUND( 6 );
	ROUND( 7 );
	ROUND( 8 );
	ROUND( 9 );
	ROUND( 10 );
	ROUND( 11 );

	for( i = 0; i < 8; ++i )
		S->h[i] = S->h[i] ^ v[i] ^ v[i + 8];

#undef G
#undef ROUND
	return 0;
}

// inlen, at least, should be ulong. Others can be size_t.
int blake2b( __private uchar *out, __private uchar *in )
{
	private blake2b_state S[1];

	clmemset( S, 0, sizeof( blake2b_state ) );
	for( int i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i];
	S->h[0] ^= 0x0000000001010020UL;

	ulong inlen = 80;
	size_t left = S->buflen;
	size_t fill = 2 * 128 - left;

	if( inlen > fill )
	{
		clmemcpy( S->buf + left, in, fill ); // Fill buffer
		S->buflen += fill;
		blake2b_compress( S, S->buf ); // Compress
		clmemcpy( S->buf, S->buf + 128, 128 ); // Shift buffer left
		S->buflen -= 128;
	}
	else
	{
		clmemcpy( S->buf + left, in, inlen );
		S->buflen += inlen; // Be lazy, do not compress
	}

	S->t[0] += S->buflen;
	S->f[0] = ~((ulong)0);
	clmemset( S->buf + S->buflen, 0, 2 * 128 - S->buflen ); // Padding
	blake2b_compress( S, S->buf );

	uchar buffer[64];
	for( int i = 0; i < 8; ++i ) // Output full hash to temp buffer
		store64( buffer + sizeof( S->h[i] ) * i, S->h[i] );

	clmemcpy( out, buffer, 32 );

	return 0;
}


Added include of pv_uint4.h

Added include of pv_uint4.h

git-svn-id: 602a26e73ae9f2a14e2bb95676f2e137fcc39919@3118 7c0444ca-5550-0410-9008-faecdbe78eb5

#include "Retina_params.h"
#include "cl_random.cl"

#include "../arch/opencl/pv_uint4.h"

#ifndef PI
#  define PI 3.1415926535897932
#endif

#ifndef __CL_PLATFORM_H

#  include <math.h>
#  define EXP expf
#  define FMOD fmodf
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_LOCAL

#else

#  define EXP exp
#  define FMOD __fmodf
#  define CL_KERNEL     __kernel
#  define CL_MEM_GLOBAL __global
#  define CL_MEM_LOCAL  __local

// This stuff needs to be moved to (or obtained from) separate file
// perhaps conversions.h

static inline int kxPos(int k, int nx, int ny, int nf)
{
   return (k/nf) % nx;
}

static inline int kyPos(int k, int nx, int ny, int nf)
{
   return k / (nx*nf);
}

static inline int kIndex(int kx, int ky, int kf, int nx, int ny, int nf)
{
   return kf + (kx + (ky * nx)) * nf;
}

static inline int featureIndex(int k, int nx, int ny, int nf)
{
   return k % nf;
}

static inline int kIndexExtended(int k, int nx, int ny, int nf, int nb)
{
   const int kx_ex = nb + kxPos(k, nx, ny, nf);
   const int ky_ex = nb + kyPos(k, nx, ny, nf);
   const int kf = featureIndex(k, nx, ny, nf);
   return kIndex(kx_ex, ky_ex, kf, nx + 2*nb, ny + 2*nb, nf);
}

#endif /* __CL_PLATFORM_H */

/*
 * Spiking method for Retina
 * Returns 1 if an event should occur, 0 otherwise. This is a stochastic model.
 *
 * REMARKS:
 *      - During ABS_REFACTORY_PERIOD a neuron does not spike
 *      - The neurons that correspond to stimuli (on Image pixels)
 *        spike with probability probStim.
 *      - The neurons that correspond to background image pixels
 *        spike with probability probBase.
 *      - After ABS_REFACTORY_PERIOD the spiking probability
 *        grows exponentially to probBase and probStim respectively.
 *      - The burst of the retina is periodic with period T set by
 *        T = 1000/burstFreq in miliseconds
 *      - When the time t is such that mT < t < mT + burstDuration, where m is
 *        an integer, the burstStatus is set to 1.
 *      - The burstStatus is also determined by the condition that
 *        beginStim < t < endStim. These parameters are set in the input
 *        params file params.stdp
 *      - sinAmp modulates the spiking probability only when burstDuration <= 0
 *        or burstFreq = 0
 *      - probSpike is set to probBase for all neurons.
 *      - for neurons exposed to Image on pixels, probSpike increases with probStim.
 *      - When the probability is negative, the neuron does not spike.
 *
 * NOTES:
 *      - time is measured in milliseconds.
 *
 */
static inline
int spike(float time, float dt,
          float prev, float stimFactor, uint4 * rnd_state, const Retina_params * params)
{
   float probSpike;
   float burstStatus = 1;
   float sinAmp = 1.0;

   // input parameters
   //
   float probBase  = params->probBase;
   float probStim  = params->probStim * stimFactor;

   // see if neuron is in a refactory period
   //
   if ((time - prev) < params->abs_refactory_period) {
      return 0;
   }
   else {
      float delta = time - prev - params->abs_refactory_period;
      float refact = 1.0f - EXP(-delta/params->refactory_period);
      refact = (refact < 0) ? 0 : refact;
      probBase *= refact;
      probStim *= refact;
   }

   if (params->burstDuration <= 0 || params->burstFreq == 0) {
      sinAmp = cos( 2*PI*time * params->burstFreq / 1000. );
   }
   else {
      burstStatus = FMOD(time, 1000. / params->burstFreq);
      burstStatus = burstStatus <= params->burstDuration;
   }

   burstStatus *= (int) ( (time >= params->beginStim) && (time < params->endStim) );
   probSpike = probBase;

   if ((int)burstStatus) {
      probSpike += probStim * sinAmp;  // negative prob is OK
   }

   int spike_flag = (cl_random_prob(*rnd_state) < probSpike);
   *rnd_state = cl_random_state(*rnd_state);
   return spike_flag;
}

//
// update the state of a retinal layer (spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void update_state (
    const float time,
    const float dt,
    const Retina_params * params,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL uint4 * rnd,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * activity,
    CL_MEM_GLOBAL float * prevTime)
{
   int k;
#ifndef PV_USE_OPENCL
for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // load local variables from global memory
   //
   uint4 l_rnd = rnd[k];
   float l_phiExc = phiExc[k];
   float l_phiInh = phiInh[k];
   float l_prev   = prevTime[kex];
   float l_activ  = activity[kex];

   l_activ = (float) spike(time, dt, l_prev, (l_phiExc - l_phiInh), &l_rnd, params);
   l_prev  = (l_activ > 0.0f) ? time : l_prev;

   l_phiExc = 0.0f;
   l_phiInh = 0.0f;

   // store local variables back to global memory
   //
   rnd[k] = l_rnd;
   phiExc[k] = l_phiExc;
   phiInh[k] = l_phiInh;
   prevTime[kex] = l_prev;
   activity[kex] = l_activ;

#ifndef PV_USE_OPENCL
   }
#endif

}

#include "Retina_params.h"
#include "cl_random.cl"

#ifndef PI
#  define PI 3.1415926535897932
#endif

#ifndef __CL_PLATFORM_H

#  include <math.h>
#  define EXP expf
#  define FMOD fmodf
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_LOCAL

#else

#  define EXP exp
#  define FMOD __fmodf
#  define CL_KERNEL     __kernel
#  define CL_MEM_GLOBAL __global
#  define CL_MEM_LOCAL  __local

// This stuff needs to be moved to (or obtained from) separate file
// perhaps conversions.h

static inline int kxPos(int k, int nx, int ny, int nf)
{
   return (k/nf) % nx;
}

static inline int kyPos(int k, int nx, int ny, int nf)
{
   return k / (nx*nf);
}

static inline int kIndex(int kx, int ky, int kf, int nx, int ny, int nf)
{
   return kf + (kx + (ky * nx)) * nf;
}

static inline int featureIndex(int k, int nx, int ny, int nf)
{
   return k % nf;
}

static inline int kIndexExtended(int k, int nx, int ny, int nf, int nb)
{
   const int kx_ex = nb + kxPos(k, nx, ny, nf);
   const int ky_ex = nb + kyPos(k, nx, ny, nf);
   const int kf = featureIndex(k, nx, ny, nf);
   return kIndex(kx_ex, ky_ex, kf, nx + 2*nb, ny + 2*nb, nf);
}

#endif /* __CL_PLATFORM_H */

/*
 * Spiking method for Retina
 * Returns 1 if an event should occur, 0 otherwise. This is a stochastic model.
 *
 * REMARKS:
 *      - During ABS_REFACTORY_PERIOD a neuron does not spike
 *      - The neurons that correspond to stimuli (on Image pixels)
 *        spike with probability probStim.
 *      - The neurons that correspond to background image pixels
 *        spike with probability probBase.
 *      - After ABS_REFACTORY_PERIOD the spiking probability
 *        grows exponentially to probBase and probStim respectively.
 *      - The burst of the retina is periodic with period T set by
 *        T = 1000/burstFreq in miliseconds
 *      - When the time t is such that mT < t < mT + burstDuration, where m is
 *        an integer, the burstStatus is set to 1.
 *      - The burstStatus is also determined by the condition that
 *        beginStim < t < endStim. These parameters are set in the input
 *        params file params.stdp
 *      - sinAmp modulates the spiking probability only when burstDuration <= 0
 *        or burstFreq = 0
 *      - probSpike is set to probBase for all neurons.
 *      - for neurons exposed to Image on pixels, probSpike increases with probStim.
 *      - When the probability is negative, the neuron does not spike.
 *
 * NOTES:
 *      - time is measured in milliseconds.
 *
 */
static inline
int spike(float time, float dt,
          float prev, float stimFactor, uint4 * rnd_state, const Retina_params * params)
{
   float probSpike;
   float burstStatus = 1;
   float sinAmp = 1.0;

   // input parameters
   //
   float probBase  = params->probBase;
   float probStim  = params->probStim * stimFactor;

   // see if neuron is in a refactory period
   //
   if ((time - prev) < params->abs_refactory_period) {
      return 0;
   }
   else {
      float delta = time - prev - params->abs_refactory_period;
      float refact = 1.0f - EXP(-delta/params->refactory_period);
      refact = (refact < 0) ? 0 : refact;
      probBase *= refact;
      probStim *= refact;
   }

   if (params->burstDuration <= 0 || params->burstFreq == 0) {
      sinAmp = cos( 2*PI*time * params->burstFreq / 1000. );
   }
   else {
      burstStatus = FMOD(time, 1000. / params->burstFreq);
      burstStatus = burstStatus <= params->burstDuration;
   }

   burstStatus *= (int) ( (time >= params->beginStim) && (time < params->endStim) );
   probSpike = probBase;

   if ((int)burstStatus) {
      probSpike += probStim * sinAmp;  // negative prob is OK
   }

   int spike_flag = (cl_random_prob(*rnd_state) < probSpike);
   *rnd_state = cl_random_state(*rnd_state);
   return spike_flag;
}

//
// update the state of a retinal layer (spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void update_state (
    const float time,
    const float dt,
    const Retina_params * params,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL uint4 * rnd,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * activity,
    CL_MEM_GLOBAL float * prevTime)
{
   int k;
#ifndef PV_USE_OPENCL
for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // load local variables from global memory
   //
   uint4 l_rnd = rnd[k];
   float l_phiExc = phiExc[k];
   float l_phiInh = phiInh[k];
   float l_prev   = prevTime[kex];
   float l_activ  = activity[kex];

   l_activ = (float) spike(time, dt, l_prev, (l_phiExc - l_phiInh), &l_rnd, params);
   l_prev  = (l_activ > 0.0f) ? time : l_prev;

   l_phiExc = 0.0f;
   l_phiInh = 0.0f;

   // store local variables back to global memory
   //
   rnd[k] = l_rnd;
   phiExc[k] = l_phiExc;
   phiInh[k] = l_phiInh;
   prevTime[kex] = l_prev;
   activity[kex] = l_activ;

#ifndef PV_USE_OPENCL
   }
#endif

}


Fix comments.

Fix comments.

; PI Common LISP example.

(defvar n 5000000)

; This likely only works in GNU Common LISP.
(if (> (list-length *args*) 0)
  (setq n (parse-integer (car *args*)))
)

(format t "Calculating PI using~%")
(format t "  ~a slices~%" n)
(format t "  1 process~%")

; get-internal-real-time gets time from some arbitrary, implementation
; dependent starting point in some fraction of a second.
(defvar start (get-internal-real-time))

(defvar sum 0.d0)
(defvar st (/ 1.d0 n))
(defvar x 0.d0)

(loop for i from 1 to n do
; In Common LISP we have to enclose multi-line dos in a block.
  (block inner
    (setq x (* (- i 0.d5) st))
    (setq sum (+ sum (/ 4.d0 (+ 1.d0 (* x x)))))
  )
)

(defvar p (* sum st))

; get-internal-real-time is in some platform specific unit which we can find in
; internal-time-units-per-second
(defvar stop (get-internal-real-time))

(defvar runt (/ (- stop start) internal-time-units-per-second))

(format t "Obtained value of PI: ~a~%" p)

; We have to print the time with ~f to force a float output.
(format t "Time taken: ~f seconds~%" runt)

; PI Common LISP example.

(defvar n 5000000)

; This likely only works in GNU Common LISP.
(if (> (list-length *args*) 0)
  (setq n (parse-integer (car *args*)))
)

(format t "Calculating PI using~%")
(format t "  ~a slices~%" n)
(format t "  1 process~%")

; get-internal-real-time gets time from some arbitrary, implementation
; dependent starting point in some fraction of a second.
(defvar start (get-internal-real-time))

(defvar sum 0.d0)
(defvar st (/ 1.d0 n))
(defvar x 0.d0)

(loop for i from 1 to n do
; In Common LISP we have to enclose multi-line dos in a block.
  (block inner
    (setq x (* (- i 0.d5) st))
    (setq sum (+ sum (/ 4.d0 (+ 1.d0 (* x x)))))
  )
)

(defvar p (* sum st))

(defvar stop (get-internal-real-time))

; get-internal-real-time is in some platform specific unit which we can find in
; internal-time-units-per-second
(defvar runt (/ (- stop start) internal-time-units-per-second))

(format t "Obtained value of PI: ~a~%" p)

; We have to print the time with ~f to force a float output.
(format t "Time taken: ~f seconds~%" runt)


Removed the illegal barrier call in the wait_group_events()
implementation.

This might cause illegal barrier usage scenarios, which seem to
still cause the WGF CFG analysis to crash in some cases.

Might Fix #543.

/* OpenCL built-in library: wait_group_events()

   Copyright (c) 2012 Pekka Jääskeläinen / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

/* The default implementation for "async copies" is a
   blocking one which doesn't actually need events for
   anything. The event waiting is therefore a dummy function. */

void _CL_OVERLOADABLE wait_group_events (int num_events,
                                         event_t *event_list)
{
}

/* OpenCL built-in library: wait_group_events()

   Copyright (c) 2012 Pekka Jääskeläinen / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

/* The default implementation for "async copies" is a
   blocking one which doesn't actually need events for
   anything. The event waiting is therefore a dummy function. */

void _CL_OVERLOADABLE wait_group_events (int num_events,
                                         event_t *event_list)
{
  barrier(CLK_GLOBAL_MEM_FENCE);
}


actual intel code

actual intel code

//intel vectorised bitonic sort
__kernel void  bitonicSort2(__global int4 * theArray,
  const uint stage,
  const uint passOfStage,
  const uint width)
{
  size_t i = get_global_id(0);
  int4 srcLeft, srcRight, mask;
  int4 imask10 = (int4)(0, 0, -1, -1);
  int4 imask11 = (int4)(0, -1, 0, -1);
  const uint dir = 0;
  if (stage > 0)
  {
    if (passOfStage > 0)    // upper level pass, exchange between two fours
    {
      size_t r = 1 << (passOfStage - 1);
      size_t lmask = r - 1;
      size_t left = ((i >> (passOfStage - 1)) << passOfStage) + (i & lmask);
      size_t right = left + r;

      srcLeft = theArray[left];
      srcRight = theArray[right];
      mask = srcLeft < srcRight;

      int4 imin = (srcLeft & mask) | (srcRight & ~mask);
      int4 imax = (srcLeft & ~mask) | (srcRight & mask);

      if (((i >> (stage - 1)) & 1) ^ dir)
      {
        theArray[left] = imin;
        theArray[right] = imax;
      }
      else
      {
        theArray[right] = imin;
        theArray[left] = imax;
      }
    }
    else    // last pass, sort inside one four
    {
      srcLeft = theArray[i];
      srcRight = srcLeft.zwxy;
      mask = (srcLeft < srcRight) ^ imask10;

      if (((i >> stage) & 1) ^ dir)
      {
        srcLeft = (srcLeft & mask) | (srcRight & ~mask);
        srcRight = srcLeft.yxwz;
        mask = (srcLeft < srcRight) ^ imask11;
        theArray[i] = (srcLeft & mask) | (srcRight & ~mask);
      }
      else
      {
        srcLeft = (srcLeft & ~mask) | (srcRight & mask);
        srcRight = srcLeft.yxwz;
        mask = (srcLeft < srcRight) ^ imask11;
        theArray[i] = (srcLeft & ~mask) | (srcRight & mask);
      }
    }
  }
  else    // first stage, sort inside one four
  {
    int4 imask0 = (int4)(0, -1, -1, 0);
    srcLeft = theArray[i];
    srcRight = srcLeft.yxwz;
    mask = (srcLeft < srcRight) ^ imask0;
    if (dir)
      srcLeft = (srcLeft & mask) | (srcRight & ~mask);
    else
      srcLeft = (srcLeft & ~mask) | (srcRight & mask);

    srcRight = srcLeft.zwxy;
    mask = (srcLeft < srcRight) ^ imask10;

    if ((i & 1) ^ dir)
    {
      srcLeft = (srcLeft & mask) | (srcRight & ~mask);
      srcRight = srcLeft.yxwz;
      mask = (srcLeft < srcRight) ^ imask11;
      theArray[i] = (srcLeft & mask) | (srcRight & ~mask);
    }
    else
    {
      srcLeft = (srcLeft & ~mask) | (srcRight & mask);
      srcRight = srcLeft.yxwz;
      mask = (srcLeft < srcRight) ^ imask11;
      theArray[i] = (srcLeft & ~mask) | (srcRight & mask);
    }
  }
}

//amds simple bitonic sort
__kernel void bitonicSort(__global uint *theArray, const uint stage, const uint passOfStage,
                          const uint width // amount of items in the array
                          ) {
  uint sortIncreasing = 1;
  uint threadId = get_global_id(0);

  uint pairDistance = 1 << (stage - passOfStage);
  uint blockWidth = 2 * pairDistance;

  uint leftId = (threadId % pairDistance) + (threadId / pairDistance) * blockWidth;

  uint rightId = leftId + pairDistance;

  uint leftElement = theArray[leftId];
  uint rightElement = theArray[rightId];

  uint sameDirectionBlockWidth = 1 << stage;

  if ((threadId / sameDirectionBlockWidth) % 2 == 1) {
    sortIncreasing = 1 - sortIncreasing;
  }

  uint greater;
  uint lesser;
  if (leftElement > rightElement) {
    greater = leftElement;
    lesser = rightElement;
  } else {
    greater = rightElement;
    lesser = leftElement;
  }

  if (sortIncreasing) {
    theArray[leftId] = lesser;
    theArray[rightId] = greater;
  } else {
    theArray[leftId] = greater;
    theArray[rightId] = lesser;
  }
}

__kernel void ParallelSelection_Local(__global const uint *in, __global uint *out,
                                      __local uint *aux) {
  int i = get_local_id(0);    // index in workgroup
  int wg = get_local_size(0); // workgroup size = block size

  // Move IN, OUT to block start
  int offset = get_group_id(0) * wg;
  in += offset;
  out += offset;

  // Load block in AUX[WG]
  uint iData = in[i];
  aux[i] = iData;
  barrier(CLK_LOCAL_MEM_FENCE);

  // Find output position of iData
  uint iKey = iData;
  int pos = 0;
  for (int j = 0; j < wg; j++) {
    uint jKey = aux[j];
    bool smaller = (jKey < iKey) || (jKey == iKey && j < i); // in[j] < in[i] ?
    pos += (smaller) ? 1 : 0;
  }

  // Store output
  out[pos] = iData;
}


sum_reduction_horizontal and gemv_reduce use value returned from w_g_r_horizontal.

#ifndef REAL
#define REAL float
#endif

#ifndef WGS
#define WGS 256
#endif

#ifndef WGSm
#define WGSm 16
#endif

#ifndef WGSn
#define WGSn 16
#endif

#ifndef TS
#define TS 32
#endif

#ifndef WPT
#define WPT 4
#endif

#define RTS (TS/WPT)

//||||||||||||||||| CLBuffer |||||||||||||||||||||||||||||||||||||||||||||||||||

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void equals_vector (__global uint* eq_flag,
                             __global const REAL* x, const uint offset_x,
                             const uint stride_x,
                             __global const REAL* y, const uint offset_y,
                             const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    if ((x[ix] != y[iy])){
        eq_flag[0]++;
    }
}

__kernel void equals_matrix (__global uint* eq_flag,
                             __global const REAL* a, const uint offset_a,
                             const uint ld_a,
                             __global const REAL* b, const uint offset_b,
                             const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    if ((a[ia] != b[ib])){
        eq_flag[0]++;
    }
}

//|||||||||||||||||       BLAS 1       |||||||||||||||||||||||||||||||||||||||||

// ================ Embarassingly parallel kernels =============================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void swap (__global REAL* x, const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    REAL temp = x[ix];
    x[ix] = y[iy];
    y[iy] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void copy (__global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    y[offset_y + get_global_id(0) * stride_y] =
        x[offset_x + get_global_id(0) * stride_x];
}

__kernel void copy_matrix (__global const REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    b[ib] = a[ia];
}

__kernel void swap_matrix (__global REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    REAL temp = a[ia];
    a[ia] = b[ia];
    b[ib] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void scal (const REAL alpha, __global REAL* x,
                    const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    x[ix] = alpha * x[ix];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpy (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y,
                    const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + y[iy];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpby (const REAL alpha, __global const REAL* x,
                     uint offset_x, const uint stride_x,
                     const REAL beta, __global REAL* y,
                     const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + beta * y[iy];
}

// ================= Reduction kernels ========================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduction (__global double* acc) {
    double sum = work_group_reduction_sum(acc[get_global_id(0)]);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== Dot product ==============================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void dot_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x,
                          __global const REAL* y,
                          const uint offset_y, const uint stride_y) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    REAL py = y[offset_y + get_global_id(0) * stride_y];
    double sum = work_group_reduction_sum((double)(px * py));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== asum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void asum_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)fabs(px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== sum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)px);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== nrm2 =====================================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void nrm2_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)(px * px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================ Max reduction ==============================================
inline void work_group_reduction_imax (__global uint* iacc,
                                       __global double* vacc,
                                       uint const ind, const double val) {

    uint local_id = get_local_id(0);
    uint local_size = get_local_size(0);

    __local uint liacc[WGS];
    __local double lvacc[WGS];
    liacc[local_id] = ind;
    lvacc[local_id] = val;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    uint index = ind;
    double value = val;

    uint i = local_size;
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_id == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            double other_value = lvacc[local_id + i + 1];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i + 1];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        if (local_id < i) {
            double other_value = lvacc[local_id + i];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    if(local_id == 0) {
        uint group_id = get_group_id(0);
        iacc[group_id] = index;
        vacc[group_id] = value;
    }

}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduction (__global uint* iacc, __global double* vacc) {
    uint gid = get_global_id(0);
    work_group_reduction_imax(iacc, vacc, iacc[gid], (double)(vacc[gid]));
}

// ================== iamax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void iamax_reduce (__global uint* iacc, __global double* vacc,
                            __global const REAL* x,
                            const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(fabs(x[ix])));
}

// ================== imax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(x[ix]));
}

// ================== imin reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imin_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(-x[ix]));
}

// ||||||||||||||||       BLAS 2      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMV =====================================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void sum_reduction_horizontal (__global REAL* acc) {

    uint ia = get_global_size(0) * get_global_id(1) + get_global_id(0);
    REAL sum = work_group_reduction_sum_horizontal(acc[ia]);
    if (get_local_id(1) == 0) {
        acc[get_global_size(0) * get_group_id(1) + get_global_id(0)] = sum;
    }
}

// ========================= gemv ==============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gemv_reduce (__global REAL* acc,
                           const REAL alpha, __global const REAL* a,
                           const uint offset_a, const uint ld_a,
                           __global const REAL* x, const uint offset_x,
                           const uint stride_x) {

    uint ia = offset_a + ld_a * get_global_id(1) + get_local_id(0);
    uint ix = offset_x + get_global_id(1) * stride_x;
    REAL sum = work_group_reduction_sum_horizontal(alpha * a[ia] * x[ix]);
    if (get_local_id(1) == 0) {
        acc[get_global_size(0) * get_group_id(1) + get_global_id(0)] = sum;
    }
}

// ========================== gerk =============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gerk (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global const REAL* y,
                    const uint offset_y, const uint stride_y,
                    __global REAL* a, const uint offset_a, const uint ld_a) {

    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(1) * stride_y;
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    a[ia] += alpha * x[ix] * y[iy];
}

// ||||||||||||||||       BLAS 3      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMM =====================================================

inline uint globalize (const uint tile_size, const uint tile_id,
                       const uint id){
    return tile_id * tile_size + id;
}

// ========================= gemm ==============================================

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled (const REAL alpha, __global const REAL* a,
                          const uint offset_a, const uint ld_a,
                          __global const REAL* b,
                          const uint offset_b, const uint ld_b,
                          const REAL beta, __global REAL* c,
                          const uint offset_c, const uint ld_c,
                          const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    // Elements that are in partial m-tiles and n-tiles need to be
    // loaded, but only if they exist.
    const bool load_row = c_row < m;
    bool load_col[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
        load_col[w] = c_col + w * RTS < n;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {

        const uint tile_row = TS * t + row;
        const uint tile_col = TS * t + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    // Compute partial k-tiles.
    const uint rest_k = k - tc * TS;
    if (0 < rest_k) {

        const uint tile_row = TS * tc + row;
        const uint tile_col = TS * tc + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < rest_k; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

    }

    //Only the elements that exist in partial c-tiles should be stored.
#pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const bool store = load_row && load_col[w];
        if (store) {
            const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
            c[ic] = alpha * acc[w] + beta * c[ic];
        }
    }

}

// Simpler version that requires dimensions that fit tiles

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled_fit (const REAL alpha, __global const REAL* a,
                              const uint offset_a, const uint ld_a,
                              __global const REAL* b,
                              const uint offset_b, const uint ld_b,
                              const REAL beta, __global REAL* c,
                              const uint offset_c, const uint ld_c,
                              const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {
        for (uint w = 0; w < WPT; w++) {
            const uint tile_row = TS * t + row;
            const uint tile_col = TS * t + col;
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = a[ia];
            b_tile[col + w * RTS][row] = b[ib];
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
        c[ic] = alpha * acc[w] + beta * c[ic];
    }

}

#ifndef REAL
#define REAL float
#endif

#ifndef WGS
#define WGS 256
#endif

#ifndef WGSm
#define WGSm 16
#endif

#ifndef WGSn
#define WGSn 16
#endif

#ifndef TS
#define TS 32
#endif

#ifndef WPT
#define WPT 4
#endif

#define RTS (TS/WPT)

//||||||||||||||||| CLBuffer |||||||||||||||||||||||||||||||||||||||||||||||||||

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void equals_vector (__global uint* eq_flag,
                             __global const REAL* x, const uint offset_x,
                             const uint stride_x,
                             __global const REAL* y, const uint offset_y,
                             const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    if ((x[ix] != y[iy])){
        eq_flag[0]++;
    }
}

__kernel void equals_matrix (__global uint* eq_flag,
                             __global const REAL* a, const uint offset_a,
                             const uint ld_a,
                             __global const REAL* b, const uint offset_b,
                             const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    if ((a[ia] != b[ib])){
        eq_flag[0]++;
    }
}

//|||||||||||||||||       BLAS 1       |||||||||||||||||||||||||||||||||||||||||

// ================ Embarassingly parallel kernels =============================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void swap (__global REAL* x, const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    REAL temp = x[ix];
    x[ix] = y[iy];
    y[iy] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void copy (__global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y, const uint offset_y, const uint stride_y) {
    y[offset_y + get_global_id(0) * stride_y] =
        x[offset_x + get_global_id(0) * stride_x];
}

__kernel void copy_matrix (__global const REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    b[ib] = a[ia];
}

__kernel void swap_matrix (__global REAL* a, const uint offset_a,
                           const uint ld_a,
                           __global REAL* b, const uint offset_b,
                           const uint ld_b) {
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    uint ib = offset_b + get_global_id(0) + get_global_id(1) * ld_b;
    REAL temp = a[ia];
    a[ia] = b[ia];
    b[ib] = temp;
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void scal (const REAL alpha, __global REAL* x,
                    const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    x[ix] = alpha * x[ix];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpy (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global REAL* y,
                    const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + y[iy];
}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void axpby (const REAL alpha, __global const REAL* x,
                     uint offset_x, const uint stride_x,
                     const REAL beta, __global REAL* y,
                     const uint offset_y, const uint stride_y) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(0) * stride_y;
    y[iy] = alpha * x[ix] + beta * y[iy];
}

// ================= Reduction kernels ========================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduction (__global double* acc) {
    double sum = work_group_reduction_sum(acc[get_global_id(0)]);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== Dot product ==============================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void dot_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x,
                          __global const REAL* y,
                          const uint offset_y, const uint stride_y) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    REAL py = y[offset_y + get_global_id(0) * stride_y];
    double sum = work_group_reduction_sum((double)(px * py));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== asum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void asum_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)fabs(px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== sum =====================================================
__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void sum_reduce (__global double* acc, __global const REAL* x,
                          const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)px);
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================== nrm2 =====================================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void nrm2_reduce (__global double* acc, __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    REAL px = x[offset_x + get_global_id(0) * stride_x];
    double sum = work_group_reduction_sum((double)(px * px));
    if (get_local_id(0) == 0) {
        acc[get_group_id(0)] = sum;
    }
}

// ================ Max reduction ==============================================
inline void work_group_reduction_imax (__global uint* iacc,
                                       __global double* vacc,
                                       uint const ind, const double val) {

    uint local_id = get_local_id(0);
    uint local_size = get_local_size(0);

    __local uint liacc[WGS];
    __local double lvacc[WGS];
    liacc[local_id] = ind;
    lvacc[local_id] = val;

    work_group_barrier(CLK_LOCAL_MEM_FENCE);

    uint index = ind;
    double value = val;

    uint i = local_size;
    while (i > 0) {
        bool include_odd = (i > ((i >> 1) << 1)) && (local_id == ((i >> 1) - 1));
        i >>= 1;
        if (include_odd) {
            double other_value = lvacc[local_id + i + 1];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i + 1];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        if (local_id < i) {
            double other_value = lvacc[local_id + i];
            if (other_value > value) {
                value = other_value;
                index = liacc[local_id + i];
                lvacc[local_id] = value;
                liacc[local_id] = index;
            }
        }
        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    if(local_id == 0) {
        uint group_id = get_group_id(0);
        iacc[group_id] = index;
        vacc[group_id] = value;
    }

}

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduction (__global uint* iacc, __global double* vacc) {
    uint gid = get_global_id(0);
    work_group_reduction_imax(iacc, vacc, iacc[gid], (double)(vacc[gid]));
}

// ================== iamax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void iamax_reduce (__global uint* iacc, __global double* vacc,
                            __global const REAL* x,
                            const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(fabs(x[ix])));
}

// ================== imax reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imax_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(x[ix]));
}

// ================== imin reduce  ============================================

__attribute__((reqd_work_group_size(WGS, 1, 1)))
__kernel void imin_reduce (__global uint* iacc, __global double* vacc,
                           __global const REAL* x,
                           const uint offset_x, const uint stride_x) {
    uint ix = offset_x + get_global_id(0) * stride_x;
    work_group_reduction_imax(iacc, vacc, ix, (double)(-x[ix]));
}

// ||||||||||||||||       BLAS 2      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMV =====================================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void sum_reduction_horizontal (__global REAL* acc) {

    uint ia = get_global_size(0) * get_global_id(1) + get_global_id(0);
    work_group_reduction_sum_horizontal(acc, acc[ia]);
}

// ========================= gemv ==============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gemv_reduce (__global REAL* acc,
                           const REAL alpha, __global const REAL* a,
                           const uint offset_a, const uint ld_a,
                           __global const REAL* x, const uint offset_x,
                           const uint stride_x) {

    uint ia = offset_a + ld_a * get_global_id(1) + get_local_id(0);
    uint ix = offset_x + get_global_id(1) * stride_x;

    work_group_reduction_sum_horizontal(acc, alpha * a[ia] * x[ix]);
}

// ========================== gerk =============================================

__attribute__((reqd_work_group_size(WGSm, WGSn, 1)))
__kernel void gerk (const REAL alpha, __global const REAL* x,
                    const uint offset_x, const uint stride_x,
                    __global const REAL* y,
                    const uint offset_y, const uint stride_y,
                    __global REAL* a, const uint offset_a, const uint ld_a) {

    uint ix = offset_x + get_global_id(0) * stride_x;
    uint iy = offset_y + get_global_id(1) * stride_y;
    uint ia = offset_a + get_global_id(0) + get_global_id(1) * ld_a;
    a[ia] += alpha * x[ix] * y[iy];
}

// ||||||||||||||||       BLAS 3      ||||||||||||||||||||||||||||||||||||||||||

// ================== GEMM =====================================================

inline uint globalize (const uint tile_size, const uint tile_id,
                       const uint id){
    return tile_id * tile_size + id;
}

// ========================= gemm ==============================================

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled (const REAL alpha, __global const REAL* a,
                          const uint offset_a, const uint ld_a,
                          __global const REAL* b,
                          const uint offset_b, const uint ld_b,
                          const REAL beta, __global REAL* c,
                          const uint offset_c, const uint ld_c,
                          const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    // Elements that are in partial m-tiles and n-tiles need to be
    // loaded, but only if they exist.
    const bool load_row = c_row < m;
    bool load_col[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
        load_col[w] = c_col + w * RTS < n;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {

        const uint tile_row = TS * t + row;
        const uint tile_col = TS * t + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    // Compute partial k-tiles.
    const uint rest_k = k - tc * TS;
    if (0 < rest_k) {

        const uint tile_row = TS * tc + row;
        const uint tile_col = TS * tc + col;

        for (uint w = 0; w < WPT; w++) {
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = load_row ? a[ia] : 0.0f;
            b_tile[col + w * RTS][row] = load_col[w] ? b[ib] : 0.0f;
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < rest_k; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

    }

    //Only the elements that exist in partial c-tiles should be stored.
#pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const bool store = load_row && load_col[w];
        if (store) {
            const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
            c[ic] = alpha * acc[w] + beta * c[ic];
        }
    }

}

// Simpler version that requires dimensions that fit tiles

__attribute__((reqd_work_group_size(TS, RTS, 1)))
__kernel void gemm_tiled_fit (const REAL alpha, __global const REAL* a,
                              const uint offset_a, const uint ld_a,
                              __global const REAL* b,
                              const uint offset_b, const uint ld_b,
                              const REAL beta, __global REAL* c,
                              const uint offset_c, const uint ld_c,
                              const uint m, const uint k, const uint n) {

    const uint row = get_local_id(0);
    const uint col = get_local_id(1);
    const uint c_row = globalize(TS, get_group_id(0), row);
    const uint c_col = globalize(TS, get_group_id(1), col);

    // Local tiles of matrices A and B
    __local REAL a_tile[TS][TS];
    __local REAL b_tile[TS][TS];

    REAL acc[WPT];

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        acc[w] = 0.0f;
    }

    // Compute full k-tiles
    const uint tc = k / TS;
    for (uint t = 0; t < tc; t++) {
        for (uint w = 0; w < WPT; w++) {
            const uint tile_row = TS * t + row;
            const uint tile_col = TS * t + col;
            const uint ia = offset_a + (tile_col + w * RTS) * ld_a + c_row;
            const uint ib = offset_b + (c_col + w * RTS) * ld_b + tile_row;
            a_tile[col + w * RTS][row] = a[ia];
            b_tile[col + w * RTS][row] = b[ib];
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);

        #pragma unroll
        for(uint i = 0; i < TS; i++) {

            #pragma unroll
            for(uint w = 0; w < WPT; w++) {
                acc[w] += a_tile[i][row] * b_tile[col + w * RTS][i];
            }
        }

        work_group_barrier(CLK_LOCAL_MEM_FENCE);
    }

    #pragma unroll
    for (uint w = 0; w < WPT; w++) {
        const uint ic = offset_c + (c_col + w * RTS) * ld_c + c_row;
        c[ic] = alpha * acc[w] + beta * c[ic];
    }

}