no need to ifdef while loop away on intel sdk

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define SIGMA_NEW_BITPOS   0x1
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define SIGMA_OLD_TO_NBH_SHIFT 0x1
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

// bit flags
#define SIGMA_NEW_F			 0x1		  //position 0
#define SIGMA_OLD_F			 0x10		  //position  1
#define NBH_F				 0x20		  //position  2
#define RLC_F				 0x80		  //position  4
#define RLC_D_POSITION_F	 0x380        //positions 6-9
#define PIXEL_F				 0x7FFF0000   //positions 10-24
#define SIGN_F				 0x2000000    //position  25

#define NOT_SIGMA_NEW_F      0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD_AND_NEW_F  0x11

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)
#define NBH(pix) ((pix) & NBH_F)
#define SIGMA_OLD(pix) ((pix) & SIGMA_OLD_F)
#define SIGMA_NEW(pix) ((pix) & SIGMA_NEW_F)
#define SIGMA_OLD_AND_NEW(pix) ((pix) & SIGMA_OLD_AND_NEW_F)
#define SIGN(pix) ((pix) & SIGN_F)

#define SET_SIGMA_NEW(pix) ( (pix) |= SIGMA_NEW_F )
#define CLEAR_SIGMA_NEW(pix) ( (pix) &= NOT_SIGMA_NEW_F )

/**

**/

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];
	LOCAL int cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between -1 and 31 - negative one value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {

		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);

		for (int i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			int absPixel = abs(pixel);
			maxVal = max(maxVal, absPixel);
			pixel = (absPixel << PIXEL_START_BITPOS) | SIGN((pixel << INPUT_TO_SIGN_SHIFT));
			statePtr[0] = pixel;

			posIn.y++;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			}
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(int i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// i) set sigma_new for strip column (sigma_old is zero)
	LOCAL int* statePtr = state + startIndex;
	for (int i = 0; i < 4; ++i) {
		int current = statePtr[0];
		current |= BIT(current);
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// ii) set nbh for strip column, and do CUP
	statePtr = state + startIndex;
	int current			= statePtr[0];
	int top				= statePtr[TOP];
	int leftTop			= statePtr[LEFT_TOP];
	int left			= statePtr[LEFT];
	int leftBottom		= statePtr[LEFT_BOTTOM];

	int rlcCount = 0;

	// first pixel in strip column
	int nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom)) << NBH_BITPOS;
	bool doRLC = !nbh && !BIT(current);
	if (doRLC) {
		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	// next two pixels in strip column
	for (int i = 0; i < 2; ++i) {

		statePtr += STATE_BUFFER_STRIDE;
		top = current;
		current		= statePtr[0];
		leftTop		= left;
		left		= leftBottom;
		leftBottom	= statePtr[LEFT_BOTTOM];

		nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom)) << NBH_BITPOS;

		// toggle doRLC flag
		doRLC = doRLC && !nbh && !BIT(current);
		if (doRLC) {

			rlcCount++;
		} else {
		   //ZC

		   if (BIT(current)) {
		       //SC
		   }
		}
	}

	// last pixel in strip column -
	// ignore leftBottom pixel, because it is in the next
	// stripe and it hasn't been processed yet
	statePtr += STATE_BUFFER_STRIDE;
	top			= current;
	current		= statePtr[0];
	leftTop		= left;
	left		= leftBottom;

	nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left)) << NBH_BITPOS;

	doRLC = doRLC && !nbh && !BIT(current);
	if (doRLC) {

		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit planes
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (int i = 0; i < 4; ++i) {
		    int current = *statePtr;
			current |=  SIGMA_NEW(current) << SIGMA_OLD_BITPOS;
			CLEAR_SIGMA_NEW(current);

			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		int top			= statePtr[TOP];
		int leftTop		= statePtr[LEFT_TOP];
		int rightTop	= statePtr[RIGHT_TOP];
		int left		= statePtr[LEFT];
		int current		= statePtr[0];
		int right		= statePtr[RIGHT];
		int leftBottom  = statePtr[LEFT_BOTTOM];
		int bottom		= statePtr[BOTTOM];
		int rightBottom = statePtr[RIGHT_BOTTOM];

		current |=  (   SIGMA_OLD(leftTop) |
						SIGMA_OLD( top) |
						SIGMA_OLD( rightTop) |
						SIGMA_OLD( left) |
						SIGMA_OLD( right) |
						SIGMA_OLD( leftBottom) |
						SIGMA_OLD( bottom) |
						SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) &&
				NBH(current) &&
					!SIGMA_OLD(current) ) {
			SET_SIGMA_NEW(current);
			blockVote = 1;

		}
		*statePtr = current;

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			 = current;
			leftTop		 = left;
			rightTop	 = right;
			left		 = leftBottom;
			current		 = statePtr[0];
			right		 = rightBottom;
			leftBottom   = statePtr[LEFT_BOTTOM];
			bottom		 = statePtr[BOTTOM];
			rightBottom  = statePtr[RIGHT_BOTTOM];

			current |=	   ( SIGMA_OLD(leftTop) |
							 SIGMA_OLD( top) |
							 SIGMA_OLD( rightTop) |
							 SIGMA_OLD( left) |
							 SIGMA_OLD( right) |
							 SIGMA_OLD( leftBottom) |
							 SIGMA_OLD( bottom) |
							 SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) &&
					 NBH(current) &&
						 !SIGMA_OLD(current) ) {
			    SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
		while (blockVote)
		{
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			//first pixel in strip column
			int top			= statePtr[TOP];
			int leftTop		= statePtr[LEFT_TOP];
			int left		= statePtr[LEFT];
			int current		= statePtr[0];
			int leftBottom  = statePtr[LEFT_BOTTOM];

			current  |= ( SIGMA_NEW(leftTop) |
				SIGMA_NEW( top) |
				SIGMA_NEW( left) |
				SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

			if (  BIT(current) &&
				        !SIGMA_OLD(current) &&
						    NBH(current)  &&
							    !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;

			}
			statePtr[0] = current;

			// next two pixels in strip column
			for (int i = 0; i < 2; ++i) {
				statePtr += STATE_BUFFER_STRIDE;

				top			= current;
				current		= statePtr[0];
				leftTop		= left;
				left		= leftBottom;
				leftBottom  = statePtr[LEFT_BOTTOM];

				current  |= ( SIGMA_NEW(leftTop) |
					SIGMA_NEW( top) |
					SIGMA_NEW( left) |
					SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

				if (  BIT(current) &&
				          !SIGMA_OLD(current) &&
						      NBH(current)  &&
							     !SIGMA_NEW(current)) {
					SET_SIGMA_NEW(current);
					blockVote = 1;

				}
				statePtr[0] = current;
			}

			// last pixel in strip column -
			// ignore leftBottom pixel, because it is in the next
			// stripe and it hasn't been processed yet
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			current		= statePtr[0];
			leftTop		= left;
			left		= leftBottom;

			current |= ( SIGMA_NEW(leftTop) |
						 SIGMA_NEW( top) |
						 SIGMA_NEW( left) ) << NBH_BITPOS;
			if (  BIT(current) &&
			        !SIGMA_OLD(current) &&
					    NBH(current)  &&
						    !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			statePtr[0] = current;

			localMemoryFence();

		}

		/////////////////////////
		// 5. bit plane processing
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop			= statePtr[LEFT_TOP];
		rightTop		= statePtr[RIGHT_TOP];
		left			= statePtr[LEFT];
		current			= statePtr[0];
		right			= statePtr[RIGHT];
		leftBottom		= statePtr[LEFT_BOTTOM];
		bottom			= statePtr[BOTTOM];
		rightBottom		= statePtr[RIGHT_BOTTOM];

		if (SIGMA_OLD(current)) {
			// MRC
		} else {
			int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
						SIGMA_OLD_AND_NEW( top) |
						SIGMA_OLD_AND_NEW( rightTop) |
						SIGMA_OLD_AND_NEW( left) |
						SIGMA_OLD_AND_NEW( right) |
						SIGMA_OLD_AND_NEW( leftBottom) |
						SIGMA_OLD_AND_NEW( bottom) |
						SIGMA_OLD_AND_NEW( rightBottom)  ) ;

			if (!nbh) {
				if ( !SIGMA_NEW(current) ) {
					//RLC
				}
			}
			else  {
				// ZC
				if (BIT(current)) {
				    //SC
				}
			}
		}

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			leftTop		= left;
			rightTop	= right;
			left		= leftBottom;
			current		= statePtr[0];
			right		= rightBottom;
			leftBottom  = statePtr[LEFT_BOTTOM];
			bottom		= statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			if (SIGMA_OLD(current)) {
				// MRC
			} else {
				int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
							SIGMA_OLD_AND_NEW( top) |
							SIGMA_OLD_AND_NEW( rightTop) |
							SIGMA_OLD_AND_NEW( left) |
							SIGMA_OLD_AND_NEW( right) |
							SIGMA_OLD_AND_NEW( leftBottom) |
							SIGMA_OLD_AND_NEW( bottom) |
							SIGMA_OLD_AND_NEW( rightBottom)  ) ;

				if (!nbh) {
					if ( !SIGMA_NEW(current) ) {
						//RLC
					}
				}
				else  {
					// ZC
					if (BIT(current)) {
						//SC
					}
				}
			}
		}

		localMemoryFence();
		bp--;
	}
}

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15

#define SIGMA_NEW_BITPOS   0x1
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define SIGMA_OLD_TO_NBH_SHIFT 0x1
#define RLC_BITPOS         0x4
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

// bit flags
#define SIGMA_NEW_F			 0x1		  //position 0
#define SIGMA_OLD_F			 0x10		  //position  1
#define NBH_F				 0x20		  //position  2
#define RLC_F				 0x80		  //position  4
#define RLC_D_POSITION_F	 0x380        //positions 6-9
#define PIXEL_F				 0x7FFF0000   //positions 10-24
#define SIGN_F				 0x2000000    //position  25

#define NOT_SIGMA_NEW_F      0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD_AND_NEW_F  0x11

#define INPUT_TO_SIGN_SHIFT 10

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

#define BIT(pix) (((pix)>>(bp))&1)
#define NBH(pix) ((pix) & NBH_F)
#define SIGMA_OLD(pix) ((pix) & SIGMA_OLD_F)
#define SIGMA_NEW(pix) ((pix) & SIGMA_NEW_F)
#define SIGMA_OLD_AND_NEW(pix) ((pix) & SIGMA_OLD_AND_NEW_F)
#define SIGN(pix) ((pix) & SIGN_F)

#define SET_SIGMA_NEW(pix) ( (pix) |= SIGMA_NEW_F )
#define CLEAR_SIGMA_NEW(pix) ( (pix) &= NOT_SIGMA_NEW_F )

#define AMD

/**

**/

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL int state[STATE_BUFFER_SIZE];
	LOCAL int cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between -1 and 31 - negative one value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {

		int maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL int* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);

		for (int i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			int absPixel = abs(pixel);
			maxVal = max(maxVal, absPixel);
			pixel = (absPixel << PIXEL_START_BITPOS) | SIGN((pixel << INPUT_TO_SIGN_SHIFT));
			statePtr[0] = pixel;

			posIn.y++;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + (getLocalId(0)/(CODEBLOCKX-1))*2; // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			for (int i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			}
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(int i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	int bp = msbScratch[0] + PIXEL_START_BITPOS;

	// i) set sigma_new for strip column (sigma_old is zero)
	LOCAL int* statePtr = state + startIndex;
	for (int i = 0; i < 4; ++i) {
		int current = statePtr[0];
		current |= BIT(current);
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// ii) set nbh for strip column, and do CUP
	statePtr = state + startIndex;
	int current			= statePtr[0];
	int top				= statePtr[TOP];
	int leftTop			= statePtr[LEFT_TOP];
	int left			= statePtr[LEFT];
	int leftBottom		= statePtr[LEFT_BOTTOM];

	int rlcCount = 0;

	// first pixel in strip column
	int nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom)) << NBH_BITPOS;
	bool doRLC = !nbh && !BIT(current);
	if (doRLC) {
		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	// next two pixels in strip column
	for (int i = 0; i < 2; ++i) {

		statePtr += STATE_BUFFER_STRIDE;
		top = current;
		current		= statePtr[0];
		leftTop		= left;
		left		= leftBottom;
		leftBottom	= statePtr[LEFT_BOTTOM];

		nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom)) << NBH_BITPOS;

		// toggle doRLC flag
		doRLC = doRLC && !nbh && !BIT(current);
		if (doRLC) {

			rlcCount++;
		} else {
		   //ZC

		   if (BIT(current)) {
		       //SC
		   }
		}
	}

	// last pixel in strip column -
	// ignore leftBottom pixel, because it is in the next
	// stripe and it hasn't been processed yet
	statePtr += STATE_BUFFER_STRIDE;
	top			= current;
	current		= statePtr[0];
	leftTop		= left;
	left		= leftBottom;

	nbh =  (SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left)) << NBH_BITPOS;

	doRLC = doRLC && !nbh && !BIT(current);
	if (doRLC) {

		rlcCount++;
	} else {
		//ZC

		if (BIT(current)) {
		    //SC
		}
	}

	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit planes
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (int i = 0; i < 4; ++i) {
		    int current = *statePtr;
			current |=  SIGMA_NEW(current) << SIGMA_OLD_BITPOS;
			CLEAR_SIGMA_NEW(current);

			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		int top			= statePtr[TOP];
		int leftTop		= statePtr[LEFT_TOP];
		int rightTop	= statePtr[RIGHT_TOP];
		int left		= statePtr[LEFT];
		int current		= statePtr[0];
		int right		= statePtr[RIGHT];
		int leftBottom  = statePtr[LEFT_BOTTOM];
		int bottom		= statePtr[BOTTOM];
		int rightBottom = statePtr[RIGHT_BOTTOM];

		current |=  (   SIGMA_OLD(leftTop) |
						SIGMA_OLD( top) |
						SIGMA_OLD( rightTop) |
						SIGMA_OLD( left) |
						SIGMA_OLD( right) |
						SIGMA_OLD( leftBottom) |
						SIGMA_OLD( bottom) |
						SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) &&
				NBH(current) &&
					!SIGMA_OLD(current) ) {
			SET_SIGMA_NEW(current);
			blockVote = 1;

		}
		*statePtr = current;

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			 = current;
			leftTop		 = left;
			rightTop	 = right;
			left		 = leftBottom;
			current		 = statePtr[0];
			right		 = rightBottom;
			leftBottom   = statePtr[LEFT_BOTTOM];
			bottom		 = statePtr[BOTTOM];
			rightBottom  = statePtr[RIGHT_BOTTOM];

			current |=	   ( SIGMA_OLD(leftTop) |
							 SIGMA_OLD( top) |
							 SIGMA_OLD( rightTop) |
							 SIGMA_OLD( left) |
							 SIGMA_OLD( right) |
							 SIGMA_OLD( leftBottom) |
							 SIGMA_OLD( bottom) |
							 SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) &&
					 NBH(current) &&
						 !SIGMA_OLD(current) ) {
			    SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
#ifdef AMD
		while (blockVote)
#endif
		{
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			//first pixel in strip column
			int top			= statePtr[TOP];
			int leftTop		= statePtr[LEFT_TOP];
			int left		= statePtr[LEFT];
			int current		= statePtr[0];
			int leftBottom  = statePtr[LEFT_BOTTOM];

			current  |= ( SIGMA_NEW(leftTop) |
				SIGMA_NEW( top) |
				SIGMA_NEW( left) |
				SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

			if (  BIT(current) &&
				        !SIGMA_OLD(current) &&
						    NBH(current)  &&
							    !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;

			}
			statePtr[0] = current;

			// next two pixels in strip column
			for (int i = 0; i < 2; ++i) {
				statePtr += STATE_BUFFER_STRIDE;

				top			= current;
				current		= statePtr[0];
				leftTop		= left;
				left		= leftBottom;
				leftBottom  = statePtr[LEFT_BOTTOM];

				current  |= ( SIGMA_NEW(leftTop) |
					SIGMA_NEW( top) |
					SIGMA_NEW( left) |
					SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

				if (  BIT(current) &&
				          !SIGMA_OLD(current) &&
						      NBH(current)  &&
							     !SIGMA_NEW(current)) {
					SET_SIGMA_NEW(current);
					blockVote = 1;

				}
				statePtr[0] = current;
			}

			// last pixel in strip column -
			// ignore leftBottom pixel, because it is in the next
			// stripe and it hasn't been processed yet
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			current		= statePtr[0];
			leftTop		= left;
			left		= leftBottom;

			current |= ( SIGMA_NEW(leftTop) |
						 SIGMA_NEW( top) |
						 SIGMA_NEW( left) ) << NBH_BITPOS;
			if (  BIT(current) &&
			        !SIGMA_OLD(current) &&
					    NBH(current)  &&
						    !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			statePtr[0] = current;

			localMemoryFence();

		}

		/////////////////////////
		// 5. bit plane processing
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop			= statePtr[LEFT_TOP];
		rightTop		= statePtr[RIGHT_TOP];
		left			= statePtr[LEFT];
		current			= statePtr[0];
		right			= statePtr[RIGHT];
		leftBottom		= statePtr[LEFT_BOTTOM];
		bottom			= statePtr[BOTTOM];
		rightBottom		= statePtr[RIGHT_BOTTOM];

		if (SIGMA_OLD(current)) {
			// MRC
		} else {
			int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
						SIGMA_OLD_AND_NEW( top) |
						SIGMA_OLD_AND_NEW( rightTop) |
						SIGMA_OLD_AND_NEW( left) |
						SIGMA_OLD_AND_NEW( right) |
						SIGMA_OLD_AND_NEW( leftBottom) |
						SIGMA_OLD_AND_NEW( bottom) |
						SIGMA_OLD_AND_NEW( rightBottom)  ) ;

			if (!nbh) {
				if ( !SIGMA_NEW(current) ) {
					//RLC
				}
			}
			else  {
				// ZC
				if (BIT(current)) {
				    //SC
				}
			}
		}

		for (int i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			leftTop		= left;
			rightTop	= right;
			left		= leftBottom;
			current		= statePtr[0];
			right		= rightBottom;
			leftBottom  = statePtr[LEFT_BOTTOM];
			bottom		= statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			if (SIGMA_OLD(current)) {
				// MRC
			} else {
				int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
							SIGMA_OLD_AND_NEW( top) |
							SIGMA_OLD_AND_NEW( rightTop) |
							SIGMA_OLD_AND_NEW( left) |
							SIGMA_OLD_AND_NEW( right) |
							SIGMA_OLD_AND_NEW( leftBottom) |
							SIGMA_OLD_AND_NEW( bottom) |
							SIGMA_OLD_AND_NEW( rightBottom)  ) ;

				if (!nbh) {
					if ( !SIGMA_NEW(current) ) {
						//RLC
					}
				}
				else  {
					// ZC
					if (BIT(current)) {
						//SC
					}
				}
			}
		}

		localMemoryFence();
		bp--;
	}
}


[OpenCL] Fix typo in test that I accidentally introduced in my previous commit.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@278235 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -cl-std=CL2.0 -ffake-address-space-map -O0 -emit-llvm -o - | FileCheck %s

typedef void (^bl_t)(local void *);

const bl_t block_G = (bl_t) ^ (local void *a) {};

kernel void device_side_enqueue(global int *a, global int *b, int i) {
  // CHECK: %default_queue = alloca %opencl.queue_t*
  queue_t default_queue;
  // CHECK: %flags = alloca i32
  unsigned flags = 0;
  // CHECK: %ndrange = alloca %opencl.ndrange_t*
  ndrange_t ndrange;
  // CHECK: %clk_event = alloca %opencl.clk_event_t*
  clk_event_t clk_event;
  // CHECK: %event_wait_list = alloca %opencl.clk_event_t*
  clk_event_t event_wait_list;
  // CHECK: %event_wait_list2 = alloca [1 x %opencl.clk_event_t*]
  clk_event_t event_wait_list2[] = {clk_event};

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[BL:%[0-9]+]] = bitcast <{ i8*, i32, i32, i8*, %struct.__block_descriptor addrspace(3)*, i32{{.*}}, i32{{.*}}, i32{{.*}} }>* %block to void ()*
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__enqueue_kernel_basic(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i8* [[BL_I8]])
  enqueue_kernel(default_queue, flags, ndrange,
                 ^(void) {
                   a[i] = b[i];
                 });

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[BL:%[0-9]+]] = bitcast <{ i8*, i32, i32, i8*, %struct.__block_descriptor addrspace(3)*, i32{{.*}}, i32{{.*}}, i32{{.*}} }>* %block3 to void ()*
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__enqueue_kernel_basic_events(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i32 2, %opencl.clk_event_t** %event_wait_list, %opencl.clk_event_t** %clk_event, i8* [[BL_I8]])
  enqueue_kernel(default_queue, flags, ndrange, 2, &event_wait_list, &clk_event,
                 ^(void) {
                   a[i] = b[i];
                 });

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i8*, i32, ...) @__enqueue_kernel_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 256)
  enqueue_kernel(default_queue, flags, ndrange,
                 ^(local void *p) {
                   return;
                 },
                 256);
  char c;
  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[SIZE:%[0-9]+]] = zext i8 {{%[0-9]+}} to i32
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i8*, i32, ...) @__enqueue_kernel_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 [[SIZE]])
  enqueue_kernel(default_queue, flags, ndrange,
                 ^(local void *p) {
                   return;
                 },
                 c);

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[AD:%arraydecay[0-9]*]] = getelementptr inbounds [1 x %opencl.clk_event_t*], [1 x %opencl.clk_event_t*]* %event_wait_list2, i32 0, i32 0
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i32, %opencl.clk_event_t**, %opencl.clk_event_t**, i8*, i32, ...) @__enqueue_kernel_events_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i32 2, %opencl.clk_event_t** [[AD]], %opencl.clk_event_t** %clk_event, i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 256)
  enqueue_kernel(default_queue, flags, ndrange, 2, event_wait_list2, &clk_event,
                 ^(local void *p) {
                   return;
                 },
                 256);

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[AD:%arraydecay[0-9]*]] = getelementptr inbounds [1 x %opencl.clk_event_t*], [1 x %opencl.clk_event_t*]* %event_wait_list2, i32 0, i32 0
  // CHECK: [[SIZE:%[0-9]+]] = zext i8 {{%[0-9]+}} to i32
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i32, %opencl.clk_event_t**, %opencl.clk_event_t**, i8*, i32, ...) @__enqueue_kernel_events_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i32 2, %opencl.clk_event_t** [[AD]], %opencl.clk_event_t** %clk_event, i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 [[SIZE]])
  enqueue_kernel(default_queue, flags, ndrange, 2, event_wait_list2, &clk_event,
                 ^(local void *p) {
                   return;
                 },
                 c);

  void (^const block_A)(void) = ^{
    return;
  };
  void (^const block_B)(local void *) = ^(local void *a) {
    return;
  };

  // CHECK: [[BL:%[0-9]+]] = load void ()*, void ()** %block_A
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_work_group_size_impl(i8* [[BL_I8]])
  unsigned size = get_kernel_work_group_size(block_A);
  // CHECK: [[BL:%[0-9]+]] = load void (i8 addrspace(2)*)*, void (i8 addrspace(2)*)** %block_B
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void (i8 addrspace(2)*)* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_work_group_size_impl(i8* [[BL_I8]])
  size = get_kernel_work_group_size(block_B);
  // CHECK: [[BL:%[0-9]+]] = load void ()*, void ()** %block_A
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_preferred_work_group_multiple_impl(i8* [[BL_I8]])
  size = get_kernel_preferred_work_group_size_multiple(block_A);
  // CHECK: [[BL:%[0-9]+]] = load void (i8 addrspace(2)*)*, void (i8 addrspace(2)*)* addrspace(1)* @block_G
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void (i8 addrspace(2)*)* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_preferred_work_group_multiple_impl(i8* [[BL_I8]])
  size = get_kernel_preferred_work_group_size_multiple(block_G);
}

// RUN: %clang_cc1 %s -cl-std=CL2.0 -ffake-address-space-map -O0 -emit-llvm -o - | FileCheck %s

typedef void (^bl_t)(local void *);

const bl_t block_G = (bl_t) ^ (local void *a) {};

kernel void device_side_enqueue(global int *a, global int *b, int i) {
  // CHECK: %deafault_queue = alloca %opencl.queue_t*
  queue_t default_queue;
  // CHECK: %flags = alloca i32
  unsigned flags = 0;
  // CHECK: %ndrange = alloca %opencl.ndrange_t*
  ndrange_t ndrange;
  // CHECK: %clk_event = alloca %opencl.clk_event_t*
  clk_event_t clk_event;
  // CHECK: %event_wait_list = alloca %opencl.clk_event_t*
  clk_event_t event_wait_list;
  // CHECK: %event_wait_list2 = alloca [1 x %opencl.clk_event_t*]
  clk_event_t event_wait_list2[] = {clk_event};

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[BL:%[0-9]+]] = bitcast <{ i8*, i32, i32, i8*, %struct.__block_descriptor addrspace(3)*, i32{{.*}}, i32{{.*}}, i32{{.*}} }>* %block to void ()*
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__enqueue_kernel_basic(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i8* [[BL_I8]])
  enqueue_kernel(default_queue, flags, ndrange,
                 ^(void) {
                   a[i] = b[i];
                 });

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[BL:%[0-9]+]] = bitcast <{ i8*, i32, i32, i8*, %struct.__block_descriptor addrspace(3)*, i32{{.*}}, i32{{.*}}, i32{{.*}} }>* %block3 to void ()*
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__enqueue_kernel_basic_events(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i32 2, %opencl.clk_event_t** %event_wait_list, %opencl.clk_event_t** %clk_event, i8* [[BL_I8]])
  enqueue_kernel(default_queue, flags, ndrange, 2, &event_wait_list, &clk_event,
                 ^(void) {
                   a[i] = b[i];
                 });

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i8*, i32, ...) @__enqueue_kernel_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 256)
  enqueue_kernel(default_queue, flags, ndrange,
                 ^(local void *p) {
                   return;
                 },
                 256);
  char c;
  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[SIZE:%[0-9]+]] = zext i8 {{%[0-9]+}} to i32
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i8*, i32, ...) @__enqueue_kernel_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 [[SIZE]])
  enqueue_kernel(default_queue, flags, ndrange,
                 ^(local void *p) {
                   return;
                 },
                 c);

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[AD:%arraydecay[0-9]*]] = getelementptr inbounds [1 x %opencl.clk_event_t*], [1 x %opencl.clk_event_t*]* %event_wait_list2, i32 0, i32 0
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i32, %opencl.clk_event_t**, %opencl.clk_event_t**, i8*, i32, ...) @__enqueue_kernel_events_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i32 2, %opencl.clk_event_t** [[AD]], %opencl.clk_event_t** %clk_event, i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 256)
  enqueue_kernel(default_queue, flags, ndrange, 2, event_wait_list2, &clk_event,
                 ^(local void *p) {
                   return;
                 },
                 256);

  // CHECK: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t*, %opencl.queue_t** %default_queue
  // CHECK: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
  // CHECK: [[NDR:%[0-9]+]] = load %opencl.ndrange_t*, %opencl.ndrange_t** %ndrange
  // CHECK: [[AD:%arraydecay[0-9]*]] = getelementptr inbounds [1 x %opencl.clk_event_t*], [1 x %opencl.clk_event_t*]* %event_wait_list2, i32 0, i32 0
  // CHECK: [[SIZE:%[0-9]+]] = zext i8 {{%[0-9]+}} to i32
  // CHECK: call i32 (%opencl.queue_t*, i32, %opencl.ndrange_t*, i32, %opencl.clk_event_t**, %opencl.clk_event_t**, i8*, i32, ...) @__enqueue_kernel_events_vaargs(%opencl.queue_t* [[DEF_Q]], i32 [[FLAGS]], %opencl.ndrange_t* [[NDR]], i32 2, %opencl.clk_event_t** [[AD]], %opencl.clk_event_t** %clk_event, i8* bitcast ({ i8**, i32, i32, i8*, %struct.__block_descriptor addrspace(3)* }* @__block_literal_global{{(.[0-9]+)?}} to i8*), i32 1, i32 [[SIZE]])
  enqueue_kernel(default_queue, flags, ndrange, 2, event_wait_list2, &clk_event,
                 ^(local void *p) {
                   return;
                 },
                 c);

  void (^const block_A)(void) = ^{
    return;
  };
  void (^const block_B)(local void *) = ^(local void *a) {
    return;
  };

  // CHECK: [[BL:%[0-9]+]] = load void ()*, void ()** %block_A
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_work_group_size_impl(i8* [[BL_I8]])
  unsigned size = get_kernel_work_group_size(block_A);
  // CHECK: [[BL:%[0-9]+]] = load void (i8 addrspace(2)*)*, void (i8 addrspace(2)*)** %block_B
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void (i8 addrspace(2)*)* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_work_group_size_impl(i8* [[BL_I8]])
  size = get_kernel_work_group_size(block_B);
  // CHECK: [[BL:%[0-9]+]] = load void ()*, void ()** %block_A
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void ()* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_preferred_work_group_multiple_impl(i8* [[BL_I8]])
  size = get_kernel_preferred_work_group_size_multiple(block_A);
  // CHECK: [[BL:%[0-9]+]] = load void (i8 addrspace(2)*)*, void (i8 addrspace(2)*)* addrspace(1)* @block_G
  // CHECK: [[BL_I8:%[0-9]+]] = bitcast void (i8 addrspace(2)*)* [[BL]] to i8*
  // CHECK: call i32 @__get_kernel_preferred_work_group_multiple_impl(i8* [[BL_I8]])
  size = get_kernel_preferred_work_group_size_multiple(block_G);
}


Reduce size_t usage some more

Reduce size_t usage some more

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, uint subblock, uint hash_lane,
       uint bw, uint bh, uint dx, uint dy, uint offset)
{
    uint index[4];
    for (uint i = 0; i < 4; i++) {
        uint bpos = (hash_lane + i * offset) % 4;
        uint x = (subblock * dy + i * dx) * bw + bpos % bw;
        uint y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, uint thread)
{
    uint subblock = (thread >> 2) & 0x7;
    uint hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    uint thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif

__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    uint lane = get_global_id(1);
    uint thread = (uint)get_global_id(2);

    uint lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 3:
        thread_input = lanes * lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
        break;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    uint skip = 2;
    for (uint pass = 0; pass < passes; ++pass) {
        for (uint slice = 0; slice < ARGON2_SYNC_POINTS; ++slice) {
            for (uint offset = 0; offset < segment_blocks; ++offset) {
                if (skip > 0) {
                    --skip;
                    continue;
                }

                uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
                uint addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
                if (addr_index == 0) {
                    if (thread == 6) {
                        ++thread_input;
                    }
                    next_addresses(thread_input, addr, curr, thread);
                }
                uint addr_index_x = addr_index % 16;
                uint addr_index_y = addr_index / 16;
                addr_index = addr_index_y * 16 +
                        (addr_index_x + (addr_index_y / 2) * 4) % 16;
                pseudo_rand_lo = addr->lo[addr_index];
                pseudo_rand_hi = addr->hi[addr_index];
#else
                pseudo_rand_lo = prev->lo[0];
                pseudo_rand_hi = prev->hi[0];
#endif

                uint ref_lane = pseudo_rand_hi % lanes;

                uint base;
                if (pass != 0) {
                    base = lane_blocks - segment_blocks;
                } else {
                    if (slice == 0) {
                        ref_lane = lane;
                    }
                    base = slice * segment_blocks;
                }

                uint ref_area_size = base + offset - 1;
                if (ref_lane != lane) {
                    ref_area_size = min(ref_area_size, base);
                }

                uint ref_index = pseudo_rand_lo;
                ref_index = mul_hi(ref_index, ref_index);
                ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

                if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                    ref_index += (slice + 1) * segment_blocks;
                    ref_index %= lane_blocks;
                }

                __global struct block_g *mem_ref = memory +
                        ref_lane * lane_blocks + ref_index;

                /* NOTE: no need to wrap fill_block in barriers, since
                 * it starts & ends in 'nicely parallel' memory operations
                 * like we do in this loop (IOW: this thread only depends on
                 * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
                fill_block(mem_ref, prev, curr, thread);
#else
                if (pass != 0) {
                    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                        ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                        curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                        curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                    }

                    fill_block_xor(mem_ref, prev, curr, thread);
                } else {
                    fill_block(mem_ref, prev, curr, thread);
                }
#endif

                for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                    uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                         curr->lo[i * THREADS_PER_LANE + pos_l]);
                    mem_curr->data[i * THREADS_PER_LANE + thread] = out;
                }

                /* swap curr and prev buffers: */
                __local struct block_l *tmp = curr;
                curr = prev;
                prev = tmp;

                ++mem_curr;
            }

            barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
            if (thread == 2) {
                ++thread_input;
            }
            if (thread == 6) {
                thread_input = 0;
            }
#endif
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0;
        }
#endif
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, uint subblock, uint hash_lane,
       uint bw, uint bh, uint dx, uint dy, uint offset)
{
    uint index[4];
    for (uint i = 0; i < 4; i++) {
        uint bpos = (hash_lane + i * offset) % 4;
        uint x = (subblock * dy + i * dx) * bw + bpos % bw;
        uint y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, uint thread)
{
    uint subblock = (thread >> 2) & 0x7;
    uint hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    uint thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif

__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    uint thread = (uint)get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 3:
        thread_input = lanes * (uint)lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
        break;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    uint skip = 2;
    for (uint pass = 0; pass < passes; ++pass) {
        for (uint slice = 0; slice < ARGON2_SYNC_POINTS; ++slice) {
            for (uint offset = 0; offset < segment_blocks; ++offset) {
                if (skip > 0) {
                    --skip;
                    continue;
                }

                uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
                uint addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
                if (addr_index == 0) {
                    if (thread == 6) {
                        ++thread_input;
                    }
                    next_addresses(thread_input, addr, curr, thread);
                }
                uint addr_index_x = addr_index % 16;
                uint addr_index_y = addr_index / 16;
                addr_index = addr_index_y * 16 +
                        (addr_index_x + (addr_index_y / 2) * 4) % 16;
                pseudo_rand_lo = addr->lo[addr_index];
                pseudo_rand_hi = addr->hi[addr_index];
#else
                pseudo_rand_lo = prev->lo[0];
                pseudo_rand_hi = prev->hi[0];
#endif

                uint ref_lane = pseudo_rand_hi % lanes;

                uint base;
                if (pass != 0) {
                    base = lane_blocks - segment_blocks;
                } else {
                    if (slice == 0) {
                        ref_lane = lane;
                    }
                    base = slice * segment_blocks;
                }

                uint ref_area_size = base + offset - 1;
                if (ref_lane != lane) {
                    ref_area_size = min(ref_area_size, base);
                }

                uint ref_index = pseudo_rand_lo;
                ref_index = mul_hi(ref_index, ref_index);
                ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

                if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                    ref_index += (slice + 1) * segment_blocks;
                    ref_index %= lane_blocks;
                }

                __global struct block_g *mem_ref = memory +
                        ref_lane * lane_blocks + ref_index;

                /* NOTE: no need to wrap fill_block in barriers, since
                 * it starts & ends in 'nicely parallel' memory operations
                 * like we do in this loop (IOW: this thread only depends on
                 * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
                fill_block(mem_ref, prev, curr, thread);
#else
                if (pass != 0) {
                    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                        ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                        curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                        curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                    }

                    fill_block_xor(mem_ref, prev, curr, thread);
                } else {
                    fill_block(mem_ref, prev, curr, thread);
                }
#endif

                for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                    uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                         curr->lo[i * THREADS_PER_LANE + pos_l]);
                    mem_curr->data[i * THREADS_PER_LANE + thread] = out;
                }

                /* swap curr and prev buffers: */
                __local struct block_l *tmp = curr;
                curr = prev;
                prev = tmp;

                ++mem_curr;
            }

            barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
            if (thread == 2) {
                ++thread_input;
            }
            if (thread == 6) {
                thread_input = 0;
            }
#endif
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0;
        }
#endif
        mem_curr = mem_lane;
    }
}