Fix bug in Backward Pooling kernel

Fix bug in Backward Pooling kernel

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint
//#define MASK_Y_SHFT (sizeof(_INT_MASK_GLOBAL) / 2)
//#define MASK_X_MASK ((1 << MASK_Y_SHFT) - 1)

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLBWD_GROUP_SZ2 1

#define MLO_POOLBWD_LCL_DATA_WIDTH  ((MLO_POOLBWD_GROUP_SZ0 *MLO_POOLBWD_N_HORIZ_OUT_PIX + MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 2) / MLO_POOLING_STRIDE0)
#define MLO_POOLBWD_LCL_DATA_HEIGHT ((MLO_POOLBWD_GROUP_SZ1 *MLO_POOLBWD_N_VERT_OUT_PIX  + MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 2) / MLO_POOLING_STRIDE1)

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingAveBwd(
       const __global _FLOAT * top_diff,
        __global _FLOAT * bot_diff
	   )
{
		__local _FLOAT lcl_top_diff[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLBWD_GROUP_LG2SZ1) + lcl_id0;
		int ob = get_global_id(2); // outputs * batch_sz
		int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;

		int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0)/ MLO_POOLING_STRIDE0 + 1;
		int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
		int top_off = b * MLO_POOLBWD_TOP_BATCH_STRIDE + o * MLO_POOLBWD_TOP_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				res[k][l] = 0;
			}
		}

// load tile
		for( int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
		{
			int top_y_act = top_y + tj;
			int top_y_off = top_y_act * MLO_POOLBWD_TOP_STRIDE;

			int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

			bool invisibleY = (top_y_act >= MLO_POOLBWD_TOP_HEIGHT);

			for(int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
			{

				int top_x_act = top_x + ti;

				bool invisibleX = (top_x_act >= MLO_POOLBWD_TOP_WIDTH);

				_FLOAT top_val = top_diff[top_off + top_y_off + top_x_act];

				top_val = (invisibleX || invisibleY)? 0 : top_val;

				lcl_top_diff[lcl_off_v + ti] = top_val;
#if 0
				if (lcl_id1==0&&o==0&&b==0)
				{
				  printf("K:in: %d %d %d   %f\n", top_off + top_y_off + top_x_act, top_y_act, top_x_act, top_val);
				}
#endif

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int bot_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
		int bot_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);

		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{

			int h = bot_y + k + MLO_POOLING_PAD1;
			int top_hstart = (h < MLO_POOLING_KERNEL_SZ1) ? 0 : (h - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
			int top_hend = min(h / MLO_POOLING_STRIDE1 + 1, MLO_POOLBWD_TOP_HEIGHT);

			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{

				int	w = bot_x + l + MLO_POOLING_PAD0;
				int top_wstart = (w < MLO_POOLING_KERNEL_SZ0) ? 0 : (w - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
				int top_wend = min(w / MLO_POOLING_STRIDE0 + 1, MLO_POOLBWD_TOP_WIDTH);

				for (int top_h = top_hstart; top_h < top_hend; ++top_h)
				{
				    int hstart = top_h * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
					int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLBWD_BOT_HEIGHT + MLO_POOLING_PAD1);

					for (int top_w =top_wstart; top_w < top_wend; ++top_w)
					{
        // figure out the pooling size
						int wstart = top_w * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
						int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLBWD_BOT_WIDTH + MLO_POOLING_PAD0);
						int pool_size = (hend - hstart) * (wend - wstart);
						int lcl_top_h = top_h - top_y;
						int lcl_top_w = top_w - top_x;
						_FLOAT add_val = (lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w] / (_FLOAT)pool_size);
						res[k][l] += add_val;
#if 0
				if (bot_x+l==6&&bot_y+k==0&&o==3&&b==0)
				{
				  printf("K:com: %d %d %d %d %d %d   %10.8f %10.8f %10.8f %d\n", k,l,top_h, top_w, lcl_top_h, lcl_top_w, res[k][l], add_val, lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w], pool_size);
				}
#endif
					}
				}

			}
		}

		int bot_off = b * MLO_POOLBWD_BOT_BATCH_STRIDE + o * MLO_POOLBWD_BOT_CHANNEL_STRIDE + bot_y * MLO_POOLBWD_BOT_STRIDE + bot_x;
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				if (bot_y + k < MLO_POOLBWD_BOT_HEIGHT && bot_x + l < MLO_POOLBWD_BOT_WIDTH)
				{
					bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l] = res[k][l];
#if 0
					if (lcl_id0==0&&lcl_id1==0&&o==0&&b==0)
					{
						printf("K:out: %d %d %d  %f\n", bot_off + k * MLO_POOLBWD_BOT_STRIDE +l, k, l, bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l]);
					}
#endif

				}
			}
		}

}

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingMaxBwd(
       const __global _FLOAT * top_df,
       __global _FLOAT * bot_df,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
	__local _FLOAT lcl_top_df[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];
	__local _INT_MASK_LOCAL lcl_mask[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

	int gid0 = get_group_id(0);
	int gid1 = get_group_id(1);
	int x = gid0 * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
	int y = gid1 * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
	int lcl_id0 = get_local_id(0);
	int lcl_id1 = get_local_id(1);
	int ob = get_global_id(2); // outputs * batch_sz
	int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
	int o = ob - b * MLO_POOLING_N_OUTPUTS;

	int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
	int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
	int top_df_off = b * MLO_POOLBWD_TOPDF_BATCH_STRIDE + o * MLO_POOLBWD_TOPDF_CHANNEL_STRIDE;

	_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];

	// load tiles
	// top df and top
	for (int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
	{
		int top_y_act = top_y + tj;
		int top_df_y_off = top_y_act * MLO_POOLBWD_TOPDF_STRIDE;

		int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

		bool visibleY = (top_y_act < MLO_POOLBWD_TOP_HEIGHT);

		for (int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
		{

			int top_x_act = top_x + ti;

			bool visible = visibleY && (top_x_act < MLO_POOLBWD_TOP_WIDTH);
			int idx = visible ? (top_df_off + top_df_y_off + top_x_act) : 0;

			_FLOAT top_df_val = top_df[idx];
			_INT_MASK_LOCAL mask_val = mask[idx];
			//top_df_val *= visible;
			mask_val = visible ? mask_val : 0xFFFFFFFF;

			int lcl_idx = lcl_off_v + ti;

			lcl_top_df[lcl_idx] = top_df_val;
			lcl_mask[lcl_idx] = mask_val;

		}
	}

	barrier(CLK_LOCAL_MEM_FENCE);

	int bt_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
	int bt_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);
	int b_idx = bt_y * MLO_POOLBWD_BOT_WIDTH + bt_x;

	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{
		int b_y = bt_y + k;

		// top most top y that can be influenced by this bot y
		int tt_y = (b_y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1) / MLO_POOLING_STRIDE1;
		tt_y = max(0, tt_y);

		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			int	b_x = bt_x + l;
			// left most top x that can be influenced by this bot x
			int lt_x = (b_x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0) / MLO_POOLING_STRIDE0;
			lt_x = max(0, lt_x);

			// find and sum up all tops that have been influenced by particular bot
			res[k][l] = 0;

			for (int th = tt_y; th < tt_y + (MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 1) / MLO_POOLING_STRIDE1; ++th)
			{
				for (int tw = lt_x; tw < lt_x + (MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 1) / MLO_POOLING_STRIDE0; ++tw)
				{
					int lcl_th = th - top_y;
					int lcl_tw = tw - top_x;

					// note, that b_idx == b_y * MLO_POOLBWD_BOT_WIDTH + b_x
					// computing b_idx instead of using (b_y * MLO_POOLBWD_BOT_WIDTH + b_x) saves VGPR
					bool visible = (lcl_th < MLO_POOLBWD_LCL_DATA_HEIGHT) && (lcl_tw < MLO_POOLBWD_LCL_DATA_WIDTH);
					int lcl_idx = visible ? (lcl_th * MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw) : 0;

					bool match = (b_idx == lcl_mask[lcl_idx]) && visible;

					_FLOAT add_val = lcl_top_df[lcl_idx] * match;
					res[k][l] += add_val;

				}
			}
			b_idx++;
		}
		b_idx += MLO_POOLBWD_BOT_WIDTH - MLO_POOLBWD_N_HORIZ_OUT_PIX;
	}

	int bot_df_off = b * MLO_POOLBWD_BOTDF_BATCH_STRIDE + o * MLO_POOLBWD_BOTDF_CHANNEL_STRIDE + bt_y * MLO_POOLBWD_BOTDF_STRIDE + bt_x;
	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{
		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			if ((bt_y + k) < MLO_POOLBWD_BOT_HEIGHT && (bt_x + l) < MLO_POOLBWD_BOT_WIDTH)
			{
				bot_df[bot_df_off + k * MLO_POOLBWD_BOTDF_STRIDE + l] = res[k][l];
			}
		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint
//#define MASK_Y_SHFT (sizeof(_INT_MASK_GLOBAL) / 2)
//#define MASK_X_MASK ((1 << MASK_Y_SHFT) - 1)

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLBWD_GROUP_SZ2 1

#define MLO_POOLBWD_LCL_DATA_WIDTH (MLO_POOLBWD_GROUP_SZ0 *MLO_POOLBWD_N_HORIZ_OUT_PIX + MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 2) / MLO_POOLING_STRIDE0
#define MLO_POOLBWD_LCL_DATA_HEIGHT (MLO_POOLBWD_GROUP_SZ1 *MLO_POOLBWD_N_VERT_OUT_PIX  + MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 2) / MLO_POOLING_STRIDE1

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingAveBwd(
       const __global _FLOAT * top_diff,
        __global _FLOAT * bot_diff
	   )
{
		__local _FLOAT lcl_top_diff[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLBWD_GROUP_LG2SZ1) + lcl_id0;
		int ob = get_global_id(2); // outputs * batch_sz
		int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;

		int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0)/ MLO_POOLING_STRIDE0 + 1;
		int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
		int top_off = b * MLO_POOLBWD_TOP_BATCH_STRIDE + o * MLO_POOLBWD_TOP_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				res[k][l] = 0;
			}
		}

// load tile
		for( int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
		{
			int top_y_act = top_y + tj;
			int top_y_off = top_y_act * MLO_POOLBWD_TOP_STRIDE;

			int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

			bool invisibleY = (top_y_act >= MLO_POOLBWD_TOP_HEIGHT);

			for(int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
			{

				int top_x_act = top_x + ti;

				bool invisibleX = (top_x_act >= MLO_POOLBWD_TOP_WIDTH);

				_FLOAT top_val = top_diff[top_off + top_y_off + top_x_act];

				top_val = (invisibleX || invisibleY)? 0 : top_val;

				lcl_top_diff[lcl_off_v + ti] = top_val;
#if 0
				if (lcl_id1==0&&o==0&&b==0)
				{
				  printf("K:in: %d %d %d   %f\n", top_off + top_y_off + top_x_act, top_y_act, top_x_act, top_val);
				}
#endif

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int bot_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
		int bot_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);

		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{

			int h = bot_y + k + MLO_POOLING_PAD1;
			int top_hstart = (h < MLO_POOLING_KERNEL_SZ1) ? 0 : (h - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
			int top_hend = min(h / MLO_POOLING_STRIDE1 + 1, MLO_POOLBWD_TOP_HEIGHT);

			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{

				int	w = bot_x + l + MLO_POOLING_PAD0;
				int top_wstart = (w < MLO_POOLING_KERNEL_SZ0) ? 0 : (w - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
				int top_wend = min(w / MLO_POOLING_STRIDE0 + 1, MLO_POOLBWD_TOP_WIDTH);

				for (int top_h = top_hstart; top_h < top_hend; ++top_h)
				{
				    int hstart = top_h * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
					int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLBWD_BOT_HEIGHT + MLO_POOLING_PAD1);

					for (int top_w =top_wstart; top_w < top_wend; ++top_w)
					{
        // figure out the pooling size
						int wstart = top_w * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
						int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLBWD_BOT_WIDTH + MLO_POOLING_PAD0);
						int pool_size = (hend - hstart) * (wend - wstart);
						int lcl_top_h = top_h - top_y;
						int lcl_top_w = top_w - top_x;
						_FLOAT add_val = (lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w] / (_FLOAT)pool_size);
						res[k][l] += add_val;
#if 0
				if (bot_x+l==6&&bot_y+k==0&&o==3&&b==0)
				{
				  printf("K:com: %d %d %d %d %d %d   %10.8f %10.8f %10.8f %d\n", k,l,top_h, top_w, lcl_top_h, lcl_top_w, res[k][l], add_val, lcl_top_diff[lcl_top_h *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_top_w], pool_size);
				}
#endif
					}
				}

			}
		}

		int bot_off = b * MLO_POOLBWD_BOT_BATCH_STRIDE + o * MLO_POOLBWD_BOT_CHANNEL_STRIDE + bot_y * MLO_POOLBWD_BOT_STRIDE + bot_x;
		for( int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
			{
				if (bot_y + k < MLO_POOLBWD_BOT_HEIGHT && bot_x + l < MLO_POOLBWD_BOT_WIDTH)
				{
					bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l] = res[k][l];
#if 0
					if (lcl_id0==0&&lcl_id1==0&&o==0&&b==0)
					{
						printf("K:out: %d %d %d  %f\n", bot_off + k * MLO_POOLBWD_BOT_STRIDE +l, k, l, bot_diff[bot_off + k * MLO_POOLBWD_BOT_STRIDE +l]);
					}
#endif

				}
			}
		}

}

__attribute__((reqd_work_group_size(MLO_POOLBWD_GROUP_SZ0,MLO_POOLBWD_GROUP_SZ1,MLO_POOLBWD_GROUP_SZ2)))
__kernel void mloPoolingMaxBwd(
       const __global _FLOAT * top_df,
       __global _FLOAT * bot_df,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
	// BUG!!!!
	//__local _FLOAT lcl_top_df[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];
	__local _FLOAT lcl_top_df[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT + 32]; //magic number to workaround bug
	__local _INT_MASK_LOCAL lcl_mask[MLO_POOLBWD_LCL_DATA_WIDTH * MLO_POOLBWD_LCL_DATA_HEIGHT];

	int gid0 = get_group_id(0);
	int gid1 = get_group_id(1);
	int x = gid0 * MLO_POOLBWD_GROUP_SZ0 * MLO_POOLBWD_N_HORIZ_OUT_PIX;
	int y = gid1 * MLO_POOLBWD_GROUP_SZ1 * MLO_POOLBWD_N_VERT_OUT_PIX;
	int lcl_id0 = get_local_id(0);
	int lcl_id1 = get_local_id(1);
	int ob = get_global_id(2); // outputs * batch_sz
	int b = (int)(float)ob / (float)MLO_POOLING_N_OUTPUTS;
	int o = ob - b * MLO_POOLING_N_OUTPUTS;

	int top_x = (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) < 0 ? 0 : (x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0) / MLO_POOLING_STRIDE0 + 1;
	int top_y = (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) < 0 ? 0 : (y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1) / MLO_POOLING_STRIDE1 + 1;
	int top_df_off = b * MLO_POOLBWD_TOPDF_BATCH_STRIDE + o * MLO_POOLBWD_TOPDF_CHANNEL_STRIDE;

	_FLOAT res[MLO_POOLBWD_N_VERT_OUT_PIX][MLO_POOLBWD_N_HORIZ_OUT_PIX];

	// load tiles
	// top df and top
	for (int tj = lcl_id1; tj < MLO_POOLBWD_LCL_DATA_HEIGHT; tj += MLO_POOLBWD_GROUP_SZ1)
	{
		int top_y_act = top_y + tj;
		int top_df_y_off = top_y_act * MLO_POOLBWD_TOPDF_STRIDE;

		int lcl_off_v = tj * MLO_POOLBWD_LCL_DATA_WIDTH;

		bool visibleY = (top_y_act < MLO_POOLBWD_TOP_HEIGHT);

		for (int ti = lcl_id0; ti < MLO_POOLBWD_LCL_DATA_WIDTH; ti += MLO_POOLBWD_GROUP_SZ0)
		{

			int top_x_act = top_x + ti;

			bool visible = visibleY && (top_x_act < MLO_POOLBWD_TOP_WIDTH);
			int idx = visible ? (top_df_off + top_df_y_off + top_x_act) : 0;

			_FLOAT top_df_val = top_df[idx];
			_INT_MASK_LOCAL mask_val = mask[idx];
			//top_df_val *= visible;
			mask_val = visible ? mask_val : 0xFFFFFFFF;

			int lcl_idx = lcl_off_v + ti;

#if 0
			if (ob == 0 && gid0 == 1 && gid1 == 0 && lcl_idx==0)
			{
				int inv = !visible;
				printf("ti%i tj%i topi%i lidx%i mask%i inv%i val%13.11f\n",
					ti, tj, top_x_act, lcl_idx, mask_val, inv, top_df_val);
			}
#endif

			lcl_top_df[lcl_idx] = top_df_val;
			lcl_mask[lcl_idx] = mask_val;

#if 0
			if (ob == 0 && gid0 == 1 && gid1 == 0 && lcl_idx == 0)
			{
				printf("lcl_mask[lcl_idx] = mask_val; lcl_idx=%i mask=%i lcl_mask[lcl_idx]=%i\n",
					lcl_idx, mask_val, lcl_mask[0]);
			}
#endif
		}
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0
	if (ob == 0 && gid0 == 1 && gid1 == 0 && lcl_id0 == 0 && lcl_id1 == 0)
	{
		printf("lcl_mask[0]=%i (%13.11f)\n", lcl_mask[0], lcl_mask[0]);
	}
#endif

	int bt_y = (y + lcl_id1 * MLO_POOLBWD_N_VERT_OUT_PIX);
	int bt_x = (x + lcl_id0 * MLO_POOLBWD_N_HORIZ_OUT_PIX);
	int b_idx = bt_y * MLO_POOLBWD_BOT_WIDTH + bt_x;

	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{
		int b_y = bt_y + k;

		// top most top y that can be influenced by this bot y
		int tt_y = (b_y + MLO_POOLING_PAD1 - MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1) / MLO_POOLING_STRIDE1;
		tt_y = max(0, tt_y);

		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			int	b_x = bt_x + l;
			// left most top x that can be influenced by this bot x
			int lt_x = (b_x + MLO_POOLING_PAD0 - MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0) / MLO_POOLING_STRIDE0;
			lt_x = max(0, lt_x);

			// find and sum up all tops that have been influenced by particular bot
			res[k][l] = 0;

#if 0
			if (ob == 0 && b_x == 26 && b_y == 1)
			{
				printf("ltx%i tty%i bt_x%i bt_y%i top_x%i top_y%i\n",
					lt_x, tt_y, bt_x, bt_y, top_x, top_y);
			}
#endif
			for (int th = tt_y; th < tt_y + (MLO_POOLING_KERNEL_SZ1 + MLO_POOLING_STRIDE1 - 1) / MLO_POOLING_STRIDE1; ++th)
			{
				for (int tw = lt_x; tw < lt_x + (MLO_POOLING_KERNEL_SZ0 + MLO_POOLING_STRIDE0 - 1) / MLO_POOLING_STRIDE0; ++tw)
				{
					int lcl_th = th - top_y;
					int lcl_tw = tw - top_x;

					// note, that b_idx == b_y * MLO_POOLBWD_BOT_WIDTH + b_x
					// computing b_idx instead of using (b_y * MLO_POOLBWD_BOT_WIDTH + b_x) saves VGPR
					bool match = (b_idx == lcl_mask[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw]);

					_FLOAT add_val = lcl_top_df[lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw] * match;
					res[k][l] += add_val;
#if 0
					if (ob == 0 && b_x == 26 && b_y == 1)
					{
						int lidx = lcl_th *  MLO_POOLBWD_LCL_DATA_WIDTH + lcl_tw;
						int mask = lcl_mask[lidx];
						_FLOAT val = lcl_top_df[lidx];
						//printf("h=%d (%d - %i) w=%i (%i - %i) x%i y%i mx: %i my: %i topy %i\n",
						//	lcl_th, th, top_y, lcl_tw, tw, top_x, get_group_id(0), get_group_id(1), mx, my,
						//	res[k][l], top_y
						//);
						printf("th%i tw%i lh%i lw%i b_idx%i lidx%i mask%i v%13.11f\n",
							th, tw, lcl_th, lcl_tw, b_idx, lidx, mask, val);
					}
#endif
				}
			}
			b_idx++;
		}
		b_idx += MLO_POOLBWD_BOT_WIDTH - MLO_POOLBWD_N_HORIZ_OUT_PIX;
	}

	int bot_df_off = b * MLO_POOLBWD_BOTDF_BATCH_STRIDE + o * MLO_POOLBWD_BOTDF_CHANNEL_STRIDE + bt_y * MLO_POOLBWD_BOTDF_STRIDE + bt_x;
	for (int k = 0; k < MLO_POOLBWD_N_VERT_OUT_PIX; k++)
	{
		for (int l = 0; l < MLO_POOLBWD_N_HORIZ_OUT_PIX; l++)
		{
			if ((bt_y + k) < MLO_POOLBWD_BOT_HEIGHT && (bt_x + l) < MLO_POOLBWD_BOT_WIDTH)
			{
				bot_df[bot_df_off + k * MLO_POOLBWD_BOTDF_STRIDE + l] = res[k][l];
#if 0
				if (b == 0 && o == 0 && bt_x + l == 9 && bt_y + k == 0)
				{
					printf("K:max: %d %d   %13.11f\n",
						k, l,
						res[k][l]
					);
				}
#endif
			}
		}
	}

}


reduced number of ALU instructions and number of GPR from 7 to 6

#pragma OPENCL EXTENSION cl_khr_3d_image_writes : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVFInit(__read_only image3d_t volume, __write_only image3d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    float f100 = read_imagef(volume, sampler, pos + (int4)(1,0,0,0)).x;
    float f_100 = read_imagef(volume, sampler, pos - (int4)(1,0,0,0)).x;
    float f010 = read_imagef(volume, sampler, pos + (int4)(0,1,0,0)).x;
    float f0_10 = read_imagef(volume, sampler, pos - (int4)(0,1,0,0)).x;
    float f001 = read_imagef(volume, sampler, pos + (int4)(0,0,1,0)).x;
    float f00_1 = read_imagef(volume, sampler, pos - (int4)(0,0,1,0)).x;

    float4 gradient = {
        0.5f*(f100-f_100),
        0.5f*(f010-f0_10),
        0.5f*(f001-f00_1),
        0};

    gradient.w = gradient.x*gradient.x + gradient.y*gradient.y + gradient.z*gradient.z;

    write_imagef(vector_field, pos, gradient);
}

__kernel void GVFIteration(__read_only image3d_t init_vector_field, __read_only image3d_t read_vector_field, __write_only image3d_t write_vector_field, __private float mu) {

    int4 writePos = {
        get_global_id(0)-(get_group_id(0)*2+1),
        get_global_id(1)-(get_group_id(1)*2+1),
        get_global_id(2)-(get_group_id(2)*2+1),
        0
    };
    int3 localPos = {get_local_id(0), get_local_id(1), get_local_id(2)};

    // Enforce mirror boundary conditions
    //int4 size = {get_global_size(0), get_global_size(1), get_global_size(2), 0};
    //int4 pos = writePos;
    //pos = select(pos, (int4)(2,2,2,0), pos == (int4)(0,0,0,0));
    //pos = select(pos, size-3, pos == size-1);

    // Allocate shared memory
    __local float3 sharedMemory[6][6][6];

    // Read into shared memory
	float4 v = read_imagef(read_vector_field, sampler, writePos);
    sharedMemory[localPos.x][localPos.y][localPos.z] = (float3)(v.x,v.y,v.z);

    int3 comp = (localPos == (int3)(0,0,0)) +
        (localPos == (int3)(get_local_size(0)-1,get_local_size(1)-1,get_local_size(2)-1));
	char c = comp.x+comp.y+comp.z;
    // Synchronize the threads in the group
    barrier(CLK_LOCAL_MEM_FENCE);

	if(c != 0)
		return;

    // Load data from shared memory and do calculations
    float3 vector = sharedMemory[localPos.x][localPos.y][localPos.z];
    float3 fx1 = sharedMemory[localPos.x+1][localPos.y][localPos.z];
    float3 fx_1 = sharedMemory[localPos.x-1][localPos.y][localPos.z];
    float3 fy1 = sharedMemory[localPos.x][localPos.y+1][localPos.z];
    float3 fy_1 = sharedMemory[localPos.x][localPos.y-1][localPos.z];
    float3 fz1 = sharedMemory[localPos.x][localPos.y][localPos.z+1];
    float3 fz_1 = sharedMemory[localPos.x][localPos.y][localPos.z-1];
    float4 init_vector = read_imagef(init_vector_field, sampler, writePos); // should read from pos

    // Update the vector field: Calculate Laplacian using a 3D central difference scheme
    float3 laplacian = -6*vector + fx1 + fx_1 + fy1 + fy_1 + fz1 + fz_1;

    vector += mu * laplacian - (vector - (float3)(init_vector.x,init_vector.y,init_vector.z))*init_vector.w;

    write_imagef(write_vector_field, writePos, (float4)(vector.x,vector.y,vector.z,0));
}

__kernel void GVFResult(__write_only image3d_t result, __read_only image3d_t vectorField) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.w = 0;
    //if(1 < length(vector))
    //    printf("%f %f %f\n", vector.x, vector.y, vector.z);
    write_imagef(result, pos, length(vector));
}

#pragma OPENCL EXTENSION cl_khr_3d_image_writes : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVFInit(__read_only image3d_t volume, __write_only image3d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    float f100 = read_imagef(volume, sampler, pos + (int4)(1,0,0,0)).x;
    float f_100 = read_imagef(volume, sampler, pos - (int4)(1,0,0,0)).x;
    float f010 = read_imagef(volume, sampler, pos + (int4)(0,1,0,0)).x;
    float f0_10 = read_imagef(volume, sampler, pos - (int4)(0,1,0,0)).x;
    float f001 = read_imagef(volume, sampler, pos + (int4)(0,0,1,0)).x;
    float f00_1 = read_imagef(volume, sampler, pos - (int4)(0,0,1,0)).x;

    float4 gradient = {
        0.5f*(f100-f_100),
        0.5f*(f010-f0_10),
        0.5f*(f001-f00_1),
        0};

    gradient.w = gradient.x*gradient.x + gradient.y*gradient.y + gradient.z*gradient.z;

    write_imagef(vector_field, pos, gradient);
}

__kernel void GVFIteration(__read_only image3d_t init_vector_field, __read_only image3d_t read_vector_field, __write_only image3d_t write_vector_field, __private float mu) {

    int4 writePos = {
        get_global_id(0)-(get_group_id(0)*2+1),
        get_global_id(1)-(get_group_id(1)*2+1),
        get_global_id(2)-(get_group_id(2)*2+1),
        0
    };
    int3 localPos = {get_local_id(0), get_local_id(1), get_local_id(2)};

    // Enforce mirror boundary conditions
    //int4 size = {get_global_size(0), get_global_size(1), get_global_size(2), 0};
    //int4 pos = writePos;
    //pos = select(pos, (int4)(2,2,2,0), pos == (int4)(0,0,0,0));
    //pos = select(pos, size-3, pos == size-1);

    // Allocate shared memory
    __local float4 sharedMemory[6][6][6];

    // Read into shared memory
    sharedMemory[localPos.x][localPos.y][localPos.z] = read_imagef(read_vector_field, sampler, writePos);

    // Synchronize the threads in the group
    barrier(CLK_LOCAL_MEM_FENCE);

    int3 comp = (localPos == (int3)(0,0,0)) +
        (localPos == (int3)(get_local_size(0)-1,get_local_size(1)-1,get_local_size(2)-1));

    if(comp.x+comp.y+comp.z == 0) {
    // Load data from shared memory and do calculations
    float4 vector = sharedMemory[localPos.x][localPos.y][localPos.z];
    float4 fx1 = sharedMemory[localPos.x+1][localPos.y][localPos.z];
    float4 fx_1 = sharedMemory[localPos.x-1][localPos.y][localPos.z];
    float4 fy1 = sharedMemory[localPos.x][localPos.y+1][localPos.z];
    float4 fy_1 = sharedMemory[localPos.x][localPos.y-1][localPos.z];
    float4 fz1 = sharedMemory[localPos.x][localPos.y][localPos.z+1];
    float4 fz_1 = sharedMemory[localPos.x][localPos.y][localPos.z-1];
    float4 init_vector = read_imagef(init_vector_field, sampler, writePos); // should read from pos

    // Update the vector field: Calculate Laplacian using a 3D central difference scheme
    float4 laplacian = -6*vector + fx1 + fx_1 + fy1 + fy_1 + fz1 + fz_1;

    vector += mu * laplacian - (vector - init_vector)*init_vector.w;

    write_imagef(write_vector_field, writePos, vector);
    }
}

__kernel void GVFResult(__write_only image3d_t result, __read_only image3d_t vectorField) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.w = 0;
    //if(1 < length(vector))
    //    printf("%f %f %f\n", vector.x, vector.y, vector.z);
    write_imagef(result, pos, length(vector));
}


Avoid some modulo computations

Avoid some modulo computations

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, uint subblock, uint hash_lane,
       uint bw, uint bh, uint dx, uint dy, uint offset)
{
    uint index[4];
    for (uint i = 0; i < 4; i++) {
        uint bpos = (hash_lane + i * offset) % 4;
        uint x = (subblock * dy + i * dx) * bw + bpos % bw;
        uint y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, uint thread)
{
    uint subblock = (thread >> 2) & 0x7;
    uint hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    uint thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif

__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    uint thread = (uint)get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 3:
        thread_input = lanes * (uint)lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
        break;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    uint skip = 2;
    for (uint pass = 0; pass < passes; ++pass) {
        for (uint slice = 0; slice < ARGON2_SYNC_POINTS; ++slice) {
            for (uint offset = 0; offset < segment_blocks; ++offset) {
                if (skip > 0) {
                    --skip;
                    continue;
                }

                uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
                uint addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
                if (addr_index == 0) {
                    if (thread == 6) {
                        ++thread_input;
                    }
                    next_addresses(thread_input, addr, curr, thread);
                }
                uint addr_index_x = addr_index % 16;
                uint addr_index_y = addr_index / 16;
                addr_index = addr_index_y * 16 +
                        (addr_index_x + (addr_index_y / 2) * 4) % 16;
                pseudo_rand_lo = addr->lo[addr_index];
                pseudo_rand_hi = addr->hi[addr_index];
#else
                pseudo_rand_lo = prev->lo[0];
                pseudo_rand_hi = prev->hi[0];
#endif

                uint ref_lane = pseudo_rand_hi % lanes;

                uint base;
                if (pass != 0) {
                    base = lane_blocks - segment_blocks;
                } else {
                    if (slice == 0) {
                        ref_lane = lane;
                    }
                    base = slice * segment_blocks;
                }

                uint ref_area_size = base + offset - 1;
                if (ref_lane != lane) {
                    ref_area_size = min(ref_area_size, base);
                }

                uint ref_index = pseudo_rand_lo;
                ref_index = mul_hi(ref_index, ref_index);
                ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

                if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                    ref_index += (slice + 1) * segment_blocks;
                    ref_index %= lane_blocks;
                }

                __global struct block_g *mem_ref = memory +
                        ref_lane * lane_blocks + ref_index;

                /* NOTE: no need to wrap fill_block in barriers, since
                 * it starts & ends in 'nicely parallel' memory operations
                 * like we do in this loop (IOW: this thread only depends on
                 * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
                fill_block(mem_ref, prev, curr, thread);
#else
                if (pass != 0) {
                    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                        ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                        curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                        curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                    }

                    fill_block_xor(mem_ref, prev, curr, thread);
                } else {
                    fill_block(mem_ref, prev, curr, thread);
                }
#endif

                for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                    uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                         curr->lo[i * THREADS_PER_LANE + pos_l]);
                    mem_curr->data[i * THREADS_PER_LANE + thread] = out;
                }

                /* swap curr and prev buffers: */
                __local struct block_l *tmp = curr;
                curr = prev;
                prev = tmp;

                ++mem_curr;
            }

            barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
            if (thread == 2) {
                ++thread_input;
            }
            if (thread == 6) {
                thread_input = 0;
            }
#endif
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0;
        }
#endif
        mem_curr = mem_lane;
    }
}

/* C compatibility For dumb IDEs: */
#ifndef __OPENCL_VERSION__
#ifndef __cplusplus
typedef int bool;
#endif
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long size_t;
typedef long ptrdiff_t;
typedef size_t uintptr_t;
typedef ptrdiff_t intptr_t;
#ifndef __kernel
#define __kernel
#endif
#ifndef __global
#define __global
#endif
#ifndef __private
#define __private
#endif
#ifndef __local
#define __local
#endif
#ifndef __constant
#define __constant const
#endif
#endif /* __OPENCL_VERSION__ */

#define ARGON2_D 0
#define ARGON2_I 1

#define ARGON2_VERSION_10 0x10
#define ARGON2_VERSION_13 0x13

#define ARGON2_BLOCK_SIZE 1024
#define ARGON2_QWORDS_IN_BLOCK (ARGON2_BLOCK_SIZE / 8)
#define ARGON2_SYNC_POINTS 4

#define THREADS_PER_LANE 32
#define QWORDS_PER_THREAD (ARGON2_QWORDS_IN_BLOCK / 32)

#ifndef ARGON2_VERSION
#define ARGON2_VERSION ARGON2_VERSION_13
#endif

#ifndef ARGON2_TYPE
#define ARGON2_TYPE ARGON2_I
#endif

#define MASK_32 0xFFFFFFFFUL

#define F(x, y) ((x) + (y) + 2 * upsample( \
    mul_hi((uint)(x), (uint)(y)), \
    (uint)(x) * (uint)(y) \
    ))

#define rotr64(x, n) rotate(x, (ulong)(64 - (n)))

struct block_g {
    ulong data[ARGON2_QWORDS_IN_BLOCK];
};

struct block_l {
    uint lo[ARGON2_QWORDS_IN_BLOCK];
    uint hi[ARGON2_QWORDS_IN_BLOCK];
};

void g(__local struct block_l *block, uint subblock, uint hash_lane,
       uint bw, uint bh, uint dx, uint dy, uint offset)
{
    uint index[4];
    for (uint i = 0; i < 4; i++) {
        uint bpos = (hash_lane + i * offset) % 4;
        uint x = (subblock * dy + i * dx) * bw + bpos % bw;
        uint y = (subblock * dx + i * dy) * bh + bpos / bw;

        index[i] = y * 16 + (x + (y / 2) * 4) % 16;
    }

    ulong a, b, c, d;
    a = upsample(block->hi[index[0]], block->lo[index[0]]);
    b = upsample(block->hi[index[1]], block->lo[index[1]]);
    c = upsample(block->hi[index[2]], block->lo[index[2]]);
    d = upsample(block->hi[index[3]], block->lo[index[3]]);

    a = F(a, b);
    d = rotr64(d ^ a, 32);
    c = F(c, d);
    b = rotr64(b ^ c, 24);
    a = F(a, b);
    d = rotr64(d ^ a, 16);
    c = F(c, d);
    b = rotr64(b ^ c, 63);

    block->lo[index[0]] = (uint)a;
    block->lo[index[1]] = (uint)b;
    block->lo[index[2]] = (uint)c;
    block->lo[index[3]] = (uint)d;

    block->hi[index[0]] = (uint)(a >> 32);
    block->hi[index[1]] = (uint)(b >> 32);
    block->hi[index[2]] = (uint)(c >> 32);
    block->hi[index[3]] = (uint)(d >> 32);
}

void shuffle_block(__local struct block_l *block, uint thread)
{
    uint subblock = (thread >> 2) & 0x7;
    uint hash_lane = (thread >> 0) & 0x3;

    g(block, subblock, hash_lane, 4, 1, 1, 0, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 4, 1, 1, 0, 1);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 0);

    barrier(CLK_LOCAL_MEM_FENCE);

    g(block, subblock, hash_lane, 2, 2, 0, 1, 1);
}

void fill_block(__global const struct block_g *restrict ref_block,
                __local struct block_l *restrict prev_block,
                __local struct block_l *restrict next_block,
                uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] = prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] = prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}

#if ARGON2_VERSION != ARGON2_VERSION_10
void fill_block_xor(__global const struct block_g *restrict ref_block,
                    __local struct block_l *restrict prev_block,
                    __local struct block_l *restrict next_block,
                    uint thread)
{
    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = ref_block->data[i * THREADS_PER_LANE + thread];
        next_block->lo[pos_l] ^= prev_block->lo[pos_l] ^= (uint)in;
        next_block->hi[pos_l] ^= prev_block->hi[pos_l] ^= (uint)(in >> 32);
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(prev_block, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = i * THREADS_PER_LANE +
                (thread & 0x10) + ((thread + i * 4) & 0xf);
        next_block->lo[pos_l] ^= prev_block->lo[pos_l];
        next_block->hi[pos_l] ^= prev_block->hi[pos_l];
    }
}
#endif

#if ARGON2_TYPE == ARGON2_I
void next_addresses(uint thread_input,
                    __local struct block_l *restrict addr,
                    __local struct block_l *restrict tmp,
                    uint thread)
{
    addr->lo[thread] = thread_input;
    addr->hi[thread] = 0;
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->hi[pos] = addr->lo[pos] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    tmp->lo[thread] = addr->lo[thread] ^= thread_input;
    tmp->hi[thread] = addr->hi[thread];
    for (uint i = 1; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        tmp->lo[pos] = addr->lo[pos];
        tmp->hi[pos] = addr->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    shuffle_block(addr, thread);

    barrier(CLK_LOCAL_MEM_FENCE);

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos = i * THREADS_PER_LANE + thread;
        addr->lo[pos] ^= tmp->lo[pos];
        addr->hi[pos] ^= tmp->hi[pos];
    }

    barrier(CLK_LOCAL_MEM_FENCE);
}
#endif

#if ARGON2_TYPE == ARGON2_I
#define SHARED_BLOCKS 3
#else
#define SHARED_BLOCKS 2
#endif

__kernel void argon2_kernel(
        __global struct block_g *memory, __local struct block_l *shared,
        uint passes, uint lanes, uint segment_blocks)
{
    size_t job_id = get_global_id(0);
    size_t lane = get_global_id(1);
    uint thread = (uint)get_global_id(2);

    size_t lane_blocks = ARGON2_SYNC_POINTS * segment_blocks;

    /* select job's memory region: */
    memory += job_id * lanes * lane_blocks;
    /* select lane's shared memory buffer: */
    shared += lane * SHARED_BLOCKS;

    __local struct block_l *restrict curr = &shared[0];
    __local struct block_l *restrict prev = &shared[1];
#if ARGON2_TYPE == ARGON2_I
    __local struct block_l *restrict addr = &shared[2];

    uint thread_input;
    switch (thread) {
    case 1:
        thread_input = lane;
        break;
    case 2:
        thread_input = segment_blocks == 2 ? 1 : 0;
        break;
    case 3:
        thread_input = lanes * (uint)lane_blocks;
        break;
    case 4:
        thread_input = passes;
        break;
    case 5:
        thread_input = ARGON2_I;
        break;
    default:
        thread_input = 0;
    }

    if (segment_blocks > 2) {
        if (thread == 6) {
            ++thread_input;
        }
        next_addresses(thread_input, addr, curr, thread);
    }
#endif

    __global struct block_g *mem_lane = memory + lane * lane_blocks;
    __global struct block_g *mem_prev = mem_lane + 1;
    __global struct block_g *mem_curr = mem_lane + 2;

    for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
        uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
        ulong in = mem_prev->data[i * THREADS_PER_LANE + thread];
        prev->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
        prev->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
    }
    for (uint pass = 0; pass < passes; ++pass) {
        for (uint index = pass == 0 ? 2 : 0; index < lane_blocks; ++index) {
            uint offset = index % segment_blocks;
            uint slice = index / segment_blocks;

            uint pseudo_rand_lo, pseudo_rand_hi;
#if ARGON2_TYPE == ARGON2_I
            uint addr_index = offset % ARGON2_QWORDS_IN_BLOCK;
            if (addr_index == 0) {
                if (thread == 6) {
                    ++thread_input;
                }
                next_addresses(thread_input, addr, curr, thread);
            }
            uint addr_index_x = addr_index % 16;
            uint addr_index_y = addr_index / 16;
            addr_index = addr_index_y * 16 +
                    (addr_index_x + (addr_index_y / 2) * 4) % 16;
            pseudo_rand_lo = addr->lo[addr_index];
            pseudo_rand_hi = addr->hi[addr_index];
#else
            pseudo_rand_lo = prev->lo[0];
            pseudo_rand_hi = prev->hi[0];
#endif

            uint ref_lane = pseudo_rand_hi % lanes;

            uint base;
            if (pass != 0) {
                base = lane_blocks - segment_blocks;
            } else {
                if (slice == 0) {
                    ref_lane = lane;
                }
                base = slice * segment_blocks;
            }

            uint ref_area_size = base + offset - 1;
            if (ref_lane != lane) {
                ref_area_size = min(ref_area_size, base);
            }

            uint ref_index = pseudo_rand_lo;
            ref_index = mul_hi(ref_index, ref_index);
            ref_index = ref_area_size - 1 - mul_hi(ref_area_size, ref_index);

            if (pass != 0 && slice != ARGON2_SYNC_POINTS - 1) {
                ref_index += (slice + 1) * segment_blocks;
                ref_index %= lane_blocks;
            }

            __global struct block_g *mem_ref = memory +
                    ref_lane * lane_blocks + ref_index;

            /* NOTE: no need to wrap fill_block in barriers, since
             * it starts & ends in 'nicely parallel' memory operations
             * like we do in this loop (IOW: this thread only depends on
             * its own data w.r.t. these boundaries) */
#if ARGON2_VERSION == ARGON2_VERSION_10
            fill_block(mem_ref, prev, curr, thread);
#else
            if (pass != 0) {
                for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                    uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                    ulong in = mem_curr->data[i * THREADS_PER_LANE + thread];
                    curr->lo[i * THREADS_PER_LANE + pos_l] = (uint)in;
                    curr->hi[i * THREADS_PER_LANE + pos_l] = (uint)(in >> 32);
                }

                fill_block_xor(mem_ref, prev, curr, thread);
            } else {
                fill_block(mem_ref, prev, curr, thread);
            }
#endif

            for (uint i = 0; i < QWORDS_PER_THREAD; i++) {
                uint pos_l = (thread & 0x10) + ((thread + i * 4) & 0xf);
                ulong out = upsample(curr->hi[i * THREADS_PER_LANE + pos_l],
                                     curr->lo[i * THREADS_PER_LANE + pos_l]);
                mem_curr->data[i * THREADS_PER_LANE + thread] = out;
            }

            /* swap curr and prev buffers: */
            __local struct block_l *tmp = curr;
            curr = prev;
            prev = tmp;

            if ((index + 1) % segment_blocks == 0) {
                barrier(CLK_GLOBAL_MEM_FENCE);
#if ARGON2_TYPE == ARGON2_I
                if (thread == 2) {
                    ++thread_input;
                }
                if (thread == 6) {
                    thread_input = 0UL;
                }
#endif
            }

            ++mem_curr;
        }
#if ARGON2_TYPE == ARGON2_I
        if (thread == 0) {
            ++thread_input;
        }
        if (thread == 2) {
            thread_input = 0UL;
        }
#endif
        mem_curr = mem_lane;
    }
}


Moved kernels directory to src/kernels.

Moved kernels directory to src/kernels.

git-svn-id: 602a26e73ae9f2a14e2bb95676f2e137fcc39919@3062 7c0444ca-5550-0410-9008-faecdbe78eb5

#ifndef PV_USE_OPENCL

#include "../../arch/opencl/pv_opencl.h"

#define EXP expf
#define FMOD fmodf

#else

#define CL_KERNEL     __kernel
#define CL_MEM_GLOBAL __global
#define CL_MEM_LOCAL  __local

#define EXP exp
#define FMOD __fmodf

// This stuff needs to be moved to (or obtained from) separate file
// perhaps conversions.h

#define pvdata_t float
#define PHI_EXC 0       // which phi buffer to use
#define PHI_INH 1       // which phi buffer to use
#define PHI_INHB 2
#define PI              3.1415926535897932

// refactory period for neurons (retina for now)
#define ABS_REFACTORY_PERIOD 3
#define REFACTORY_PERIOD     5

#define PV_RANDOM_MAX       0x7fffffff
#define PV_INV_RANDOM_MAX   (1.0 / (double) PV_RANDOM_MAX)

static inline double pv_random_prob()
{
//   return (double) random() * PV_INV_RANDOM_MAX;
   return 1.0;
}

static inline int kxPos(int k, int nx, int ny, int nf)
{
   return (k/nf) % nx;
}

static inline int kyPos(int k, int nx, int ny, int nf)
{
   return k / (nx*nf);
}

static inline int kIndex(int kx, int ky, int kf, int nx, int ny, int nf)
{
   return kf + (kx + (ky * nx)) * nf;
}

static inline int featureIndex(int k, int nx, int ny, int nf)
{
   return k % nf;
}

static inline int kIndexExtended(int k, int nx, int ny, int nf, int nb)
{
   const int kx_ex = nb + kxPos(k, nx, ny, nf);
   const int ky_ex = nb + kyPos(k, nx, ny, nf);
   const int kf = featureIndex(k, nx, ny, nf);
   return kIndex(kx_ex, ky_ex, kf, nx + 2*nb, ny + 2*nb, nf);
}

#endif // PV_USE_OPENCL

static int spike(float time, float dt, float prev, float probBase, float probStim)
{
   float probSpike;
   float burstStatus = 1;
   float sinAmp = 1.0;

   // input parameters
   // TODO - how to get these into kernel
   //
   const float beginStim = 0.0f;
   const float endStim   = 9000000.0f;
   const float burstFreq = 40.0f;
   const float burstDuration = 7.5f;

   // see if neuron is in a refactory period
   //
   if ((time - prev) < ABS_REFACTORY_PERIOD) {
      return 0;
   }
   else {
      float delta = time - prev - ABS_REFACTORY_PERIOD;
      float refact = 1.0f - EXP(-delta/REFACTORY_PERIOD);
      refact = (refact < 0) ? 0 : refact;
      probBase *= refact;
      probStim *= refact;
   }

   if (burstDuration <= 0 || burstFreq == 0) {
      sinAmp = cos( 2 * PI * time * burstFreq / 1000. );
   }
   else {
      burstStatus = FMOD(time, 1000. / burstFreq);
      burstStatus = burstStatus <= burstDuration;
   }

   burstStatus *= (int) ( (time >= beginStim) && (time < endStim) );
   probSpike = probBase;

   if ((int)burstStatus) {
      probSpike += probStim * sinAmp;  // negative prob is OK
   }
   return ( pv_random_prob() < probSpike );
}

//
// update the state of a retinal layer (spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void update_state (
    const float time,
    const float dt,
    const float probStim,
    const float probBase,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * activity,
    CL_MEM_GLOBAL float * prevTime)
{
#ifndef PV_USE_OPENCL
for (register int k = 0; k < nx*ny*nf; k++) {
#else
   register int k = get_global_id(0);
#endif

   register int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // load local copies
   //
   register float l_phiExc = phiExc[k];
   register float l_phiInh = phiInh[k];
   register float l_prev   = prevTime[kex];
   register float l_activ  = activity[kex];

   l_activ = spike(time, dt, l_prev, probBase, (l_phiExc - l_phiInh)*probStim);
   l_prev  = (l_activ > 0.0f) ? time : l_prev;

   l_phiExc = 0.0f;
   l_phiInh = 0.0f;

   // store local copies
   //
   phiExc[k] = l_phiExc;
   phiInh[k] = l_phiInh;
   prevTime[kex] = l_prev;
   activity[kex] = l_activ;

#ifndef PV_USE_OPENCL
   }
#endif

}


lpr-wrapper.cl: commented out debugging printout statements.

#!/bin/bash

# lpr-wrapper: calls the user's preferred lpr, building the
# argument string from environment variables

opts=$( printenv | grep '^LPR_OPT_' )
space=${IFS:- }
while read opt; do
	opt_key=$( echo "$opt" | sed 's/=.*//' )
	opt_val=$( echo "$opt" | sed 's/.*=//' )
	case "$opt_key" in
		LPR_OPT_p) argstring="${argstring}${space}-p ${opt_val}" ;;
		LPR_OPT_staple) argstring="${argstring}${space}-staple" ;;
		LPR_OPT_printer|LPR_OPT_Printer) argstring="${argstring}${space}-P ${opt_val}" ;;
		LPR_OPT_filename*) #filenames="$filenames"$'\n'"$opt_val" ;;
			;;
		LPR_OPT_fullcmd) ;; # ignore this
		*)	echo "Unrecognised option key: $opt_key" 1>&2 ;;
	esac
done <<<"$opts"

#echo "$filenames" | echo xargs lpr ${argstring}
#printenv 1>&2
#echo "$@" 1>&2
lpr ${argstring} "$@"

#!/bin/bash

# lpr-wrapper: calls the user's preferred lpr, building the
# argument string from environment variables

opts=$( printenv | grep '^LPR_OPT_' )
space=${IFS:- }
while read opt; do
	opt_key=$( echo "$opt" | sed 's/=.*//' )
	opt_val=$( echo "$opt" | sed 's/.*=//' )
	case "$opt_key" in
		LPR_OPT_p) argstring="${argstring}${space}-p ${opt_val}" ;;
		LPR_OPT_staple) argstring="${argstring}${space}-staple" ;;
		LPR_OPT_printer|LPR_OPT_Printer) argstring="${argstring}${space}-P ${opt_val}" ;;
		LPR_OPT_filename*) #filenames="$filenames"$'\n'"$opt_val" ;;
			;;
		LPR_OPT_fullcmd) ;; # ignore this
		*)	echo "Unrecognised option key: $opt_key" 1>&2 ;;
	esac
done <<<"$opts"

#echo "$filenames" | echo xargs lpr ${argstring}
printenv 1>&2
echo "$@" 1>&2
lpr ${argstring} "$@"