corrected 3x3

corrected 3x3

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define DBG_OUT_OF_RNGE 0

#define MLO_HW_WAVE_ID_SETTING 1
// FIXME Conduct enabling from the host code.
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);

#define MLO_IN_LCL_WIDTH (MLO_IN_STRIDE + 2 * MLO_FILTER_PAD0)

static inline int getWaveId()
{
	int wave_id = 0;

#if MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_HSAIL==1
		wave_id = __hsail_get_dynwave_id();
		wave_id &= MLO_N_WAVES_MASK;
#elif MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_LC==1 && MLO_GRP_SZ1==1 && MLO_GRP_SZ2==1 && (MLO_GRP_SZ % (1 << MLO_LG2_WAVE_SZ))==0
		// (local_id/wavesize) has the same value in all workitems.
		// Make it scalar to enable scalarization optimizations.
		wave_id = __llvm_amdgcn_readfirstlane((uint)(get_local_id(0) >> MLO_LG2_WAVE_SZ));
		// Alternate implementation:
		//__asm__ ("v_readfirstlane_b32 %0, %1" : "=s" (wave_id) : "v" ((int)(get_local_id(0) >> MLO_LG2_WAVE_SZ)) );
#else
		wave_id = (get_local_id(0) >> MLO_LG2_WAVE_SZ);
#endif
	return(wave_id);
}

static inline int getWaveLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_WAVE_SZ) << MLO_LG2_WAVE_SZ);;
	return(lcl_wave_id);
}

static inline int getWaveLocalIds(int * lcl_wave_id1, int * lcl_wave_id0)
{
	int lcl_wave_id = getWaveLocalId();
	*lcl_wave_id1 = (lcl_wave_id >> MLO_LG2_WAVE_SZ0);
	*lcl_wave_id0 = lcl_wave_id - ((*lcl_wave_id1) << MLO_LG2_WAVE_SZ0);
	return(lcl_wave_id);
}

static inline int getLocalIds(int * lcl_id1, int * lcl_id0)
{
    int lcl_wave_id = getWaveLocalId();
	int wave_id = getWaveId();
	int lcl_id = (wave_id << MLO_LG2_WAVE_SZ) + lcl_wave_id; //get_local_id(0);
	*lcl_id1 = (lcl_id >> MLO_LG2ALU_EXTENT_X);
	*lcl_id0 = lcl_id - ((*lcl_id1) << MLO_LG2ALU_EXTENT_X);
	return(lcl_id);
}

/*********************************************************************************************************

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvD3x3_WSR0(
	const __global _FLOAT * __restrict in_ptr,
	const __global _FLOAT * __restrict wei_ptr,
#if MLO_CONV_BIAS
	const __global _FLOAT * __restrict bias,
#endif
	__global _FLOAT * __restrict out_ptr,
	_FLOAT alpha // LRU fusion
	)
{
// all other allocation should be before this one since we may exceed lcl mem range
	__local _FLOAT in_lcl[MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1)];

	__private _FLOAT pvt_accum[MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0];

	int grp_input_id = get_group_id(0); // tile id inside the input map
	int grp_gbl_offset = (grp_input_id == 0) ? 0 : (grp_input_id*MLO_OUT_EXTENT1 - MLO_FILTER_PAD1)*MLO_IN_STRIDE;
	int gpr_lcl_off = (grp_input_id != 0) ? MLO_FILTER_PAD0 : MLO_IN_LCL_WIDTH*MLO_FILTER_PAD1 + MLO_FILTER_PAD0;

	int lcl_id1;
	int lcl_id0;
	int lcl_id = getLocalIds(&lcl_id1, &lcl_id0);
	grp_gbl_offset += lcl_id1*MLO_IN_STRIDE;
	gpr_lcl_off += lcl_id1*MLO_IN_LCL_WIDTH;

	int n_vert_reads = (grp_input_id == 0)? (MLO_OUT_EXTENT1 + MLO_FILTER_PAD1) :  (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1);
	n_vert_reads = (grp_input_id == (get_num_groups(0) - 1)) ? (MLO_IN_HEIGHT - grp_input_id*MLO_OUT_EXTENT1 + MLO_FILTER_PAD1) : n_vert_reads;

// output
	int o_block = get_group_id(1);

	int weave_id = getWaveId();
	//assumption is MLO_N_OUTPUTS is multiple of MLO_N_LCL_OUT_MAPS*MLO_N_WAVES
	int o_base = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES + weave_id * MLO_N_LCL_OUT_MAPS;

	__private int weights_base_offsets[MLO_N_LCL_OUT_MAPS];

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
	{
		weights_base_offsets[i] = (o_base + i)*MLO_WEI_BATCH_STRIDE;
	}

// batch
	int b_block = get_group_id(2);
	// batch
	int b_base = b_block * MLO_N_LCL_BATCHS;

	grp_gbl_offset += b_base * MLO_IN_BATCH_STRIDE;

// padding

	for (uint i = lcl_id; i < MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1); i+=MLO_GRP_SZ)
	{
		in_lcl[i] = 0;
	}

	// local read and relative output adresses and offsets
	int lcl_wave_id1;
	int lcl_wave_id0;
	getWaveLocalIds(&lcl_wave_id1, &lcl_wave_id0);
	int lcl_off1 = lcl_wave_id1 * MLO_OUT_TILE1;
	int lcl_off0 = lcl_wave_id0 * MLO_OUT_TILE0;
	int lcl_read_off = lcl_off1 * MLO_IN_LCL_WIDTH + lcl_off0;

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0; ++i)
	{
		pvt_accum[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0

	if (get_group_id(0) == 0 && lcl_id0 == 1)
	{
		printf("k:s: %d %d %d %d %d\n",
			getWaveId(),
			lcl_id,
			lcl_id1,
			grp_gbl_offset,
			gpr_lcl_off
			);
	}
#endif

	// outer loop over all input maps
	for (uint c = 0, wei_offset = 0;
		c < MLO_N_INPUTS;
		++c,
		grp_gbl_offset += MLO_IN_CHANNEL_STRIDE, wei_offset += MLO_WEI_CHANNEL_STRIDE)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

// this kernel assumes with is multiple of MLO_READ_UNIT
// number of inputs has been guarded above

		int gbl_scan_off = grp_gbl_offset;
		int lcl_scan_off = gpr_lcl_off;
		for (uint j = lcl_id1; j < n_vert_reads; j += MLO_ALU_EXTENT_Y, gbl_scan_off += MLO_IN_STRIDE*MLO_ALU_EXTENT_Y, lcl_scan_off += MLO_IN_LCL_WIDTH*MLO_ALU_EXTENT_Y)
		{
			for(int k = 0; k < MLO_READ_UNIT; ++k)
			{
				in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + k] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + k];
			}
#if 0

			if (get_group_id(0) == 0 && j < 3 && lcl_id0 == 0)
			{
				printf("k:i: %d %d %d  %f %f %f\n",
					n_vert_reads,
					lcl_scan_off + lcl_id0*MLO_READ_UNIT,
					gbl_scan_off + lcl_id0*MLO_READ_UNIT,
					in_lcl[lcl_scan_off - 1], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1]
					);
			}
#endif
		}
// finish reading
		barrier(CLK_LOCAL_MEM_FENCE);

// read weights
		__private _FLOAT pvt_wei_stage[MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE0 * MLO_FILTER_SIZE0];
		for (uint j = 0; j < MLO_N_LCL_OUT_MAPS; ++j)
		{
			for (uint i = 0; i < MLO_WEI_CHANNEL_STRIDE; ++i)
			{
				int wei_gbl_off = weights_base_offsets[j] + wei_offset;

#if MLO_OUTPUTS_ALIGNED ==0
		        if (o_base + j >= MLO_N_OUTPUTS)
				{
					pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = 0;
				}
				else
#endif
				{
					pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = wei_ptr[wei_gbl_off + i];
				}
			}
		}

// read data
		__private _FLOAT pvt_in_stage[(MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0)];

		int lcl_read_scan_off = lcl_read_off;
		for (uint j = 0; j < (MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1); ++j, lcl_read_scan_off += MLO_IN_LCL_WIDTH)
		{
			for (uint i = 0; i < (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0); ++i)
			{
				pvt_in_stage[j * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i]
					= in_lcl[lcl_read_scan_off + i];

			}

		}

		// actual conv

		for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc)
		{
			for (uint j = 0; j < MLO_OUT_TILE1; ++j)
			{
				for (uint i = 0; i < MLO_OUT_TILE0; ++i)
				{
					for (uint k = 0; k < MLO_FILTER_SIZE1; ++k)
					{
						for (uint l = 0; l < MLO_FILTER_SIZE0; ++l)
						{

							pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								+= pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l]
									* pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l];
#if 0

							if (get_group_id(0) == 0 && lcl_id == 0 && j == 0 && i == 0)
							{
								printf("k:c: %d %d %d  %f %f %f\n",
									oc,
									k,
									l,
									pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i],
									pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l],
									pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l]
								);
							}
#endif

						} //for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
					}

				} // for (int i = 0; i < MLO_OUT_TILE0; ++i)

			} // for (int j = 0; j < MLO_OUT_TILE1; ++j)

		}
	}

	// send it out
	// TODO: batch loop

	int grp_x = 0;
	int grp_y = grp_input_id*MLO_OUT_EXTENT1;
	int out_off = b_base*MLO_OUT_BATCH_STRIDE + o_base* MLO_OUT_CHANNEL_STRIDE + (grp_y + lcl_off1)*MLO_OUT_STRIDE + grp_x + lcl_off0;

	int out_off1 = out_off;
	for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc, out_off1 += MLO_OUT_CHANNEL_STRIDE)
	{
#if MLO_OUTPUTS_ALIGNED ==0
		if (o_base + oc < MLO_N_OUTPUTS)
#endif
		{
			int out_off2 = out_off1;
			for (uint j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
			// input height == output heiht
				if ( grp_y + lcl_off1 + j < MLO_IN_HEIGHT)
				{
					for (uint i = 0; i < MLO_OUT_TILE0 / MLO_READ_UNIT; ++i)
					{
					int gbl_out_off = out_off2 + i*MLO_READ_UNIT;
						for(int k = 0; k < MLO_READ_UNIT; ++k)
						{
							out_ptr[gbl_out_off + k] = pvt_accum[/*ib*MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE_SZ + */(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i*MLO_READ_UNIT + k];
#if DBG_OUT_OF_RNGE
							if(gbl_out_off + k >  MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
							{
									printf("k:err:out-of-range\n");
							}
#endif
						}
					}
				}
			}
		}
	}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define DBG_OUT_OF_RNGE 0

#define MLO_HW_WAVE_ID_SETTING 1
// FIXME Conduct enabling from the host code.
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);

#define MLO_IN_LCL_WIDTH (MLO_IN_STRIDE + 2 * MLO_FILTER_PAD0)

static inline int getWaveId()
{
	int wave_id = 0;

#if MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_HSAIL==1
		wave_id = __hsail_get_dynwave_id();
		wave_id &= MLO_N_WAVES_MASK;
#elif MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_LC==1 && MLO_GRP_SZ1==1 && MLO_GRP_SZ2==1 && (MLO_GRP_SZ % (1 << MLO_LG2_WAVE_SZ))==0
		// (local_id/wavesize) has the same value in all workitems.
		// Make it scalar to enable scalarization optimizations.
		wave_id = __llvm_amdgcn_readfirstlane((uint)(get_local_id(0) >> MLO_LG2_WAVE_SZ));
		// Alternate implementation:
		//__asm__ ("v_readfirstlane_b32 %0, %1" : "=s" (wave_id) : "v" ((int)(get_local_id(0) >> MLO_LG2_WAVE_SZ)) );
#else
		wave_id = (get_local_id(0) >> MLO_LG2_WAVE_SZ);
#endif
	return(wave_id);
}

static inline int getWaveLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_WAVE_SZ) << MLO_LG2_WAVE_SZ);;
	return(lcl_wave_id);
}

static inline int getWaveLocalIds(int * lcl_wave_id1, int * lcl_wave_id0)
{
	int lcl_wave_id = getWaveLocalId();
	*lcl_wave_id1 = (lcl_wave_id >> MLO_LG2_WAVE_SZ0);
	*lcl_wave_id0 = lcl_wave_id - ((*lcl_wave_id1) << MLO_LG2_WAVE_SZ0);
	return(lcl_wave_id);
}

static inline int getLocalIds(int * lcl_id1, int * lcl_id0)
{
    int lcl_wave_id = getWaveLocalId();
	int wave_id = getWaveId();
	int lcl_id = (wave_id << MLO_LG2_WAVE_SZ) + lcl_wave_id; //get_local_id(0);
	*lcl_id1 = (lcl_id >> MLO_LG2ALU_EXTENT_X);
	*lcl_id0 = lcl_id - ((*lcl_id1) << MLO_LG2ALU_EXTENT_X);
	return(lcl_id);
}

/*********************************************************************************************************

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvD3x3_WSR0(
	const __global _FLOAT * __restrict in_ptr,
	const __global _FLOAT * __restrict wei_ptr,
#if MLO_CONV_BIAS
	const __global _FLOAT * __restrict bias,
#endif
	__global _FLOAT * __restrict out_ptr,
	_FLOAT alpha // LRU fusion
	)
{
// all other allocation should be before this one since we may exceed lcl mem range
	__local _FLOAT in_lcl[MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1)];

	__private _FLOAT pvt_accum[MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0];

	int grp_input_id = get_group_id(0); // tile id inside the input map
	int grp_gbl_offset = (grp_input_id == 0) ? 0 : (grp_input_id*MLO_OUT_EXTENT1 - 1)*MLO_IN_STRIDE;
	int gpr_lcl_off = (grp_input_id != 0) ? MLO_FILTER_PAD0 : MLO_IN_LCL_WIDTH*MLO_FILTER_PAD1 + MLO_FILTER_PAD0;

	int lcl_id1;
	int lcl_id0;
	int lcl_id = getLocalIds(&lcl_id1, &lcl_id0);
	grp_gbl_offset += lcl_id1*MLO_IN_STRIDE;
	gpr_lcl_off += lcl_id1*MLO_IN_LCL_WIDTH;

	int n_vert_reads = (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1);
	n_vert_reads -= (grp_input_id == 0 || grp_input_id == get_num_groups(0) - 1) ? 1 : 0;

// output
	int o_block = get_group_id(1);

	int weave_id = getWaveId();
	//assumption is MLO_N_OUTPUTS is multiple of MLO_N_LCL_OUT_MAPS*MLO_N_WAVES
	int o_base = o_block*MLO_N_LCL_OUT_MAPS*MLO_N_WAVES + weave_id * MLO_N_LCL_OUT_MAPS;

	__private int weights_base_offsets[MLO_N_LCL_OUT_MAPS];

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS; ++i)
	{
		weights_base_offsets[i] = (o_base + i)*MLO_WEI_BATCH_STRIDE;
	}

// batch
	int b_block = get_group_id(2);
	// batch
	int b_base = b_block * MLO_N_LCL_BATCHS;

	grp_gbl_offset += b_base * MLO_IN_BATCH_STRIDE;

// padding
// ATT: padding == 1
	for (uint i = lcl_id; i < MLO_IN_LCL_WIDTH; i+=MLO_GRP_SZ)
	{
		in_lcl[i] = 0;
		in_lcl[i + MLO_IN_LCL_WIDTH * (MLO_OUT_EXTENT1 + MLO_FILTER_PAD1)] = 0;
	}
	for (uint i = lcl_id; i < (MLO_OUT_EXTENT1 + 2 * MLO_FILTER_PAD1); i += MLO_GRP_SZ)
	{
		in_lcl[i*MLO_IN_LCL_WIDTH] = 0;
		in_lcl[(i+1)*MLO_IN_LCL_WIDTH - 1] = 0;
	}

	// local read and relative output adresses and offsets
	int lcl_wave_id1;
	int lcl_wave_id0;
	getWaveLocalIds(&lcl_wave_id1, &lcl_wave_id0);
	int lcl_off1 = lcl_wave_id1 * MLO_OUT_TILE1;
	int lcl_off0 = lcl_wave_id0 * MLO_OUT_TILE0;
	int lcl_read_off = lcl_off1 * MLO_IN_LCL_WIDTH + lcl_off0;

	for (uint i = 0; i < MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE1*MLO_OUT_TILE0; ++i)
	{
		pvt_accum[i] = 0;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0

	if (get_group_id(0) == 0 && lcl_id0 == 1)
	{
		printf("k:s: %d %d %d %d %d\n",
			getWaveId(),
			lcl_id,
			lcl_id1,
			grp_gbl_offset,
			gpr_lcl_off
			);
	}
#endif

	// outer loop over all input maps
	for (uint c = 0, wei_offset = 0;
		c < MLO_N_INPUTS;
		++c,
		grp_gbl_offset += MLO_IN_CHANNEL_STRIDE, wei_offset += MLO_WEI_CHANNEL_STRIDE)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

// this kernel assumes with is multiple of MLO_READ_UNIT
// number of inputs has been guarded above

		int gbl_scan_off = grp_gbl_offset;
		int lcl_scan_off = gpr_lcl_off;
		for (uint j = lcl_id1; j < n_vert_reads; j += MLO_ALU_EXTENT_Y, gbl_scan_off += MLO_IN_STRIDE*MLO_ALU_EXTENT_Y, lcl_scan_off += MLO_IN_LCL_WIDTH*MLO_ALU_EXTENT_Y)
		{
			for(int k = 0; k < MLO_READ_UNIT; ++k)
			{
				in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + k] = in_ptr[gbl_scan_off + lcl_id0*MLO_READ_UNIT + k];
			}
#if 0

			if (get_group_id(0) == 0 && j < 3 && lcl_id0 == 0)
			{
				printf("k:i: %d %d %d  %f %f %f\n",
					n_vert_reads,
					lcl_scan_off + lcl_id0*MLO_READ_UNIT,
					gbl_scan_off + lcl_id0*MLO_READ_UNIT,
					in_lcl[lcl_scan_off - 1], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT], in_lcl[lcl_scan_off + lcl_id0*MLO_READ_UNIT + 1]
					);
			}
#endif
		}
// finish reading
		barrier(CLK_LOCAL_MEM_FENCE);

// read weights
		__private _FLOAT pvt_wei_stage[MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE0 * MLO_FILTER_SIZE0];
		for (uint j = 0; j < MLO_N_LCL_OUT_MAPS; ++j)
		{
			for (uint i = 0; i < MLO_WEI_CHANNEL_STRIDE; ++i)
			{
				int wei_gbl_off = weights_base_offsets[j] + wei_offset;

#if MLO_OUTPUTS_ALIGNED ==0
		        if (o_base + j >= MLO_N_OUTPUTS)
				{
					pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = 0;
				}
				else
#endif
				{
					pvt_wei_stage[j*MLO_WEI_CHANNEL_STRIDE + i] = wei_ptr[wei_gbl_off + i];
				}
			}
		}

// read data
		__private _FLOAT pvt_in_stage[(MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0)];

		int lcl_read_scan_off = lcl_read_off;
		for (uint j = 0; j < (MLO_OUT_TILE1 + 2 * MLO_FILTER_PAD1); ++j, lcl_read_scan_off += MLO_IN_LCL_WIDTH)
		{
			for (uint i = 0; i < (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0); ++i)
			{
				pvt_in_stage[j * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i]
					= in_lcl[lcl_read_scan_off + i];

			}

		}

		// actual conv

		for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc)
		{
			for (uint j = 0; j < MLO_OUT_TILE1; ++j)
			{
				for (uint i = 0; i < MLO_OUT_TILE0; ++i)
				{
					for (uint k = 0; k < MLO_FILTER_SIZE1; ++k)
					{
						for (uint l = 0; l < MLO_FILTER_SIZE0; ++l)
						{

							pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i]
								+= pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l]
									* pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l];
#if 0

							if (get_group_id(0) == 0 && lcl_id == 0 && j == 0 && i == 0)
							{
								printf("k:c: %d %d %d  %f %f %f\n",
									oc,
									k,
									l,
									pvt_accum[(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i],
									pvt_in_stage[(j + k) * (MLO_OUT_TILE0 + 2 * MLO_FILTER_PAD0) + i + l],
									pvt_wei_stage[oc * MLO_WEI_CHANNEL_STRIDE + k * MLO_FILTER_SIZE0 + l]
								);
							}
#endif

						} //for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
					}

				} // for (int i = 0; i < MLO_OUT_TILE0; ++i)

			} // for (int j = 0; j < MLO_OUT_TILE1; ++j)

		}
	}

	// send it out
	// TODO: batch loop

	int grp_x = 0;
	int grp_y = grp_input_id*MLO_OUT_EXTENT1;
	int out_off = b_base*MLO_OUT_BATCH_STRIDE + o_base* MLO_OUT_CHANNEL_STRIDE + (grp_y + lcl_off1)*MLO_OUT_STRIDE + grp_x + lcl_off0;

	int out_off1 = out_off;
	for (uint oc = 0; oc < MLO_N_LCL_OUT_MAPS; ++oc, out_off1 += MLO_OUT_CHANNEL_STRIDE)
	{
#if MLO_OUTPUTS_ALIGNED ==0
		if (o_base + oc < MLO_N_OUTPUTS)
#endif
		{
			int out_off2 = out_off1;
			for (uint j = 0; j < MLO_OUT_TILE1; ++j, out_off2 += MLO_OUT_STRIDE)
			{
			// input height == output heiht
				if ( grp_y + lcl_off1 + j < MLO_IN_HEIGHT)
				{
					for (uint i = 0; i < MLO_OUT_TILE0 / MLO_READ_UNIT; ++i)
					{
					int gbl_out_off = out_off2 + i*MLO_READ_UNIT;
						for(int k = 0; k < MLO_READ_UNIT; ++k)
						{
							out_ptr[gbl_out_off + k] = pvt_accum[/*ib*MLO_N_LCL_OUT_MAPS * MLO_OUT_TILE_SZ + */(oc * MLO_OUT_TILE1 + j) * MLO_OUT_TILE0 + i*MLO_READ_UNIT + k];
#if DBG_OUT_OF_RNGE
							if(gbl_out_off + k >  MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
							{
									printf("k:err:out-of-range\n");
							}
#endif
						}
					}
				}
			}
		}
	}

}


Generalized the way I'm handling reflections. Need to do some soul searching on how I'm accumulating colors

float DistanceBetweenPoints(float3 a, float3 b) {
	return fast_distance(a, b);
	//return sqrt(pow(a.x - b.x, 2) + pow(a.y - b.y, 2) + pow(a.z - b.z, 2));
}

float Distance(float3 a) {
	return fast_length(a);
	//return sqrt(pow(a.x, 2) + pow(a.y, 2) + pow(a.z, 2));
}

// Naive incident ray light
float4 white_light(float4 input, float3 light, int3 mask) {

	input.w = input.w + acos(
		dot(
			normalize(light),
			normalize(convert_float3(mask * (-mask)))
			)
		) / 32;

	input.w += 0.25f;

	return input;

}

// Phong + diffuse lighting function for g
//  0  1  2  3  4  5  6  7   8   9
// {r, g, b, i, x, y, z, x', y', z'}

float4 view_light(float4 in_color, float3 light, float4 light_color, float3 view, int3 mask) {

	if (all(light == (0.0f,0.0f,0.0f)))
		return (0,0,0,0);

	float d = Distance(light) / 100.0f;
	d *= d;

	float diffuse = max(dot(normalize(convert_float3(mask)), normalize(light)), 0.0f);
	in_color += diffuse * light_color * 0.5f / d;

	if (dot(light, normalize(convert_float3(mask))) > 0.0f)
	{
		// Small dots of light are caused by floating point error
		// flipping bits on the face mask and screwing up this calculation
		float3 halfwayVector = normalize(normalize(light) + normalize(view));
		float specTmp = max(dot(normalize(convert_float3(mask)), halfwayVector), 0.0f);
		in_color += pow(specTmp, 8.0f) * light_color * 0.5f / d;
	}
	if (in_color.w > 1.0f){
		in_color.xyz *= in_color.w;
	}

	return in_color;
}

int rand(int* seed) // 1 <= *seed < m
{
	int const a = 16807; //ie 7**5
	int const m = 2147483647; //ie 2**31-1

	*seed = ((*seed) * a) % m;
	return(*seed);
}

bool get_oct_vox(
	int3 position,
	global ulong *octree_descriptor_buffer,
	global uint *octree_attachment_lookup_buffer,
	global ulong *octree_attachment_buffer,
	global ulong *settings_buffer
){

	// (X, Y, Z) mask for the idx
	const uchar idx_set_x_mask = 0x1;
	const uchar idx_set_y_mask = 0x2;
	const uchar idx_set_z_mask = 0x4;

	const uchar mask_8[8] = {
		0x1,  0x2,  0x4,  0x8,
		0x10, 0x20, 0x40, 0x80
	};

	// Mask for counting the previous valid bits
	const uchar count_mask_8[8] = {
		0x1,  0x3,  0x7,  0xF,
		0x1F, 0x3F, 0x7F, 0xFF
	};

		// uint64_t manipulation masks
	const ulong child_pointer_mask = 0x0000000000007fff;
	const ulong far_bit_mask = 0x8000;
	const ulong valid_mask = 0xFF0000;
	const ulong leaf_mask = 0xFF000000;
	const ulong contour_pointer_mask = 0xFFFFFF00000000;
	const ulong contour_mask = 0xFF00000000000000;

	// push the root node to the parent stack
	ulong current_index = *settings_buffer;
	ulong head = octree_descriptor_buffer[current_index];

	uint parent_stack_position = 0;
	ulong parent_stack[32];

	uchar scale = 0;
	uchar idx_stack[32];

	ulong current_descriptor = 0;

	bool found = false;

	parent_stack[parent_stack_position] = head;

	// Set our initial dimension and the position at the corner of the oct to keep track of our position
	int dimension = 128;
	int3 quad_position = (0, 0, 0);

	// While we are not at the required resolution
	//		Traverse down by setting the valid/leaf mask to the subvoxel
	//		Check to see if it is valid
	//			Yes?
	//				Check to see if it is a leaf
	//				No? Break
	//				Yes? Scale down to the next hierarchy, push the parent to the stack
	//
	//			No?
	//				Break
	while (dimension > 1) {

		// So we can be a little bit tricky here and increment our
		// array index that holds our masks as we build the idx.
		// Adding 1 for X, 2 for Y, and 4 for Z
		int mask_index = 0;

		// Do the logic steps to find which sub oct we step down into
		if (position.x >= (dimension / 2) + quad_position.x) {

			// Set our voxel position to the (0,0) of the correct oct
			quad_position.x += (dimension / 2);

			// increment the mask index and mentioned above
			mask_index += 1;

			// Set the idx to represent the move
			idx_stack[scale] |= idx_set_x_mask;

		}
		if (position.y >= (dimension / 2) + quad_position.y) {

			quad_position.y |= (dimension / 2);

			mask_index += 2;

			// TODO What is up with the binary operator on this one?
			idx_stack[scale] ^= idx_set_y_mask;

		}
		if (position.z >= (dimension / 2) + quad_position.z) {

			quad_position.z += (dimension / 2);

			mask_index += 4;

			idx_stack[scale] |= idx_set_z_mask;

		}

		// Check to see if we are on a valid oct
		if ((head >> 16) & mask_8[mask_index]) {

			// Check to see if it is a leaf
			if ((head >> 24) & mask_8[mask_index]) {

				// If it is, then we cannot traverse further as CP's won't have been generated
				found = true;
				return found;
			}

			// If all went well and we found a valid non-leaf oct then we will traverse further down the hierarchy
			scale++;
			dimension /= 2;

			// Count the number of valid octs that come before and add it to the index to get the position
			// Negate it by one as it counts itself
			int count = popcount((uchar)(head >> 16) & count_mask_8[mask_index]) - 1;

			// access the element at which head points to and then add the specified number of indices
			// to get to the correct child descriptor
			current_index = current_index + (head & child_pointer_mask) + count;
			head = octree_descriptor_buffer[current_index];

			// Increment the parent stack position and put the new oct node as the parent
			parent_stack_position++;
			parent_stack[parent_stack_position] = head;

		}
		else {
			// If the oct was not valid, then no CP's exists any further
			// This implicitly says that if it's non-valid then it must be a leaf!!

			// It appears that the traversal is now working but I need
			// to focus on how to now take care of the end condition.
			// Currently it adds the last parent on the second to lowest
			// oct CP. Not sure if thats correct
			found = 0;
			return found;
		}
	}

	found = 1;
	return found;
}

// =================================== Boolean ray intersection ============================
// =========================================================================================

bool cast_light_intersection_ray(
	global char* map,
	global int3* map_dim,
	float3 ray_dir,
    float3 ray_pos,
	global float* lights,
	global int* light_count

	){

	float distance_to_light = DistanceBetweenPoints(ray_pos, (float3)(lights[4], lights[5], lights[6]));
	//if (distance_to_light > 200.0f){
	//	return false;
	//}

	// Setup the voxel step based on what direction the ray is pointing
	int3 voxel_step = { 1, 1, 1 };
	voxel_step *= (ray_dir > 0) - (ray_dir < 0);

	if (any(ray_dir == (0.0f,0.0f,0.0f)))
		return false;

	// Setup the voxel coords from the camera origin
	int3 voxel = convert_int3(ray_pos);

	// Delta T is the units a ray must travel along an axis in order to traverse an integer split
	float3 delta_t = fabs(1.0f / ray_dir);

	// Compute intersection_t and add in the offset
	float3 intersection_t = delta_t * ((ray_pos)-floor(ray_pos)) * convert_float3(voxel_step);

	// for negative values, wrap around the delta_t
	intersection_t += delta_t * -convert_float3(isless(intersection_t, 0));

	int3 face_mask = { 0, 0, 0 };
	int length_cutoff = 0;

	// Andrew Woo's raycasting algo
	do {

		// Fancy no branch version of the logic step
		face_mask = intersection_t.xyz <= min(intersection_t.yzx, intersection_t.zxy);
		intersection_t += delta_t * fabs(convert_float3(face_mask.xyz));
		voxel.xyz += voxel_step.xyz * face_mask.xyz;

		if (any(voxel >= *map_dim) ||
			any(voxel < 0)) {
			return false;
		}

		// If we hit a voxel
		int voxel_data = map[voxel.x + (*map_dim).x * (voxel.y + (*map_dim).z * (voxel.z))];

		if (voxel_data != 0)
			return true;

		if (++length_cutoff > 300)
			return false;

	} while (any(isless(intersection_t, (float3)(distance_to_light - 1))));

	return false;
}

// ====================================== Raycaster entry point =====================================
// ==================================================================================================

constant float4 fog_color = { 0.73f, 0.81f, 0.89f, 0.8f };
constant float4 overshoot_color = { 0.00f, 0.00f, 0.00f, 0.00f };
constant float4 overshoot_color_2 = { 0.00f, 0.00f, 0.00f, 0.00f };

__kernel void raycaster(
	global char* map,
	global int3* map_dim,
	global int2* resolution,
	global float3* projection_matrix,
	global float2* cam_dir,
	global float3* cam_pos,
	global float* lights,
	global int* light_count,
	__write_only image2d_t image,
	//global int* seed_memory,
	__read_only image2d_t texture_atlas,
	global int2 *atlas_dim,
	global int2 *tile_dim,
	global ulong *octree_descriptor_buffer,
	global uint *octree_attachment_lookup_buffer,
	global ulong *octree_attachment_buffer,
	global ulong *settings_buffer
){
	//	int global_id = x * y;

	// Get and set the random seed from seed memory
	//int seed = seed_memory[global_id];
	//int random_number = rand(&seed);
	//seed_memory[global_id] = seed;

	// Get the pixel on the viewport, and find the view matrix ray that matches it
	int2 pixel = (int2)(get_global_id(0), get_global_id(1));
    float3 ray_dir = projection_matrix[pixel.x + (*resolution).x * pixel.y];

	// Pitch
	ray_dir = (float3)(
		ray_dir.z * sin((*cam_dir).x) + ray_dir.x * cos((*cam_dir).x),
		ray_dir.y,
		ray_dir.z * cos((*cam_dir).x) - ray_dir.x * sin((*cam_dir).x)
		);

	// Yaw
    ray_dir = (float3)(
        ray_dir.x * cos((*cam_dir).y) - ray_dir.y * sin((*cam_dir).y),
        ray_dir.x * sin((*cam_dir).y) + ray_dir.y * cos((*cam_dir).y),
        ray_dir.z
    );

	// Setup the voxel step based on what direction the ray is pointing
    int3 voxel_step = {1, 1, 1};
	voxel_step *= (ray_dir > 0) - (ray_dir < 0);

    // Setup the voxel coords from the camera origin
	int3 voxel = convert_int3(*cam_pos);

    // Delta T is the units a ray must travel along an axis in order to
    // traverse an integer split
	if (any(ray_dir == (0.0f,0.0f,0.0f)))
		return;
	float3 delta_t = fabs(1.0f / ray_dir);

	// Intersection T is the collection of the next intersection points
	// for all 3 axis XYZ. We take the full positive cardinality when
	// subtracting the floor, so we must transfer the sign over from
	// the voxel step
	float3 intersection_t = delta_t * ((*cam_pos) - ceil(*cam_pos)) * convert_float3(voxel_step);

	// When we transfer the sign over, we get the correct direction of
	// the offset, but we merely transposed over the value instead of mirroring
	// it over the axis like we want. So here, isless returns a boolean if intersection_t
	// is less than 0 which dictates whether or not we subtract the delta which in effect
	// mirrors the offset
	intersection_t -= delta_t * convert_float3(isless(intersection_t, 0));

	int distance_traveled = 0;
	int max_distance = 700;
	uint bounce_count = 0;
	int3 face_mask = { 0, 0, 0 };
	int voxel_data = 0;
	float3 face_position = (0,0,0);
	float4 voxel_color= (0,0,0,0);
	float2 tile_face_position = (0,0);
	float3 sign = (0,0,0);
	float4 first_strike = (0,0,0,0);

	bool shadow_ray = false;

	// Andrew Woo's raycasting algo
    while (distance_traveled < max_distance && bounce_count < 2) {

		// Fancy no branch version of the logic step
		face_mask = intersection_t.xyz <= min(intersection_t.yzx, intersection_t.zxy);
		intersection_t += delta_t * fabs(convert_float3(face_mask.xyz));
		voxel.xyz += voxel_step.xyz * face_mask.xyz;

		// Test for out of bounds contions, add fog
		if (any(voxel >= *map_dim) || any(voxel < 0)){
			voxel_data = 5;
			voxel.xyz -= voxel_step.xyz * face_mask.xyz;
			first_strike = mix(fog_color, voxel_color, 1.0 - max(distance_traveled / 700.0f, (float)0));
		}

        // If we hit a voxel
		// if (voxel.x < 128 && voxel.y < 128 && voxel.z < 128){
		// 	if (get_oct_vox(
		// 		voxel,
		// 		octree_descriptor_buffer,
		// 		octree_attachment_lookup_buffer,
		// 		octree_attachment_buffer,
		// 		settings_buffer
		// 		)){
		// 			voxel_data = 1;
		// 		} else {
		// 			voxel_data = 0;
		// 		}
		// } else {
			voxel_data = map[voxel.x + (*map_dim).x * (voxel.y + (*map_dim).z * (voxel.z))];
		//}

		if (voxel_data != 0) {

			// Determine where on the 2d plane the ray intersected
			face_position = (float3)(0);
			tile_face_position = (float2)(0);
			sign = (float3)(1.0f, 1.0f, 1.0f);

			// First determine the percent of the way the ray is towards the next intersection_t
			// in relation to the xyz position on the plane
			if (face_mask.x == -1) {

				sign.x *= -1.0;

				// the next intersection for this plane - the last intersection of the passed plane / delta of this plane
				// basically finds how far in on the other 2 axis we are when the ray traversed the plane
				float z_percent = (intersection_t.z - (intersection_t.x - delta_t.x)) / delta_t.z;
				float y_percent = (intersection_t.y - (intersection_t.x - delta_t.x)) / delta_t.y;

				// Since we intersected face x, we know that we are at the face (1.0)
				// I think the 1.001f rendering bug is the ray thinking it's within the voxel
				// even though it's sitting on the very edge
				face_position = (float3)(1.0001f, y_percent, z_percent);
				tile_face_position = (float2)(y_percent, z_percent);
			}
			else if (face_mask.y == -1) {

				sign.y *= -1.0;
				float x_percent = (intersection_t.x - (intersection_t.y - delta_t.y)) / delta_t.x;
				float z_percent = (intersection_t.z - (intersection_t.y - delta_t.y)) / delta_t.z;

				face_position = (float3)(x_percent, 1.0001f, z_percent);
				tile_face_position = (float2)(x_percent, z_percent);
			}

			else if (face_mask.z == -1) {

				sign.z *= -1.0;
				float x_percent = (intersection_t.x - (intersection_t.z - delta_t.z)) / delta_t.x;
				float y_percent = (intersection_t.y - (intersection_t.z - delta_t.z)) / delta_t.y;

				face_position = (float3)(x_percent, y_percent, 1.0001f);
				tile_face_position = (float2)(x_percent, y_percent);

			}

			// Because the raycasting process is agnostic to the quadrant
			// it's working in, we need to transpose the sign over to the face positions.
			// If we don't it will think that it is always working in the (1, 1, 1) quadrant
			// and will just "copy" the quadrant. This includes shadows as they use the face_position
			// in order to cast the intersection ray!!

			face_position.x = select((float)(face_position.x), (float)(-face_position.x + 1.0f), (int)(ray_dir.x > 0));
			tile_face_position.x = select((float)(tile_face_position.x), (float)(-tile_face_position.x + 1.0f), (int)(ray_dir.x < 0));

			if (ray_dir.y > 0){
				face_position.y =  - face_position.y + 1;
			} else {
				tile_face_position.x = 1.0 - tile_face_position.x;

				// We run into the Hairy ball problem, so we need to define
				// a special case for the zmask
				if (face_mask.z == -1) {
					tile_face_position.x = 1.0 - tile_face_position.x;
					tile_face_position.y = 1.0 - tile_face_position.y;
				}
			}

			face_position.z = select((float)(face_position.z), (float)(-face_position.z + 1.0f), (int)(ray_dir.z > 0));
			tile_face_position.y = select((float)(tile_face_position.y), (float)(-tile_face_position.y + 1.0f), (int)(ray_dir.z < 0));

			// Now either use the face position to retrieve a texture sample, or
			// just a plain color for the voxel color. Notice the JANK -1 after the
			// conditionals in the select statement. That's because select works on negs
			// and pos's. So a false equality will still eval as true as it is technically
			// a positive result (0)
			// voxel_color = select(
			// 	(float4)(0.25f, 0.64f, 0.87f, 0.0f),
			// 	(float4)voxel_color,
			//  	(int4)((voxel_data == 5) - 1)
			// );

			// SHADOWING
			if (voxel_data == 5 && !shadow_ray){

				shadow_ray = true;
				voxel_color.xyz += (float3)read_imagef(
						 texture_atlas,
						 convert_int2(tile_face_position * convert_float2(*atlas_dim / *tile_dim)) +
						 convert_int2((float2)(3, 0) * convert_float2(*atlas_dim / *tile_dim))
				).xyz/2;

				//voxel_color.w = 0.0f;
				first_strike = view_light(
							voxel_color,
							(convert_float3(voxel) + face_position) - (float3)(lights[4], lights[5], lights[6]),
							(float4)(lights[0], lights[1], lights[2], lights[3]),
							(convert_float3(voxel) + face_position) - (*cam_pos),
							face_mask * voxel_step
				);

				max_distance = DistanceBetweenPoints(convert_float3(voxel), (float3)(lights[4], lights[5], lights[6]));
				distance_traveled = 0;

				float3 hit_pos = convert_float3(voxel) + face_position;
				ray_dir = normalize((float3)(lights[4], lights[5], lights[6]) - hit_pos);
				if (any(ray_dir == (0.0f,0.0f,0.0f)))
					return;

				voxel -= voxel_step * face_mask;
				voxel_step = ( 1, 1, 1 );
				voxel_step *= (ray_dir > 0) - (ray_dir < 0);

				//voxel = convert_int3(hit_pos);

				delta_t = fabs(1.0f / ray_dir);
				intersection_t = delta_t * ((hit_pos)-floor(hit_pos)) * convert_float3(voxel_step);
				intersection_t += delta_t * -convert_float3(isless(intersection_t, 0));

			// REFLECTION
			} else if (voxel_data == 6 && !shadow_ray) {

				voxel_color.xyz += (float3)read_imagef(
						 texture_atlas,
						 convert_int2(tile_face_position * convert_float2(*atlas_dim / *tile_dim)) +
						 convert_int2((float2)(3, 4) * convert_float2(*atlas_dim / *tile_dim))
				).xyz/2;

				voxel_color.w += 0.3f;
				max_distance = 500;
				distance_traveled = 0;

				float3 hit_pos = convert_float3(voxel) + face_position;
				ray_dir *= sign;
				if (any(ray_dir == (0.0f,0.0f,0.0f)))
					return;

				voxel -= voxel_step * face_mask;
				voxel_step = ( 1, 1, 1 );
				voxel_step *= (ray_dir > 0) - (ray_dir < 0);

				//voxel = convert_int3(hit_pos);

				delta_t = fabs(1.0f / ray_dir);
				intersection_t = delta_t * ((hit_pos)-floor(hit_pos)) * convert_float3(voxel_step);
				intersection_t += delta_t * -convert_float3(isless(intersection_t, 0));

				bounce_count += 1;

			// SHADOW RAY HIT
			} else {
				max_distance = 0;
				distance_traveled = 1;
			}
		}
		distance_traveled++;
    }
	write_imagef(
		image,
		pixel,
		first_strike
	);

    return;
}

float DistanceBetweenPoints(float3 a, float3 b) {
	return fast_distance(a, b);
	//return sqrt(pow(a.x - b.x, 2) + pow(a.y - b.y, 2) + pow(a.z - b.z, 2));
}

float Distance(float3 a) {
	return fast_length(a);
	//return sqrt(pow(a.x, 2) + pow(a.y, 2) + pow(a.z, 2));
}

// Naive incident ray light
float4 white_light(float4 input, float3 light, int3 mask) {

	input.w = input.w + acos(
		dot(
			normalize(light),
			normalize(convert_float3(mask * (-mask)))
			)
		) / 32;

	input.w += 0.25f;

	return input;

}

// Phong + diffuse lighting function for g
//  0  1  2  3  4  5  6  7   8   9
// {r, g, b, i, x, y, z, x', y', z'}

float4 view_light(float4 in_color, float3 light, float4 light_color, float3 view, int3 mask) {

	if (all(light == (0.0f,0.0f,0.0f)))
		return (0,0,0,0);

	float d = Distance(light) / 100.0f;
	d *= d;

	float diffuse = max(dot(normalize(convert_float3(mask)), normalize(light)), 0.0f);
	in_color += diffuse * light_color * 0.5f / d;

	if (dot(light, normalize(convert_float3(mask))) > 0.0f)
	{
		// Small dots of light are caused by floating point error
		// flipping bits on the face mask and screwing up this calculation
		float3 halfwayVector = normalize(normalize(light) + normalize(view));
		float specTmp = max(dot(normalize(convert_float3(mask)), halfwayVector), 0.0f);
		in_color += pow(specTmp, 8.0f) * light_color * 0.5f / d;
	}
	if (in_color.w > 1.0f){
		in_color.xyz *= in_color.w;
	}

	return in_color;
}

int rand(int* seed) // 1 <= *seed < m
{
	int const a = 16807; //ie 7**5
	int const m = 2147483647; //ie 2**31-1

	*seed = ((*seed) * a) % m;
	return(*seed);
}

bool get_oct_vox(
	int3 position,
	global ulong *octree_descriptor_buffer,
	global uint *octree_attachment_lookup_buffer,
	global ulong *octree_attachment_buffer,
	global ulong *settings_buffer
){

	// (X, Y, Z) mask for the idx
	const uchar idx_set_x_mask = 0x1;
	const uchar idx_set_y_mask = 0x2;
	const uchar idx_set_z_mask = 0x4;

	const uchar mask_8[8] = {
		0x1,  0x2,  0x4,  0x8,
		0x10, 0x20, 0x40, 0x80
	};

	// Mask for counting the previous valid bits
	const uchar count_mask_8[8] = {
		0x1,  0x3,  0x7,  0xF,
		0x1F, 0x3F, 0x7F, 0xFF
	};

		// uint64_t manipulation masks
	const ulong child_pointer_mask = 0x0000000000007fff;
	const ulong far_bit_mask = 0x8000;
	const ulong valid_mask = 0xFF0000;
	const ulong leaf_mask = 0xFF000000;
	const ulong contour_pointer_mask = 0xFFFFFF00000000;
	const ulong contour_mask = 0xFF00000000000000;

	// push the root node to the parent stack
	ulong current_index = *settings_buffer;
	ulong head = octree_descriptor_buffer[current_index];

	uint parent_stack_position = 0;
	ulong parent_stack[32];

	uchar scale = 0;
	uchar idx_stack[32];

	ulong current_descriptor = 0;

	bool found = false;

	parent_stack[parent_stack_position] = head;

	// Set our initial dimension and the position at the corner of the oct to keep track of our position
	int dimension = 128;
	int3 quad_position = (0, 0, 0);

	// While we are not at the required resolution
	//		Traverse down by setting the valid/leaf mask to the subvoxel
	//		Check to see if it is valid
	//			Yes?
	//				Check to see if it is a leaf
	//				No? Break
	//				Yes? Scale down to the next hierarchy, push the parent to the stack
	//
	//			No?
	//				Break
	while (dimension > 1) {

		// So we can be a little bit tricky here and increment our
		// array index that holds our masks as we build the idx.
		// Adding 1 for X, 2 for Y, and 4 for Z
		int mask_index = 0;

		// Do the logic steps to find which sub oct we step down into
		if (position.x >= (dimension / 2) + quad_position.x) {

			// Set our voxel position to the (0,0) of the correct oct
			quad_position.x += (dimension / 2);

			// increment the mask index and mentioned above
			mask_index += 1;

			// Set the idx to represent the move
			idx_stack[scale] |= idx_set_x_mask;

		}
		if (position.y >= (dimension / 2) + quad_position.y) {

			quad_position.y |= (dimension / 2);

			mask_index += 2;

			// TODO What is up with the binary operator on this one?
			idx_stack[scale] ^= idx_set_y_mask;

		}
		if (position.z >= (dimension / 2) + quad_position.z) {

			quad_position.z += (dimension / 2);

			mask_index += 4;

			idx_stack[scale] |= idx_set_z_mask;

		}

		// Check to see if we are on a valid oct
		if ((head >> 16) & mask_8[mask_index]) {

			// Check to see if it is a leaf
			if ((head >> 24) & mask_8[mask_index]) {

				// If it is, then we cannot traverse further as CP's won't have been generated
				found = true;
				return found;
			}

			// If all went well and we found a valid non-leaf oct then we will traverse further down the hierarchy
			scale++;
			dimension /= 2;

			// Count the number of valid octs that come before and add it to the index to get the position
			// Negate it by one as it counts itself
			int count = popcount((uchar)(head >> 16) & count_mask_8[mask_index]) - 1;

			// access the element at which head points to and then add the specified number of indices
			// to get to the correct child descriptor
			current_index = current_index + (head & child_pointer_mask) + count;
			head = octree_descriptor_buffer[current_index];

			// Increment the parent stack position and put the new oct node as the parent
			parent_stack_position++;
			parent_stack[parent_stack_position] = head;

		}
		else {
			// If the oct was not valid, then no CP's exists any further
			// This implicitly says that if it's non-valid then it must be a leaf!!

			// It appears that the traversal is now working but I need
			// to focus on how to now take care of the end condition.
			// Currently it adds the last parent on the second to lowest
			// oct CP. Not sure if thats correct
			found = 0;
			return found;
		}
	}

	found = 1;
	return found;
}

// =================================== Boolean ray intersection ============================
// =========================================================================================

bool cast_light_intersection_ray(
	global char* map,
	global int3* map_dim,
	float3 ray_dir,
    float3 ray_pos,
	global float* lights,
	global int* light_count

	){

	float distance_to_light = DistanceBetweenPoints(ray_pos, (float3)(lights[4], lights[5], lights[6]));
	//if (distance_to_light > 200.0f){
	//	return false;
	//}

	// Setup the voxel step based on what direction the ray is pointing
	int3 voxel_step = { 1, 1, 1 };
	voxel_step *= (ray_dir > 0) - (ray_dir < 0);

	if (any(ray_dir == (0.0f,0.0f,0.0f)))
		return false;

	// Setup the voxel coords from the camera origin
	int3 voxel = convert_int3(ray_pos);

	// Delta T is the units a ray must travel along an axis in order to traverse an integer split
	float3 delta_t = fabs(1.0f / ray_dir);

	// Compute intersection_t and add in the offset
	float3 intersection_t = delta_t * ((ray_pos)-floor(ray_pos)) * convert_float3(voxel_step);

	// for negative values, wrap around the delta_t
	intersection_t += delta_t * -convert_float3(isless(intersection_t, 0));

	int3 face_mask = { 0, 0, 0 };
	int length_cutoff = 0;

	// Andrew Woo's raycasting algo
	do {

		// Fancy no branch version of the logic step
		face_mask = intersection_t.xyz <= min(intersection_t.yzx, intersection_t.zxy);
		intersection_t += delta_t * fabs(convert_float3(face_mask.xyz));
		voxel.xyz += voxel_step.xyz * face_mask.xyz;

		if (any(voxel >= *map_dim) ||
			any(voxel < 0)) {
			return false;
		}

		// If we hit a voxel
		int voxel_data = map[voxel.x + (*map_dim).x * (voxel.y + (*map_dim).z * (voxel.z))];

		if (voxel_data != 0)
			return true;

		if (++length_cutoff > 300)
			return false;

	} while (any(isless(intersection_t, (float3)(distance_to_light - 1))));

	return false;
}

// ====================================== Raycaster entry point =====================================
// ==================================================================================================

constant float4 fog_color = { 0.73f, 0.81f, 0.89f, 0.8f };
constant float4 overshoot_color = { 0.00f, 0.00f, 0.00f, 0.00f };
constant float4 overshoot_color_2 = { 0.00f, 0.00f, 0.00f, 0.00f };

__kernel void raycaster(
	global char* map,
	global int3* map_dim,
	global int2* resolution,
	global float3* projection_matrix,
	global float2* cam_dir,
	global float3* cam_pos,
	global float* lights,
	global int* light_count,
	__write_only image2d_t image,
	//global int* seed_memory,
	__read_only image2d_t texture_atlas,
	global int2 *atlas_dim,
	global int2 *tile_dim,
	global ulong *octree_descriptor_buffer,
	global uint *octree_attachment_lookup_buffer,
	global ulong *octree_attachment_buffer,
	global ulong *settings_buffer
){
	//	int global_id = x * y;

	// Get and set the random seed from seed memory
	//int seed = seed_memory[global_id];
	//int random_number = rand(&seed);
	//seed_memory[global_id] = seed;

	// Get the pixel on the viewport, and find the view matrix ray that matches it
	int2 pixel = (int2)(get_global_id(0), get_global_id(1));
    float3 ray_dir = projection_matrix[pixel.x + (*resolution).x * pixel.y];

	// Pitch
	ray_dir = (float3)(
		ray_dir.z * sin((*cam_dir).x) + ray_dir.x * cos((*cam_dir).x),
		ray_dir.y,
		ray_dir.z * cos((*cam_dir).x) - ray_dir.x * sin((*cam_dir).x)
		);

	// Yaw
    ray_dir = (float3)(
        ray_dir.x * cos((*cam_dir).y) - ray_dir.y * sin((*cam_dir).y),
        ray_dir.x * sin((*cam_dir).y) + ray_dir.y * cos((*cam_dir).y),
        ray_dir.z
    );

	// Setup the voxel step based on what direction the ray is pointing
    int3 voxel_step = {1, 1, 1};
	voxel_step *= (ray_dir > 0) - (ray_dir < 0);

    // Setup the voxel coords from the camera origin
	int3 voxel = convert_int3(*cam_pos);

    // Delta T is the units a ray must travel along an axis in order to
    // traverse an integer split
	if (any(ray_dir == (0.0f,0.0f,0.0f)))
		return;
	float3 delta_t = fabs(1.0f / ray_dir);

	// Intersection T is the collection of the next intersection points
	// for all 3 axis XYZ. We take the full positive cardinality when
	// subtracting the floor, so we must transfer the sign over from
	// the voxel step
	float3 intersection_t = delta_t * ((*cam_pos) - ceil(*cam_pos)) * convert_float3(voxel_step);

	// When we transfer the sign over, we get the correct direction of
	// the offset, but we merely transposed over the value instead of mirroring
	// it over the axis like we want. So here, isless returns a boolean if intersection_t
	// is less than 0 which dictates whether or not we subtract the delta which in effect
	// mirrors the offset
	intersection_t -= delta_t * convert_float3(isless(intersection_t, 0));

	int distance_traveled = 0;
	int max_distance = 700;
	uint bounce_count = 0;
	int3 face_mask = { 0, 0, 0 };
	int voxel_data = 0;
	float3 face_position = (0,0,0);
	float4 voxel_color= (0,0,0,0);
	float2 tile_face_position = (0,0);
	float3 sign = (0,0,0);
	float4 first_strike = (0,0,0,0);

	// Andrew Woo's raycasting algo
    while (distance_traveled < max_distance && bounce_count < 2) {

		// Fancy no branch version of the logic step
		face_mask = intersection_t.xyz <= min(intersection_t.yzx, intersection_t.zxy);
		intersection_t += delta_t * fabs(convert_float3(face_mask.xyz));
		voxel.xyz += voxel_step.xyz * face_mask.xyz;

		// Test for out of bounds contions, add fog
		if (any(voxel >= *map_dim)){
			write_imagef(image, pixel, white_light(mix(fog_color, overshoot_color, 1.0 - max(distance_traveled / 700.0f, (float)0)), (float3)(lights[7], lights[8], lights[9]), face_mask));
			return;
		}
		if (any(voxel < 0)) {
			write_imagef(image, pixel, white_light(mix(fog_color, overshoot_color_2, 1.0 - max(distance_traveled / 700.0f, (float)0)), (float3)(lights[7], lights[8], lights[9]), face_mask));
			return;
		}

        // If we hit a voxel
		// if (voxel.x < 128 && voxel.y < 128 && voxel.z < 128){
		// 	if (get_oct_vox(
		// 		voxel,
		// 		octree_descriptor_buffer,
		// 		octree_attachment_lookup_buffer,
		// 		octree_attachment_buffer,
		// 		settings_buffer
		// 		)){
		// 			voxel_data = 1;
		// 		} else {
		// 			voxel_data = 0;
		// 		}
		// } else {
			voxel_data = map[voxel.x + (*map_dim).x * (voxel.y + (*map_dim).z * (voxel.z))];
		//}

		if (voxel_data != 0) {

			// Determine where on the 2d plane the ray intersected
			face_position = (float3)(0);
			tile_face_position = (float2)(0);
			sign = (float3)(1.0f, 1.0f, 1.0f);

			// First determine the percent of the way the ray is towards the next intersection_t
			// in relation to the xyz position on the plane
			if (face_mask.x == -1) {

				sign.x *= -1.0;

				// the next intersection for this plane - the last intersection of the passed plane / delta of this plane
				// basically finds how far in on the other 2 axis we are when the ray traversed the plane
				float z_percent = (intersection_t.z - (intersection_t.x - delta_t.x)) / delta_t.z;
				float y_percent = (intersection_t.y - (intersection_t.x - delta_t.x)) / delta_t.y;

				// Since we intersected face x, we know that we are at the face (1.0)
				// I think the 1.001f rendering bug is the ray thinking it's within the voxel
				// even though it's sitting on the very edge
				face_position = (float3)(1.0001f, y_percent, z_percent);
				tile_face_position = (float2)(y_percent, z_percent);
			}
			else if (face_mask.y == -1) {

				sign.y *= -1.0;
				float x_percent = (intersection_t.x - (intersection_t.y - delta_t.y)) / delta_t.x;
				float z_percent = (intersection_t.z - (intersection_t.y - delta_t.y)) / delta_t.z;

				face_position = (float3)(x_percent, 1.0001f, z_percent);
				tile_face_position = (float2)(x_percent, z_percent);
			}

			else if (face_mask.z == -1) {

				sign.z *= -1.0;
				float x_percent = (intersection_t.x - (intersection_t.z - delta_t.z)) / delta_t.x;
				float y_percent = (intersection_t.y - (intersection_t.z - delta_t.z)) / delta_t.y;

				face_position = (float3)(x_percent, y_percent, 1.0001f);
				tile_face_position = (float2)(x_percent, y_percent);

			}

			// Because the raycasting process is agnostic to the quadrant
			// it's working in, we need to transpose the sign over to the face positions.
			// If we don't it will think that it is always working in the (1, 1, 1) quadrant
			// and will just "copy" the quadrant. This includes shadows as they use the face_position
			// in order to cast the intersection ray!!

			face_position.x = select((float)(face_position.x), (float)(-face_position.x + 1.0f), (int)(ray_dir.x > 0));
			tile_face_position.x = select((float)(tile_face_position.x), (float)(-tile_face_position.x + 1.0f), (int)(ray_dir.x < 0));

			if (ray_dir.y > 0){
				face_position.y =  - face_position.y + 1;
			} else {
				tile_face_position.x = 1.0 - tile_face_position.x;

				// We run into the Hairy ball problem, so we need to define
				// a special case for the zmask
				if (face_mask.z == -1) {
					tile_face_position.x = 1.0 - tile_face_position.x;
					tile_face_position.y = 1.0 - tile_face_position.y;
				}
			}

			face_position.z = select((float)(face_position.z), (float)(-face_position.z + 1.0f), (int)(ray_dir.z > 0));
			tile_face_position.y = select((float)(tile_face_position.y), (float)(-tile_face_position.y + 1.0f), (int)(ray_dir.z < 0));

			// Now either use the face position to retrieve a texture sample, or
			// just a plain color for the voxel color. Notice the JANK -1 after the
			// conditionals in the select statement. That's because select works on negs
			// and pos's. So a false equality will still eval as true as it is technically
			// a positive result (0)
			// voxel_color = select(
			// 	(float4)(0.25f, 0.64f, 0.87f, 0.0f),
			// 	(float4)voxel_color,
			//  	(int4)((voxel_data == 5) - 1)
			// );

			if (bounce_count == 0){
				voxel_color = (float4)read_imagef(
						 texture_atlas,
						 convert_int2(tile_face_position * convert_float2(*atlas_dim / *tile_dim)) +
						 convert_int2((float2)(3, 0) * convert_float2(*atlas_dim / *tile_dim))
				);

				voxel_color.w = 0.0f;
				first_strike = view_light(
			 			voxel_color,
			 			(convert_float3(voxel) + face_position) - (float3)(lights[4], lights[5], lights[6]),
			 			(float4)(lights[0], lights[1], lights[2], lights[3]),
			 			(convert_float3(voxel) + face_position) - (*cam_pos),
			 			face_mask * voxel_step
			 	);
				max_distance = 10;
				distance_traveled = 0;
			} else {
				// voxel_color = (float4)read_imagef(
				// 		 texture_atlas,
				// 		 convert_int2(tile_face_position * convert_float2(*atlas_dim / *tile_dim)) +
				// 		 convert_int2((float2)(3, 0) * convert_float2(*atlas_dim / *tile_dim))
				// );

				voxel_color.w = 0.2f;
				first_strike = white_light(voxel_color, (float3)(1.0f, 1.0f, 1.0f), face_mask);
			}
			//
			// voxel_color = select(
			// 	(float4)(0.0f, 0.239f, 0.419f, 0.0f),
			// 	(float4)read_imagef(
			// 		 texture_atlas,
			// 		 convert_int2(tile_face_position * convert_float2(*atlas_dim / *tile_dim)) +
			// 		 convert_int2((float2)(3, 0) * convert_float2(*atlas_dim / *tile_dim))
			// 	 ),
			// 	 (int4)((voxel_data == 6) - 1)
			// );

			// The new direction of the ray
			// The list of lights and their distances

			// a way to accumulate color and light intensity through multiple bounces??
			//		Only for indirect lighting and refraction
			//

			float3 hit_pos = convert_float3(voxel) + face_position;
			ray_dir = normalize((float3)(lights[4], lights[5], lights[6]) - hit_pos);
			if (any(ray_dir == (0.0f,0.0f,0.0f)))
				return;

			voxel -= voxel_step * face_mask;
			voxel_step = ( 1, 1, 1 );
			voxel_step *= (ray_dir > 0) - (ray_dir < 0);

			//voxel = convert_int3(hit_pos);

			delta_t = fabs(1.0f / ray_dir);
			intersection_t = delta_t * ((hit_pos)-floor(hit_pos)) * convert_float3(voxel_step);
			intersection_t += delta_t * -convert_float3(isless(intersection_t, 0));

			bounce_count += 1;

			// max_distance = DistanceBetweenPoints(convert_float3(voxel) + face_position, (float3)(lights[4], lights[5], lights[6]));

			// // This has a very large performance hit, I assume CL doesn't really
			// like calling into other functions with lots of state.
			// if (cast_light_intersection_ray(
			// 	map,
			// 	map_dim,
			// 	normalize((float3)(lights[4], lights[5], lights[6]) - (convert_float3(voxel) + face_position)),
			// 	(convert_float3(voxel) + face_position),
			// 	lights,
			// 	light_count
			// )) {
			//
			// 	// If the light ray intersected an object on the way to the light point
			// 	write_imagef(image, pixel, white_light(voxel_color, (float3)(1.0f, 1.0f, 1.0f), face_mask));
			// 	return;
			// }

			//return;
		}

		distance_traveled++;

    }
	write_imagef(
		image,
		pixel,
		first_strike
		);
	// write_imagef(
	// 	image,
	// 	pixel,
	// 	view_light(
	// 		voxel_color,
	// 		(convert_float3(voxel) + face_position) - (float3)(lights[4], lights[5], lights[6]),
	// 		(float4)(lights[0], lights[1], lights[2], lights[3]),
	// 		(convert_float3(voxel) + face_position) - (*cam_pos),
	// 		face_mask * voxel_step
	// 		)
	// 	);
    return;
}