Implement packet-base BVH traversal kernel

#ifndef INTERSECT_CL
#define INTERSECT_CL

#define MAX_ITERATIONS 10000

#define INTERSECTION_EPSILON 0.001f

#define BVH_MAX_STACK_SIZE 32

#define BVH_IS_LEAF(node) (node.leftChild.w <= 0)
#define BVH_LEFT_CHILD(node) (node.leftChild.w)
#define BVH_RIGHT_CHILD(node) (node.rightChild.w)
#define BVH_TRIANGLE_INDEX(node) (-node.firstTriIndex.w)
#define BVH_TRIANGLE_COUNT(node) (node.numTriangles.w)
#define BVH_MESH_INSTANCE_ID(node) (-node.meshInstance.w)

#define RAY_PACKET_SIZE 64
#define HALF_RAY_PACKET_SIZE RAY_PACKET_SIZE / 2

#define RAY_VISIT_NO_NODE 0
#define RAY_VISIT_LEFT_NODE 1
#define RAY_VISIT_RIGHT_NODE 2
#define RAY_VISIT_BOTH_NODES 3

// Test for ray intersections with scene geometry and set an ouput flag to indicate
// intersections. This method does not calculate any intersection details so its
// cheaper to use for general intersection queries (e.g light occlusion)
__kernel void rayIntersectionTest(
		__global Ray* rays,
		__global BvhNode* bvhNodes,
		__global MeshInstance* meshInstances,
		__global float4* vertexList,
		__global int* hitFlag
		){

	Intersection intersection;
	intersection.wuvt.w = FLT_MAX;

	uint globalId = get_global_id(0);

	int stackIndex;
	int meshBvhStackStartIndex;
	uint nodeStack[BVH_MAX_STACK_SIZE];
	BvhNode curNode;
	BvhNode childNodes[2];
	int meshInstanceId;
	MeshInstance meshInstance;

	// triangle intersection vars
	float3 v0, edge01, edge02;

	// Node bbox and leaf primitive intersection vars
	float3 invDir, tmin, tmax, rmin, rmax;
	float minmax, maxmin;
	int triStartIndex, numTriangles;

	// Fetch ray
	Ray	ray = rays[globalId];
	float3 origRayOrigin = ray.origin.xyz;
	float3 origRayDir = ray.dir.xyz;

	// Setup stack
	stackIndex = 0;
	meshBvhStackStartIndex = -1;
	curNode = bvhNodes[0];

	for(int iteration=0;iteration<MAX_ITERATIONS;iteration++){
		if(BVH_IS_LEAF(curNode)){
			numTriangles = BVH_TRIANGLE_COUNT(curNode);

			// If this is a top BVH leaf we need to load the mesh instance
			// and transform all rays using its matrix.
			if( numTriangles == 0 ){
				meshInstanceId = BVH_MESH_INSTANCE_ID(curNode);
				meshInstance = meshInstances[meshInstanceId];

				// Push bottom BVH root to the stack and keep a record
				// of the current stack so that we know when we exit the
				// bottom BVH
				meshBvhStackStartIndex = stackIndex;
				nodeStack[stackIndex++] = meshInstance.bvhRoot;

				// Transform rays without translating ray direction vector
				ray.origin.xyz = mul4x1(ray.origin.xyz, meshInstance.transformMat0, meshInstance.transformMat1, meshInstance.transformMat2, meshInstance.transformMat3);
				ray.dir.xyz = mul3x1(ray.dir.xyz, meshInstance.transformMat0.xyz, meshInstance.transformMat1.xyz, meshInstance.transformMat2.xyz);
			} else {
				// Intersect with all triangles using the Moller-Trumbore algorithm
				triStartIndex = BVH_TRIANGLE_INDEX(curNode);
				for(int vIndex = triStartIndex * 3; vIndex < (triStartIndex + numTriangles)*3;vIndex+=3){
					v0 = vertexList[vIndex].xyz;
					edge01 = vertexList[vIndex+1].xyz - v0;
					edge02 = vertexList[vIndex+2].xyz - v0;

					float3 pVec = cross(ray.dir.xyz, edge02);
					float det = dot(edge01, pVec);

					if (fabs(det) < INTERSECTION_EPSILON){
						continue;
					}

					float invDet = native_recip(det);

					// Calculate barycentric coords
					float3 tVec = ray.origin.xyz - v0;
					float u = dot(tVec, pVec) * invDet;
					if( u < 0.0f || u > 1.0f ){
						continue;
					}

					float3 qVec = cross(tVec, edge01);
					float v = dot(ray.dir.xyz, qVec) * invDet;
					if( v < 0.0f || u+v > 1.0f ){
						continue;
					}

					float t = dot(edge02, qVec) * invDet;
					if (t > INTERSECTION_EPSILON && t < intersection.wuvt.w){
						hitFlag[globalId] = 1;
						return;
					}
				}
			}
		} else {
			// Read children
			childNodes[0] = bvhNodes[BVH_LEFT_CHILD(curNode)];
			childNodes[1] = bvhNodes[BVH_RIGHT_CHILD(curNode)];

			// Check for intersection with first child
			invDir = native_recip(ray.dir.xyz);
			tmin = (childNodes[0].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[0].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float lHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			// Check for intersection with second child
			tmin = (childNodes[1].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[1].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float rHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			int wantLeft = lHitDist < FLT_MAX ? 1 : 0;
			int wantRight = rHitDist < FLT_MAX ? 1 : 0;

			if( wantLeft && wantRight ){
				nodeStack[stackIndex++] = wantLeft ? BVH_RIGHT_CHILD(curNode) : BVH_LEFT_CHILD(curNode);
				curNode = wantLeft ? childNodes[0] : childNodes[1];
				continue;
			} else if(wantLeft || wantRight){
				curNode = wantLeft ? childNodes[0] : childNodes[1];
				continue;
			}
		}

 		if(stackIndex == 0){
			// We are done
			hitFlag[globalId] = 0;
			return;
		} else if(stackIndex == meshBvhStackStartIndex){
			// If we exited from a bottom bvh tree we need to restore our ray
			ray.origin.xyz = origRayOrigin;
			ray.dir.xyz = origRayDir;
			meshBvhStackStartIndex = -1;
		}

		// Pop the next node off the stack
		curNode = bvhNodes[nodeStack[--stackIndex]];
	}
}

// Test for ray intersections with scene geometry. Sets an ouput flag to indicate
// intersections and also emits intersection data for any found intersections.
__kernel void rayIntersectionQuery(
		__global Ray* rays,
		__global BvhNode* bvhNodes,
		__global MeshInstance* meshInstances,
		__global float4* vertexList,
		__global int* hitFlag,
		__global Intersection* intersections
		){

	Intersection intersection;
	intersection.wuvt.w = FLT_MAX;

	uint globalId = get_global_id(0);

	int stackIndex;
	int meshBvhStackStartIndex;
	uint nodeStack[BVH_MAX_STACK_SIZE];
	BvhNode curNode;
	BvhNode childNodes[2];
	int meshInstanceId;
	MeshInstance meshInstance;

	// triangle intersection vars
	 float3 v0, edge01, edge02;

	// Node bbox and leaf primitive intersection vars
	float3 invDir, tmin, tmax, rmin, rmax;
	float minmax, maxmin;
	int triStartIndex, numTriangles;

	// Fetch ray
	Ray	ray = rays[globalId];
	float3 origRayOrigin = ray.origin.xyz;
	float3 origRayDir = ray.dir.xyz;

	// Setup stack
	stackIndex = 0;
	meshBvhStackStartIndex = -1;
	curNode = bvhNodes[0];

	for(int iteration=0;iteration<MAX_ITERATIONS;iteration++){
		if(BVH_IS_LEAF(curNode)){
			numTriangles = BVH_TRIANGLE_COUNT(curNode);

			// If this is a top BVH leaf we need to load the mesh instance
			// and transform all rays using its matrix.
			if( numTriangles == 0 ){
				meshInstanceId = BVH_MESH_INSTANCE_ID(curNode);
				meshInstance = meshInstances[meshInstanceId];

				// Push bottom BVH root to the stack and keep a record
				// of the current stack so that we know when we exit the
				// bottom BVH
				meshBvhStackStartIndex = stackIndex;
				nodeStack[stackIndex++] = meshInstance.bvhRoot;

				// Transform rays without translating ray direction vector
				ray.origin.xyz = mul4x1(ray.origin.xyz, meshInstance.transformMat0, meshInstance.transformMat1, meshInstance.transformMat2, meshInstance.transformMat3);
				ray.dir.xyz = mul3x1(ray.dir.xyz, meshInstance.transformMat0.xyz, meshInstance.transformMat1.xyz, meshInstance.transformMat2.xyz);
			} else {
				// Intersect with all triangles using the Moller-Trumbore algorithm
				triStartIndex = BVH_TRIANGLE_INDEX(curNode);
				for(int vIndex = triStartIndex * 3; vIndex < (triStartIndex + numTriangles)*3;vIndex+=3){
					v0 = vertexList[vIndex].xyz;
					edge01 = vertexList[vIndex+1].xyz - v0;
					edge02 = vertexList[vIndex+2].xyz - v0;

					float3 pVec = cross(ray.dir.xyz, edge02);
					float det = dot(edge01, pVec);

					if (fabs(det) < INTERSECTION_EPSILON){
						continue;
					}

					float invDet = native_recip(det);

					// Calculate barycentric coords
					float3 tVec = ray.origin.xyz - v0;
					float u = dot(tVec, pVec) * invDet;
					if( u < 0.0f || u > 1.0f ){
						continue;
					}

					float3 qVec = cross(tVec, edge01);
					float v = dot(ray.dir.xyz, qVec) * invDet;
					if( v < 0.0f || u+v > 1.0f ){
						continue;
					}

					float t = dot(edge02, qVec) * invDet;
					if (t > INTERSECTION_EPSILON && t < intersection.wuvt.w){
						intersection.wuvt = (float4)(
								1.0f - (u+v),
								u,
								v,
								t
						);
						intersection.triIndex = vIndex / 3;
						intersection.meshInstance = meshInstanceId;
					}
				}
			}
		} else {
			// Read children
			childNodes[0] = bvhNodes[BVH_LEFT_CHILD(curNode)];
			childNodes[1] = bvhNodes[BVH_RIGHT_CHILD(curNode)];

			// Check for intersection with first child
			invDir = native_recip(ray.dir.xyz);
			tmin = (childNodes[0].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[0].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float lHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			// Check for intersection with second child
			tmin = (childNodes[1].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[1].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float rHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			int wantLeft = lHitDist < FLT_MAX ? 1 : 0;
			int wantRight = rHitDist < FLT_MAX ? 1 : 0;

			if( wantLeft && wantRight ){
				nodeStack[stackIndex++] = wantLeft ? BVH_RIGHT_CHILD(curNode) : BVH_LEFT_CHILD(curNode);
				curNode = wantLeft ? childNodes[0] : childNodes[1];

				continue;
			} else if(wantLeft || wantRight){
				curNode = wantLeft ? childNodes[0] : childNodes[1];
				continue;
			}
		}

 		if(stackIndex == 0){
			// We are done
			hitFlag[globalId] = intersection.wuvt.w < FLT_MAX ? 1 : 0;
			intersections[globalId] = intersection;
			return;
		} else if(stackIndex == meshBvhStackStartIndex){
			// If we exited from a bottom bvh tree we need to restore our ray
			ray.origin.xyz = origRayOrigin;
			ray.dir.xyz = origRayDir;
			meshBvhStackStartIndex = -1;
		}

		// Pop the next node off the stack
		curNode = bvhNodes[nodeStack[--stackIndex]];
	}
}

// Test for ray packet intersections with scene geometry. Sets an ouput flag to
// indicate intersections and also emits intersection data for any found intersections.
// This kernel operates on a bundle of RAY_PACKET_SIZE rays in parallel. Stack
// operations are handled by the first thread in the local thread group.
__kernel void packetIntersectionQuery(
		__global Ray* rays,
		__global BvhNode* bvhNodes,
		__global MeshInstance* meshInstances,
		__global float4* vertexList,
		__global int* hitFlag,
		__global Intersection* intersections
		){

	Intersection intersection;
	intersection.wuvt.w = FLT_MAX;

	uint globalId = get_global_id(0);
	uint localId = get_local_id(0);

	// Shared data for all threads
	__local int stackIndex;
	__local int meshBvhStackStartIndex;
	__local uint nodeStack[BVH_MAX_STACK_SIZE];
	__local BvhNode curNode;
	__local  BvhNode childNodes[2];
	__local int scratchMemory[RAY_PACKET_SIZE];
	__local int meshInstanceId;
	__local MeshInstance meshInstance;

	// Shared triangle intersection vars
	__local float3 v0, edge01, edge02;

	// Node bbox and leaf primitive intersection vars
	float3 invDir, tmin, tmax, rmin, rmax;
	float minmax, maxmin;
	int triStartIndex, numTriangles;

	// Fetch ray
	Ray	ray = rays[globalId];
	float3 origRayOrigin = ray.origin.xyz;
	float3 origRayDir = ray.dir.xyz;

	// Thread 0 manages the stack; set initial values
	if(localId == 0){
		stackIndex = 0;
		meshBvhStackStartIndex = -1;
		curNode = bvhNodes[0];
	}

	for(int iteration=0;iteration<MAX_ITERATIONS;iteration++){
		if(BVH_IS_LEAF(curNode)){
			numTriangles = BVH_TRIANGLE_COUNT(curNode);

			// If this is a top BVH leaf we need to load the mesh instance
			// and transform all rays using its matrix.
			if( numTriangles == 0 ){
				if( localId == 0 ){
					meshInstanceId = BVH_MESH_INSTANCE_ID(curNode);
					meshInstance = meshInstances[meshInstanceId];

					// Push bottom BVH root to the stack and keep a record
					// of the current stack so that we know when we exit the
					// bottom BVH
					meshBvhStackStartIndex = stackIndex;
					nodeStack[stackIndex++] = meshInstance.bvhRoot;
				}
				barrier(CLK_LOCAL_MEM_FENCE);

				// Transform rays without translating ray direction vector
				ray.origin.xyz = mul4x1(ray.origin.xyz, meshInstance.transformMat0, meshInstance.transformMat1, meshInstance.transformMat2, meshInstance.transformMat3);
				ray.dir.xyz = mul3x1(ray.dir.xyz, meshInstance.transformMat0.xyz, meshInstance.transformMat1.xyz, meshInstance.transformMat2.xyz);
			} else {
				// Intersect with all triangles using the Moller-Trumbore algorithm
				triStartIndex = BVH_TRIANGLE_INDEX(curNode);
				for(int vIndex = triStartIndex * 3; vIndex < (triStartIndex + numTriangles)*3;vIndex+=3){
					// Thread 0 fetches triangle data into local memory
					if(localId == 0 ){
						v0 = vertexList[vIndex].xyz;
						edge01 = vertexList[vIndex+1].xyz - v0;
						edge02 = vertexList[vIndex+2].xyz - v0;
					}
					barrier(CLK_LOCAL_MEM_FENCE);

					float3 pVec = cross(ray.dir.xyz, edge02);
					float det = dot(edge01, pVec);

					if (fabs(det) >= INTERSECTION_EPSILON){
						float invDet = native_recip(det);

						// Calculate barycentric coords
						float3 tVec = ray.origin.xyz - v0;
						float u = dot(tVec, pVec) * invDet;
						float3 qVec = cross(tVec, edge01);
						float v = dot(ray.dir.xyz, qVec) * invDet;
						float t = dot(edge02, qVec) * invDet;

						if (u >= 0.0f &&
							u <= 1.0f &&
							v >= 0.0f &&
							u+v <= 1.0f &&
							t > INTERSECTION_EPSILON &&
							t < intersection.wuvt.w){
							intersection.wuvt = (float4)(
									1.0f - (u+v),
									u,
									v,
									t
							);
							intersection.triIndex = vIndex / 3;
							intersection.meshInstance = meshInstanceId;
						}
					}
					barrier(CLK_LOCAL_MEM_FENCE);
				}
			}
		} else {
			// Read children in parallel
			if( localId < 2 ){
				childNodes[localId] = bvhNodes[localId == 0 ? BVH_LEFT_CHILD(curNode) : BVH_RIGHT_CHILD(curNode)];

				// Clear the first 4 scratchMemory locations
				scratchMemory[localId] = 0;
				scratchMemory[localId+2] = 0;
			}

			barrier(CLK_LOCAL_MEM_FENCE);

			// Check for intersection with first child
			invDir = native_recip(ray.dir.xyz);
			tmin = (childNodes[0].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[0].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float lHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			// Check for intersection with second child
			tmin = (childNodes[1].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[1].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float rHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			// If scratchMemory[i] is TRUE then at least one ray wants to:
			// [0] visit none of the nodes
			// [1] visit only left node
			// [2] visit only right node
			// [3] visit both nodes
			int wantLeft = lHitDist < FLT_MAX ? 1 : 0;
			int wantRight = rHitDist < FLT_MAX ? 1 : 0;
			//printf("[%d] wantLeft: %d, wantRight: %d\n", localId, wantLeft, wantRight);
			scratchMemory[2*wantRight + wantLeft] = 1;

			// Wait for all threads to encode their traversal preference
			barrier(CLK_LOCAL_MEM_FENCE);

			if( scratchMemory[RAY_VISIT_BOTH_NODES] || (scratchMemory[RAY_VISIT_LEFT_NODE] && scratchMemory[RAY_VISIT_RIGHT_NODE])){
				// For each thread, set its scratchMemory location to -1 if
				// ray prefers the left node and to 1 if the ray prefers the right node.
				scratchMemory[localId] = wantLeft || lHitDist < rHitDist ? -1 : 1;
				//printf("[%2d] scratchMemory -> %d\n", localId, scratchMemory[localId]);
				barrier(CLK_LOCAL_MEM_FENCE);

				// run a parallel reduction on scratchMemory. We are using
				// sequential addressing to avoid bank conflicts
				// see: https://docs.nvidia.com/cuda/samples/6_Advanced/reduction/doc/reduction.pdf
				for (unsigned int s=HALF_RAY_PACKET_SIZE; s>0; s>>=1) {
					if (localId < s) {
						scratchMemory[localId] += scratchMemory[localId + s];
					}
					barrier(CLK_LOCAL_MEM_FENCE);
				}

				// scratchMemory[0] sign indicates which node should we visit first
				if( localId == 0 ){
					nodeStack[stackIndex++] = scratchMemory[0] < 1 ? BVH_RIGHT_CHILD(curNode) : BVH_LEFT_CHILD(curNode);
					curNode = scratchMemory[0] < 1 ? childNodes[0] : childNodes[1];
				}

				barrier(CLK_LOCAL_MEM_FENCE);
				continue;
			} else if(scratchMemory[RAY_VISIT_LEFT_NODE] || scratchMemory[RAY_VISIT_RIGHT_NODE]) {
				if( localId == 0 ){
					curNode = scratchMemory[1] ? childNodes[0] : childNodes[1];
				}

				barrier(CLK_LOCAL_MEM_FENCE);
				continue;
			}
		}

 		if(stackIndex == 0){
			// We are done
			hitFlag[globalId] = intersection.wuvt.w < FLT_MAX ? 1 : 0;
			intersections[globalId] = intersection;
			return;
		} else if(stackIndex == meshBvhStackStartIndex){
			// If we exited from a bottom bvh tree we need to restore our ray
			ray.origin.xyz = origRayOrigin;
			ray.dir.xyz = origRayDir;

			if(localId == 0 ){
				meshBvhStackStartIndex = -1;
			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		// Pop the next node pointer off the stack and read it into local memory
		if( localId == 0 ){
			curNode = bvhNodes[nodeStack[--stackIndex]];
		}

		barrier(CLK_LOCAL_MEM_FENCE);
	}
}

#endif

#ifndef INTERSECT_CL
#define INTERSECT_CL

#define MAX_ITERATIONS 10000

#define INTERSECTION_EPSILON 0.001f

#define BVH_MAX_STACK_SIZE 32

#define BVH_IS_LEAF(node) (node.leftChild.w <= 0)
#define BVH_LEFT_CHILD(node) (node.leftChild.w)
#define BVH_RIGHT_CHILD(node) (node.rightChild.w)
#define BVH_TRIANGLE_INDEX(node) (-node.firstTriIndex.w)
#define BVH_TRIANGLE_COUNT(node) (node.numTriangles.w)
#define BVH_MESH_INSTANCE_ID(node) (-node.meshInstance.w)

// Test for ray intersections with scene geometry and set an ouput flag to indicate
// intersections. This method does not calculate any intersection details so its
// cheaper to use for general intersection queries (e.g light occlusion)
__kernel void rayIntersectionTest(
		__global Ray* rays,
		__global BvhNode* bvhNodes,
		__global MeshInstance* meshInstances,
		__global float4* vertexList,
		__global int* hitFlag
		){

	Intersection intersection;
	intersection.wuvt.w = FLT_MAX;

	uint globalId = get_global_id(0);

	int stackIndex;
	int meshBvhStackStartIndex;
	uint nodeStack[BVH_MAX_STACK_SIZE];
	BvhNode curNode;
	BvhNode childNodes[2];
	int meshInstanceId;
	MeshInstance meshInstance;

	// triangle intersection vars
	float3 v0, edge01, edge02;

	// Node bbox and leaf primitive intersection vars
	float3 invDir, tmin, tmax, rmin, rmax;
	float minmax, maxmin;
	int triStartIndex, numTriangles;

	// Fetch ray
	Ray	ray = rays[globalId];
	float3 origRayOrigin = ray.origin.xyz;
	float3 origRayDir = ray.dir.xyz;

	// Setup stack
	stackIndex = 0;
	meshBvhStackStartIndex = -1;
	curNode = bvhNodes[0];

	for(int iteration=0;iteration<MAX_ITERATIONS;iteration++){
		if(BVH_IS_LEAF(curNode)){
			numTriangles = BVH_TRIANGLE_COUNT(curNode);

			// If this is a top BVH leaf we need to load the mesh instance
			// and transform all rays using its matrix.
			if( numTriangles == 0 ){
				meshInstanceId = BVH_MESH_INSTANCE_ID(curNode);
				meshInstance = meshInstances[meshInstanceId];

				// Push bottom BVH root to the stack and keep a record
				// of the current stack so that we know when we exit the
				// bottom BVH
				meshBvhStackStartIndex = stackIndex;
				nodeStack[stackIndex++] = meshInstance.bvhRoot;

				// Transform rays without translating ray direction vector
				ray.origin.xyz = mul4x1(ray.origin.xyz, meshInstance.transformMat0, meshInstance.transformMat1, meshInstance.transformMat2, meshInstance.transformMat3);
				ray.dir.xyz = mul3x1(ray.dir.xyz, meshInstance.transformMat0.xyz, meshInstance.transformMat1.xyz, meshInstance.transformMat2.xyz);
			} else {
				// Intersect with all triangles using the Moller-Trumbore algorithm
				triStartIndex = BVH_TRIANGLE_INDEX(curNode);
				for(int vIndex = triStartIndex * 3; vIndex < (triStartIndex + numTriangles)*3;vIndex+=3){
					v0 = vertexList[vIndex].xyz;
					edge01 = vertexList[vIndex+1].xyz - v0;
					edge02 = vertexList[vIndex+2].xyz - v0;

					float3 pVec = cross(ray.dir.xyz, edge02);
					float det = dot(edge01, pVec);

					if (fabs(det) < INTERSECTION_EPSILON){
						continue;
					}

					float invDet = native_recip(det);

					// Calculate barycentric coords
					float3 tVec = ray.origin.xyz - v0;
					float u = dot(tVec, pVec) * invDet;
					if( u < 0.0f || u > 1.0f ){
						continue;
					}

					float3 qVec = cross(tVec, edge01);
					float v = dot(ray.dir.xyz, qVec) * invDet;
					if( v < 0.0f || u+v > 1.0f ){
						continue;
					}

					float t = dot(edge02, qVec) * invDet;
					if (t > INTERSECTION_EPSILON && t < intersection.wuvt.w){
						hitFlag[globalId] = 1;
						return;
					}
				}
			}
		} else {
			// Read children
			childNodes[0] = bvhNodes[BVH_LEFT_CHILD(curNode)];
			childNodes[1] = bvhNodes[BVH_RIGHT_CHILD(curNode)];

			// Check for intersection with first child
			invDir = native_recip(ray.dir.xyz);
			tmin = (childNodes[0].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[0].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float lHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			// Check for intersection with second child
			tmin = (childNodes[1].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[1].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float rHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			int wantLeft = lHitDist < FLT_MAX ? 1 : 0;
			int wantRight = rHitDist < FLT_MAX ? 1 : 0;

			if( wantLeft && wantRight ){
				nodeStack[stackIndex++] = wantLeft ? BVH_RIGHT_CHILD(curNode) : BVH_LEFT_CHILD(curNode);
				curNode = wantLeft ? childNodes[0] : childNodes[1];
				continue;
			} else if(wantLeft || wantRight){
				curNode = wantLeft ? childNodes[0] : childNodes[1];
				continue;
			}
		}

 		if(stackIndex == 0){
			// We are done
			hitFlag[globalId] = 0;
			return;
		} else if(stackIndex == meshBvhStackStartIndex){
			// If we exited from a bottom bvh tree we need to restore our ray
			ray.origin.xyz = origRayOrigin;
			ray.dir.xyz = origRayDir;
			meshBvhStackStartIndex = -1;
		}

		// Pop the next node off the stack
		curNode = bvhNodes[nodeStack[--stackIndex]];
	}
}

// Test for ray intersections with scene geometry. Sets an ouput flag to indicate
// intersections and also emits intersection data for any found intersections.
__kernel void rayIntersectionQuery(
		__global Ray* rays,
		__global BvhNode* bvhNodes,
		__global MeshInstance* meshInstances,
		__global float4* vertexList,
		__global int* hitFlag,
		__global Intersection* intersections
		){

	Intersection intersection;
	intersection.wuvt.w = FLT_MAX;

	uint globalId = get_global_id(0);

	int stackIndex;
	int meshBvhStackStartIndex;
	uint nodeStack[BVH_MAX_STACK_SIZE];
	BvhNode curNode;
	BvhNode childNodes[2];
	int meshInstanceId;
	MeshInstance meshInstance;

	// triangle intersection vars
	 float3 v0, edge01, edge02;

	// Node bbox and leaf primitive intersection vars
	float3 invDir, tmin, tmax, rmin, rmax;
	float minmax, maxmin;
	int triStartIndex, numTriangles;

	// Fetch ray
	Ray	ray = rays[globalId];
	float3 origRayOrigin = ray.origin.xyz;
	float3 origRayDir = ray.dir.xyz;

	// Setup stack
	stackIndex = 0;
	meshBvhStackStartIndex = -1;
	curNode = bvhNodes[0];

	for(int iteration=0;iteration<MAX_ITERATIONS;iteration++){
		if(BVH_IS_LEAF(curNode)){
			numTriangles = BVH_TRIANGLE_COUNT(curNode);

			// If this is a top BVH leaf we need to load the mesh instance
			// and transform all rays using its matrix.
			if( numTriangles == 0 ){
				meshInstanceId = BVH_MESH_INSTANCE_ID(curNode);
				meshInstance = meshInstances[meshInstanceId];

				// Push bottom BVH root to the stack and keep a record
				// of the current stack so that we know when we exit the
				// bottom BVH
				meshBvhStackStartIndex = stackIndex;
				nodeStack[stackIndex++] = meshInstance.bvhRoot;

				// Transform rays without translating ray direction vector
				ray.origin.xyz = mul4x1(ray.origin.xyz, meshInstance.transformMat0, meshInstance.transformMat1, meshInstance.transformMat2, meshInstance.transformMat3);
				ray.dir.xyz = mul3x1(ray.dir.xyz, meshInstance.transformMat0.xyz, meshInstance.transformMat1.xyz, meshInstance.transformMat2.xyz);
			} else {
				// Intersect with all triangles using the Moller-Trumbore algorithm
				triStartIndex = BVH_TRIANGLE_INDEX(curNode);
				for(int vIndex = triStartIndex * 3; vIndex < (triStartIndex + numTriangles)*3;vIndex+=3){
					v0 = vertexList[vIndex].xyz;
					edge01 = vertexList[vIndex+1].xyz - v0;
					edge02 = vertexList[vIndex+2].xyz - v0;

					float3 pVec = cross(ray.dir.xyz, edge02);
					float det = dot(edge01, pVec);

					if (fabs(det) < INTERSECTION_EPSILON){
						continue;
					}

					float invDet = native_recip(det);

					// Calculate barycentric coords
					float3 tVec = ray.origin.xyz - v0;
					float u = dot(tVec, pVec) * invDet;
					if( u < 0.0f || u > 1.0f ){
						continue;
					}

					float3 qVec = cross(tVec, edge01);
					float v = dot(ray.dir.xyz, qVec) * invDet;
					if( v < 0.0f || u+v > 1.0f ){
						continue;
					}

					float t = dot(edge02, qVec) * invDet;
					if (t > INTERSECTION_EPSILON && t < intersection.wuvt.w){
						intersection.wuvt = (float4)(
								1.0f - (u+v),
								u,
								v,
								t
						);
						intersection.triIndex = vIndex / 3;
						intersection.meshInstance = meshInstanceId;
					}
				}
			}
		} else {
			// Read children
			childNodes[0] = bvhNodes[BVH_LEFT_CHILD(curNode)];
			childNodes[1] = bvhNodes[BVH_RIGHT_CHILD(curNode)];

			// Check for intersection with first child
			invDir = native_recip(ray.dir.xyz);
			tmin = (childNodes[0].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[0].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float lHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			// Check for intersection with second child
			tmin = (childNodes[1].minExtent.xyz - ray.origin.xyz) * invDir;
			tmax = (childNodes[1].maxExtent.xyz - ray.origin.xyz) * invDir;
			rmin = fmin(tmin, tmax);
			rmax = fmax(tmin, tmax);
			minmax = fmin( fmin(rmax.x, rmax.y), rmax.z);
			maxmin = fmax( fmax(rmin.x, rmin.y), rmin.z);
			float rHitDist = minmax < 0 || maxmin > minmax ? FLT_MAX : (maxmin >= ray.origin.w ? FLT_MAX : maxmin);

			int wantLeft = lHitDist < FLT_MAX ? 1 : 0;
			int wantRight = rHitDist < FLT_MAX ? 1 : 0;

			if( wantLeft && wantRight ){
				nodeStack[stackIndex++] = wantLeft ? BVH_RIGHT_CHILD(curNode) : BVH_LEFT_CHILD(curNode);
				curNode = wantLeft ? childNodes[0] : childNodes[1];

				continue;
			} else if(wantLeft || wantRight){
				curNode = wantLeft ? childNodes[0] : childNodes[1];
				continue;
			}
		}

 		if(stackIndex == 0){
			// We are done
			hitFlag[globalId] = intersection.wuvt.w < FLT_MAX ? 1 : 0;
			intersections[globalId] = intersection;
			return;
		} else if(stackIndex == meshBvhStackStartIndex){
			// If we exited from a bottom bvh tree we need to restore our ray
			ray.origin.xyz = origRayOrigin;
			ray.dir.xyz = origRayDir;
			meshBvhStackStartIndex = -1;
		}

		// Pop the next node off the stack
		curNode = bvhNodes[nodeStack[--stackIndex]];
	}
}

#endif


fix bugs in vertically looping

fix bugs in vertically looping

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0) * MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
//

#define MLO_IN_VERT_READS MLO_IN_EXTENT1

#if MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS - 2)
#else
#define MLO_N_GENERIC_LOOPS 0
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF \
    (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS / MLO_READ_UNIT) * MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS * MLO_N_LCL_IN_MAPS * MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = v / d; //(uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24((uint)u, (uint)d);
    return (r);
}

#if 0
__attribute__((always_inline)) void ReduceKernel(__local _FLOAT* lcl_blob,
                                                 __private _FLOAT* weights_accum,
                                                 uint lcl_id,
                                                 uint scan_lcl,
                                                 uint sum_stride,
                                                 uint unit_len,
                                                 UNUSED bool debug)
{
    // read first half
    if(scan_lcl < (sum_stride >> 1))
    {
        for(uint i = 0; i < unit_len; ++i)
        {
            weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];
        }
    }
    // add second half
    // appload accumulated value so far
    for(uint j = (sum_stride >> 1); j > 0; j >>= 1)
    {
        barrier(CLK_LOCAL_MEM_FENCE);
        if(scan_lcl < j)
        {
            for(uint i = 0; i < unit_len; ++i)
            {
                weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

                lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
            }
        }
    }
}
#endif

/*
        group cooperative read
        read by MLO_READ_UNIT
        handle out of range both horizontally and vertically (by fixed number of veryical reads)

        no guard against number of inputs
*/
__attribute__((always_inline)) void readInput(uint lcl_id,
                                              uint gbl_in_scan_off,
                                              uint n_v_reads,
                                              const __global _FLOAT* __restrict bot,
                                              __local _FLOAT* __restrict lcl_bot)
{
    for(uint p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
        p4 += MLO_GRP_SZ)
    {
        __private _FLOAT in_rd_data[MLO_READ_UNIT];
        // TODO : more than 1 input
        uint c    = 0;
        uint t_p4 = p4;
#if MLO_N_LCL_IN_MAPS > 1
        c    = p4 / (MLO_N_IN_HORIZ_READS * n_v_reads);
        t_p4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS * n_v_reads));
#endif

        uint c_scan = t_p4 / (MLO_N_IN_HORIZ_READS);

#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
        uint c_pix4 = iMod(t_p4, c_scan, (MLO_N_IN_HORIZ_READS));
#else
        uint c_pix4 = t_p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

        //		if (c < MLO_N_INPUTS)

        {
            uint bot_off = gbl_in_scan_off + c * MLO_IN_CHANNEL_STRIDE + c_scan * MLO_IN_STRIDE +
                           c_pix4 * MLO_READ_UNIT;
            const __global _FLOAT* bot_p = &bot[bot_off];
#if MLO_IN_N_PIXS_OFF > 0

            if(c_pix4 == MLO_N_IN_HORIZ_READS - 1)
            {
                for(uint i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
                {
                    in_rd_data[i] = bot_p[i];

#if DBG_OUT_OF_RNGE
                    if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                    {
                        printf("k:err:out-of-range\n");
                    }
#endif
                }

                for(uint i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
                {
                    in_rd_data[i] = 0;
                }
            }
            else
#endif
            {

                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    in_rd_data[i] = bot_p[i];

#if DBG_OUT_OF_RNGE
                    if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                    {
                        printf("k:err:out-of-range\n");
                    }
#endif
                }
            }

            // MLO_N_LCL_IN_MAPS inputs
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                int lcl_in_off = c * MLO_IN_LCL_SZ + c_scan * MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 +
                                 c_pix4 * MLO_READ_UNIT + i;
                lcl_bot[lcl_in_off] = in_rd_data[i];
            }
        }

    } // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

    barrier(CLK_LOCAL_MEM_FENCE);
}

/*
        core processing loop
        bot - input, from local (1 span)
        top - output diff, from global (array of spans, filters vertical size)

        loop over filter vertical size

*/
__attribute__((always_inline)) void Processing(UNUSED uint sc,
                                               uint sc_lcl_off,
                                               uint top_lim,
                                               int bot_lim,
                                               __private _FLOAT* __restrict pvt_accum,
                                               __local _FLOAT* __restrict lcl_bot,
                                               __private _FLOAT* __restrict top_dat)
{
    for(int l = top_lim; l >= bot_lim; --l)
    {
        for(uint m = 0; m < MLO_IN_TILE0; ++m)
        {
            for(uint n = 0; n < MLO_FILTER_SIZE0; ++n)
            {
                for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
                {
                    int bot_off = sc_lcl_off + c * MLO_IN_LCL_SZ + n + m;

                    _FLOAT bot_val = lcl_bot[bot_off];

                    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
                    {
                        uint pvt_top_off =
                            k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
                        uint pvt_accum_off =
                            (k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0 +
                            l * MLO_FILTER_SIZE0 + n;

                        _FLOAT top_val = top_dat[pvt_top_off];

                        pvt_accum[pvt_accum_off]
                            // each wk-it process an input
                            += bot_val * top_val;
                    }
                }
            }
        }
    }
}

__attribute__((always_inline)) void moveOutputUp(__private _FLOAT* __restrict top_dat)
{
    // move up output to reduce overfetch
    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
    {
        for(uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
        {
            for(uint i = 0; i < MLO_IN_TILE0; ++i)
            {
                uint pvt_off_n = k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j * MLO_IN_TILE0 + i;
                uint pvt_off_o = k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) * MLO_IN_TILE0 + i;
                top_dat[pvt_off_n] = top_dat[pvt_off_o];
            }
        }
    }
}

__attribute__((always_inline)) void spanReadingOutput(int spn,
                                                      int k,
                                                      int j,
                                                      int top_df_off,
                                                      _FLOAT mask,
                                                      __private _FLOAT* __restrict top_dat,
                                                      const __global _FLOAT* __restrict top_df)
{
    int pvt_off                     = k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j * MLO_IN_TILE0;
    const __global _FLOAT* top_df_p = &top_df[top_df_off];
#if MLO_OUT_N_PIXS_OFF > 0
    if(spn == MLO_N_SPANS_PER_SCAN - 1)
    {
        uint i = 0;
        for(; i < MLO_OUT_N_PIXS_OFF; ++i)
        {
            top_dat[pvt_off + i] = top_df_p[i] * mask;

#if DBG_OUT_OF_RNGE
            if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
            {
                printf("k:err:out-of-range\n");
            }
#endif
        }
        for(; i < MLO_IN_TILE0; ++i)
        {
            top_dat[pvt_off + i] = 0;
        }
    }
    else
#else
    (void)spn;
#endif
    {
        for(uint i = 0; i < MLO_IN_TILE0; ++i)
        {
            top_dat[pvt_off + i] = top_df_p[i] * mask;

#if DBG_OUT_OF_RNGE
            if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
            {
                printf("k:err:out-of-range\n");
            }
#endif
        }
    }
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenCvBwdWrW(const __global _FLOAT* __restrict top_df,
               const __global _FLOAT* __restrict bot,
               __global _FLOAT* __restrict weights_df,
#if MLO_CONV_BIAS
               __global _FLOAT* __restrict bias_df,
#endif
               UNUSED _FLOAT padding_val)
{

    // input/output tiles + reduce buffer

    __local _FLOAT lcl[(MLO_LCL_SZ) + 1];
    __local _FLOAT* lcl_bot = lcl;

    uint lcl_id = get_local_id(0);

    uint c_idx_base = get_group_id(0); // input map index base

    uint o_idx_base = get_group_id(1); // output map index base

    uint ib_base = get_group_id(2);

    uint ib = ib_base * (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS);

    uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

    uint o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

    uint gbl_in_off  = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
    uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
    // 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
    // TODO: more than 1 input
    uint o = lcl_id / MLO_N_SPANS_PER_SCAN;
#if MLO_N_SPANS_PER_SCAN & (MLO_N_SPANS_PER_SCAN - 1)
    uint spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);
#else
    uint spn         = lcl_id & (MLO_N_SPANS_PER_SCAN - 1);
#endif
    //	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);

    uint lcl_bot_off     = spn * MLO_IN_TILE0;
    uint out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
    gbl_out_off += out_wk_item_off;
    // no output out of range
    gbl_out_off = (o_idx + o < MLO_N_OUTPUTS && o < MLO_OUT_STACKS) ? gbl_out_off : 0;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

    __private _FLOAT top_dat[MLO_TOP_DAT_SZ];

    for(uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
    {
        top_dat[i] = 0;
    }

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_N_LCL_IN_MAPS * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0)

    __private _FLOAT pvt_accum[MLO_ACCUM_SZ];

    for(uint i = 0; i < MLO_ACCUM_SZ; ++i)
    {
        pvt_accum[i] = 0;
    }

    // zero out LDS
    for(uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
    {
        lcl[i] = 0;
    }

    // over all batches
    uint bend = ib + MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS;
    bend      = bend > MLO_BATCH_SZ ? MLO_BATCH_SZ : bend;

    for(uint b = ib; b < bend; ++b,
             gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
             gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE)
    {
        barrier(CLK_LOCAL_MEM_FENCE);

        // top border input block
        uint gbl_in_scan_off  = gbl_in_off;
        uint gbl_out_scan_off = gbl_out_off;

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 0: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[MLO_FILTER_SIZE1], pvt_accum[2*MLO_FILTER_SIZE1]);
        }
#endif

        // read input map
        readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

        // move input pointer
        gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

        for(uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
        {
            top_dat[i] = 0;
        }

        // prefetch output
        int gbl_out_scan_off1 = gbl_out_scan_off;
        for(uint k = 0; k < MLO_N_LCL_OUT_MAPS;
            ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
        {
            for(uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
            {
                // loop around all output maps
                uint top_df_off = gbl_out_scan_off1 + j * MLO_OUT_STRIDE;
                _FLOAT mask     = 1;
#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
                top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
                mask       = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

                spanReadingOutput(spn, k, j, top_df_off, mask, top_dat, top_df);
            }
        }

        gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

        uint sc         = 0;
        uint sc_lcl_off = lcl_bot_off;

        // prolog
        // handling padding

        // top padding
        for(; sc < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
        {
            Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
        }

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 1: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[MLO_FILTER_SIZE1], pvt_accum[2*MLO_FILTER_SIZE1]);
        }
#endif

#ifdef __AMDGCN__
#pragma unroll 2
#endif
        for(; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
                       -
                       MLO_FILTER_PAD1
#endif
            ;
            ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
        {

            for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
            {
                uint top_df_off = gbl_out_scan_off + k * MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
                _FLOAT mask     = 1;

#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
                top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
                mask       = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

                spanReadingOutput(
                    spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask, top_dat, top_df);
            }

            // processing
            Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

            // move up output to reduce overfetch
            moveOutputUp(top_dat);
        }

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 2: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[MLO_FILTER_SIZE1], pvt_accum[2*MLO_FILTER_SIZE1]);
        }
#endif

        // non-border input blocks
#if 0
//bug
        for(uint i_loop = 1; i_loop < MLO_N_GENERIC_LOOPS;
#else
//fix
        for(uint i_loop = 1; i_loop < MLO_N_GENERIC_LOOPS+1;
#endif
            ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
        {
            barrier(CLK_LOCAL_MEM_FENCE);

            readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

            // point to the start of the local buffer

            sc_lcl_off = lcl_bot_off;

            for(; sc < (i_loop + 1) * MLO_IN_EXTENT1;
                ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
            {

                for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
                {
                    uint top_df_off =
                        gbl_out_scan_off + k * MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
                    _FLOAT mask = 1;

#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT
                    top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
                    mask       = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

                    spanReadingOutput(
                        spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask, top_dat, top_df);
                }

                // processing
                Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

                // move up output to reduce overfetch
                moveOutputUp(top_dat);
            }
#if 1
            if(get_local_id(0) == 0)
            {
                printf("inside batch loop: 3: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f, Generic %d\n",
                    get_group_id(0), get_group_id(1), get_group_id(2),
                    get_local_id(0),
                    gbl_in_scan_off,
                    pvt_accum[0], pvt_accum[MLO_FILTER_SIZE1], pvt_accum[2*MLO_FILTER_SIZE1],
                    MLO_N_GENERIC_LOOPS);
            }
#endif
        }

        // bottom border block

        for(int i_loop = 1; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS);
            ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
        {
            barrier(CLK_LOCAL_MEM_FENCE);

// read 1 scan line less
// padding processing takes care of the bottom border.

#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))

            readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

            // point to the start of the local buffer
            sc_lcl_off = lcl_bot_off;

#pragma unroll 3
            for(; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1;
                ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
            {

                for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
                {
                    uint top_df_off =
                        gbl_out_scan_off + k * MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
                    _FLOAT mask = 1;

#if 0 // chao: not necessary
#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT
                    top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
                    mask       = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
#endif
                    spanReadingOutput(
                        spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask, top_dat, top_df);
                }

                // processing
                Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

                // move up output to reduce overfetch
                //
                //
                //
                moveOutputUp(top_dat);
            }

#if 1
            if(get_local_id(0) == 0)
            {
                printf("inside batch loop: 4: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                    get_group_id(0), get_group_id(1), get_group_id(2),
                    get_local_id(0),
                    gbl_in_scan_off,
                    pvt_accum[0], pvt_accum[MLO_FILTER_SIZE1], pvt_accum[2*MLO_FILTER_SIZE1]);
            }
#endif
        }

        // epilog
        // handling padding

        for(; sc < MLO_IN_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
        {

            // processing
            Processing(sc,
                       sc_lcl_off,
                       MLO_FILTER_SIZE1 - 1,
#if 0
// bug
                       (MLO_FILTER_PAD1 + 1 - (MLO_IN_HEIGHT - sc)),
#else
// fix
                       MLO_FILTER_SIZE1 - (MLO_IN_HEIGHT + MLO_FILTER_PAD1 - sc),
#endif
                       pvt_accum,
                       lcl_bot,
                       top_dat);
            // move up output to reduce overfetch
            moveOutputUp(top_dat);

        } // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off +=
          // MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 5: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[MLO_FILTER_SIZE1], pvt_accum[2*MLO_FILTER_SIZE1]);
        }
#endif

    } // 	for (int b = 0;

#if 1
  //if(get_local_id(0) == 0)
    {
        printf("outside batch 1: grp(%10d %10d %10d), lid(%10d), pvt_accum %f %f %f\n",
            get_group_id(0), get_group_id(1), get_group_id(2),
            get_local_id(0),
            pvt_accum[0], pvt_accum[MLO_FILTER_SIZE1], pvt_accum[2*MLO_FILTER_SIZE1]);
    }
#endif

    // final summation over all output maps and each filter row
    // this coudl be done with log but it negligeble anyway
    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
    {
        for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
        {

            for(uint l = 0; l < MLO_FILTER_SIZE1; ++l)
            {
                barrier(CLK_LOCAL_MEM_FENCE);

                for(uint n = 0; n < MLO_FILTER_SIZE0; ++n)
                {
                    uint pvt_off =
                        (k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0 +
                        l * MLO_FILTER_SIZE0 + n;

                    lcl[lcl_id * MLO_FILTER_SIZE0 + n] = pvt_accum[pvt_off];
                }

                barrier(CLK_LOCAL_MEM_FENCE);

                if(spn == 0)
                {
                    for(uint s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
                    {

                        for(uint n = 0; n < MLO_FILTER_SIZE0; ++n)
                        {
                            uint pvt_off =
                                (k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0 +
                                l * MLO_FILTER_SIZE0 + n;
                            pvt_accum[pvt_off] += lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
                        }
                    }
                }
            }
        }
    }

    // output
    // inputs are outputs
    // TODO : for more than 1 input

    uint wei_df_off =
        (((ib / MLO_N_BATCH_LOOPS) * MLO_N_OUTPUTS + o_idx + o) * (uint)MLO_WEI_BATCH_STRIDE)
        // this input channel
        + mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);
    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
    {
        if(spn == 0 && o_idx + o + k * MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
                {
                    weights_df[wei_df_off + k * MLO_OUT_STACKS * MLO_WEI_BATCH_STRIDE +
                               c * MLO_WEI_CHANNEL_STRIDE + i] =
                        pvt_accum[(k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 *
                                      MLO_FILTER_SIZE0 +
                                  i];
                }
            }
        }
    }
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1))) __kernel void
MIOpenCvBwdWrW_rdc(const __global _FLOAT* __restrict weight_df_tmp,
                   __global _FLOAT* __restrict weights_df)
{
    uint gbl_id   = get_global_id(0);
    uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
    uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
    uint wei_idx     = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
    uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
    uint wei_idx     = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

    _FLOAT pvt_accum_wei[MLO_UT_READ_UNIT] = {MLO_UT_READ_UNIT * (_FLOAT)0};
    //	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
    //	{
    //		pvt_accum_wei[i] = 0;
    //	}

    int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS) - 1) /
                     (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS);

    for(uint i = 0; i < batch_loop; ++i)
    {
        for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
        {
            pvt_accum_wei[j] += weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE +
                                               i * MLO_N_OUTPUTS * MLO_WEI_BATCH_STRIDE) +
                                              wei_idx + j];
        }
    }

    for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
    {
        weights_df[wei_idx0 + j] = pvt_accum_wei[j];
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0) * MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
#if MLO_FILTER_PAD1 > 0

#if MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS - 2)
#define MLO_IN_VERT_READS (MLO_IN_EXTENT1 + MLO_FILTER_PAD1)
#else
#define MLO_N_GENERIC_LOOPS 0
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

#else
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS)
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF \
    (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS / MLO_READ_UNIT) * MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS * MLO_N_LCL_IN_MAPS * MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = v / d; //(uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24((uint)u, (uint)d);
    return (r);
}

#if 0
__attribute__((always_inline)) void ReduceKernel(__local _FLOAT* lcl_blob,
                                                 __private _FLOAT* weights_accum,
                                                 uint lcl_id,
                                                 uint scan_lcl,
                                                 uint sum_stride,
                                                 uint unit_len,
                                                 UNUSED bool debug)
{
    // read first half
    if(scan_lcl < (sum_stride >> 1))
    {
        for(uint i = 0; i < unit_len; ++i)
        {
            weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];
        }
    }
    // add second half
    // appload accumulated value so far
    for(uint j = (sum_stride >> 1); j > 0; j >>= 1)
    {
        barrier(CLK_LOCAL_MEM_FENCE);
        if(scan_lcl < j)
        {
            for(uint i = 0; i < unit_len; ++i)
            {
                weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

                lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
            }
        }
    }
}
#endif

/*
        group cooperative read
        read by MLO_READ_UNIT
        handle out of range both horizontally and vertically (by fixed number of veryical reads)

        no guard against number of inputs
*/
__attribute__((always_inline)) void readInput(uint lcl_id,
                                              uint gbl_in_scan_off,
                                              uint n_v_reads,
                                              const __global _FLOAT* __restrict bot,
                                              __local _FLOAT* __restrict lcl_bot)
{
#if 0
    if(get_local_id(0) == 0)
    {
        printf("grp(%10d %10d %10d), lid(%10d), n_v_reads %d\n",
            get_group_id(0), get_group_id(1), get_group_id(2),
            get_local_id(0),
            n_v_reads);
    }

#endif

    for(uint p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
        p4 += MLO_GRP_SZ)
    {
        __private _FLOAT in_rd_data[MLO_READ_UNIT];
        // TODO : more than 1 input
        uint c    = 0;
        uint t_p4 = p4;
#if MLO_N_LCL_IN_MAPS > 1
        c    = p4 / (MLO_N_IN_HORIZ_READS * n_v_reads);
        t_p4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS * n_v_reads));
#endif

        uint c_scan = t_p4 / (MLO_N_IN_HORIZ_READS);

#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
        uint c_pix4 = iMod(t_p4, c_scan, (MLO_N_IN_HORIZ_READS));
#else
        uint c_pix4 = t_p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

        //		if (c < MLO_N_INPUTS)

        {
            uint bot_off = gbl_in_scan_off + c * MLO_IN_CHANNEL_STRIDE + c_scan * MLO_IN_STRIDE +
                           c_pix4 * MLO_READ_UNIT;
            const __global _FLOAT* bot_p = &bot[bot_off];
#if MLO_IN_N_PIXS_OFF > 0

            if(c_pix4 == MLO_N_IN_HORIZ_READS - 1)
            {
                for(uint i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
                {
                    in_rd_data[i] = bot_p[i];

#if DBG_OUT_OF_RNGE
                    if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                    {
                        printf("k:err:out-of-range\n");
                    }
#endif
                }

                for(uint i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
                {
                    in_rd_data[i] = 0;
                }
            }
            else
#endif
            {

                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    in_rd_data[i] = bot_p[i];

#if DBG_OUT_OF_RNGE
                    if(bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                    {
                        printf("k:err:out-of-range\n");
                    }
#endif
                }
            }

            // MLO_N_LCL_IN_MAPS inputs
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                int lcl_in_off = c * MLO_IN_LCL_SZ + c_scan * MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 +
                                 c_pix4 * MLO_READ_UNIT + i;
                lcl_bot[lcl_in_off] = in_rd_data[i];
            }
        }

    } // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

    barrier(CLK_LOCAL_MEM_FENCE);
}

/*
        core processing loop
        bot - input, from local (1 span)
        top - output diff, from global (array of spans, filters vertical size)

        loop over filter vertical size

*/
__attribute__((always_inline)) void Processing(UNUSED uint sc,
                                               uint sc_lcl_off,
                                               uint top_lim,
                                               int bot_lim,
                                               __private _FLOAT* __restrict pvt_accum,
                                               __local _FLOAT* __restrict lcl_bot,
                                               __private _FLOAT* __restrict top_dat)
{
    for(int l = top_lim; l >= bot_lim; --l)
    {
        for(uint m = 0; m < MLO_IN_TILE0; ++m)
        {
            for(uint n = 0; n < MLO_FILTER_SIZE0; ++n)
            {
                for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
                {
                    int bot_off = sc_lcl_off + c * MLO_IN_LCL_SZ + n + m;

                    _FLOAT bot_val = lcl_bot[bot_off];

                    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
                    {
                        uint pvt_top_off =
                            k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
                        uint pvt_accum_off =
                            (k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0 +
                            l * MLO_FILTER_SIZE0 + n;

                        _FLOAT top_val = top_dat[pvt_top_off];

                        pvt_accum[pvt_accum_off]
                            // each wk-it process an input
                            += bot_val * top_val;
                    }
                }
            }
        }
    }
}

__attribute__((always_inline)) void moveOutputUp(__private _FLOAT* __restrict top_dat)
{
    // move up output to reduce overfetch
    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
    {
        for(uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
        {
            for(uint i = 0; i < MLO_IN_TILE0; ++i)
            {
                uint pvt_off_n = k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j * MLO_IN_TILE0 + i;
                uint pvt_off_o = k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) * MLO_IN_TILE0 + i;
                top_dat[pvt_off_n] = top_dat[pvt_off_o];
            }
        }
    }
}

__attribute__((always_inline)) void spanReadingOutput(int spn,
                                                      int k,
                                                      int j,
                                                      int top_df_off,
                                                      _FLOAT mask,
                                                      __private _FLOAT* __restrict top_dat,
                                                      const __global _FLOAT* __restrict top_df)
{
    int pvt_off                     = k * MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j * MLO_IN_TILE0;
    const __global _FLOAT* top_df_p = &top_df[top_df_off];
#if MLO_OUT_N_PIXS_OFF > 0
    if(spn == MLO_N_SPANS_PER_SCAN - 1)
    {
        uint i = 0;
        for(; i < MLO_OUT_N_PIXS_OFF; ++i)
        {
            top_dat[pvt_off + i] = top_df_p[i] * mask;

#if DBG_OUT_OF_RNGE
            if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
            {
                printf("k:err:out-of-range\n");
            }
#endif
        }
        for(; i < MLO_IN_TILE0; ++i)
        {
            top_dat[pvt_off + i] = 0;
        }
    }
    else
#else
    (void)spn;
#endif
    {
        for(uint i = 0; i < MLO_IN_TILE0; ++i)
        {
            top_dat[pvt_off + i] = top_df_p[i] * mask;

#if DBG_OUT_OF_RNGE
            if(top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
            {
                printf("k:err:out-of-range\n");
            }
#endif
        }
    }
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenCvBwdWrW(const __global _FLOAT* __restrict top_df,
               const __global _FLOAT* __restrict bot,
               __global _FLOAT* __restrict weights_df,
#if MLO_CONV_BIAS
               __global _FLOAT* __restrict bias_df,
#endif
               UNUSED _FLOAT padding_val)
{

    // input/output tiles + reduce buffer

    __local _FLOAT lcl[(MLO_LCL_SZ) + 1];
    __local _FLOAT* lcl_bot = lcl;

    uint lcl_id = get_local_id(0);

    uint c_idx_base = get_group_id(0); // input map index base

    uint o_idx_base = get_group_id(1); // output map index base

    uint ib_base = get_group_id(2);

    uint ib = ib_base * (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS);

    uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

    uint o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

    uint gbl_in_off  = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
    uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
    // 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
    // TODO: more than 1 input
    uint o = lcl_id / MLO_N_SPANS_PER_SCAN;
#if MLO_N_SPANS_PER_SCAN & (MLO_N_SPANS_PER_SCAN - 1)
    uint spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);
#else
    uint spn         = lcl_id & (MLO_N_SPANS_PER_SCAN - 1);
#endif
    //	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);

    uint lcl_bot_off     = spn * MLO_IN_TILE0;
    uint out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
    gbl_out_off += out_wk_item_off;
    // no output out of range
    gbl_out_off = (o_idx + o < MLO_N_OUTPUTS && o < MLO_OUT_STACKS) ? gbl_out_off : 0;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

    __private _FLOAT top_dat[MLO_TOP_DAT_SZ];

    for(uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
    {
        top_dat[i] = 0;
    }

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_N_LCL_IN_MAPS * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0)

    __private _FLOAT pvt_accum[MLO_ACCUM_SZ];

    for(uint i = 0; i < MLO_ACCUM_SZ; ++i)
    {
        pvt_accum[i] = 0;
    }

    // zero out LDS
    for(uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
    {
        lcl[i] = 0;
    }

    // over all batches
    uint bend = ib + MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS;
    bend      = bend > MLO_BATCH_SZ ? MLO_BATCH_SZ : bend;

    for(uint b = ib; b < bend; ++b,
             gbl_in_off += MLO_N_LCL_BATCHS * MLO_IN_BATCH_STRIDE,
             gbl_out_off += MLO_N_LCL_BATCHS * MLO_OUT_BATCH_STRIDE)
    {
        barrier(CLK_LOCAL_MEM_FENCE);

        // top border input block
        uint gbl_in_scan_off  = gbl_in_off;
        uint gbl_out_scan_off = gbl_out_off;

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 0: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[3], pvt_accum[6]);
        }
#endif

        // read input map
        readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

        // move input pointer
        gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

        for(uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
        {
            top_dat[i] = 0;
        }

        // prefetch output
        int gbl_out_scan_off1 = gbl_out_scan_off;
        for(uint k = 0; k < MLO_N_LCL_OUT_MAPS;
            ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
        {
            for(uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
            {
                // loop around all output maps
                uint top_df_off = gbl_out_scan_off1 + j * MLO_OUT_STRIDE;
                _FLOAT mask     = 1;
#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
                top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
                mask       = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

                spanReadingOutput(spn, k, j, top_df_off, mask, top_dat, top_df);
            }
        }

        gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

        uint sc         = 0;
        uint sc_lcl_off = lcl_bot_off;

        // prolog
        // handling padding

        // top padding
        for(; sc < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
        {
            Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
        }

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 1: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[3], pvt_accum[6]);
        }
#endif

#ifdef __AMDGCN__
#pragma unroll 2
#endif
        for(; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
                       -
                       MLO_FILTER_PAD1
#endif
            ;
            ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
        {

            for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
            {
                uint top_df_off = gbl_out_scan_off + k * MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
                _FLOAT mask     = 1;

#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
                top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
                mask       = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

                spanReadingOutput(
                    spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask, top_dat, top_df);
            }

            // processing
            Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

            // move up output to reduce overfetch
            moveOutputUp(top_dat);
        }

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 2: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[3], pvt_accum[6]);
        }
#endif

        // non-border input blocks
        for(uint i_loop = 1; i_loop < MLO_N_GENERIC_LOOPS;
            ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
        {
            barrier(CLK_LOCAL_MEM_FENCE);

            readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

            // point to the start of the local buffer

            sc_lcl_off = lcl_bot_off;

            for(; sc < (i_loop + 1) * MLO_IN_EXTENT1;
                ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
            {

                for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
                {
                    uint top_df_off =
                        gbl_out_scan_off + k * MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
                    _FLOAT mask = 1;

#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT
                    top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
                    mask       = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

                    spanReadingOutput(
                        spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask, top_dat, top_df);
                }

                // processing
                Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

                // move up output to reduce overfetch
                moveOutputUp(top_dat);
            }
#if 1
            if(get_local_id(0) == 0)
            {
                printf("inside batch loop: 3: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f, Generic %d\n",
                    get_group_id(0), get_group_id(1), get_group_id(2),
                    get_local_id(0),
                    gbl_in_scan_off,
                    pvt_accum[0], pvt_accum[3], pvt_accum[6],
                    MLO_N_GENERIC_LOOPS);
            }
#endif
        }

        // bottom border block

        for(int i_loop = 1; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS);
            ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
        {
            barrier(CLK_LOCAL_MEM_FENCE);

// read 1 scan line less
// padding processing takes care of the bottom border.

#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))

            readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

            // point to the start of the local buffer
            sc_lcl_off = lcl_bot_off;

#pragma unroll 3
            for(; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1;
                ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
            {

                for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
                {
                    uint top_df_off =
                        gbl_out_scan_off + k * MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
                    _FLOAT mask = 1;

#if 0 // chao: not necessary
#if MLO_IN_HEIGHT != MLO_OUT_HEIGHT
                    top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
                    mask       = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
#endif
                    spanReadingOutput(
                        spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask, top_dat, top_df);
                }

                // processing
                Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

                // move up output to reduce overfetch
                //
                //
                //
                moveOutputUp(top_dat);
            }

#if 1
            if(get_local_id(0) == 0)
            {
                printf("inside batch loop: 4: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                    get_group_id(0), get_group_id(1), get_group_id(2),
                    get_local_id(0),
                    gbl_in_scan_off,
                    pvt_accum[0], pvt_accum[3], pvt_accum[6]);
            }
#endif
        }

        // epilog
        // handling padding

        for(; sc < MLO_IN_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
        {

            // processing
            Processing(sc,
                       sc_lcl_off,
                       MLO_FILTER_SIZE1 - 1,
#if 0
// bug
                       (MLO_FILTER_PAD1 + 1 - (MLO_IN_HEIGHT - sc)),
#else
// fix
                       MLO_FILTER_SIZE1 - (MLO_IN_HEIGHT + MLO_FILTER_PAD1 - sc),
#endif
                       pvt_accum,
                       lcl_bot,
                       top_dat);
            // move up output to reduce overfetch
            moveOutputUp(top_dat);

        } // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off +=
          // MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

#if 1
        if(get_local_id(0) == 0)
        {
            printf("inside batch loop: 5: grp(%10d %10d %10d), lid(%10d), gbl_in_scan_off, %10d, pvt_accum %f %f %f\n",
                get_group_id(0), get_group_id(1), get_group_id(2),
                get_local_id(0),
                gbl_in_scan_off,
                pvt_accum[0], pvt_accum[3], pvt_accum[6]);
        }
#endif

    } // 	for (int b = 0;

#if 0
    if(get_local_id(0) == 0 || get_local_id(0) == 1)
    {
        printf("2: grp(%10d %10d %10d), lid(%10d), pvt_accum[0] %f\n",
            get_group_id(0), get_group_id(1), get_group_id(2),
            get_local_id(0),
            pvt_accum[0]);
    }
#endif

    // final summation over all output maps and each filter row
    // this coudl be done with log but it negligeble anyway
    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
    {
        for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
        {

            for(uint l = 0; l < MLO_FILTER_SIZE1; ++l)
            {
                barrier(CLK_LOCAL_MEM_FENCE);

                for(uint n = 0; n < MLO_FILTER_SIZE0; ++n)
                {
                    uint pvt_off =
                        (k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0 +
                        l * MLO_FILTER_SIZE0 + n;

                    lcl[lcl_id * MLO_FILTER_SIZE0 + n] = pvt_accum[pvt_off];
                }

                barrier(CLK_LOCAL_MEM_FENCE);

                if(spn == 0)
                {
                    for(uint s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
                    {

                        for(uint n = 0; n < MLO_FILTER_SIZE0; ++n)
                        {
                            uint pvt_off =
                                (k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0 +
                                l * MLO_FILTER_SIZE0 + n;
                            pvt_accum[pvt_off] += lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
                        }
                    }
                }
            }
        }
    }

#if 0
    if(get_local_id(0) == 0 || get_local_id(0) == 1)
    {
        printf("3: grp(%10d %10d %10d), lid(%10d), pvt_accum[0] %f\n",
            get_group_id(0), get_group_id(1), get_group_id(2),
            get_local_id(0),
            pvt_accum[0]);
    }
#endif

    // output
    // inputs are outputs
    // TODO : for more than 1 input

    uint wei_df_off =
        (((ib / MLO_N_BATCH_LOOPS) * MLO_N_OUTPUTS + o_idx + o) * (uint)MLO_WEI_BATCH_STRIDE)
        // this input channel
        + mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);
    for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
    {
        if(spn == 0 && o_idx + o + k * MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
                {
                    weights_df[wei_df_off + k * MLO_OUT_STACKS * MLO_WEI_BATCH_STRIDE +
                               c * MLO_WEI_CHANNEL_STRIDE + i] =
                        pvt_accum[(k * MLO_N_LCL_IN_MAPS + c) * MLO_FILTER_SIZE1 *
                                      MLO_FILTER_SIZE0 +
                                  i];
                }
            }
        }
    }
}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1))) __kernel void
MIOpenCvBwdWrW_rdc(const __global _FLOAT* __restrict weight_df_tmp,
                   __global _FLOAT* __restrict weights_df)
{
    uint gbl_id   = get_global_id(0);
    uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
    uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
    uint wei_idx     = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
    uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
    uint wei_idx     = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

    _FLOAT pvt_accum_wei[MLO_UT_READ_UNIT] = {MLO_UT_READ_UNIT * (_FLOAT)0};
    //	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
    //	{
    //		pvt_accum_wei[i] = 0;
    //	}

    int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS) - 1) /
                     (MLO_N_BATCH_LOOPS * MLO_N_LCL_BATCHS);

    for(uint i = 0; i < batch_loop; ++i)
    {
        for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
        {
#if 0
            if(get_local_id(0) == 0 || get_local_id(0) == 1)
            {
                printf("inside batch loop: 0: grp(%10d %10d %10d), lid(%10d), batch_lopp, %10d, %f %f\n",
                    get_group_id(0), get_group_id(1), get_group_id(2),
                    get_local_id(0),
                    batch_loop,
                    pvt_accum_wei[j], weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE +
                                               i * MLO_N_OUTPUTS * MLO_WEI_BATCH_STRIDE) +
                                              wei_idx + j]);
            }
#endif
            pvt_accum_wei[j] += weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE +
                                               i * MLO_N_OUTPUTS * MLO_WEI_BATCH_STRIDE) +
                                              wei_idx + j];
        }
    }

    for(uint j = 0; j < MLO_UT_READ_UNIT; ++j)
    {
        weights_df[wei_idx0 + j] = pvt_accum_wei[j];
    }
}


maybe solve intel opencl compiler build error?

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [imageid][upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
// assumes filter is small, so filtersize * filterSize * inputPlanes * 4 < about 3KB
//                            eg 5 * 5 * 32 * 4 = 3.2KB => ok :-)
//                           but 28 * 28 * 32 * 4 = 100KB => less good :-P
void kernel propagate_2_by_outplane( const int batchSize,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube ) {
    const int globalId = get_global_id(0);

//    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int outPlane = workgroupId;

    const int localId = get_local_id(0);
    const int outputRow = localId / gOutputBoardSize;
    const int outputCol = localId % gOutputBoardSize;

    #if gPadZeros == 1
        const int minu = max( -gHalfFilterSize, -outputRow );
        const int maxu = min( gHalfFilterSize, gOutputBoardSize - 1 - outputRow ) - gEven;
        const int minv = max( -gHalfFilterSize, -outputCol );
        const int maxv = min( gHalfFilterSize, gOutputBoardSize - 1 - outputCol ) - gEven;
    #else
        const int minu = -gHalfFilterSize;
        const int maxu = gHalfFilterSize - gEven;
        const int minv = -gHalfFilterSize;
        const int maxv = gHalfFilterSize - gEven;
    #endif

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;

    const int filterCubeLength = gInputPlanes * gFilterSizeSquared;
    const int filterCubeGlobalOffset = outPlane * filterCubeLength;
    const int numPixelsPerThread = ( filterCubeLength + workgroupSize - 1 ) / workgroupSize;
    for( int i = 0; i < numPixelsPerThread; i++ ) {
        int thisOffset = localId + i * workgroupSize;
        if( thisOffset < filterCubeLength ) {
            _filterCube[thisOffset] = filters[filterCubeGlobalOffset + thisOffset];
        }
    }
    // dont need a barrier, since we'll just run behind the barrier from the upstream board download

    for( int n = 0; n < batchSize; n++ ) {
        float sum = 0;
        for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
            int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
            barrier(CLK_LOCAL_MEM_FENCE);
            for( int i = 0; i < numUpstreamsPerThread; i++ ) {
                int thisOffset = workgroupSize * i + localId;
                if( thisOffset < gInputBoardSizeSquared ) {
                    _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);
            int filterBoardOffset = upstreamPlane * gFilterSizeSquared;
            if( localId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u;
                    #if gPadZeros == 0
                         inputRow += gHalfFilterSize;
                    #endif
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = filterBoardOffset + (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v;
                        #if gPadZeros == 0
                             inputCol += gHalfFilterSize;
                        #endif
                        sum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
        }
        #ifdef BIASED
            sum += biases[outPlane];
        #endif
        // results are organized like [imageid][filterid][row][col]
        int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + localId;
        if( localId < gOutputBoardSizeSquared ) {
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
        }
    }
}
#endif
#endif

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [imageid][upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
// assumes filter is small, so filtersize * filterSize * inputPlanes * 4 < about 3KB
//                            eg 5 * 5 * 32 * 4 = 3.2KB => ok :-)
//                           but 28 * 28 * 32 * 4 = 100KB => less good :-P
void kernel propagate_2_by_outplane( const int batchSize,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube ) {
    const int globalId = get_global_id(0);

//    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int outPlane = workgroupId;

    const int localId = get_local_id(0);
    const int outputRow = localId / gOutputBoardSize;
    const int outputCol = localId % gOutputBoardSize;

    #if gPadZeros == 1
        const int minu = max( -gHalfFilterSize, -outputRow );
        const int maxu = min( gHalfFilterSize, gOutputBoardSize - 1 - outputRow ) - gEven;
        const int minv = max( -gHalfFilterSize, -outputCol );
        const int maxv = min( gHalfFilterSize, gOutputBoardSize - 1 - outputCol ) - gEven;
    #else
        const int minu = -gHalfFilterSize;
        const int maxu = gHalfFilterSize - gEven;
        const int minv = -gHalfFilterSize;
        const int maxv = gHalfFilterSize - gEven;
    #endif

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;

    const int filterCubeLength = gInputPlanes * gFilterSizeSquared;
    const int filterCubeGlobalOffset = outPlane * filterCubeLength;
    const int numPixelsPerThread = ( filterCubeLength + workgroupSize - 1 ) / workgroupSize;
    for( int i = 0; i < numPixelsPerThread; i++ ) {
        int thisOffset = localId + i * workgroupSize;
        if( thisOffset < filterCubeLength ) {
            _filterCube[thisOffset] = filters[filterCubeGlobalOffset + thisOffset];
        }
    }
    // dont need a barrier, since we'll just run behind the barrier from the upstream board download

    for( int n = 0; n < batchSize; n++ ) {
        float sum = 0;
        for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
            int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
            barrier(CLK_LOCAL_MEM_FENCE);
            for( int i = 0; i < numUpstreamsPerThread; i++ ) {
                int thisOffset = workgroupSize * i + localId;
                if( thisOffset < gInputBoardSizeSquared ) {
                    _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
                }
            }
            barrier(CLK_LOCAL_MEM_FENCE);
            int filterBoardOffset = upstreamPlane * gFilterSizeSquared;
            if( localId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u + ( gPadZeros ? 0 : gHalfFilterSize );
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = filterBoardOffset + (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v + ( gPadZeros ? 0 : gHalfFilterSize );
                        sum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
        }
        #ifdef BIASED
            sum += biases[outPlane];
        #endif
        // results are organized like [imageid][filterid][row][col]
        int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + localId;
        if( localId < gOutputBoardSizeSquared ) {
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
        }
    }
}
#endif
#endif