10 pixel unpack/pack removed

10 pixel unpack/pack removed

Redundant code that can't be compatible linear space processing (would
lose about 1 bit in shadows)

/* Deathray - An Avisynth plug-in filter for spatial/temporal non-local means de-noising.
 *
 * version 1.01
 *
 * Copyright 2013, Jawed Ashraf - Deathray@cupidity.f9.co.uk
 */

#define TILE_SIDE 53
#define USE_SRGB_GAMMA_CURVE 0

__kernel void Zero(__global float4 *A) {
    int pos = get_global_id(0);
	A[pos] = 0.f;
}

// gamma_decode
// Converts input from gamma space into linear space.
float gamma_decode(float x) {
#if USE_SRGB_GAMMA_CURVE
	if (x <= 0.081f) return x/4.5f;
	return native_powr(((x + 0.099f)/1.099f), 2.22222222f);
#else
	return native_powr(x, 2.22222222f);
#endif
}

float4 gamma_decode4(float4 x) {
	float4 de_gamma;

	de_gamma.x = gamma_decode(x.x);
	de_gamma.y = gamma_decode(x.y);
	de_gamma.z = gamma_decode(x.z);
	de_gamma.w = gamma_decode(x.w);

	return de_gamma;
}

// gamma_encode
// Converts input from linear space into gamma space.
float gamma_encode(float x) {
#if USE_SRGB_GAMMA_CURVE
	if (x <= 0.018f) return 4.5f * x;
	return 1.099f * native_powr(x, 0.45f) - 0.099f;
#else
	return native_powr(x, 0.45f);
#endif
}

float4 gamma_encode4(float4 x) {
	float4 en_gamma;

	en_gamma.x = gamma_encode(x.x);
	en_gamma.y = gamma_encode(x.y);
	en_gamma.z = gamma_encode(x.z);
	en_gamma.w = gamma_encode(x.w);

	return en_gamma;
}

float4 ReadPixel4(
	read_only 	image2d_t 	plane,
	const		int2		coordinates,
	const		int			linear) {

	const sampler_t frame = CLK_NORMALIZED_COORDS_FALSE |
							CLK_ADDRESS_CLAMP |
							CLK_FILTER_NEAREST;

	float4 pixel = read_imagef(plane, frame, coordinates);
	if (linear) pixel = gamma_decode4(pixel);
	return pixel;
}

void WritePixel4(
	const		float4 		pixel,
	const		int2		coordinates,
	const		int			linear,
	write_only 	image2d_t 	plane) {

	float4 write_pixel = pixel;
	if (linear) write_pixel = gamma_encode4(pixel);
	write_imagef(plane, coordinates, write_pixel);
}

void WriteTile4(
	float4 store,
	int x,					// coordinates of a strip ...
	int y,					// ... of 4 pixels to be written
	local float *tile) {	// 48x48 portion of plane in local memory

	vstore4(store, 0, tile + ((y * TILE_SIDE) + x));
	write_mem_fence(CLK_LOCAL_MEM_FENCE);
}

float4 ReadTile4(
	int x,					// coordinates of a strip ...
	int y,					// ... of 4 pixels to be fetched
	local float *tile) {	// 48x48 portion of plane in local memory

	float4 result;

	result = vload4(0, tile + ((y * TILE_SIDE) + x));
	read_mem_fence(CLK_LOCAL_MEM_FENCE);
	return result;
}

float16 ReadTile16(
	int x,					// coordinates of a strip ...
	int y,					// ... of 16 pixels to be fetched
	local float *tile) {	// 48x48 portion of plane in local memory

	float16 result;

	result = vload16(0, tile + ((y * TILE_SIDE) + x));
	read_mem_fence(CLK_LOCAL_MEM_FENCE);
	return result;
}

void Coordinates32x32(
	int2 *local_id,	// Work item
	int2 *source) {	// Each work item processes 4 pixels from source as a single row uchar4/float4

	// Generate coordinates for use by a variety of kernels based upon
	// a tile size of 32x32 pixels.

	// The 32x32 tile's top-left position in the plane, in uchar4s
	int2 tile_top_left = (int2)(get_group_id(0) << 3, get_group_id(1) << 5);

	*local_id = (int2)(get_local_id(0), get_local_id(1));
	*source = tile_top_left + *local_id;
}

void FetchAndMirror48x48(
	read_only 	image2d_t 	target_plane,	// input image
	const		int			width,			// width in pixels
	const		int			height,			// height in pixels
	const		int2		local_id,		// Work item
	const		int2		source,			// coordinates
	const		int			linear,			// process plane in linear space instead of gamma space
	local		float		*tile) {		// existing block of local memory populated with 48x48 pixels
	// Fetch pixels from source and populate a 48x48 tile of pixels.
	//
	// The tile is defined as a 32x32 region to be filtered plus an
	// apron of 8 pixels on all four sides, making 48x48.
	//
	// The apron is filled with pixels from adjacent regions. At the
	// four edges of the frame the apron is filled, instead, with
	// pixels mirrored from just inside the frame.

	float4 source_pixels = ReadPixel4(target_plane, source, linear);

	write_mem_fence(CLK_LOCAL_MEM_FENCE);

	// Scalar address for left-most pixel of 4 pixels that are put into tile
	int2 target = (int2)((local_id.x << 2) + 8, local_id.y + 8);

	WriteTile4(source_pixels, target.x, target.y, tile);

	if (local_id.y < 8 || local_id.y > 23) { // 8 top and bottom rows of apron are filled from adjacent tiles
		int offset = (local_id.y < 8) ? -8 : 8;
		source_pixels = ReadPixel4(target_plane, source + (int2)(0, offset), linear);
		WriteTile4(source_pixels, target.x, target.y + offset, tile);
	}
	if (local_id.x < 2 || local_id.x > 5) { // 8 columns at left and right
		int offset = (local_id.x < 2) ? -2 : 2;
		source_pixels = ReadPixel4(target_plane, source + (int2)(offset, 0), linear);
		WriteTile4(source_pixels, target.x + (offset << 2), target.y, tile);
	}
	if (local_id.y < 8 || local_id.y > 23) { // 4 corners, each 8x8
		int2 offset;
		offset.y = (local_id.y < 8) ? -8 : 8;
		if (local_id.x < 2 || local_id.x > 5) {
			offset.x = (local_id.x < 2) ? -2 : 2;
			source_pixels = ReadPixel4(target_plane, source + offset, linear);
			WriteTile4(source_pixels, target.x + (offset.x << 2), target.y + offset.y, tile);
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	// Mirror into apron when at frame border
	float4 mirror;
	bool left_group  = (get_group_id(0) == 0);
	bool right_group = (get_group_id(0) == (get_num_groups(0) - 1));
	bool top_group   = (get_group_id(1) == 0);
	bool bot_group   = (get_group_id(1) == (get_num_groups(1) - 1));

	if (top_group && local_id.y < 8) { 	// top 8 rows
		mirror = ReadTile4(target.x, target.y + 1, tile);
		WriteTile4(mirror, target.x, 8 - local_id.y - 1, tile);
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	if (bot_group && local_id.y < 8) { // bottom 8 rows
		int centre = ((height - 1) & 31) + 8;

		mirror = ReadTile4(target.x, centre - local_id.y - 1, tile);
		WriteTile4(mirror, target.x, centre + local_id.y + 1, tile);
	}
	barrier(CLK_LOCAL_MEM_FENCE);
	// left 8 columns + top-left and bottom-left apron corners per tile
	int read_x  = (local_id.x == 1) ? 13 : 9;
	int write_x = (local_id.x == 1) ? 0 : 4;
	if (local_id.x < 2) {
		if (left_group) { // left 8 columns, 32 rows
			mirror = ReadTile4(read_x, target.y, tile);
			WriteTile4(mirror.wzyx, write_x, target.y, tile);
		}

		// TODO - cleanup IF structure - not sure if the compiler will jinx this
		if (local_id.y < 8) {
			if (left_group) { // left edge of frame apron corners, top and bottom
				mirror = ReadTile4(read_x, local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, local_id.y, tile);
				mirror = ReadTile4(read_x, 40 + local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, 40 + local_id.y, tile);
			}
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);
	if (local_id.x < 2 && local_id.y < 8) {
		int x = (local_id.x == 1) ? 4 : 0;
		if (top_group) { // apron at top edge of frame, left corners
			mirror = ReadTile4(x, 8 + local_id.y + 1, tile);
			WriteTile4(mirror, x, 8 - local_id.y - 1, tile);
		}
		if (bot_group) { // apron at bottom edge of frame, left corners
			int bottom = ((height - 1) & 31) + 8;
			mirror = ReadTile4(x, bottom - local_id.y - 1, tile);
			WriteTile4(mirror, x, bottom + local_id.y + 1, tile);
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	if (local_id.x > 5) { // right 8 columns + top-right and bottom-right apron corners per tile

		if (right_group) { // right edge of frame
			int centre  = ((width - 1) & 31) + 8;
			int read_x  = (local_id.x == 7) ? centre - 4 : centre - 8;
			int write_x = (local_id.x == 7) ? centre + 1 : centre + 5;

			mirror = ReadTile4(read_x, target.y, tile);
			WriteTile4(mirror.wzyx, write_x, target.y, tile);

			if (local_id.y < 8) { // top right and bottom right corner of every tile on right edge
				mirror = ReadTile4(read_x, local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, local_id.y, tile);
				mirror = ReadTile4(read_x, 40 + local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, 40 + local_id.y, tile);

				if (bot_group) { // bottom right corner of frame
					int bottom = ((height - 1) & 31) + 8;
					mirror = ReadTile4(read_x, bottom + local_id.y + 1, tile);
					WriteTile4(mirror.wzyx, write_x, bottom + local_id.y + 1, tile);
				}
			}
		} else if ((top_group || bot_group) && local_id.y < 8) {
			if (top_group) {
				int x = (local_id.x == 7) ? 44 : 40; // ATI 2.3: compiler bug requires this here if TILE_SIDE is not 48
				mirror = ReadTile4(x, 8 + local_id.y + 1, tile);
				WriteTile4(mirror, x, 8 - local_id.y - 1, tile);
			}
			if (bot_group) {
				int x = (local_id.x == 7) ? 44 : 40; // ATI 2.3: compiler bug requires this here if TILE_SIDE is not 48
				int bottom = ((height - 1) & 31) + 8;
				mirror = ReadTile4(x, bottom - local_id.y - 1, tile);
				WriteTile4(mirror, x, bottom + local_id.y + 1, tile);
			}
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);
}

/* Deathray - An Avisynth plug-in filter for spatial/temporal non-local means de-noising.
 *
 * version 1.01
 *
 * Copyright 2013, Jawed Ashraf - Deathray@cupidity.f9.co.uk
 */

#define TILE_SIDE 53
#define USE_SRGB_GAMMA_CURVE 0

__kernel void Zero(__global float4 *A) {
    int pos = get_global_id(0);
	A[pos] = 0.f;
}

// gamma_decode
// Converts input from gamma space into linear space.
float gamma_decode(float x) {
#if USE_SRGB_GAMMA_CURVE
	if (x <= 0.081f) return x/4.5f;
	return native_powr(((x + 0.099f)/1.099f), 2.22222222f);
#else
	return native_powr(x, 2.22222222f);
#endif
}

float4 gamma_decode4(float4 x) {
	float4 de_gamma;

	de_gamma.x = gamma_decode(x.x);
	de_gamma.y = gamma_decode(x.y);
	de_gamma.z = gamma_decode(x.z);
	de_gamma.w = gamma_decode(x.w);

	return de_gamma;
}

// gamma_encode
// Converts input from linear space into gamma space.
float gamma_encode(float x) {
#if USE_SRGB_GAMMA_CURVE
	if (x <= 0.018f) return 4.5f * x;
	return 1.099f * native_powr(x, 0.45f) - 0.099f;
#else
	return native_powr(x, 0.45f);
#endif
}

float4 gamma_encode4(float4 x) {
	float4 en_gamma;

	en_gamma.x = gamma_encode(x.x);
	en_gamma.y = gamma_encode(x.y);
	en_gamma.z = gamma_encode(x.z);
	en_gamma.w = gamma_encode(x.w);

	return en_gamma;
}

float4 ReadPixel4(
	read_only 	image2d_t 	plane,
	const		int2		coordinates,
	const		int			linear) {

	const sampler_t frame = CLK_NORMALIZED_COORDS_FALSE |
							CLK_ADDRESS_CLAMP |
							CLK_FILTER_NEAREST;

	float4 pixel = read_imagef(plane, frame, coordinates);
	if (linear) pixel = gamma_decode4(pixel);
	return pixel;
}

void WritePixel4(
	const		float4 		pixel,
	const		int2		coordinates,
	const		int			linear,
	write_only 	image2d_t 	plane) {

	float4 write_pixel = pixel;
	if (linear) write_pixel = gamma_encode4(pixel);
	write_imagef(plane, coordinates, write_pixel);
}

void WriteTile4(
	float4 store,
	int x,					// coordinates of a strip ...
	int y,					// ... of 4 pixels to be written
	local float *tile) {	// 48x48 portion of plane in local memory

	vstore4(store, 0, tile + ((y * TILE_SIDE) + x));
	write_mem_fence(CLK_LOCAL_MEM_FENCE);
}

float4 ReadTile4(
	int x,					// coordinates of a strip ...
	int y,					// ... of 4 pixels to be fetched
	local float *tile) {	// 48x48 portion of plane in local memory

	float4 result;

	result = vload4(0, tile + ((y * TILE_SIDE) + x));
	read_mem_fence(CLK_LOCAL_MEM_FENCE);
	return result;
}

float16 ReadTile16(
	int x,					// coordinates of a strip ...
	int y,					// ... of 16 pixels to be fetched
	local float *tile) {	// 48x48 portion of plane in local memory

	float16 result;

	result = vload16(0, tile + ((y * TILE_SIDE) + x));
	read_mem_fence(CLK_LOCAL_MEM_FENCE);
	return result;
}

float16 Unpack10PixelsAndNormalise(uint4 sixteen_pixels) {
	// Unpack 10 pixels, which are packed into 128 bits as uchars,
	// into 10 normalised pixels as floats.
	//
	// Only 10 pixels are required from each row of a window.

	uchar4 four_pixels;
	float16 result;

	four_pixels = as_uchar4(sixteen_pixels.x);

	result.s0 = (float)four_pixels.x * 0.0039215686f;
	result.s1 = (float)four_pixels.y * 0.0039215686f;
	result.s2 = (float)four_pixels.z * 0.0039215686f;
	result.s3 = (float)four_pixels.w * 0.0039215686f;

	four_pixels = as_uchar4(sixteen_pixels.y);

	result.s4 = (float)four_pixels.x * 0.0039215686f;
	result.s5 = (float)four_pixels.y * 0.0039215686f;
	result.s6 = (float)four_pixels.z * 0.0039215686f;
	result.s7 = (float)four_pixels.w * 0.0039215686f;

	four_pixels = as_uchar4(sixteen_pixels.z);

	result.s8 = (float)four_pixels.x * 0.0039215686f;
	result.s9 = (float)four_pixels.y * 0.0039215686f;

	return result;
}

uint4 Pack10PixelsIntoTargetRow(float16 sixteen_pixels) {
	// Pack 10 float pixels into 10 uchars, stored as a uint4.
	//
	// Only 10 pixels are required for target window's row.

	uchar4 four_pixels;
	uint4 result;

	four_pixels.x = (uchar)(sixteen_pixels.s0 * 255.f);
	four_pixels.y = (uchar)(sixteen_pixels.s1 * 255.f);
	four_pixels.z = (uchar)(sixteen_pixels.s2 * 255.f);
	four_pixels.w = (uchar)(sixteen_pixels.s3 * 255.f);

	result.x = as_uint(four_pixels);

	four_pixels.x = (uchar)(sixteen_pixels.s4 * 255.f);
	four_pixels.y = (uchar)(sixteen_pixels.s5 * 255.f);
	four_pixels.z = (uchar)(sixteen_pixels.s6 * 255.f);
	four_pixels.w = (uchar)(sixteen_pixels.s7 * 255.f);

	result.y = as_uint(four_pixels);

	four_pixels.x = (uchar)(sixteen_pixels.s8 * 255.f);
	four_pixels.y = (uchar)(sixteen_pixels.s9 * 255.f);

	result.z = as_uint(four_pixels);

	return result;
}

void Coordinates32x32(
	int2 *local_id,	// Work item
	int2 *source) {	// Each work item processes 4 pixels from source as a single row uchar4/float4

	// Generate coordinates for use by a variety of kernels based upon
	// a tile size of 32x32 pixels.

	// The 32x32 tile's top-left position in the plane, in uchar4s
	int2 tile_top_left = (int2)(get_group_id(0) << 3, get_group_id(1) << 5);

	*local_id = (int2)(get_local_id(0), get_local_id(1));
	*source = tile_top_left + *local_id;
}

void FetchAndMirror48x48(
	read_only 	image2d_t 	target_plane,	// input image
	const		int			width,			// width in pixels
	const		int			height,			// height in pixels
	const		int2		local_id,		// Work item
	const		int2		source,			// coordinates
	const		int			linear,			// process plane in linear space instead of gamma space
	local		float		*tile) {		// existing block of local memory populated with 48x48 pixels
	// Fetch pixels from source and populate a 48x48 tile of pixels.
	//
	// The tile is defined as a 32x32 region to be filtered plus an
	// apron of 8 pixels on all four sides, making 48x48.
	//
	// The apron is filled with pixels from adjacent regions. At the
	// four edges of the frame the apron is filled, instead, with
	// pixels mirrored from just inside the frame.

	float4 source_pixels = ReadPixel4(target_plane, source, linear);

	write_mem_fence(CLK_LOCAL_MEM_FENCE);

	// Scalar address for left-most pixel of 4 pixels that are put into tile
	int2 target = (int2)((local_id.x << 2) + 8, local_id.y + 8);

	WriteTile4(source_pixels, target.x, target.y, tile);

	if (local_id.y < 8 || local_id.y > 23) { // 8 top and bottom rows of apron are filled from adjacent tiles
		int offset = (local_id.y < 8) ? -8 : 8;
		source_pixels = ReadPixel4(target_plane, source + (int2)(0, offset), linear);
		WriteTile4(source_pixels, target.x, target.y + offset, tile);
	}
	if (local_id.x < 2 || local_id.x > 5) { // 8 columns at left and right
		int offset = (local_id.x < 2) ? -2 : 2;
		source_pixels = ReadPixel4(target_plane, source + (int2)(offset, 0), linear);
		WriteTile4(source_pixels, target.x + (offset << 2), target.y, tile);
	}
	if (local_id.y < 8 || local_id.y > 23) { // 4 corners, each 8x8
		int2 offset;
		offset.y = (local_id.y < 8) ? -8 : 8;
		if (local_id.x < 2 || local_id.x > 5) {
			offset.x = (local_id.x < 2) ? -2 : 2;
			source_pixels = ReadPixel4(target_plane, source + offset, linear);
			WriteTile4(source_pixels, target.x + (offset.x << 2), target.y + offset.y, tile);
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	// Mirror into apron when at frame border
	float4 mirror;
	bool left_group  = (get_group_id(0) == 0);
	bool right_group = (get_group_id(0) == (get_num_groups(0) - 1));
	bool top_group   = (get_group_id(1) == 0);
	bool bot_group   = (get_group_id(1) == (get_num_groups(1) - 1));

	if (top_group && local_id.y < 8) { 	// top 8 rows
		mirror = ReadTile4(target.x, target.y + 1, tile);
		WriteTile4(mirror, target.x, 8 - local_id.y - 1, tile);
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	if (bot_group && local_id.y < 8) { // bottom 8 rows
		int centre = ((height - 1) & 31) + 8;

		mirror = ReadTile4(target.x, centre - local_id.y - 1, tile);
		WriteTile4(mirror, target.x, centre + local_id.y + 1, tile);
	}
	barrier(CLK_LOCAL_MEM_FENCE);
	// left 8 columns + top-left and bottom-left apron corners per tile
	int read_x  = (local_id.x == 1) ? 13 : 9;
	int write_x = (local_id.x == 1) ? 0 : 4;
	if (local_id.x < 2) {
		if (left_group) { // left 8 columns, 32 rows
			mirror = ReadTile4(read_x, target.y, tile);
			WriteTile4(mirror.wzyx, write_x, target.y, tile);
		}

		// TODO - cleanup IF structure - not sure if the compiler will jinx this
		if (local_id.y < 8) {
			if (left_group) { // left edge of frame apron corners, top and bottom
				mirror = ReadTile4(read_x, local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, local_id.y, tile);
				mirror = ReadTile4(read_x, 40 + local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, 40 + local_id.y, tile);
			}
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);
	if (local_id.x < 2 && local_id.y < 8) {
		int x = (local_id.x == 1) ? 4 : 0;
		if (top_group) { // apron at top edge of frame, left corners
			mirror = ReadTile4(x, 8 + local_id.y + 1, tile);
			WriteTile4(mirror, x, 8 - local_id.y - 1, tile);
		}
		if (bot_group) { // apron at bottom edge of frame, left corners
			int bottom = ((height - 1) & 31) + 8;
			mirror = ReadTile4(x, bottom - local_id.y - 1, tile);
			WriteTile4(mirror, x, bottom + local_id.y + 1, tile);
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);

	if (local_id.x > 5) { // right 8 columns + top-right and bottom-right apron corners per tile

		if (right_group) { // right edge of frame
			int centre  = ((width - 1) & 31) + 8;
			int read_x  = (local_id.x == 7) ? centre - 4 : centre - 8;
			int write_x = (local_id.x == 7) ? centre + 1 : centre + 5;

			mirror = ReadTile4(read_x, target.y, tile);
			WriteTile4(mirror.wzyx, write_x, target.y, tile);

			if (local_id.y < 8) { // top right and bottom right corner of every tile on right edge
				mirror = ReadTile4(read_x, local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, local_id.y, tile);
				mirror = ReadTile4(read_x, 40 + local_id.y, tile);
				WriteTile4(mirror.wzyx, write_x, 40 + local_id.y, tile);

				if (bot_group) { // bottom right corner of frame
					int bottom = ((height - 1) & 31) + 8;
					mirror = ReadTile4(read_x, bottom + local_id.y + 1, tile);
					WriteTile4(mirror.wzyx, write_x, bottom + local_id.y + 1, tile);
				}
			}
		} else if ((top_group || bot_group) && local_id.y < 8) {
			if (top_group) {
				int x = (local_id.x == 7) ? 44 : 40; // ATI 2.3: compiler bug requires this here if TILE_SIDE is not 48
				mirror = ReadTile4(x, 8 + local_id.y + 1, tile);
				WriteTile4(mirror, x, 8 - local_id.y - 1, tile);
			}
			if (bot_group) {
				int x = (local_id.x == 7) ? 44 : 40; // ATI 2.3: compiler bug requires this here if TILE_SIDE is not 48
				int bottom = ((height - 1) & 31) + 8;
				mirror = ReadTile4(x, bottom - local_id.y - 1, tile);
				WriteTile4(mirror, x, bottom + local_id.y + 1, tile);
			}
		}
	}
	barrier(CLK_LOCAL_MEM_FENCE);
}


Typo.

Typo.

<%
from tectosaur.nearfield.nearfield_op import pairs_func_name
from tectosaur.kernels import elastic_kernels, kernels

K = elastic_kernels[kernel_name]

def dn(dim):
    return ['x', 'y', 'z'][dim]

%>
#pragma OPENCL EXTENSION cl_khr_fp64: enable

#define Real ${float_type}

<%namespace name="prim" file="../integral_primitives.cl"/>

<%def name="integrate_pair(K, limit, check0)">
    ${prim.tri_info("obs", "nobs")}
    ${prim.tri_info("src", "nsrc")}

    Real result_temp[81];
    Real kahanC[81];

    for (int iresult = 0; iresult < 81; iresult++) {
        result_temp[iresult] = 0;
        kahanC[iresult] = 0;
    }

    ${K.constants_code}

    for (int iq = 0; iq < n_quad_pts; iq++) {
        <%
        qd = 4
        if limit:
            qd = 5
        %>
        Real obsxhat = quad_pts[iq * ${qd} + 0];
        Real obsyhat = quad_pts[iq * ${qd} + 1];
        Real srcxhat = quad_pts[iq * ${qd} + 2];
        Real srcyhat = quad_pts[iq * ${qd} + 3];
        % if limit:
            Real eps = quad_pts[iq * ${qd} + 4];
        % endif
        Real quadw = quad_wts[iq];

        % for which, ptname in [("obs", "x"), ("src", "y")]:
            ${prim.basis(which)}
            ${prim.pts_from_basis(
                ptname, which,
                lambda b, d: which + "_tri[" + str(b) + "][" + str(d) + "]", 3
            )}
        % endfor

        % if limit:
            % for dim in range(3):
                x${dn(dim)} -= eps * sqrt(obs_jacobian) * nobs${dn(dim)};
            % endfor
        % endif

        Real Dx = yx - xx;
        Real Dy = yy - xy;
        Real Dz = yz - xz;
        Real r2 = Dx * Dx + Dy * Dy + Dz * Dz;

        % if check0:
        if (r2 == 0.0) {
            continue;
        }
        % endif

        Real K[9];
        ${K.tensor_code}
        obsb[0] *= quadw;
        obsb[1] *= quadw;
        obsb[2] *= quadw;

        Real val, y, t;
        int idx = 0;
        for (int b_obs = 0; b_obs < 3; b_obs++) {
        for (int d_obs = 0; d_obs < 3; d_obs++) {
        for (int b_src = 0; b_src < 3; b_src++) {
        for (int d_src = 0; d_src < 3; d_src++, idx++) {
            Real val = obsb[b_obs] * srcb[b_src] * K[d_obs * 3 + d_src];
            Real y = val - kahanC[idx];
            Real t = result_temp[idx] + y;
            kahanC[idx] = (t - result_temp[idx]) - y;
            result_temp[idx] = t;
        }
        }
        }
        }
    }
</%def>

<%def name="single_pairs(K, limit, check0)">
__kernel
void ${pairs_func_name(limit, check0)}(__global Real* result,
    int n_quad_pts, __global Real* quad_pts, __global Real* quad_wts,
    __global Real* pts, __global int* obs_tris, __global int* src_tris,
    __global Real* params)
{
    const int i = get_global_id(0);

    ${prim.get_triangle("obs_tri", "obs_tris", "i")}
    ${prim.get_triangle("src_tri", "src_tris", "i")}
    ${integrate_pair(K, limit, check0)}

    for (int iresult = 0; iresult < 81; iresult++) {
        result[i * 81 + iresult] = obs_jacobian * src_jacobian * result_temp[iresult];
    }
}
</%def>

<%def name="farfield_tris(K)">
__kernel
void farfield_tris(__global Real* result,
    int n_quad_pts, __global Real* quad_pts, __global Real* quad_wts,
    __global Real* pts, int n_obs_tris, __global int* obs_tris,
    int n_src_tris, __global int* src_tris,
    __global Real* params)
{
    const int i = get_global_id(0);
    const int j = get_global_id(1);

    ${prim.get_triangle("obs_tri", "obs_tris", "i")}
    ${prim.get_triangle("src_tri", "src_tris", "j")}
    ${integrate_pair(K, limit = False, check0 = False)}

    for (int b_obs = 0; b_obs < 3; b_obs++) {
    for (int d_obs = 0; d_obs < 3; d_obs++) {
    for (int b_src = 0; b_src < 3; b_src++) {
    for (int d_src = 0; d_src < 3; d_src++) {

        result[
            (i * 9 + b_obs * 3 + d_obs) * n_src_tris * 9 +
            (j * 9 + b_src * 3 + d_src)
            ] = obs_jacobian * src_jacobian *
                result_temp[b_obs * 27 + d_obs * 9 + b_src * 3 + d_src];
    }
    }
    }
    }
}
</%def>

${prim.geometry_fncs()}
${single_pairs(K, limit = True, check0 = True)}
${single_pairs(K, limit = True, check0 = False)}
${single_pairs(K, limit = False, check0 = True)}
${single_pairs(K, limit = False, check0 = False)}
${farfield_tris(K)}

<%
from tectosaur.nearfield.nearfield_op import pairs_func_name
from tectosaur.kernels import elastic_kernels, kernels

K = elastic_kernels[kernel_name]

def dn(dim):
    return ['x', 'y', 'z'][dim]

%>
#pragma OPENCL EXTENSION cl_khr_fp64: enable

#define Real ${float_type}

<%namespace name="prim" file="../integral_primitives.cl"/>

<%def name="integrate_pair(K, limit, check0)">
    ${prim.tri_info("obs", "nobs")}
    ${prim.tri_info("src", "nsrc")}

    Real result_temp[81];
    Real kahanC[81];

    for (int iresult = 0; iresult < 81; iresult++) {
        result_temp[iresult] = 0;
        kahanC[iresult] = 0;
    }

    ${K.constants_code}

    for (int iq = 0; iq < n_quad_pts; iq++) {
        <%
        qd = 4
        if limit:
            qd = 5
        %>
        Real obsxhat = quad_pts[iq * ${qd} + 0];
        Real obsyhat = quad_pts[iq * ${qd} + 1];
        Real srcxhat = quad_pts[iq * ${qd} + 2];
        Real srcyhat = quad_pts[iq * ${qd} + 3];
        % if limit:
            Real eps = quad_pts[iq * ${qd} + 4];
        % endif
        Real quadw = quad_wts[iq];

        % for which, ptname in [("obs", "x"), ("src", "y")]:
            ${prim.basis(which)}
            ${prim.pts_from_basis(
                ptname, which,
                lambda b, d: which + "_tri[" + str(b) + "][" + str(d) + "]", 3
            )}
        % endfor

        % if limit:
            % for dim in range(3):
                x${dn(dim)} -= eps * sqrt(obs_jacobian) * nobs${dn(dim)};
            % endfor
        % endif

        Real Dx = yx - xx;
        Real Dy = yy - xy;
        Real Dz = yz - xz;
        Real r2 = Dx * Dx + Dy * Dy + Dz * Dz;

        % if check0:
        if (r2 == 0.0) {
            continue;
        }
        % endif

        Real K[9];
        ${K.tensor_code}
        obsb[0] *= quadw;
        obsb[1] *= quadw;
        obsb[2] *= quadw;

        Real val, y, t;
        int idx = 0;
        for (int b_obs = 0; b_obs < 3; b_obs++) {
        for (int d_obs = 0; d_obs < 3; d_obs++) {
        for (int b_src = 0; b_src < 3; b_src++) {
        for (int d_src = 0; d_src < 3; d_src++, idx++) {
            Real val = val = obsb[b_obs] * srcb[b_src] * K[d_obs * 3 + d_src];
            Real y = val - kahanC[idx];
            Real t = result_temp[idx] + y;
            kahanC[idx] = (t - result_temp[idx]) - y;
            result_temp[idx] = t;
        }
        }
        }
        }
    }
</%def>

<%def name="single_pairs(K, limit, check0)">
__kernel
void ${pairs_func_name(limit, check0)}(__global Real* result,
    int n_quad_pts, __global Real* quad_pts, __global Real* quad_wts,
    __global Real* pts, __global int* obs_tris, __global int* src_tris,
    __global Real* params)
{
    const int i = get_global_id(0);

    ${prim.get_triangle("obs_tri", "obs_tris", "i")}
    ${prim.get_triangle("src_tri", "src_tris", "i")}
    ${integrate_pair(K, limit, check0)}

    for (int iresult = 0; iresult < 81; iresult++) {
        result[i * 81 + iresult] = obs_jacobian * src_jacobian * result_temp[iresult];
    }
}
</%def>

<%def name="farfield_tris(K)">
__kernel
void farfield_tris(__global Real* result,
    int n_quad_pts, __global Real* quad_pts, __global Real* quad_wts,
    __global Real* pts, int n_obs_tris, __global int* obs_tris,
    int n_src_tris, __global int* src_tris,
    __global Real* params)
{
    const int i = get_global_id(0);
    const int j = get_global_id(1);

    ${prim.get_triangle("obs_tri", "obs_tris", "i")}
    ${prim.get_triangle("src_tri", "src_tris", "j")}
    ${integrate_pair(K, limit = False, check0 = False)}

    for (int b_obs = 0; b_obs < 3; b_obs++) {
    for (int d_obs = 0; d_obs < 3; d_obs++) {
    for (int b_src = 0; b_src < 3; b_src++) {
    for (int d_src = 0; d_src < 3; d_src++) {

        result[
            (i * 9 + b_obs * 3 + d_obs) * n_src_tris * 9 +
            (j * 9 + b_src * 3 + d_src)
            ] = obs_jacobian * src_jacobian *
                result_temp[b_obs * 27 + d_obs * 9 + b_src * 3 + d_src];
    }
    }
    }
    }
}
</%def>

${prim.geometry_fncs()}
${single_pairs(K, limit = True, check0 = True)}
${single_pairs(K, limit = True, check0 = False)}
${single_pairs(K, limit = False, check0 = True)}
${single_pairs(K, limit = False, check0 = False)}
${farfield_tris(K)}


remove unneccessary barrier in pooling

remove unneccessary barrier in pooling

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);

				int pool_size = (hend - hstart) * (wend - wstart);
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFFFFFFFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							int src_x = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0 + i;
							int src_y = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1 + j;
							//mask_private[k][l].x = src_x;
							//mask_private[k][l].y = src_y;
							mask_private[k][l] = src_x + src_y * MLO_POOLING_BOT_WIDTH;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		ushort
#define _INT_MASK_LOCAL			uint

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

// BUG!!!!
			barrier(CLK_LOCAL_MEM_FENCE);

				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);

				int pool_size = (hend - hstart) * (wend - wstart);
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFFFFFFFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							int src_x = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0 + i;
							int src_y = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1 + j;
							//mask_private[k][l].x = src_x;
							//mask_private[k][l].y = src_y;
							mask_private[k][l] = src_x + src_y * MLO_POOLING_BOT_WIDTH;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}


tcleanup: fix transmission-filename-to-tracker

Test that the torrent file exists before operating on it.  For some
reason I was seeing it disappear between the `directory' and
bdecode-file opening it.  Installed a warning, which I've not seen for
a while.  The machine transmission runs on was behaving really
strangely and I rebooted it, so maybe it was related to that.

(in-package :user)

(defvar *transmission-directory*
    "~/Library/Application Support/Transmission/Torrents/")

(defparameter *filename-to-torrent-cache* nil)

(defun transmission-filename-to-tracker (given-filename
					 &key debug hash
					 &aux temp)
  (when (null *filename-to-torrent-cache*)
    (setq *filename-to-torrent-cache*
      (make-hash-table :size 777 :test #'equal)))
  (when (setq temp (gethash given-filename *filename-to-torrent-cache*))
    (return-from transmission-filename-to-tracker temp))
  (dolist (torrent-file (directory
			 (merge-pathnames "*.torrent"
					  *transmission-directory*)))
    (if* (probe-file torrent-file)
       then (let* ((dict (bdecode-file torrent-file))
		   (announce (dict-get "announce" dict))
		   (info (dict-get "info" dict))
		   (name (dict-get "name" info))
		   ;; The hash prefix at the end of the .torrent filename
		   hash-prefix)
	      ;; If the filename was renamed in Transmission, the simple
	      ;; `name' to `given-filename' test will fail.  We can salvage
	      ;; this by looking at the hash included in the filename.
	      (when (=~ "\\.([0-9a-fA-F]+)\\.torrent$"
			(file-namestring torrent-file))
		(setq hash-prefix $1))
	      (when (or (string= name given-filename)
			(and hash-prefix
			     hash
			     (match-re hash-prefix hash :return nil)))
		(setf (gethash given-filename *filename-to-torrent-cache*)
		  (setq temp
		    (net.uri:uri-host (net.uri:parse-uri announce))))
		(and debug (format t "~a: ~s~%" given-filename temp))
		(return temp)))
       else ;; For reasons I don't understand, torrent-file sometimes
	    ;; disappears. I'm sure it's not a race condition, because the
	    ;; files that disappear always finished seeding some time
	    ;; before we are executed.
	    (warn "torrent file disappeared! ~a" torrent-file))))

(in-package :user)

(defvar *transmission-directory*
    "~/Library/Application Support/Transmission/Torrents/")

(defparameter *filename-to-torrent-cache* nil)

(defun transmission-filename-to-tracker (given-filename
					 &key debug hash
					 &aux temp)
  (when (null *filename-to-torrent-cache*)
    (setq *filename-to-torrent-cache*
      (make-hash-table :size 777 :test #'equal)))
  (when (setq temp (gethash given-filename *filename-to-torrent-cache*))
    (return-from transmission-filename-to-tracker temp))
  (dolist (torrent-file (directory
			 (merge-pathnames "*.torrent"
					  *transmission-directory*)))

    (let* ((dict (bdecode-file torrent-file))
	   (announce (dict-get "announce" dict))
	   (info (dict-get "info" dict))
	   (name (dict-get "name" info))
	   ;; The hash prefix at the end of the .torrent filename
	   hash-prefix)
      ;; If the filename was renamed in Transmission, the simple
      ;; `name' to `given-filename' test will fail.  We can salvage this by
      ;; looking at the hash included in the filename.
      (when (=~ "\\.([0-9a-fA-F]+)\\.torrent$" (file-namestring torrent-file))
	(setq hash-prefix $1))
      (when (or (string= name given-filename)
		(and hash-prefix hash (match-re hash-prefix hash :return nil)))
	(setf (gethash given-filename *filename-to-torrent-cache*)
	  (setq temp
	    (net.uri:uri-host (net.uri:parse-uri announce))))
	(and debug (format t "~a: ~s~%" given-filename temp))
	(return temp)))))