turns out local atomics not supported on mac gpu 9600m

//////////////////////////////////////////////////////////////////////////////////////
// Private constants
typedef uint2 KeyValuePair;

#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable

// May want to factor BITS_PER_PASS out
#define BITS_PER_PASS 4

#define GROUP_SIZE 128
#define ELEMENTS_PER_WORK_ITEM 4
#define SCAN_WORKGROUP_SIZE 128
#define BLOCK_SUM_WORKGROUP_SIZE 128
#define NUM_BLOCK_SUM_ELEMENTS_PER_ITEM ((GRID_SIZE+255)/BLOCK_SUM_WORKGROUP_SIZE)

#define LOG_NUM_BANKS 5
#define NUM_LOCAL_BANKS (1<<LOG_NUM_BANKS)

// Remove the top one and replace with the others, then can put the tuned versions back in
#if 1
#define CONVERT_CONFLICT_FREE(in) (in)
#define CONVERT_CONFLICT_FREE_2VEC(in) (in)
#define CONVERT_CONFLICT_FREE_4VEC(in) (in)
#else
#define CONVERT_CONFLICT_FREE(in) (in + (in>>LOG_NUM_BANKS))
#define CONVERT_CONFLICT_FREE_2VEC(in) (intermediate.x = in.x>>LOG_NUM_BANKS, intermediate.y = in.y>>LOG_NUM_BANKS, in + intermediate)
#define CONVERT_CONFLICT_FREE_4VEC(in) (intermediate.x = in.x>>LOG_NUM_BANKS, intermediate.y = in.y>>LOG_NUM_BANKS, intermediate.z = in.z>>LOG_NUM_BANKS, intermediate.w = in.w>>LOG_NUM_BANKS, in + intermediate)
#endif

#define BANK_CONFLICT_RESOLUTION_PADDING (CONVERT_CONFLICT_FREE(GROUP_SIZE*ELEMENTS_PER_WORK_ITEM)-GROUP_SIZE*ELEMENTS_PER_WORK_ITEM)

////////////////////////////////////////////////////////////////////////////////////////
// Prefixsum functions

/**
 * Perform work efficient prefix sum in local memory in place.
 */
void localPrefixSum(  __local unsigned *prefixSums, unsigned numElements )
{
    // Iterate over halving sizes of the element set performing reduction phase of scan

    int offset = 1;
    //for( int level = get_local_size(0); level > 0; level >>= 1 )
    for( int level = numElements>>1; level > 0; level >>= 1 )
    //for( int level = 1; level > 0; level >>= 1 )
    {
        barrier(CLK_LOCAL_MEM_FENCE);

        for( int sumElement = get_local_id(0); sumElement < level; sumElement += get_local_size(0) )
        {
            //int ai = offset*(2*sumElement+1)-1;
            //int bi = offset*(2*sumElement+2)-1;

            int i = 2*offset*sumElement;
            int ai = i + offset - 1;
            int bi = ai + offset;

            ai = CONVERT_CONFLICT_FREE(ai);
            bi = CONVERT_CONFLICT_FREE(bi);
            prefixSums[bi] += prefixSums[ai];
        }
        offset <<= 1;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    // Need to clear the last element
    if( get_local_id(0) == 0 )
        prefixSums[ CONVERT_CONFLICT_FREE(numElements-1) ] = 0;

    for( int level = 1; level < numElements; level <<= 1 )
    {
        offset >>= 1;
        barrier(CLK_LOCAL_MEM_FENCE);

        for( int sumElement = get_local_id(0); sumElement < level; sumElement += get_local_size(0) )
        {
            int ai = offset*(2*sumElement+1)-1;
            int bi = offset*(2*sumElement+2)-1;

            ai = CONVERT_CONFLICT_FREE(ai);
            bi = CONVERT_CONFLICT_FREE(bi);
            unsigned temporary = prefixSums[ai];
            prefixSums[ai] = prefixSums[bi];
            prefixSums[bi] += temporary;
        }
    }
}

// Barrier free because we know that this never gets wider than a WF
void BClocalPF(  __local unsigned *prefixSums )
{
    // Iterate over halving sizes of the element set performing reduction phase of scan
    int offset = 1;

    for( int level = 8; level > 0; level >>= 1 )
    {
        for( int sumElement = get_local_id(0); sumElement < level; sumElement += get_local_size(0) )
        {
            int ai = offset*(2*sumElement+1)-1;
            int bi = offset*(2*sumElement+2)-1;
            prefixSums[bi] += prefixSums[ai];
        }
        offset <<= 1;
    }

    // Need to clear the last element
    if( get_local_id(0) == 0 )
        prefixSums[ 15 ] = 0;

    for( int level = 1; level < 16; level <<= 1 )
    {
        offset >>= 1;

        for( int sumElement = get_local_id(0); sumElement < level; sumElement += get_local_size(0) )
        {
            int ai = offset*(2*sumElement+1)-1;
            int bi = offset*(2*sumElement+2)-1;
            unsigned temporary = prefixSums[ai];
            prefixSums[ai] = prefixSums[bi];
            prefixSums[bi] += temporary;
        }
    }
}

/**
 * Optimised prefix sum that takes 4 elements per WI and assumes 128 WIs in
 * a group.
 * Also depends on WF size of 64 elements. Will need minor adaptation to scale down.
 */
uint4 localPrefixSumBlock( uint4 prefixSumData, __local unsigned *prefixSums )
{
    uint4 originalData = prefixSumData;

    // Do sum across vector in two stages
    prefixSumData.y += prefixSumData.x;
    prefixSumData.w += prefixSumData.z;

    prefixSumData.z += prefixSumData.y;
    prefixSumData.w += prefixSumData.y;

    // Now just 128 values, each sum of a block of 4
    prefixSums[get_local_id(0)] = 0;
    prefixSums[get_local_id(0)+128] = prefixSumData.w;

    // TODO: Could get rid of these barriers if get both WFs working but on different halves of data
    barrier(CLK_LOCAL_MEM_FENCE);

    // Do for single WF as we only have 128 values to deal with = 64*2
/*
	//	lh.todo. th. doesn't work on v2.2 because compiler mess this up. Update of prefixSums[idx] is only done at the end of this {}.
	//	1. orig ver
    if( get_local_id(0) < 64 )
    {
        int idx = 2*get_local_id(0) + 129;
        prefixSums[idx] += prefixSums[idx-1];
        prefixSums[idx] += prefixSums[idx-2];
        prefixSums[idx] += prefixSums[idx-4];
        prefixSums[idx] += prefixSums[idx-8];
        prefixSums[idx] += prefixSums[idx-16];
        prefixSums[idx] += prefixSums[idx-32];
        prefixSums[idx] += prefixSums[idx-64];
        // Propagate intermediate values through
        prefixSums[idx-1] += prefixSums[idx-2];
    }
*/
	//	2. using mem fence
    if( get_local_id(0) < 64 )
    {
        int idx = 2*get_local_id(0) + 129;
        prefixSums[idx] += prefixSums[idx-1];
		mem_fence(CLK_LOCAL_MEM_FENCE);
        prefixSums[idx] += prefixSums[idx-2];
		mem_fence(CLK_LOCAL_MEM_FENCE);
        prefixSums[idx] += prefixSums[idx-4];
		mem_fence(CLK_LOCAL_MEM_FENCE);
        prefixSums[idx] += prefixSums[idx-8];
		mem_fence(CLK_LOCAL_MEM_FENCE);
        prefixSums[idx] += prefixSums[idx-16];
		mem_fence(CLK_LOCAL_MEM_FENCE);
        prefixSums[idx] += prefixSums[idx-32];
		mem_fence(CLK_LOCAL_MEM_FENCE);
        prefixSums[idx] += prefixSums[idx-64];
		mem_fence(CLK_LOCAL_MEM_FENCE);
        // Propagate intermediate values through
        prefixSums[idx-1] += prefixSums[idx-2];
    }
/*
	//	this should work on v2.2 too. Checked assembly but doesnt run as I expect.
	//	3. using volatile (doesn't work somehow)
	volatile __local unsigned* ps = prefixSums;
    if( get_local_id(0) < 64 )
    {
        int idx = 2*get_local_id(0) + 129;
        ps[idx] += ps[idx-1];
        ps[idx] += ps[idx-2];
        ps[idx] += ps[idx-4];
        ps[idx] += ps[idx-8];
        ps[idx] += ps[idx-16];
        ps[idx] += ps[idx-32];
        ps[idx] += ps[idx-64];
        // Propagate intermediate values through
        ps[idx-1] += ps[idx-2];
    }
*/
/*
	//	4. using barrier
    {
        int idx = 2*get_local_id(0) + 129;
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx] += prefixSums[idx-1];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx] += prefixSums[idx-2];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx] += prefixSums[idx-4];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx] += prefixSums[idx-8];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx] += prefixSums[idx-16];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx] += prefixSums[idx-32];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx] += prefixSums[idx-64];
        // Propagate intermediate values through
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	    if( get_local_id(0) < 64 )
		{
	        prefixSums[idx-1] += prefixSums[idx-2];
		}
    }
*/

    barrier(CLK_LOCAL_MEM_FENCE);

    // Grab and propagate for whole WG - loading the - 1 value
    uint addValue = prefixSums[get_local_id(0)+127];

    // Propagate item level sum across vector
    prefixSumData += (uint4)(addValue, addValue, addValue, addValue);

    // And return the final value which is the final sum
    return prefixSumData - originalData;
}

////////////////////////////////////////////////////////////////////////////////////////
// Public functions

/**
 * Generate a local histogram from the sorted data counting how many entries of each set of 4 local digits we have.
 * This version just uses atomics to do summation because the number should be relatively small.
 *
 * alternative: Histogram has 32(local mem banks)*2^numBits entries which are reduced into the first 2^numBits elements in this function.
 * The rest of the histogram array is temporary data.
 * There are 32 histogram bins for each value to avoid the need for atomics.
 *
 * @param histogramOutputRadixMajor is ordered by radices - inefficient read and write but easily prefix summed.
 * This will be the global set of offsets.
 * @param histogramOutputGroupMajor is ordered by groups. This is the local set of offsets for the radix sort scatter to use
 * later that will not be globally summed.
 */
void generateHistogram(
		uint4 sortedData,
		__local unsigned *histogram,
		__global unsigned *histogramOutputRadixMajor,
		__global unsigned *histogramOutputGroupMajor,
		unsigned startBit,
        unsigned numGroups)
{
    uint4 addresses;
    addresses = (uint4)(get_local_id(0), get_local_id(0), get_local_id(0), get_local_id(0));
    // This bit is best unvectorised as we can't write more than 2 values to local per WI anyway
    if( get_local_id(0) < (1<<BITS_PER_PASS) )
    {
    	histogram[addresses.x] = 0;
    }

    addresses = addresses * (unsigned)4;
    addresses.y = addresses.y+1;
    addresses.z = addresses.z+2;
    addresses.w = addresses.w+3;

	// Obtain correct histogram address using only the BITS_PER_PASS bits that we're sorting by in this pass
	sortedData.x >>= startBit;
	sortedData.y >>= startBit;
	sortedData.z >>= startBit;
	sortedData.w >>= startBit;

	int andValue = ((1<<BITS_PER_PASS)-1);
	sortedData &= (uint4)(andValue, andValue, andValue, andValue);

	// Alternative approach counting differences
	// Doesn't work yet, and will probably never be faster anyway due to ifs
	// only makes sense if atomics are *very* slow
#if 0
	histogram[16+get_local_id(0)] = sortedData.w;
#endif

    // Perform the atomic histogram updates
	barrier(CLK_LOCAL_MEM_FENCE);

	// Alternative approach counting differences
	// Doesn't work yet, and will probably never be faster anyway due to ifs
	// only makes sense if atomics are *very* slow
#if 0
	uint previousValue = histogram[16-1+get_local_id(0)];
	if(sortedData.x != previousValue && get_local_id(0) > 0)
		histogram[previousValue] = get_local_id(0)*4;
	if(sortedData.z != sortedData.y)
		histogram[sortedData.y] = get_local_id(0)*4;
	if(sortedData.w != sortedData.z)
		histogram[sortedData.z] = get_local_id(0)*4;
#else
    /*
     * changed by enjalot
	atomic_inc( &(histogram[sortedData.x]) );
	atomic_inc( &(histogram[sortedData.y]) );
	atomic_inc( &(histogram[sortedData.z]) );
	atomic_inc( &(histogram[sortedData.w]) );
    */
    atom_inc( &(histogram[sortedData.x]) );
	atom_inc( &(histogram[sortedData.y]) );
	atom_inc( &(histogram[sortedData.z]) );
	atom_inc( &(histogram[sortedData.w]) );
#endif

	barrier(CLK_LOCAL_MEM_FENCE);

    // No need to vectorise this with only 16 values to process
    if( get_local_id(0) < 16 )
    {
     	uint histValues;

     	histValues = histogram[get_local_id(0)];

     	unsigned globalOffset = 16*get_group_id(0);
     	uint globalAddresses = get_local_id(0) + globalOffset;

     	uint globalAddressesRadixMajor = numGroups;
     	globalAddressesRadixMajor = globalAddressesRadixMajor * get_local_id(0);
     	globalAddressesRadixMajor = globalAddressesRadixMajor + get_group_id(0);

     	histogramOutputGroupMajor[globalAddresses] = histValues;
     	histogramOutputRadixMajor[globalAddressesRadixMajor] = histValues;
     }
}

#define LOCAL_SORT_WG_SIZE 128

/*
 * Perform radix sort operation on a local array of unsigned ints
 * based on a particular set of bit entries of BITS_PER_PASS in size at
 * startbit in the data.
 * Outputs the data and a histogram representing the counts of all
 * BITS_PER_PASS sized values within the local memory array.
 * histogramOutput is stored as all the 0 bins, then all the 1 bins
 * and so on. Inefficient write out in here - simple prefix sum.
 */
__attribute__((reqd_work_group_size(LOCAL_SORT_WG_SIZE,1,1)))
__kernel void RadixSortLocal(
        // uint8 - this is really 4 KeyValuePairs packed together
		// There is probably a cleaner way to represent this
		__global uint8 *dataToSort,

		__global unsigned *histogramOutputGlobalRadixMajor,
		__global unsigned *histogramOutputLocalGroupMajor,
		int startBit,
        int numGroups,
		__local unsigned *sorterSharedMemory
        )
{
	int numLocalElements = get_local_size(0)*ELEMENTS_PER_WORK_ITEM;

    uint4 plainLocalAddress = (uint4)(get_local_id(0), get_local_id(0), get_local_id(0), get_local_id(0));
    uint4 addValues = (uint4)(0,1,2,3);
    plainLocalAddress = plainLocalAddress * (unsigned)4;
    plainLocalAddress = plainLocalAddress + addValues;

    // localAddress that's been converted to avoid conflicts
    uint4 localAddress = CONVERT_CONFLICT_FREE_4VEC(plainLocalAddress);

    uint4 localKeys;
    uint4 localValues;
	{
        uint4 globalAddress =  (uint4)(get_group_id(0), get_group_id(0), get_group_id(0), get_group_id(0));
        uint4 localElementsCount = (uint4)(numLocalElements, numLocalElements, numLocalElements, numLocalElements);
        globalAddress = globalAddress*localElementsCount+localAddress;

        // uint8 - this is really 4 KeyValuePairs packed together
        // Probably a cleaner way to represent this
        uint8 localData;
		localData = dataToSort[globalAddress.x/4];
		localKeys.s0 = localData.s0;
		localKeys.s1 = localData.s2;
		localKeys.s2 = localData.s4;
		localKeys.s3 = localData.s6;
		localValues.s0 = localData.s1;
		localValues.s1 = localData.s3;
		localValues.s2 = localData.s5;
		localValues.s3 = localData.s7;
	}

	// Iterate over the block of bits we are sorting internally in this kernel
	//for( int bitIndex = startBit; bitIndex < (startBit+BITS_PER_PASS ); ++bitIndex )
	int bitIndex = startBit;
	do
	{
		// Write a local mem value just after the range to be processed by the prefix sum

		// TODO: 512??
        if( get_local_id(0) == (get_local_size(0)-1) )
        	sorterSharedMemory[256] = localKeys.w;

        unsigned compare = (1<<bitIndex);
        uint4 compareVec = (uint4)(compare, compare, compare, compare);

        uint4 localCompareVec = localKeys & compareVec;

        uint4 prefixSum;
        prefixSum = select( (uint4)(1,1,1,1), (uint4)(0,0,0,0), localCompareVec != (uint4)(0,0,0,0) );

        prefixSum = localPrefixSumBlock(prefixSum, sorterSharedMemory);

        // Need to get totalFalses from somewhere - that's the value in [255]. Could just assume 255 of sorterSharedMemory is ok, but
        // that's hardly clean
        uint totalFalses = sorterSharedMemory[255];

		/////////////////
		// Perform the local sort
        // Rearrange data using local memory
        {
        	uint4 localCompareVec = localKeys & compareVec;

        	uint4 newAddress = plainLocalAddress - prefixSum;
        	newAddress += (uint4)(totalFalses, totalFalses, totalFalses, totalFalses);

        	newAddress = select( prefixSum, newAddress, localCompareVec != (uint4)(0, 0, 0, 0) );

        	newAddress = CONVERT_CONFLICT_FREE_4VEC(newAddress);

        	// Internally sort keys and then values
        	// May be more efficient to do this in one go using the pairs
        	barrier(CLK_LOCAL_MEM_FENCE);

        	sorterSharedMemory[newAddress.x] = localKeys.x;
        	sorterSharedMemory[newAddress.y] = localKeys.y;
        	sorterSharedMemory[newAddress.z] = localKeys.z;
        	sorterSharedMemory[newAddress.w] = localKeys.w;

        	barrier(CLK_LOCAL_MEM_FENCE);

            // Back up sort values to give thread-local store of current state of local mem
        	localKeys.x = sorterSharedMemory[localAddress.x];
        	localKeys.y = sorterSharedMemory[localAddress.y];
        	localKeys.z = sorterSharedMemory[localAddress.z];
        	localKeys.w = sorterSharedMemory[localAddress.w];

    		barrier(CLK_LOCAL_MEM_FENCE);

            sorterSharedMemory[newAddress.x] = localValues.x;
            sorterSharedMemory[newAddress.y] = localValues.y;
            sorterSharedMemory[newAddress.z] = localValues.z;
            sorterSharedMemory[newAddress.w] = localValues.w;

            barrier(CLK_LOCAL_MEM_FENCE);

            // Back up sort values to give thread-local store of current state of local mem
            localValues.x = sorterSharedMemory[localAddress.x];
            localValues.y = sorterSharedMemory[localAddress.y];
            localValues.z = sorterSharedMemory[localAddress.z];
            localValues.w = sorterSharedMemory[localAddress.w];

            barrier(CLK_LOCAL_MEM_FENCE);
        }

		bitIndex = bitIndex + 1;
	} while( bitIndex < (startBit+BITS_PER_PASS ) );

	// Generate local histogram and output to global histogram storage array
	generateHistogram( localKeys, sorterSharedMemory, histogramOutputGlobalRadixMajor, histogramOutputLocalGroupMajor, startBit, numGroups );

	{
        uint4 globalAddress =  (uint4)(get_group_id(0), get_group_id(0), get_group_id(0), get_group_id(0));
        uint4 localElementsCount = (uint4)(numLocalElements, numLocalElements, numLocalElements, numLocalElements);
        globalAddress = globalAddress*localElementsCount+plainLocalAddress;

        uint8 localData;
        localData.s0 = localKeys.s0;
        localData.s2 = localKeys.s1;
        localData.s4 = localKeys.s2;
        localData.s6 = localKeys.s3;
        localData.s1 = localValues.s0;
        localData.s3 = localValues.s1;
        localData.s5 = localValues.s2;
        localData.s7 = localValues.s3;

        // 2* because we have key/value pairs now
		dataToSort[globalAddress.x/4] = localData;
	}
}

/*
 * ScanLargeArrays : Scan is done for each block and the sum of each
 * block is stored in separate array (sumBuffer). SumBuffer is scanned
 * and results are added to every value of next corresponding block to
 * compute the scan of a large array.(not limited to 2*MAX_GROUP_SIZE)
 * Scan uses a balanced tree algorithm. See Belloch, 1990 "Prefix Sums
 * and Their Applications"
 * @param output output data
 * @param input  input data
 * @param block  local memory used in the kernel
 * @param length length of the input data
 * @param sumBuffer  sum of blocks
 */
__attribute__((reqd_work_group_size(128,1,1)))
__kernel
void ScanLargeArrays(
		__global unsigned int *output,
        __global unsigned int *input,
        __local  unsigned int *block,	 // Size : block_size
        const uint block_size,	 // size of block
        const uint length,	 	 // no of elements
        __global unsigned int *sumBuffer)  // sum of blocks

{
	int tid = get_local_id(0);
	int gid = get_global_id(0);
	int bid = get_group_id(0);

	int offset = 1;

    /* Cache the computational window in shared memory */
    if( (2*gid + 1) < length )
    {
	   block[2*tid]     = input[2*gid];
	   block[2*tid + 1] = input[2*gid + 1];
    }  else {
       block[2*tid]     = 0;
       block[2*tid + 1] = 0;
    }

    /* build the sum in place up the tree */
	for(int d = block_size>>1; d > 0; d >>=1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		if(tid<d)
		{
			int ai = offset*(2*tid + 1) - 1;
			int bi = offset*(2*tid + 2) - 1;

			block[bi] += block[ai];
		}
		offset *= 2;
	}

	barrier(CLK_LOCAL_MEM_FENCE);

    /* store the value in sum buffer before making it to 0 */
	sumBuffer[bid] = block[block_size - 1];

	barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);

    /* scan back down the tree */

    /* clear the last element */
	block[block_size - 1] = 0;

    /* traverse down the tree building the scan in the place */
	for(int d = 1; d < block_size ; d *= 2)
	{
		offset >>=1;
		barrier(CLK_LOCAL_MEM_FENCE);

		if(tid < d)
		{
			int ai = offset*(2*tid + 1) - 1;
			int bi = offset*(2*tid + 2) - 1;

			unsigned int t = block[ai];
			block[ai] = block[bi];
			block[bi] += t;
		}
	}

	barrier(CLK_LOCAL_MEM_FENCE);

    /*write the results back to global memory */

	if( (2*gid + 1) < length )
    {
        output[2*gid]     = block[2*tid];
        output[2*gid + 1] = block[2*tid + 1];
    }

}

/**
  * ScanPropagateBlockSums : Takes as input blocks of data,
 * each individually prefix summed, as well as a set of block
 * sums that summarise the entire set.
 * Propagates the block sums into the prefix sum blocks.
 * @param scanArray the set of prefix summed blocks
 * @param block_size the number of data elements dealt with
 * by each work group. Need not relate directly to the
 * number of work items in a group, but should align for efficiency.
 * @param length length of the input data
 * @param sumBuffer  block sum array
 */
__attribute__((reqd_work_group_size(128,1,1)))
__kernel
void ScanPropagateBlockSums(
		__global unsigned int *scanArray,
        const uint block_size,	 // size of block
        const uint length,	 	 // no of elements
        __global unsigned int *sumBuffer)  // sum of blocks
{
	// Get the appropriate block sum for this group
	unsigned int blockSum = sumBuffer[get_group_id(0)+1];

	// For entire block, add the sum to it

    //enjalot
    uint left = (get_group_id(0)+2)*(block_size);
    int endValue = min(left, length);
//	int endValue = min((get_group_id(0)+2)*(block_size), length);
	for(
			int i = (get_group_id(0)+1)*block_size + get_local_id(0);
			i < endValue;
			i += get_local_size(0))
	{
		scanArray[i] += blockSum;
	}
}

/**
 * Do a scan on the block sum values which can then be propagated back to the blocks.
 * Currently this does not deal correctly with non power-of-two sizes (I think
 * certainly some sizes of data set fail to sort correctly, problem could be elsewhere).
 */
__attribute__((reqd_work_group_size(128,1,1)))
__kernel
void SumBlockSums(__global unsigned *blockSums, unsigned numElements, __local unsigned *localMemory)
{
	int element;
	for( element = get_local_id(0); element < numElements; element += get_local_size(0) )
	{
		localMemory[CONVERT_CONFLICT_FREE(element)] = blockSums[element];
	}
/*
	for( ; element < 2048; element += get_local_size(0) )
	{
		localMemory[element + CONFLICT_FREE_OFFSET(element)] = 0;
	}*/

	barrier(CLK_LOCAL_MEM_FENCE);

	//localPrefixSum(localMemory, 2048);
	localPrefixSum(localMemory, numElements);

	barrier(CLK_LOCAL_MEM_FENCE);

	for( int element = get_local_id(0); element < (numElements); element += get_local_size(0) )
	{
		blockSums[element] = localMemory[CONVERT_CONFLICT_FREE(element)];
	}
}

/**
 * RadixSortGlobal: Perform the global radix sort scatter from
 * pre-sorted local buffers and both local and global prefix-sum
 * offset information.
 * @param dataToSort contains the buffer of locally sorted blocks
 * @param histogramGlobalRadixMajor is radix major arranged
 * global prefix sum offset information for the bit block values.
 * @param histogramLocalGroupMajor contains per-group
 * local prefix sum information.
 * @param startBit is the first of BITS_PER_PASS bits in the key this pass is
 * sorting by.
 * @param sorterSharedMemory is the shared memory storage region,
 * divided up as necessary.
 */
__attribute__((reqd_work_group_size(128,1,1)))
__kernel
void RadixSortGlobal(
        // uint8 - this is really 4 KeyValuePairs packed together
        // There is probably a cleaner way to represent this
		__global uint8 *dataToSort,

		__global unsigned *histogramGlobalRadixMajor,
		__global unsigned *histogramLocalGroupMajor,
		__global KeyValuePair *destinationArray,
		unsigned startBit,
		__local unsigned *sorterLocalMemory)
{
	// Move local histogram far enough through memory to allow for
	// efficient local prefix sum
	__local unsigned *localHistogram = sorterLocalMemory + 2*(1<<BITS_PER_PASS);
	__local unsigned *globalHistogram = sorterLocalMemory;

	// First load the local and global histogram data into local memory
	// And do prefix sum of local histogram
	// No point in vectorising this with only 16 bins
	if( get_local_id(0) < ((1<<BITS_PER_PASS)/2) )
	{
		uint2 histElement = (uint2)(get_local_id(0), get_local_id(0)+8);

		uint2 localValues;
		globalHistogram[histElement.x] = histogramGlobalRadixMajor[get_num_groups(0)*histElement.x + get_group_id(0)];
		globalHistogram[histElement.y] = histogramGlobalRadixMajor[get_num_groups(0)*histElement.y + get_group_id(0)];

		localValues.x = histogramLocalGroupMajor[(1<<BITS_PER_PASS)*get_group_id(0) + histElement.x];
		localValues.y = histogramLocalGroupMajor[(1<<BITS_PER_PASS)*get_group_id(0) + histElement.y];
		localHistogram[histElement.x] = localValues.x;
		localHistogram[histElement.y] = localValues.y;

		localHistogram[histElement.x-(1<<BITS_PER_PASS)] = 0;
		localHistogram[histElement.y-(1<<BITS_PER_PASS)] = 0;

		int idx = 2*get_local_id(0);
		localHistogram[idx] += localHistogram[idx-1];
		mem_fence(CLK_LOCAL_MEM_FENCE);
		localHistogram[idx] += localHistogram[idx-2];
		mem_fence(CLK_LOCAL_MEM_FENCE);
		localHistogram[idx] += localHistogram[idx-4];
		mem_fence(CLK_LOCAL_MEM_FENCE);
		localHistogram[idx] += localHistogram[idx-8];
		mem_fence(CLK_LOCAL_MEM_FENCE);

		// Propagate intermediate values through
		localHistogram[idx-1] += localHistogram[idx-2];
		mem_fence(CLK_LOCAL_MEM_FENCE);

		// Grab and propagate for whole WG - loading the - 1 value
		localValues.x = localHistogram[histElement.x-1];
		localValues.y = localHistogram[histElement.y-1];

		localHistogram[histElement.x] = localValues.x;
		localHistogram[histElement.y] = localValues.y;

	}

	barrier(CLK_LOCAL_MEM_FENCE);

    const int numLocalElements = 512;
    uint4 localAddress = (uint4)(get_local_id(0), get_local_id(0), get_local_id(0), get_local_id(0));
    localAddress = localAddress*(unsigned)4;
    uint4 addValues = (uint4)(0,1,2,3);
    localAddress = localAddress + addValues;
    //uint4 globalAddress = get_group_id(0)*numLocalElements + localAddress;
    //enjalot
    uint4 globalAddress = get_group_id(0);
    globalAddress *= numLocalElements + localAddress;

    // uint8 - this is really 4 KeyValuePairs packed together
    // There is probably a cleaner way to represent this
	uint8 sortValue;
	sortValue = dataToSort[globalAddress.x/4];

	uint cmpValue = ((1<<BITS_PER_PASS)-1);
	uint4 cmpValueVector = (uint4)(cmpValue, cmpValue, cmpValue, cmpValue);
	uint4 radix;
	// s0,2,4,6 are keys, 1,3,5,7 are values
	radix.x = (sortValue.s0>>startBit);
	radix.y = (sortValue.s2>>startBit);
	radix.z = (sortValue.s4>>startBit);
	radix.w = (sortValue.s6>>startBit);

	radix = radix & cmpValueVector;

	uint4 localOffsetIntoRadixSet;
	localOffsetIntoRadixSet = localAddress;
	localOffsetIntoRadixSet.x = localOffsetIntoRadixSet.x - localHistogram[radix.x];
	localOffsetIntoRadixSet.y = localOffsetIntoRadixSet.y - localHistogram[radix.y];
	localOffsetIntoRadixSet.z = localOffsetIntoRadixSet.z - localHistogram[radix.z];
	localOffsetIntoRadixSet.w = localOffsetIntoRadixSet.w - localHistogram[radix.w];

	uint4 globalOffset = localOffsetIntoRadixSet;
	globalOffset.x = globalOffset.x + globalHistogram[radix.x];
	globalOffset.y = globalOffset.y + globalHistogram[radix.y];
	globalOffset.z = globalOffset.z + globalHistogram[radix.z];
	globalOffset.w = globalOffset.w + globalHistogram[radix.w];

	// s0,2,4,6 are keys, 1,3,5,7 are values
	destinationArray[globalOffset.x] = (KeyValuePair)(sortValue.s0, sortValue.s1);
	destinationArray[globalOffset.y] = (KeyValuePair)(sortValue.s2, sortValue.s3);
	destinationArray[globalOffset.z] = (KeyValuePair)(sortValue.s4, sortValue.s5);
	destinationArray[globalOffset.w] = (KeyValuePair)(sortValue.s6, sortValue.s7);
}


Change the thread cutoff to match the C version.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

#ifdef FILTER_OFFSET
__constant short sub_pel_filters[8][6] = {
    { 0, 0, 128, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0},
    { 2, -11, 108, 36, -8, 1}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0},
    { 3, -16, 77, 77, -16, 3}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0},
    { 1, -8, 36, 108, -11, 2}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0},
};
#endif

__kernel void vp8_filter_block2d_first_pass_kernel(
    __global unsigned char *src_ptr,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
#ifdef FILTER_OFFSET
    int filter_offset)
#else
    __global short *vp8_filter)
#endif
{
    uint i = get_global_id(0);

    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

#ifdef FILTER_OFFSET
    __constant short *vp8_filter = sub_pel_filters[filter_offset];
#endif

    if (i < (output_width*output_height)){
        src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

        Temp = ((int)*(src_ptr+src_offset - PS2)      * vp8_filter[0]) +
           ((int)*(src_ptr+src_offset - (int)pixel_step) * vp8_filter[1]) +
           ((int)*(src_ptr+src_offset)                * vp8_filter[2]) +
           ((int)*(src_ptr+src_offset + pixel_step)   * vp8_filter[3]) +
           ((int)*(src_ptr+src_offset + PS2)          * vp8_filter[4]) +
           ((int)*(src_ptr+src_offset + PS3)          * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if ( Temp > 255 )
            Temp = 255;

        output_ptr[i] = Temp;
    }
}

__kernel void vp8_filter_block2d_second_pass_kernel
(
    __global int *src_ptr,
    unsigned int offset,
    __global unsigned char *output_ptr,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    __global const short *vp8_filter
) {

    int out_offset;
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);
    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment) + offset;
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

#ifdef FILTER_OFFSET
__constant short sub_pel_filters[8][6] = {
    { 0, 0, 128, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0},
    { 2, -11, 108, 36, -8, 1}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0},
    { 3, -16, 77, 77, -16, 3}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0},
    { 1, -8, 36, 108, -11, 2}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0},
};
#endif

__kernel void vp8_filter_block2d_first_pass_kernel(
    __global unsigned char *src_ptr,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
#ifdef FILTER_OFFSET
    int filter_offset)
#else
    __global short *vp8_filter)
#endif
{
    uint i = get_global_id(0);

    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

#ifdef FILTER_OFFSET
    __constant short *vp8_filter = sub_pel_filters[filter_offset];
#endif

    if (i < (output_width*output_height)){
        src_offset = i + (i/output_width * (src_pixels_per_line - output_width)) + PS2;

        Temp = ((int)*(src_ptr+src_offset - PS2)      * vp8_filter[0]) +
           ((int)*(src_ptr+src_offset - (int)pixel_step) * vp8_filter[1]) +
           ((int)*(src_ptr+src_offset)                * vp8_filter[2]) +
           ((int)*(src_ptr+src_offset + pixel_step)   * vp8_filter[3]) +
           ((int)*(src_ptr+src_offset + PS2)          * vp8_filter[4]) +
           ((int)*(src_ptr+src_offset + PS3)          * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if ( Temp > 255 )
            Temp = 255;

        output_ptr[i] = Temp;
    }
}

__kernel void vp8_filter_block2d_second_pass_kernel
(
    __global int *src_ptr,
    unsigned int offset,
    __global unsigned char *output_ptr,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    __global const short *vp8_filter
) {

    int out_offset;
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);
    if (i < output_width +(output_pitch*(output_height-1))){

        out_offset = i/output_width;
        src_offset = out_offset;
        //src_ptr = &src_ptr[offset];
        //src_offset = out_offset;

        src_offset = i + (src_offset * src_increment) + offset;
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}


Auto-detect if local<global size. Hopefully this allows successful 1-pass CPU runs by auto-selected the looped 1st pass.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant int bilinear_filters[8][2] = {
    { 128, 0},
    { 112, 16},
    { 96, 32},
    { 80, 48},
    { 64, 64},
    { 48, 80},
    { 32, 96},
    { 16, 112}
};

__constant short sub_pel_filters[8][8] = {
    //These were originally 8x6, but are padded for vector ops
    { 0, 0, 128, 0, 0, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0, 0, 0},
    { 2, -11, 108, 36, -8, 1, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0, 0, 0},
    { 3, -16, 77, 77, -16, 3, 0, 0}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0, 0, 0},
    { 1, -8, 36, 108, -11, 2, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0, 0, 0},
};

kernel void vp8_filter_block2d_first_pass_kernel(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we use it now

    int Temp;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        src_offset = tid + (tid/output_width * (src_pixels_per_line - output_width));

        Temp = (int)(src_ptr[src_offset - 2] * vp8_filter[0]) +
           (int)(src_ptr[src_offset - 1] * vp8_filter[1]) +
           (int)(src_ptr[src_offset]     * vp8_filter[2]) +
           (int)(src_ptr[src_offset + 1] * vp8_filter[3]) +
           (int)(src_ptr[src_offset + 2] * vp8_filter[4]) +
           (int)(src_ptr[src_offset + 3] * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;

        if (Temp < 0)
            Temp = 0;
        else if ( Temp > 255 )
            Temp = 255;

        output_ptr[tid] = Temp;
    }

}

kernel void vp8_filter_block2d_second_pass_kernel
(
    __global int *src_base,
    int src_offset,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    uint i = get_global_id(0);

    global int *src_ptr = &src_base[src_offset];
    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

kernel void vp8_filter_block2d_bil_first_pass_kernel(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
)
{
    uint tid = get_global_id(0);

    if (tid < output_width * output_height){
        global unsigned char *src_ptr = &src_base[src_offset];

        unsigned int i, j;
        __constant int *vp8_filter = bilinear_filters[filter_offset];

        unsigned int out_row,out_offset;
        int src_increment = src_pixels_per_line - output_width;

        i = tid / output_width;
        j = tid % output_width;

        src_offset = i*(output_width+src_increment) + j;
        out_row = output_width * i;

        out_offset = out_row + j;

        /* Apply bilinear filter */
        output_ptr[out_offset] = (((int)src_ptr[src_offset]   * vp8_filter[0]) +
                 ((int)src_ptr[src_offset+1] * vp8_filter[1]) +
                 (VP8_FILTER_WEIGHT / 2)) >> VP8_FILTER_SHIFT;
    }
}

kernel void vp8_filter_block2d_bil_second_pass_kernel
(
    __global int *src_ptr,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
)
{

    uint tid = get_global_id(0);

    if (tid < output_width * output_height){
        global unsigned char *output_ptr = &output_base[output_offset];

        unsigned int i, j;
        int Temp;
        __constant int *vp8_filter = bilinear_filters[filter_offset];

        int out_offset;
        int src_offset;

        i = tid / output_width;
        j = tid % output_width;

        src_offset = i*(output_width) + j;
        out_offset = i*output_pitch + j;

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset]         * vp8_filter[0]) +
               ((int)src_ptr[src_offset+output_width] * vp8_filter[1]) +
               (VP8_FILTER_WEIGHT / 2);

        output_ptr[out_offset++] = (unsigned int)(Temp >> VP8_FILTER_SHIFT);
    }
}

//Called from reconinter_cl.c
kernel void vp8_memcpy_kernel(
    global unsigned char *src_base,
    int src_offset,
    int src_stride,
    global unsigned char *dst_base,
    int dst_offset,
    int dst_stride,
    int num_bytes,
    int num_iter
){

    int i,r;
    global unsigned char *src = &src_base[src_offset];
    global unsigned char *dst = &dst_base[dst_offset];
    src_offset = dst_offset = 0;

    r = get_global_id(1);
    if (r < get_global_size(1)){
        i = get_global_id(0);
        if (i < get_global_size(0)){
            src_offset = r*src_stride + i;
            dst_offset = r*dst_stride + i;
            dst[dst_offset] = src[src_offset];
        }
    }
}

//Not used currently.
void vp8_memset_short(
    global short *mem,
    int offset,
    short newval,
    unsigned int size
)
{
    int tid = get_global_id(0);

    if (tid < (size/2)){
        mem[offset+tid/2] = newval;
    }
}

//Here to EOF is old code that should be removed at end of project.

__kernel void vp8_bilinear_predict4x4_kernel
(
        __global unsigned char *src_base,
        int src_offset,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_base,
        int dst_offset,
        int dst_pitch,
        __global int *int_mem
)
{
    int Height = 4, Width = 4;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict8x8_kernel
(
    __global unsigned char *src_base,
    int src_offset,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *int_mem
)
{
    int Height = 8, Width = 8;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);

}

__kernel void vp8_bilinear_predict8x4_kernel
(
    __global unsigned char *src_base,
    int src_offset,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *int_mem
)
{
    int Height = 4, Width = 8;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict16x16_kernel
(
    __global unsigned char *src_base,
    int src_offset,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *int_mem
)
{

    int Height = 16, Width = 16;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);

}

void vp8_filter_block2d_first_pass(
    global unsigned char *src_base,
    int src_offset,
    local int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    int nthreads = get_global_size(0);
    int ngroups = nthreads / get_local_size(0);

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we capture it now

    int Temp;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        short filter0 = vp8_filter[0];
        short filter1 = vp8_filter[1];
        short filter2 = vp8_filter[2];
        short filter3 = vp8_filter[3];
        short filter4 = vp8_filter[4];
        short filter5 = vp8_filter[5];

        if (ngroups > 1){
            for (i=0; i < output_width*output_height; i++){
                src_offset = i + (i/output_width * (src_pixels_per_line - output_width));

                Temp = (int)(src_ptr[src_offset - 2] * filter0) +
                       (int)(src_ptr[src_offset - 1] * filter1) +
                       (int)(src_ptr[src_offset]     * filter2) +
                       (int)(src_ptr[src_offset + 1] * filter3) +
                       (int)(src_ptr[src_offset + 2] * filter4) +
                       (int)(src_ptr[src_offset + 3] * filter5) +
                       (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

                /* Normalize back to 0-255 */
                Temp >>= VP8_FILTER_SHIFT;

                if (Temp < 0)
                    Temp = 0;
                else if ( Temp > 255 )
                    Temp = 255;

                output_ptr[i] = Temp;
            }
        } else {
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width));

            Temp = (int)(src_ptr[src_offset - 2] * filter0) +
                   (int)(src_ptr[src_offset - 1] * filter1) +
                   (int)(src_ptr[src_offset]     * filter2) +
                   (int)(src_ptr[src_offset + 1] * filter3) +
                   (int)(src_ptr[src_offset + 2] * filter4) +
                   (int)(src_ptr[src_offset + 3] * filter5) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

            /* Normalize back to 0-255 */
            Temp >>= VP8_FILTER_SHIFT;

            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
        }
    }

    //Add a fence so that no 2nd pass stuff starts before 1st pass writes are done.
    //write_mem_fence(CLK_GLOBAL_MEM_FENCE);
    //barrier(CLK_GLOBAL_MEM_FENCE);
    if (ngroups > 1){
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

void vp8_filter_block2d_second_pass
(
    local int *src_ptr,
    global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]            * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

__kernel void vp8_sixtap_predict_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{

    local int FData[9*4];

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 9, 4, xoffset);

    /* then filter vertically... */
    vp8_filter_block2d_second_pass(&FData[8], dst_ptr, dst_offset, dst_pitch, 4, 4, 4, 4, yoffset);
}

__kernel void vp8_sixtap_predict8x8_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    local int FData[13*16];   /* Temp data bufffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 13, 8, xoffset);

    /* then filter vertically... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 8, 8, yoffset);

}

__kernel void vp8_sixtap_predict8x4_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    local int FData[13*16];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 9, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 4, 8, yoffset);
}

__kernel void vp8_sixtap_predict16x16_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    local int FData[21*24];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 21, 16, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[32], dst_ptr, dst_offset, dst_pitch, 16, 16, 16, 16, yoffset);

    return;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

__constant int bilinear_filters[8][2] = {
    { 128, 0},
    { 112, 16},
    { 96, 32},
    { 80, 48},
    { 64, 64},
    { 48, 80},
    { 32, 96},
    { 16, 112}
};

__constant short sub_pel_filters[8][8] = {
    //These were originally 8x6, but are padded for vector ops
    { 0, 0, 128, 0, 0, 0, 0, 0}, /* note that 1/8 pel positions are just as per alpha -0.5 bicubic */
    { 0, -6, 123, 12, -1, 0, 0, 0},
    { 2, -11, 108, 36, -8, 1, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -9, 93, 50, -6, 0, 0, 0},
    { 3, -16, 77, 77, -16, 3, 0, 0}, /* New 1/2 pel 6 tap filter */
    { 0, -6, 50, 93, -9, 0, 0, 0},
    { 1, -8, 36, 108, -11, 2, 0, 0}, /* New 1/4 pel 6 tap filter */
    { 0, -1, 12, 123, -6, 0, 0, 0},
};

kernel void vp8_filter_block2d_first_pass_kernel(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we use it now

    int Temp;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        src_offset = tid + (tid/output_width * (src_pixels_per_line - output_width));

        Temp = (int)(src_ptr[src_offset - 2] * vp8_filter[0]) +
           (int)(src_ptr[src_offset - 1] * vp8_filter[1]) +
           (int)(src_ptr[src_offset]     * vp8_filter[2]) +
           (int)(src_ptr[src_offset + 1] * vp8_filter[3]) +
           (int)(src_ptr[src_offset + 2] * vp8_filter[4]) +
           (int)(src_ptr[src_offset + 3] * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;

        if (Temp < 0)
            Temp = 0;
        else if ( Temp > 255 )
            Temp = 255;

        output_ptr[tid] = Temp;
    }

}

kernel void vp8_filter_block2d_second_pass_kernel
(
    __global int *src_base,
    int src_offset,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    uint i = get_global_id(0);

    global int *src_ptr = &src_base[src_offset];
    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]       * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

kernel void vp8_filter_block2d_bil_first_pass_kernel(
    __global unsigned char *src_base,
    int src_offset,
    __global int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
)
{
    uint tid = get_global_id(0);

    if (tid < output_width * output_height){
        global unsigned char *src_ptr = &src_base[src_offset];

        unsigned int i, j;
        __constant int *vp8_filter = bilinear_filters[filter_offset];

        unsigned int out_row,out_offset;
        int src_increment = src_pixels_per_line - output_width;

        i = tid / output_width;
        j = tid % output_width;

        src_offset = i*(output_width+src_increment) + j;
        out_row = output_width * i;

        out_offset = out_row + j;

        /* Apply bilinear filter */
        output_ptr[out_offset] = (((int)src_ptr[src_offset]   * vp8_filter[0]) +
                 ((int)src_ptr[src_offset+1] * vp8_filter[1]) +
                 (VP8_FILTER_WEIGHT / 2)) >> VP8_FILTER_SHIFT;
    }
}

kernel void vp8_filter_block2d_bil_second_pass_kernel
(
    __global int *src_ptr,
    __global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
)
{

    uint tid = get_global_id(0);

    if (tid < output_width * output_height){
        global unsigned char *output_ptr = &output_base[output_offset];

        unsigned int i, j;
        int Temp;
        __constant int *vp8_filter = bilinear_filters[filter_offset];

        int out_offset;
        int src_offset;

        i = tid / output_width;
        j = tid % output_width;

        src_offset = i*(output_width) + j;
        out_offset = i*output_pitch + j;

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset]         * vp8_filter[0]) +
               ((int)src_ptr[src_offset+output_width] * vp8_filter[1]) +
               (VP8_FILTER_WEIGHT / 2);

        output_ptr[out_offset++] = (unsigned int)(Temp >> VP8_FILTER_SHIFT);
    }
}

//Called from reconinter_cl.c
kernel void vp8_memcpy_kernel(
    global unsigned char *src_base,
    int src_offset,
    int src_stride,
    global unsigned char *dst_base,
    int dst_offset,
    int dst_stride,
    int num_bytes,
    int num_iter
){

    int i,r;
    global unsigned char *src = &src_base[src_offset];
    global unsigned char *dst = &dst_base[dst_offset];
    src_offset = dst_offset = 0;

    r = get_global_id(1);
    if (r < get_global_size(1)){
        i = get_global_id(0);
        if (i < get_global_size(0)){
            src_offset = r*src_stride + i;
            dst_offset = r*dst_stride + i;
            dst[dst_offset] = src[src_offset];
        }
    }
}

//Not used currently.
void vp8_memset_short(
    global short *mem,
    int offset,
    short newval,
    unsigned int size
)
{
    int tid = get_global_id(0);

    if (tid < (size/2)){
        mem[offset+tid/2] = newval;
    }
}

//Here to EOF is old code that should be removed at end of project.

__kernel void vp8_bilinear_predict4x4_kernel
(
        __global unsigned char *src_base,
        int src_offset,
        int src_pixels_per_line,
        int xoffset,
        int yoffset,
        __global unsigned char *dst_base,
        int dst_offset,
        int dst_pitch,
        __global int *int_mem
)
{
    int Height = 4, Width = 4;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict8x8_kernel
(
    __global unsigned char *src_base,
    int src_offset,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *int_mem
)
{
    int Height = 8, Width = 8;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);

}

__kernel void vp8_bilinear_predict8x4_kernel
(
    __global unsigned char *src_base,
    int src_offset,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *int_mem
)
{
    int Height = 4, Width = 8;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);
}

__kernel void vp8_bilinear_predict16x16_kernel
(
    __global unsigned char *src_base,
    int src_offset,
    int src_pixels_per_line,
    int xoffset,
    int yoffset,
    __global unsigned char *dst_base,
    int dst_offset,
    int dst_pitch,
    __global int *int_mem
)
{

    int Height = 16, Width = 16;

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_bil_first_pass_kernel(src_base, src_offset, int_mem, src_pixels_per_line, Height + 1, Width, xoffset);

    /* then 1-D vertically... */
    vp8_filter_block2d_bil_second_pass_kernel(int_mem, dst_base, dst_offset, dst_pitch, Height, Width, yoffset);

}

void vp8_filter_block2d_first_pass(
    global unsigned char *src_base,
    int src_offset,
    local int *output_ptr,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
){
    uint tid = get_global_id(0);
    uint i = tid;

    global unsigned char *src_ptr = &src_base[src_offset];
    //Note that src_offset will be reset later, which is why we capture it now

    int Temp;

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (tid < (output_width*output_height)){
        short filter0 = vp8_filter[0];
        short filter1 = vp8_filter[1];
        short filter2 = vp8_filter[2];
        short filter3 = vp8_filter[3];
        short filter4 = vp8_filter[4];
        short filter5 = vp8_filter[5];

        for (i=0; i < output_width*output_height; i++){
            src_offset = i + (i/output_width * (src_pixels_per_line - output_width));

            Temp = (int)(src_ptr[src_offset - 2] * filter0) +
                   (int)(src_ptr[src_offset - 1] * filter1) +
                   (int)(src_ptr[src_offset]     * filter2) +
                   (int)(src_ptr[src_offset + 1] * filter3) +
                   (int)(src_ptr[src_offset + 2] * filter4) +
                   (int)(src_ptr[src_offset + 3] * filter5) +
                   (VP8_FILTER_WEIGHT >> 1);      /* Rounding */

            /* Normalize back to 0-255 */
            Temp >>= VP8_FILTER_SHIFT;

            if (Temp < 0)
                Temp = 0;
            else if ( Temp > 255 )
                Temp = 255;

            output_ptr[i] = Temp;
        }
    }

    //Add a fence so that no 2nd pass stuff starts before 1st pass writes are done.
    //write_mem_fence(CLK_GLOBAL_MEM_FENCE);
    //barrier(CLK_GLOBAL_MEM_FENCE);
    barrier(CLK_LOCAL_MEM_FENCE);
}

void vp8_filter_block2d_second_pass
(
    local int *src_ptr,
    global unsigned char *output_base,
    int output_offset,
    int output_pitch,
    unsigned int src_pixels_per_line,
    unsigned int pixel_step,
    unsigned int output_height,
    unsigned int output_width,
    int filter_offset
) {

    global unsigned char *output_ptr = &output_base[output_offset];

    int out_offset; //Not same as output_offset...
    int src_offset;
    int Temp;
    int PS2 = 2*(int)pixel_step;
    int PS3 = 3*(int)pixel_step;

    unsigned int src_increment = src_pixels_per_line - output_width;

    uint i = get_global_id(0);

    __constant short *vp8_filter = sub_pel_filters[filter_offset];

    if (i < (output_width * output_height)){
        out_offset = i/output_width;
        src_offset = out_offset;

        src_offset = i + (src_offset * src_increment);
        out_offset = i%output_width + (out_offset * output_pitch);

        /* Apply filter */
        Temp = ((int)src_ptr[src_offset - PS2] * vp8_filter[0]) +
           ((int)src_ptr[src_offset -(int)pixel_step] * vp8_filter[1]) +
           ((int)src_ptr[src_offset]                  * vp8_filter[2]) +
           ((int)src_ptr[src_offset + pixel_step]     * vp8_filter[3]) +
           ((int)src_ptr[src_offset + PS2]            * vp8_filter[4]) +
           ((int)src_ptr[src_offset + PS3]       * vp8_filter[5]) +
           (VP8_FILTER_WEIGHT >> 1);   /* Rounding */

        /* Normalize back to 0-255 */
        Temp = Temp >> VP8_FILTER_SHIFT;
        if (Temp < 0)
            Temp = 0;
        else if (Temp > 255)
            Temp = 255;

        output_ptr[out_offset] = (unsigned char)Temp;
    }
}

__kernel void vp8_sixtap_predict_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
        ) {

    local int FData[9*4];

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 9, 4, xoffset);

    /* then filter vertically... */
    vp8_filter_block2d_second_pass(&FData[8], dst_ptr, dst_offset, dst_pitch, 4, 4, 4, 4, yoffset);
}

__kernel void vp8_sixtap_predict8x8_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    local int FData[13*16];   /* Temp data bufffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 13, 8, xoffset);

    /* then filter vertically... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 8, 8, yoffset);

}

__kernel void vp8_sixtap_predict8x4_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    local int FData[13*16];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 9, 8, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[16], dst_ptr, dst_offset, dst_pitch, 8, 8, 4, 8, yoffset);
}

__kernel void vp8_sixtap_predict16x16_kernel
(
    __global unsigned char  *src_ptr,
    int src_offset,
    int  src_pixels_per_line,
    int  xoffset,
    int  yoffset,
    __global unsigned char *dst_ptr,
    int dst_offset,
    int  dst_pitch
)
{
    local int FData[21*24];   /* Temp data buffer used in filtering */

    /* First filter 1-D horizontally... */
    vp8_filter_block2d_first_pass(src_ptr, src_offset, FData, src_pixels_per_line, 1, 21, 16, xoffset);

    /* then filter verticaly... */
    vp8_filter_block2d_second_pass(&FData[32], dst_ptr, dst_offset, dst_pitch, 16, 16, 16, 16, yoffset);

    return;
}