HSAIL kernel library: update sqrt.cl with new macros

/* OpenCL built-in library: sqrt()

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "hsail_templates.h"

/* TODO:

   Clang does not recognize __builtin_sqrt* in CGBuiltin.cpp (around line 1285)
   to be a call to sqrt and cannot selectively convert it to the llvm.sqrt.*
   intrinsics with looser math flags. Therefore we have to call it as a libcall
   so it regonized it. For the double we still call the __builtin_sqrt() to
   disambiguate from the sqrt() to avoid infinite recursion. Probably the
   correct fix is to patch CGBuiltin.cpp to recognize also the call via
   __builtin_sqrt*. */

/* TODO HSAIL:
   related to previous TODO, neither __builtin_sqrt() nor any
  "high-level" intrinsics (llvm.sqrt, llvm.hsail.sqrt) works.

  I have implemented a workaround using bitcode in sqrt_default.ll, this however
  uses intrinsics with fixed behaviour (rounding mode etc.) so it's not a long
  term solution.
*/

#undef sqrt

float _CL_OVERLOADABLE _sqrt_default_f32(float a);
IMPLEMENT_VECWITHSCALARS(_sqrt_default_f32, V_V, float, float)
double _CL_OVERLOADABLE _sqrt_default_f64(double a);
IMPLEMENT_VECWITHSCALARS(_sqrt_default_f64, V_V, double, double)

IMPLEMENT_EXPR_VECS_AND_SCALAR(_cl_sqrt, V_V, EXPR_, float, int,_sqrt_default_f32(a) )
IMPLEMENT_EXPR_VECS_AND_SCALAR(_cl_sqrt, V_V, EXPR_, double, long, _sqrt_default_f64(a) )

/* OpenCL built-in library: sqrt()

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "hsail_templates.h"

/* TODO:

   Clang does not recognize __builtin_sqrt* in CGBuiltin.cpp (around line 1285)
   to be a call to sqrt and cannot selectively convert it to the llvm.sqrt.*
   intrinsics with looser math flags. Therefore we have to call it as a libcall
   so it regonized it. For the double we still call the __builtin_sqrt() to
   disambiguate from the sqrt() to avoid infinite recursion. Probably the
   correct fix is to patch CGBuiltin.cpp to recognize also the call via
   __builtin_sqrt*. */

/* TODO HSAIL:
   related to previous TODO, neither __builtin_sqrt() nor any
  "high-level" intrinsics (llvm.sqrt, llvm.hsail.sqrt) works.

  I have implemented a workaround using bitcode in sqrt_default.ll, this however
  uses intrinsics with fixed behaviour (rounding mode etc.) so it's not a long
  term solution.
*/

#undef sqrt

float __attribute__ ((overloadable)) _sqrt_default_f32(float a);
IMPLEMENT_BUILTIN_TYPE_ALL_VECS(_sqrt_default_f32, V_V, float, float)
double __attribute__ ((overloadable)) _sqrt_default_f64(double a);
IMPLEMENT_BUILTIN_TYPE_ALL_VECS(_sqrt_default_f64, V_V, double, double)

IMPLEMENT_EXPR_VECS_AND_SCALAR(_cl_sqrt, V_V, EXPR_, float, int,_sqrt_default_f32(a) )
IMPLEMENT_EXPR_VECS_AND_SCALAR(_cl_sqrt, V_V, EXPR_, double, long, _sqrt_default_f64(a) )


Fixed bug in OpenCL kernel, where the uninitialized verification array was being updated instead of just written to. This follows a similar bug fix made to RSBench.

// Structures
typedef struct{
	double energy;
	double total_xs;
	double elastic_xs;
	double absorbtion_xs;
	double fission_xs;
	double nu_fission_xs;
} NuclideGridPoint;

typedef struct{
	int nthreads;
	long n_isotopes;
	long n_gridpoints;
	int lookups;
	int HM;
	int grid_type; // 0: Unionized Grid (default)    1: Nuclide Grid
	int hash_bins;
	int particles;
	int simulation_method;
	int binary_mode;
	int kernel_id;
} Inputs;

// Grid types
#define UNIONIZED 0
#define NUCLIDE 1
#define HASH 2

// Simulation types
#define HISTORY_BASED 1
#define EVENT_BASED 2

// Starting Seed
#define STARTING_SEED 1070

unsigned long fast_forward_LCG(unsigned long seed, unsigned long n)
{
	// LCG parameters
	const unsigned long m = 9223372036854775808ULL; // 2^63
	unsigned long a = 2806196910506780709ULL;
	unsigned long c = 1ULL;

	n = n % m;

	unsigned long a_new = 1;
	unsigned long c_new = 0;

	while(n > 0)
	{
		if(n & 1)
		{
			a_new *= a;
			c_new = c_new * a + c;
		}
		c *= (a + 1);
		a *= a;

		n >>= 1;
	}

	return (a_new * seed + c_new) % m;

}

double LCG_random_double(unsigned long * seed)
{
	// LCG parameters
	const unsigned long m = 9223372036854775808ULL; // 2^63
	const unsigned long a = 2806196910506780709ULL;
	const unsigned long c = 1ULL;
	*seed = (a * (*seed) + c) % m;
	return (double) (*seed) / (double) m;
}

// picks a material based on a probabilistic distribution
int pick_mat( unsigned long * seed )
{
	// I have a nice spreadsheet supporting these numbers. They are
	// the fractions (by volume) of material in the core. Not a
	// *perfect* approximation of where XS lookups are going to occur,
	// but this will do a good job of biasing the system nonetheless.

	// Also could be argued that doing fractions by weight would be
	// a better approximation, but volume does a good enough job for now.

	double dist[12];
	dist[0]  = 0.140;	// fuel
	dist[1]  = 0.052;	// cladding
	dist[2]  = 0.275;	// cold, borated water
	dist[3]  = 0.134;	// hot, borated water
	dist[4]  = 0.154;	// RPV
	dist[5]  = 0.064;	// Lower, radial reflector
	dist[6]  = 0.066;	// Upper reflector / top plate
	dist[7]  = 0.055;	// bottom plate
	dist[8]  = 0.008;	// bottom nozzle
	dist[9]  = 0.015;	// top nozzle
	dist[10] = 0.025;	// top of fuel assemblies
	dist[11] = 0.013;	// bottom of fuel assemblies

	double roll = LCG_random_double(seed);

	// makes a pick based on the distro
	for( int i = 0; i < 12; i++ )
	{
		double running = 0;
		for( int j = i; j > 0; j-- )
			running += dist[j];
		if( roll < running )
			return i;
	}

	return 0;
}

// (fixed) binary search for energy on unionized energy grid
// returns lower index
long grid_search( long n, double quarry, __global const double * A)
{
	long lowerLimit = 0;
	long upperLimit = n-1;
	long examinationPoint;
	long length = upperLimit - lowerLimit;

	while( length > 1 )
	{
		examinationPoint = lowerLimit + ( length / 2 );

		if( A[examinationPoint] > quarry )
			upperLimit = examinationPoint;
		else
			lowerLimit = examinationPoint;

		length = upperLimit - lowerLimit;
	}

	return lowerLimit;
}

// binary search for energy on nuclide energy grid
long grid_search_nuclide( long n, double quarry, __global const NuclideGridPoint * A, long low, long high)
{
	long lowerLimit = low;
	long upperLimit = high;
	long examinationPoint;
	long length = upperLimit - lowerLimit;

	while( length > 1 )
	{
		examinationPoint = lowerLimit + ( length / 2 );

		if( A[examinationPoint].energy > quarry )
			upperLimit = examinationPoint;
		else
			lowerLimit = examinationPoint;

		length = upperLimit - lowerLimit;
	}

	return lowerLimit;
}

// Calculates the microscopic cross section for a given nuclide & energy
void calculate_micro_xs(   double p_energy, int nuc, long n_isotopes,
                           long n_gridpoints,
                           __global const double * egrid, __global const int * index_data,
                           __global const NuclideGridPoint * nuclide_grids,
                           long idx, double * xs_vector, int grid_type, int hash_bins ){
	// Variables
	double f;
	__global const NuclideGridPoint * low, * high;

	// If using only the nuclide grid, we must perform a binary search
	// to find the energy location in this particular nuclide's grid.
	if( grid_type == NUCLIDE )
	{
		// Perform binary search on the Nuclide Grid to find the index
		idx = grid_search_nuclide( n_gridpoints, p_energy, &nuclide_grids[nuc*n_gridpoints], 0, n_gridpoints-1);

		// pull ptr from nuclide grid and check to ensure that
		// we're not reading off the end of the nuclide's grid
		if( idx == n_gridpoints - 1 )
			low = &nuclide_grids[nuc*n_gridpoints + idx - 1];
		else
			low = &nuclide_grids[nuc*n_gridpoints + idx];
	}
	else if( grid_type == UNIONIZED) // Unionized Energy Grid - we already know the index, no binary search needed.
	{
		// pull ptr from energy grid and check to ensure that
		// we're not reading off the end of the nuclide's grid
		if( index_data[idx * n_isotopes + nuc] == n_gridpoints - 1 )
			low = &nuclide_grids[nuc*n_gridpoints + index_data[idx * n_isotopes + nuc] - 1];
		else
			low = &nuclide_grids[nuc*n_gridpoints + index_data[idx * n_isotopes + nuc]];
	}
	else // Hash grid
	{
		// load lower bounding index
		int u_low = index_data[idx * n_isotopes + nuc];

		// Determine higher bounding index
		int u_high;
		if( idx == hash_bins - 1 )
			u_high = n_gridpoints - 1;
		else
			u_high = index_data[(idx+1)*n_isotopes + nuc] + 1;

		// Check edge cases to make sure energy is actually between these
		// Then, if things look good, search for gridpoint in the nuclide grid
		// within the lower and higher limits we've calculated.
		double e_low  = nuclide_grids[nuc*n_gridpoints + u_low].energy;
		double e_high = nuclide_grids[nuc*n_gridpoints + u_high].energy;
		int lower;
		if( p_energy <= e_low )
			lower = 0;
		else if( p_energy >= e_high )
			lower = n_gridpoints - 1;
		else
			lower = grid_search_nuclide( n_gridpoints, p_energy, &nuclide_grids[nuc*n_gridpoints], u_low, u_high);

		if( lower == n_gridpoints - 1 )
			low = &nuclide_grids[nuc*n_gridpoints + lower - 1];
		else
			low = &nuclide_grids[nuc*n_gridpoints + lower];
	}

	high = low + 1;

	// calculate the re-useable interpolation factor
	f = (high->energy - p_energy) / (high->energy - low->energy);

	// Total XS
	xs_vector[0] = high->total_xs - f * (high->total_xs - low->total_xs);

	// Elastic XS
	xs_vector[1] = high->elastic_xs - f * (high->elastic_xs - low->elastic_xs);

	// Absorbtion XS
	xs_vector[2] = high->absorbtion_xs - f * (high->absorbtion_xs - low->absorbtion_xs);

	// Fission XS
	xs_vector[3] = high->fission_xs - f * (high->fission_xs - low->fission_xs);

	// Nu Fission XS
	xs_vector[4] = high->nu_fission_xs - f * (high->nu_fission_xs - low->nu_fission_xs);
}

// Calculates macroscopic cross section based on a given material & energy
void calculate_macro_xs( double p_energy, int mat, long n_isotopes,
                         long n_gridpoints, __global const int * num_nucs,
                         __global const double * restrict concs,
                         __global const double * egrid, __global const int * index_data,
                         __global const NuclideGridPoint * nuclide_grids,
                         __global const int * mats,
                         double * macro_xs_vector, int grid_type, int hash_bins, int max_num_nucs ){
	int p_nuc; // the nuclide we are looking up
	long idx = -1;
	double conc; // the concentration of the nuclide in the material

	// cleans out macro_xs_vector
	for( int k = 0; k < 5; k++ )
		macro_xs_vector[k] = 0;

	// If we are using the unionized energy grid (UEG), we only
	// need to perform 1 binary search per macroscopic lookup.
	// If we are using the nuclide grid search, it will have to be
	// done inside of the "calculate_micro_xs" function for each different
	// nuclide in the material.
	if( grid_type == UNIONIZED )
		idx = grid_search( n_isotopes * n_gridpoints, p_energy, egrid);
	else if( grid_type == HASH )
	{
		double du = 1.0 / hash_bins;
		idx = p_energy / du;
	}

	// Once we find the pointer array on the UEG, we can pull the data
	// from the respective nuclide grids, as well as the nuclide
	// concentration data for the material
	// Each nuclide from the material needs to have its micro-XS array
	// looked up & interpolatied (via calculate_micro_xs). Then, the
	// micro XS is multiplied by the concentration of that nuclide
	// in the material, and added to the total macro XS array.
	// (Independent -- though if parallelizing, must use atomic operations
	//  or otherwise control access to the xs_vector and macro_xs_vector to
	//  avoid simulataneous writing to the same data structure)
	for( int j = 0; j < num_nucs[mat]; j++ )
	{
		double xs_vector[5];
		p_nuc = mats[mat*max_num_nucs + j];
		conc = concs[mat*max_num_nucs + j];
		calculate_micro_xs( p_energy, p_nuc, n_isotopes,
		                    n_gridpoints, egrid, index_data,
		                    nuclide_grids, idx, xs_vector, grid_type, hash_bins );
		for( int k = 0; k < 5; k++ )
			macro_xs_vector[k] += xs_vector[k] * conc;
	}
}

__kernel void macro_xs_lookup_kernel(		Inputs in,
											int max_num_nucs,
											__global const int *num_nucs,
											__global const double *concs,
											__global const double * unionized_energy_array,
											__global const int * index_grid,
											__global const NuclideGridPoint * nuclide_grid,
											__global const int * mats,
											__global int * verification_array )
{
	// Get the index of the current element to be processed
	int i = get_global_id(0);

	// Set the initial seed value
	unsigned long seed = STARTING_SEED;

	// Forward seed to lookup index (we need 2 samples per lookup)
	seed = fast_forward_LCG(seed, 2*i);

	// Randomly pick an energy and material for the particle
	double p_energy = LCG_random_double(&seed);
	int mat         = pick_mat(&seed);

	double macro_xs_vector[5] = {0};

	// Perform macroscopic Cross Section Lookup
	calculate_macro_xs(
			p_energy,        // Sampled neutron energy (in lethargy)
			mat,             // Sampled material type index neutron is in
			in.n_isotopes,   // Total number of isotopes in simulation
			in.n_gridpoints, // Number of gridpoints per isotope in simulation
			num_nucs,     // 1-D array with number of nuclides per material
			concs,        // Flattened 2-D array with concentration of each nuclide in each material
			unionized_energy_array, // 1-D Unionized energy array
			index_grid,   // Flattened 2-D grid holding indices into nuclide grid for each unionized energy level
			nuclide_grid, // Flattened 2-D grid holding energy levels and XS_data for all nuclides in simulation
			mats,         // Flattened 2-D array with nuclide indices defining composition of each type of material
			macro_xs_vector, // 1-D array with result of the macroscopic cross section (5 different reaction channels)
			in.grid_type,    // Lookup type (nuclide, hash, or unionized)
			in.hash_bins,    // Number of hash bins used (if using hash lookup type)
			max_num_nucs  // Maximum number of nuclides present in any material
			);

	// For verification, and to prevent the compiler from optimizing
	// all work out, we interrogate the returned macro_xs_vector array
	// to find its maximum value index, then increment the verification
	// value by that index. In this implementation, we prevent thread
	// contention by using an OMP reduction on the verification value.
	// For accelerators, a different approach might be required
	// (e.g., atomics, reduction of thread-specific values in large
	// array via CUDA thrust, etc).
	double max = -1.0;
	int max_idx = 0;
	for(int j = 0; j < 5; j++ )
	{
		if( macro_xs_vector[j] > max )
		{
			max = macro_xs_vector[j];
			max_idx = j;
		}
	}
	verification_array[i] = max_idx+1;

}

// Structures
typedef struct{
	double energy;
	double total_xs;
	double elastic_xs;
	double absorbtion_xs;
	double fission_xs;
	double nu_fission_xs;
} NuclideGridPoint;

typedef struct{
	int nthreads;
	long n_isotopes;
	long n_gridpoints;
	int lookups;
	int HM;
	int grid_type; // 0: Unionized Grid (default)    1: Nuclide Grid
	int hash_bins;
	int particles;
	int simulation_method;
	int binary_mode;
	int kernel_id;
} Inputs;

// Grid types
#define UNIONIZED 0
#define NUCLIDE 1
#define HASH 2

// Simulation types
#define HISTORY_BASED 1
#define EVENT_BASED 2

// Starting Seed
#define STARTING_SEED 1070

unsigned long fast_forward_LCG(unsigned long seed, unsigned long n)
{
	// LCG parameters
	const unsigned long m = 9223372036854775808ULL; // 2^63
	unsigned long a = 2806196910506780709ULL;
	unsigned long c = 1ULL;

	n = n % m;

	unsigned long a_new = 1;
	unsigned long c_new = 0;

	while(n > 0)
	{
		if(n & 1)
		{
			a_new *= a;
			c_new = c_new * a + c;
		}
		c *= (a + 1);
		a *= a;

		n >>= 1;
	}

	return (a_new * seed + c_new) % m;

}

double LCG_random_double(unsigned long * seed)
{
	// LCG parameters
	const unsigned long m = 9223372036854775808ULL; // 2^63
	const unsigned long a = 2806196910506780709ULL;
	const unsigned long c = 1ULL;
	*seed = (a * (*seed) + c) % m;
	return (double) (*seed) / (double) m;
}

// picks a material based on a probabilistic distribution
int pick_mat( unsigned long * seed )
{
	// I have a nice spreadsheet supporting these numbers. They are
	// the fractions (by volume) of material in the core. Not a
	// *perfect* approximation of where XS lookups are going to occur,
	// but this will do a good job of biasing the system nonetheless.

	// Also could be argued that doing fractions by weight would be
	// a better approximation, but volume does a good enough job for now.

	double dist[12];
	dist[0]  = 0.140;	// fuel
	dist[1]  = 0.052;	// cladding
	dist[2]  = 0.275;	// cold, borated water
	dist[3]  = 0.134;	// hot, borated water
	dist[4]  = 0.154;	// RPV
	dist[5]  = 0.064;	// Lower, radial reflector
	dist[6]  = 0.066;	// Upper reflector / top plate
	dist[7]  = 0.055;	// bottom plate
	dist[8]  = 0.008;	// bottom nozzle
	dist[9]  = 0.015;	// top nozzle
	dist[10] = 0.025;	// top of fuel assemblies
	dist[11] = 0.013;	// bottom of fuel assemblies

	double roll = LCG_random_double(seed);

	// makes a pick based on the distro
	for( int i = 0; i < 12; i++ )
	{
		double running = 0;
		for( int j = i; j > 0; j-- )
			running += dist[j];
		if( roll < running )
			return i;
	}

	return 0;
}

// (fixed) binary search for energy on unionized energy grid
// returns lower index
long grid_search( long n, double quarry, __global const double * A)
{
	long lowerLimit = 0;
	long upperLimit = n-1;
	long examinationPoint;
	long length = upperLimit - lowerLimit;

	while( length > 1 )
	{
		examinationPoint = lowerLimit + ( length / 2 );

		if( A[examinationPoint] > quarry )
			upperLimit = examinationPoint;
		else
			lowerLimit = examinationPoint;

		length = upperLimit - lowerLimit;
	}

	return lowerLimit;
}

// binary search for energy on nuclide energy grid
long grid_search_nuclide( long n, double quarry, __global const NuclideGridPoint * A, long low, long high)
{
	long lowerLimit = low;
	long upperLimit = high;
	long examinationPoint;
	long length = upperLimit - lowerLimit;

	while( length > 1 )
	{
		examinationPoint = lowerLimit + ( length / 2 );

		if( A[examinationPoint].energy > quarry )
			upperLimit = examinationPoint;
		else
			lowerLimit = examinationPoint;

		length = upperLimit - lowerLimit;
	}

	return lowerLimit;
}

// Calculates the microscopic cross section for a given nuclide & energy
void calculate_micro_xs(   double p_energy, int nuc, long n_isotopes,
                           long n_gridpoints,
                           __global const double * egrid, __global const int * index_data,
                           __global const NuclideGridPoint * nuclide_grids,
                           long idx, double * xs_vector, int grid_type, int hash_bins ){
	// Variables
	double f;
	__global const NuclideGridPoint * low, * high;

	// If using only the nuclide grid, we must perform a binary search
	// to find the energy location in this particular nuclide's grid.
	if( grid_type == NUCLIDE )
	{
		// Perform binary search on the Nuclide Grid to find the index
		idx = grid_search_nuclide( n_gridpoints, p_energy, &nuclide_grids[nuc*n_gridpoints], 0, n_gridpoints-1);

		// pull ptr from nuclide grid and check to ensure that
		// we're not reading off the end of the nuclide's grid
		if( idx == n_gridpoints - 1 )
			low = &nuclide_grids[nuc*n_gridpoints + idx - 1];
		else
			low = &nuclide_grids[nuc*n_gridpoints + idx];
	}
	else if( grid_type == UNIONIZED) // Unionized Energy Grid - we already know the index, no binary search needed.
	{
		// pull ptr from energy grid and check to ensure that
		// we're not reading off the end of the nuclide's grid
		if( index_data[idx * n_isotopes + nuc] == n_gridpoints - 1 )
			low = &nuclide_grids[nuc*n_gridpoints + index_data[idx * n_isotopes + nuc] - 1];
		else
			low = &nuclide_grids[nuc*n_gridpoints + index_data[idx * n_isotopes + nuc]];
	}
	else // Hash grid
	{
		// load lower bounding index
		int u_low = index_data[idx * n_isotopes + nuc];

		// Determine higher bounding index
		int u_high;
		if( idx == hash_bins - 1 )
			u_high = n_gridpoints - 1;
		else
			u_high = index_data[(idx+1)*n_isotopes + nuc] + 1;

		// Check edge cases to make sure energy is actually between these
		// Then, if things look good, search for gridpoint in the nuclide grid
		// within the lower and higher limits we've calculated.
		double e_low  = nuclide_grids[nuc*n_gridpoints + u_low].energy;
		double e_high = nuclide_grids[nuc*n_gridpoints + u_high].energy;
		int lower;
		if( p_energy <= e_low )
			lower = 0;
		else if( p_energy >= e_high )
			lower = n_gridpoints - 1;
		else
			lower = grid_search_nuclide( n_gridpoints, p_energy, &nuclide_grids[nuc*n_gridpoints], u_low, u_high);

		if( lower == n_gridpoints - 1 )
			low = &nuclide_grids[nuc*n_gridpoints + lower - 1];
		else
			low = &nuclide_grids[nuc*n_gridpoints + lower];
	}

	high = low + 1;

	// calculate the re-useable interpolation factor
	f = (high->energy - p_energy) / (high->energy - low->energy);

	// Total XS
	xs_vector[0] = high->total_xs - f * (high->total_xs - low->total_xs);

	// Elastic XS
	xs_vector[1] = high->elastic_xs - f * (high->elastic_xs - low->elastic_xs);

	// Absorbtion XS
	xs_vector[2] = high->absorbtion_xs - f * (high->absorbtion_xs - low->absorbtion_xs);

	// Fission XS
	xs_vector[3] = high->fission_xs - f * (high->fission_xs - low->fission_xs);

	// Nu Fission XS
	xs_vector[4] = high->nu_fission_xs - f * (high->nu_fission_xs - low->nu_fission_xs);
}

// Calculates macroscopic cross section based on a given material & energy
void calculate_macro_xs( double p_energy, int mat, long n_isotopes,
                         long n_gridpoints, __global const int * num_nucs,
                         __global const double * restrict concs,
                         __global const double * egrid, __global const int * index_data,
                         __global const NuclideGridPoint * nuclide_grids,
                         __global const int * mats,
                         double * macro_xs_vector, int grid_type, int hash_bins, int max_num_nucs ){
	int p_nuc; // the nuclide we are looking up
	long idx = -1;
	double conc; // the concentration of the nuclide in the material

	// cleans out macro_xs_vector
	for( int k = 0; k < 5; k++ )
		macro_xs_vector[k] = 0;

	// If we are using the unionized energy grid (UEG), we only
	// need to perform 1 binary search per macroscopic lookup.
	// If we are using the nuclide grid search, it will have to be
	// done inside of the "calculate_micro_xs" function for each different
	// nuclide in the material.
	if( grid_type == UNIONIZED )
		idx = grid_search( n_isotopes * n_gridpoints, p_energy, egrid);
	else if( grid_type == HASH )
	{
		double du = 1.0 / hash_bins;
		idx = p_energy / du;
	}

	// Once we find the pointer array on the UEG, we can pull the data
	// from the respective nuclide grids, as well as the nuclide
	// concentration data for the material
	// Each nuclide from the material needs to have its micro-XS array
	// looked up & interpolatied (via calculate_micro_xs). Then, the
	// micro XS is multiplied by the concentration of that nuclide
	// in the material, and added to the total macro XS array.
	// (Independent -- though if parallelizing, must use atomic operations
	//  or otherwise control access to the xs_vector and macro_xs_vector to
	//  avoid simulataneous writing to the same data structure)
	for( int j = 0; j < num_nucs[mat]; j++ )
	{
		double xs_vector[5];
		p_nuc = mats[mat*max_num_nucs + j];
		conc = concs[mat*max_num_nucs + j];
		calculate_micro_xs( p_energy, p_nuc, n_isotopes,
		                    n_gridpoints, egrid, index_data,
		                    nuclide_grids, idx, xs_vector, grid_type, hash_bins );
		for( int k = 0; k < 5; k++ )
			macro_xs_vector[k] += xs_vector[k] * conc;
	}
}

__kernel void macro_xs_lookup_kernel(		Inputs in,
											int max_num_nucs,
											__global const int *num_nucs,
											__global const double *concs,
											__global const double * unionized_energy_array,
											__global const int * index_grid,
											__global const NuclideGridPoint * nuclide_grid,
											__global const int * mats,
											__global int * verification_array )
{
	// Get the index of the current element to be processed
	int i = get_global_id(0);

	// Set the initial seed value
	unsigned long seed = STARTING_SEED;

	// Forward seed to lookup index (we need 2 samples per lookup)
	seed = fast_forward_LCG(seed, 2*i);

	// Randomly pick an energy and material for the particle
	double p_energy = LCG_random_double(&seed);
	int mat         = pick_mat(&seed);

	double macro_xs_vector[5] = {0};

	// Perform macroscopic Cross Section Lookup
	calculate_macro_xs(
			p_energy,        // Sampled neutron energy (in lethargy)
			mat,             // Sampled material type index neutron is in
			in.n_isotopes,   // Total number of isotopes in simulation
			in.n_gridpoints, // Number of gridpoints per isotope in simulation
			num_nucs,     // 1-D array with number of nuclides per material
			concs,        // Flattened 2-D array with concentration of each nuclide in each material
			unionized_energy_array, // 1-D Unionized energy array
			index_grid,   // Flattened 2-D grid holding indices into nuclide grid for each unionized energy level
			nuclide_grid, // Flattened 2-D grid holding energy levels and XS_data for all nuclides in simulation
			mats,         // Flattened 2-D array with nuclide indices defining composition of each type of material
			macro_xs_vector, // 1-D array with result of the macroscopic cross section (5 different reaction channels)
			in.grid_type,    // Lookup type (nuclide, hash, or unionized)
			in.hash_bins,    // Number of hash bins used (if using hash lookup type)
			max_num_nucs  // Maximum number of nuclides present in any material
			);

	// For verification, and to prevent the compiler from optimizing
	// all work out, we interrogate the returned macro_xs_vector array
	// to find its maximum value index, then increment the verification
	// value by that index. In this implementation, we prevent thread
	// contention by using an OMP reduction on the verification value.
	// For accelerators, a different approach might be required
	// (e.g., atomics, reduction of thread-specific values in large
	// array via CUDA thrust, etc).
	double max = -1.0;
	int max_idx = 0;
	for(int j = 0; j < 5; j++ )
	{
		if( macro_xs_vector[j] > max )
		{
			max = macro_xs_vector[j];
			max_idx = j;
		}
	}
	verification_array[i] += max_idx+1;

}


a number of small improvements

a number of small improvements

#include "ocl_platform.cl"

// number of banks in local memory
// LDS_BANKS

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

/// Returns index ranging from 0 to num work items, such that first half
/// of the work items get even indices and others get odd indices. Each work item
/// gets a different index.
/// Example: (for WIN_SiZE_X == 8)   getLocalId(0):   0  1  2  3  4  5  6  7
///									 parityIdx:		  0  2  4  6  1  3  5  7
/// @return parity-separated index of work item
inline int parityIdx() {
	return (getLocalId(0) * 2) - (WIN_SIZE_X - 1) * (getLocalId(0) / (WIN_SIZE_X / 2));
}

// scratch buffer (in local memory of GPU) where block of input image is stored.

/// Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
/// All even columns are stored first, then all odd columns.
/// All operations expect WIN_SIZE_X threads.

// BOUNDARY_X boundary columns for even or odd columns

 /**

Layout for scratch buffer:

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns

 **/

// two vertical neighbours: pointer diff:
#define VERTICAL_STRIDE_EVEN 66			// 2*BOUNDARY_X + (WIN_SIZE_X / 2)

#define VERTICAL_STRIDE_ODD  65          //  BOUNDARY_X  + (WIN_SIZE_X / 2)

// size of one of two buffers (odd or even)
#define BUFFER_SIZE_EVEN     528          // VERTICAL_STRIDE_EVEN * WIN_SIZE_Y
#define BUFFER_SIZE_ODD     528          // VERTICAL_STRIDE_ODD * WIN_SIZE_Y

/// padding between even and odd buffers: used to reduce bank conflicts
#define PADDING			32           // LDS_BANKS - ((BUFFER_SIZE + LDS_BANKS / 2) % LDS_BANKS);

// offset of the odd columns buffer from the beginning of data buffer
#define ODD_OFFSET 560              // BUFFER_SIZE + PADDING

// size of buffer for both even and odd columns
#define TOTAL_BUFFER_SIZE  1088     // 2 * BUFFER_SIZE + PADDING;
#define TOTAL_BUFFER_SIZE_X2  2176
#define TOTAL_BUFFER_SIZE_X3  3264
///////////////////////////////////////////////////////////////////////

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

int4 readPixel( LOCAL int*  restrict  src) {
	return (int4)(*src, *(src+TOTAL_BUFFER_SIZE),  *(src+TOTAL_BUFFER_SIZE_X2),  *(src+TOTAL_BUFFER_SIZE_X3)) ;
}

void writePixel(int4 pix, LOCAL int*  restrict  dest) {
	*dest = pix.x;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.y;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.z;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.w;
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {

	if (getGlobalId(0) >= width)
	    return;

    const unsigned int halfHeight = height >> 1;
	LOCAL int scratch[TOTAL_BUFFER_SIZE << 2];
	float yDelta = 1.0/height;
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

    /////////////////////////////////////////////////////////////////////////////////
	//fetch first pixel (and 2 top boundary pixels)
	// read -1 point
	const float2 posIn = (float2)(getGlobalId(0), firstY - 1) /  (float2)(width, height);
	int4 minusOne = read_imagei(idata, sampler, posIn);

	// read 0 point
	posIn.y += yDelta;
	int4 current = read_imagei(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= ( read_imagei(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) >> 1;
	////////////////////////////////////////////////////////////////////////////////

	for (int i = 0; i < steps; ++i) {

	   LOCAL int* currentScratch = scratch + getLocalId(0) + BOUNDARY_X;

	   //read pixels two at a time, and store in local buffer
	   for (int j = 0; j < WIN_SIZE_Y>>1; ++j) {

	   		// jump two pixels
	   		posIn.y += yDelta;
			if (posIn.y >= 1)
			   break;

			///////////////////////////////////////////////////////////////////////////////////////////
			// fetch next two pixels, and write last (even) and current (odd)
			// read current plus one (odd) point
			int4 currentPlusOne = read_imagei(idata, sampler, posIn);

			// read next (even) point
			posIn.y += yDelta;
			int4 currentPlusTwo = read_imagei(idata, sampler, posIn);

			// transform current (odd) point
			currentPlusOne -= (current + currentPlusTwo) >> 1;

			// transform previous (even) point
			current += (minusOne + currentPlusOne + (int4)(2,2,2,2)) >> 2;

			//write even
			writePixel(current, currentScratch);

			//write odd
			currentScratch += WIN_SIZE_X;
			writePixel(currentPlusOne, currentScratch);

			//advance scratch pointer
			currentScratch += WIN_SIZE_X;

			minusOne = currentPlusOne;
			current = currentPlusTwo;

			////////////////////////////////////////////////////////////////////////////////////////////

		}

		// write points to destination
		int2 posOut = {getGlobalId(0), firstY>>1};
		currentScratch = scratch +  getLocalId(0) + BOUNDARY_X;
		for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	        // even
	        if (posOut.y >= halfHeight)
				break;

			write_imagei(odata, posOut,readPixel(currentScratch));

			currentScratch += WIN_SIZE_X ;
			posOut.y+= halfHeight + 1;

			// odd
			if (posOut.y >= height)
				break;

			write_imagei(odata, posOut,readPixel(currentScratch));

			currentScratch += WIN_SIZE_X;
			posOut.y -= halfHeight;
		}
		firstY += WIN_SIZE_Y;
	}
}

#include "ocl_platform.cl"

// number of banks in local memory
// LDS_BANKS

//////////////////////////
// dimensions of window
// WIN_SIZE_X	assume this equals number of work items in work group
// WIN_SIZE_Y
///////////////////////////

/// Returns index ranging from 0 to num work items, such that first half
/// of the work items get even indices and others get odd indices. Each work item
/// gets a different index.
/// Example: (for WIN_SiZE_X == 8)   getLocalId(0):   0  1  2  3  4  5  6  7
///									 parityIdx:		  0  2  4  6  1  3  5  7
/// @return parity-separated index of work item
inline int parityIdx() {
	return (getLocalId(0) * 2) - (WIN_SIZE_X - 1) * (getLocalId(0) / (WIN_SIZE_X / 2));
}

// scratch buffer (in local memory of GPU) where block of input image is stored.

/// Odd and even columns are separated. (Generates less bank conflicts when using lifting scheme.)
/// All even columns are stored first, then all odd columns.
/// All operations expect WIN_SIZE_X threads.

// BOUNDARY_X boundary columns for even or odd columns

 /**

Layout for scratch buffer:

Left (even) boundary column
Even Columns
Right (even) boundary column
Left (odd) boundary column
Odd Columns

 **/

// two vertical neighbours: pointer diff:
#define VERTICAL_STRIDE_EVEN 66			// 2*BOUNDARY_X + (WIN_SIZE_X / 2)

#define VERTICAL_STRIDE_ODD  65          //  BOUNDARY_X  + (WIN_SIZE_X / 2)

// size of one of two buffers (odd or even)
#define BUFFER_SIZE_EVEN     528          // VERTICAL_STRIDE_EVEN * WIN_SIZE_Y
#define BUFFER_SIZE_ODD     528          // VERTICAL_STRIDE_ODD * WIN_SIZE_Y

/// padding between even and odd buffers: used to reduce bank conflicts
#define PADDING			32           // LDS_BANKS - ((BUFFER_SIZE + LDS_BANKS / 2) % LDS_BANKS);

// offset of the odd columns buffer from the beginning of data buffer
#define ODD_OFFSET 560              // BUFFER_SIZE + PADDING

// size of buffer for both even and odd columns
#define TOTAL_BUFFER_SIZE  1088     // 2 * BUFFER_SIZE + PADDING;
#define TOTAL_BUFFER_SIZE_X2  2176
#define TOTAL_BUFFER_SIZE_X3  3264
///////////////////////////////////////////////////////////////////////

CONSTANT int4 twoVec = (int4)(2,2,2,2);

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_MIRRORED_REPEAT  | CLK_FILTER_NEAREST;

int4 readPixel( LOCAL int*  restrict  src) {
	return (int4)(*src, *(src+TOTAL_BUFFER_SIZE),  *(src+TOTAL_BUFFER_SIZE_X2),  *(src+TOTAL_BUFFER_SIZE_X3)) ;
}

void writePixel(int4 pix, LOCAL int*  restrict  dest) {
	*dest = pix.x;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.y;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.z;
	dest += TOTAL_BUFFER_SIZE;
	*dest = pix.w;
}

// assumptions: width and height are both even
// (we will probably have to relax these assumptions in the future)

//1. initialize column (and optionaly boundary column)
//2. read column(s) into scratch
//3. vertically transform column(s)
//4. horizontally transform all rows corresponding to this column(s)
//5. write into destination
void KERNEL run(__read_only image2d_t idata, __write_only image2d_t odata,
                       const unsigned int  width, const unsigned int  height, const unsigned int steps) {
	LOCAL int scratch[TOTAL_BUFFER_SIZE << 2];
	float yDelta = 1.0/height;
	int firstY = getGlobalId(1) * (steps * WIN_SIZE_Y);

	// move to left boundary position and initialize
	const float2 posIn = {getGlobalId(0)/(float)width, (firstY - 1)*yDelta};

    /////////////////////////////////////////////////////////////////////////////////
	//fetch first pixel (and 2 top boundary pixels)
	// -2  -1 0 1 2

	// read -1 point
	int4 minusOne = read_imagei(idata, sampler, posIn);
	// read 0 point
	posIn.y += yDelta;
	int4 current = read_imagei(idata, sampler, posIn);

	// transform -1 point (no need to write to local memory)
	minusOne -= (read_imagei(idata, sampler, (float2)(posIn.x, (firstY - 2)*yDelta)) + current) >> 1;
	////////////////////////////////////////////////////////////////////////////////

	for (int i = 0; i < steps; ++i) {

	   LOCAL int* currentScratch = scratch + getLocalId(0) + BOUNDARY_X;

	   //read pixels two at a time, and store in local buffer
	   for (int j = 0; j < WIN_SIZE_Y>>1; ++j) {

	   		// jump two pixels
	   		posIn.y += yDelta;
			if (posIn.y >= 1)
			   break;

			///////////////////////////////////////////////////////////////////////////////////////////
			// fetch next two pixels, and write last (even) and current (odd)
			// read current plus one (odd) point
			int4 currentPlusOne = read_imagei(idata, sampler, posIn);

			// read next (even) point
			posIn.y += yDelta;
			int4 currentPlusTwo = read_imagei(idata, sampler, posIn);

			// transform current (odd) point
			currentPlusOne -= (current + currentPlusTwo) >> 1;

			// transform previous (even) point
			current += (minusOne + currentPlusOne + twoVec) >> 2;

			//write even
			writePixel(current, currentScratch);

			//write odd
			currentScratch += WIN_SIZE_X;
			writePixel(currentPlusOne, currentScratch);

			//advance scratch pointer
			currentScratch += WIN_SIZE_X;

			minusOne = currentPlusOne;
			current = currentPlusTwo;

			////////////////////////////////////////////////////////////////////////////////////////////

		}

	    localMemoryFence();

		// write even points to destination
		const int2 posOut = {getGlobalId(0), firstY>>1};
		currentScratch = scratch +  getLocalId(0) + BOUNDARY_X;
		for (int j = 0; j < WIN_SIZE_Y; j+=2) {

	        if (posOut.x >= width || posOut.y >= (height >> 1))
				break;

			write_imagei(odata, posOut,readPixel(currentScratch));

			currentScratch += WIN_SIZE_X << 1;
			posOut.y++;
		}
		//write odd points to destination
		posOut.y =  ((height + firstY)>>1) + 1;
		currentScratch = scratch +  getLocalId(0) + BOUNDARY_X + WIN_SIZE_X;
		for (int j = 0; j < WIN_SIZE_Y; j+=2) {

			 if (posOut.x >= width || posOut.y >= height)
				break;

			write_imagei(odata, posOut,readPixel(currentScratch));

			currentScratch += WIN_SIZE_X << 1;
			posOut.y++;
		}

		firstY += WIN_SIZE_Y;
		localMemoryFence();
	}

}


Add target triple to CodeGenOpenCL/pipe_types.cl test case

The test is failing on SystemZ since different IR is being
generated due to platform ABI differences.  Add a target triple.

Fix suggested by Anastasia Stulova.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@259183 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -O0 -cl-std=CL2.0 -o - %s | FileCheck %s

// CHECK: %opencl.pipe_t = type opaque
typedef unsigned char __attribute__((ext_vector_type(3))) uchar3;
typedef int __attribute__((ext_vector_type(4))) int4;

void test1(read_only pipe int p) {
// CHECK: define void @test1(%opencl.pipe_t* %p)
  reserve_id_t rid;
// CHECK: %rid = alloca %opencl.reserve_id_t
}

void test2(write_only pipe float p) {
// CHECK: define void @test2(%opencl.pipe_t* %p)
}

void test3(read_only pipe const int p) {
// CHECK: define void @test3(%opencl.pipe_t* %p)
}

void test4(read_only pipe uchar3 p) {
// CHECK: define void @test4(%opencl.pipe_t* %p)
}

void test5(read_only pipe int4 p) {
// CHECK: define void @test5(%opencl.pipe_t* %p)
}

// RUN: %clang_cc1 -emit-llvm -O0 -cl-std=CL2.0 -o - %s | FileCheck %s

// CHECK: %opencl.pipe_t = type opaque
typedef unsigned char __attribute__((ext_vector_type(3))) uchar3;
typedef int __attribute__((ext_vector_type(4))) int4;

void test1(read_only pipe int p) {
// CHECK: define void @test1(%opencl.pipe_t* %p)
  reserve_id_t rid;
// CHECK: %rid = alloca %opencl.reserve_id_t
}

void test2(write_only pipe float p) {
// CHECK: define void @test2(%opencl.pipe_t* %p)
}

void test3(read_only pipe const int p) {
// CHECK: define void @test3(%opencl.pipe_t* %p)
}

void test4(read_only pipe uchar3 p) {
// CHECK: define void @test4(%opencl.pipe_t* %p)
}

void test5(read_only pipe int4 p) {
// CHECK: define void @test5(%opencl.pipe_t* %p)
}


Change buffer cols count from 64 to 32

Change buffer cols count from 64 to 32

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the copyright holders or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#ifdef DOUBLE_SUPPORT
#ifdef cl_amd_fp64
#pragma OPENCL EXTENSION cl_amd_fp64:enable
#elif defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#endif
#endif

#if ddepth == 0
#define MIN_VAL 0
#define MAX_VAL 255
#elif ddepth == 1
#define MIN_VAL -128
#define MAX_VAL 127
#elif ddepth == 2
#define MIN_VAL 0
#define MAX_VAL 65535
#elif ddepth == 3
#define MIN_VAL -32768
#define MAX_VAL 32767
#elif ddepth == 4
#define MIN_VAL INT_MIN
#define MAX_VAL INT_MAX
#elif ddepth == 5
#define MIN_VAL (-FLT_MAX)
#define MAX_VAL FLT_MAX
#elif ddepth == 6
#define MIN_VAL (-DBL_MAX)
#define MAX_VAL DBL_MAX
#else
#error "Unsupported depth"
#endif

#define noconvert

#ifdef OCL_CV_REDUCE_SUM
#define INIT_VALUE 0
#define PROCESS_ELEM(acc, value) acc += value
#elif defined(OCL_CV_REDUCE_MAX)
#define INIT_VALUE MIN_VAL
#define PROCESS_ELEM(acc, value) acc = value > acc ? value : acc
#elif defined(OCL_CV_REDUCE_MIN)
#define INIT_VALUE MAX_VAL
#define PROCESS_ELEM(acc, value) acc = value < acc ? value : acc
#elif defined(OCL_CV_REDUCE_AVG)
#error "This operation should be implemented through OCL_CV_REDUCE_SUM"
#else
#error "No operation is specified"
#endif

#ifndef BUF_COLS
#define BUF_COLS  32
#endif

__kernel void reduce_horz_pre(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * bufptr, int buf_step, int buf_offset)
{
    int x = get_global_id(0);
    int y = get_global_id(1);
    if (x < BUF_COLS)
    {
        int src_index = mad24(y, src_step, mad24(x, (int)sizeof(srcT) * cn, src_offset));
        int buf_index = mad24(y, buf_step, mad24(x, (int)sizeof(dstT) * cn, buf_offset));

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * buf = (__global dstT *)(bufptr + buf_index);
        dstT tmp[cn] = { INIT_VALUE };

        int src_step_mul = BUF_COLS * cn;
        for (int x = 0; x < cols; x += BUF_COLS, src += src_step_mul)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            buf[c] = tmp[c];
    }
}

__kernel void reduce(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * dstptr, int dst_step, int dst_offset)
{
#if dim == 0 // reduce to a single row
    int x = get_global_id(0);
    if (x < cols)
    {
        int src_index = mad24(x, (int)sizeof(srcT) * cn, src_offset);
        __global dstT * dst = (__global dstT *)(dstptr + dst_offset) + x * cn;
        dstT tmp[cn] = { INIT_VALUE };

        for (int y = 0; y < rows; ++y, src_index += src_step)
        {
            __global const srcT * src = (__global const srcT *)(srcptr + src_index);
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#elif dim == 1 // reduce to a single column
    int y = get_global_id(0);
    if (y < rows)
    {
        int src_index = mad24(y, src_step, src_offset);
        int dst_index = mad24(y, dst_step, dst_offset);

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * dst = (__global dstT *)(dstptr + dst_index);
        dstT tmp[cn] = { INIT_VALUE };

        for (int x = 0; x < cols; ++x, src += cn)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#else
#error "Dims must be either 0 or 1"
#endif
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Copyright (C) 2013, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the copyright holders or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#ifdef DOUBLE_SUPPORT
#ifdef cl_amd_fp64
#pragma OPENCL EXTENSION cl_amd_fp64:enable
#elif defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#endif
#endif

#if ddepth == 0
#define MIN_VAL 0
#define MAX_VAL 255
#elif ddepth == 1
#define MIN_VAL -128
#define MAX_VAL 127
#elif ddepth == 2
#define MIN_VAL 0
#define MAX_VAL 65535
#elif ddepth == 3
#define MIN_VAL -32768
#define MAX_VAL 32767
#elif ddepth == 4
#define MIN_VAL INT_MIN
#define MAX_VAL INT_MAX
#elif ddepth == 5
#define MIN_VAL (-FLT_MAX)
#define MAX_VAL FLT_MAX
#elif ddepth == 6
#define MIN_VAL (-DBL_MAX)
#define MAX_VAL DBL_MAX
#else
#error "Unsupported depth"
#endif

#define noconvert

#ifdef OCL_CV_REDUCE_SUM
#define INIT_VALUE 0
#define PROCESS_ELEM(acc, value) acc += value
#elif defined(OCL_CV_REDUCE_MAX)
#define INIT_VALUE MIN_VAL
#define PROCESS_ELEM(acc, value) acc = value > acc ? value : acc
#elif defined(OCL_CV_REDUCE_MIN)
#define INIT_VALUE MAX_VAL
#define PROCESS_ELEM(acc, value) acc = value < acc ? value : acc
#elif defined(OCL_CV_REDUCE_AVG)
#error "This operation should be implemented through OCL_CV_REDUCE_SUM"
#else
#error "No operation is specified"
#endif

#ifndef BUF_COLS
#define BUF_COLS  64
#endif

__kernel void reduce_horz_pre(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * bufptr, int buf_step, int buf_offset)
{
    int x = get_global_id(0);
    int y = get_global_id(1);
    if (x < BUF_COLS)
    {
        int src_index = mad24(y, src_step, mad24(x, (int)sizeof(srcT) * cn, src_offset));
        int buf_index = mad24(y, buf_step, mad24(x, (int)sizeof(dstT) * cn, buf_offset));

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * buf = (__global dstT *)(bufptr + buf_index);
        dstT tmp[cn] = { INIT_VALUE };

        int src_step_mul = BUF_COLS * cn;
        for (int x = 0; x < cols; x += BUF_COLS, src += src_step_mul)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            buf[c] = tmp[c];
    }
}

__kernel void reduce(__global const uchar * srcptr, int src_step, int src_offset, int rows, int cols,
                     __global uchar * dstptr, int dst_step, int dst_offset)
{
#if dim == 0 // reduce to a single row
    int x = get_global_id(0);
    if (x < cols)
    {
        int src_index = mad24(x, (int)sizeof(srcT) * cn, src_offset);
        __global dstT * dst = (__global dstT *)(dstptr + dst_offset) + x * cn;
        dstT tmp[cn] = { INIT_VALUE };

        for (int y = 0; y < rows; ++y, src_index += src_step)
        {
            __global const srcT * src = (__global const srcT *)(srcptr + src_index);
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#elif dim == 1 // reduce to a single column
    int y = get_global_id(0);
    if (y < rows)
    {
        int src_index = mad24(y, src_step, src_offset);
        int dst_index = mad24(y, dst_step, dst_offset);

        __global const srcT * src = (__global const srcT *)(srcptr + src_index);
        __global dstT * dst = (__global dstT *)(dstptr + dst_index);
        dstT tmp[cn] = { INIT_VALUE };

        for (int x = 0; x < cols; ++x, src += cn)
        {
            #pragma unroll
            for (int c = 0; c < cn; ++c)
            {
                dstT value = convertToDT(src[c]);
                PROCESS_ELEM(tmp[c], value);
            }
        }

        #pragma unroll
        for (int c = 0; c < cn; ++c)
            dst[c] = tmp[c];
    }
#else
#error "Dims must be either 0 or 1"
#endif
}


removed arithm_absdiff.cl

removed arithm_absdiff.cl

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Jia Haipeng, jiahaipeng95@gmail.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other oclMaterials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#if defined (DOUBLE_SUPPORT)
#ifdef cl_khr_fp64
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#elif defined (cl_amd_fp64)
#pragma OPENCL EXTENSION cl_amd_fp64:enable
#endif
#endif

//////////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////absdiff////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////////
/**************************************adddiff *************************************/
__kernel void arithm_absdiff_D0 (__global uchar *src1, int src1_step, int src1_offset,
                                 __global uchar *src2, int src2_step, int src2_offset,
                                 __global uchar *dst,  int dst_step,  int dst_offset,
                                 int rows, int cols, int dst_step1)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 2;

#ifdef dst_align
#undef dst_align
#endif
        #define dst_align (dst_offset & 3)
        int src1_index = mad24(y, src1_step, x + src1_offset - dst_align);
        int src2_index = mad24(y, src2_step, x + src2_offset - dst_align);

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + x & (int)0xfffffffc);
        int src1_index_fix = src1_index < 0 ? 0 : src1_index;
        int src2_index_fix = src2_index < 0 ? 0 : src2_index;
        uchar4 src1_data = vload4(0, src1 + src1_index_fix);
        uchar4 src2_data = vload4(0, src2 + src2_index_fix);
        if(src1_index < 0)
        {
            uchar4 tmp;
            tmp.xyzw = (src1_index == -2) ? src1_data.zwxy:src1_data.yzwx;
            src1_data.xyzw = (src1_index == -1) ? src1_data.wxyz:tmp.xyzw;
        }
        if(src2_index < 0)
        {
            uchar4 tmp;
            tmp.xyzw = (src2_index == -2) ? src2_data.zwxy:src2_data.yzwx;
            src2_data.xyzw = (src2_index == -1) ? src2_data.wxyz:tmp.xyzw;
        }

        uchar4 dst_data = *((__global uchar4 *)(dst + dst_index));
        uchar4 tmp_data = abs_diff(src1_data, src2_data);

        dst_data.x = ((dst_index + 0 >= dst_start) && (dst_index + 0 < dst_end)) ? tmp_data.x : dst_data.x;
        dst_data.y = ((dst_index + 1 >= dst_start) && (dst_index + 1 < dst_end)) ? tmp_data.y : dst_data.y;
        dst_data.z = ((dst_index + 2 >= dst_start) && (dst_index + 2 < dst_end)) ? tmp_data.z : dst_data.z;
        dst_data.w = ((dst_index + 3 >= dst_start) && (dst_index + 3 < dst_end)) ? tmp_data.w : dst_data.w;

        *((__global uchar4 *)(dst + dst_index)) = dst_data;
    }
}
__kernel void arithm_absdiff_D2 (__global ushort *src1, int src1_step, int src1_offset,
                                 __global ushort *src2, int src2_step, int src2_offset,
                                 __global ushort *dst,  int dst_step,  int dst_offset,
                                 int rows, int cols, int dst_step1)

{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 2;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align ((dst_offset >> 1) & 3)
        int src1_index = mad24(y, src1_step, (x << 1) + src1_offset - (dst_align << 1));
        int src2_index = mad24(y, src2_step, (x << 1) + src2_offset - (dst_align << 1));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x << 1) & (int)0xfffffff8);

        ushort4 src1_data = vload4(0, (__global ushort *)((__global char *)src1 + src1_index));
        ushort4 src2_data = vload4(0, (__global ushort *)((__global char *)src2 + src2_index));

        ushort4 dst_data = *((__global ushort4 *)((__global char *)dst + dst_index));
        ushort4 tmp_data = abs_diff(src1_data, src2_data);

        dst_data.x = ((dst_index + 0 >= dst_start) && (dst_index + 0 < dst_end)) ? tmp_data.x : dst_data.x;
        dst_data.y = ((dst_index + 2 >= dst_start) && (dst_index + 2 < dst_end)) ? tmp_data.y : dst_data.y;
        dst_data.z = ((dst_index + 4 >= dst_start) && (dst_index + 4 < dst_end)) ? tmp_data.z : dst_data.z;
        dst_data.w = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end)) ? tmp_data.w : dst_data.w;

        *((__global ushort4 *)((__global char *)dst + dst_index)) = dst_data;
    }
}
__kernel void arithm_absdiff_D3 (__global short *src1, int src1_step, int src1_offset,
                                 __global short *src2, int src2_step, int src2_offset,
                                 __global short *dst,  int dst_step,  int dst_offset,
                                 int rows, int cols, int dst_step1)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 2;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align ((dst_offset >> 1) & 3)
        int src1_index = mad24(y, src1_step, (x << 1) + src1_offset - (dst_align << 1));
        int src2_index = mad24(y, src2_step, (x << 1) + src2_offset - (dst_align << 1));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x << 1) & (int)0xfffffff8);

        short4 src1_data = vload4(0, (__global short *)((__global char *)src1 + src1_index));
        short4 src2_data = vload4(0, (__global short *)((__global char *)src2 + src2_index));

        short4  dst_data = *((__global short4 *)((__global char *)dst + dst_index));
        ushort4 tmp = abs_diff(src1_data, src2_data);
        short4  tmp_data = convert_short4_sat(tmp);

        dst_data.x = ((dst_index + 0 >= dst_start) && (dst_index + 0 < dst_end)) ? tmp_data.x : dst_data.x;
        dst_data.y = ((dst_index + 2 >= dst_start) && (dst_index + 2 < dst_end)) ? tmp_data.y : dst_data.y;
        dst_data.z = ((dst_index + 4 >= dst_start) && (dst_index + 4 < dst_end)) ? tmp_data.z : dst_data.z;
        dst_data.w = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end)) ? tmp_data.w : dst_data.w;

        *((__global short4 *)((__global char *)dst + dst_index)) = dst_data;
    }
}

__kernel void arithm_absdiff_D4 (__global int *src1, int src1_step, int src1_offset,
                                 __global int *src2, int src2_step, int src2_offset,
                                 __global int *dst,  int dst_step,  int dst_offset,
                                 int rows, int cols, int dst_step1)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 2) + src1_offset);
        int src2_index = mad24(y, src2_step, (x << 2) + src2_offset);
        int dst_index  = mad24(y, dst_step,  (x << 2) + dst_offset);

        int data1 = *((__global int *)((__global char *)src1 + src1_index));
        int data2 = *((__global int *)((__global char *)src2 + src2_index));
        uint tmp = abs_diff(data1, data2);
        int  tmp_data = convert_int_sat(tmp);

        *((__global int *)((__global char *)dst + dst_index)) = tmp_data;
    }
}
__kernel void arithm_absdiff_D5 (__global float *src1, int src1_step, int src1_offset,
                                 __global float *src2, int src2_step, int src2_offset,
                                 __global float *dst,  int dst_step,  int dst_offset,
                                 int rows, int cols, int dst_step1)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 2) + src1_offset);
        int src2_index = mad24(y, src2_step, (x << 2) + src2_offset);
        int dst_index  = mad24(y, dst_step,  (x << 2) + dst_offset);

        float data1 = *((__global float *)((__global char *)src1 + src1_index));
        float data2 = *((__global float *)((__global char *)src2 + src2_index));
        float tmp = fabs(data1 - data2);

        *((__global float *)((__global char *)dst + dst_index)) = tmp;
    }
}

#if defined (DOUBLE_SUPPORT)
__kernel void arithm_absdiff_D6 (__global double *src1, int src1_step, int src1_offset,
                                 __global double *src2, int src2_step, int src2_offset,
                                 __global double *dst,  int dst_step,  int dst_offset,
                                 int rows, int cols, int dst_step1)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 3) + src1_offset);
        int src2_index = mad24(y, src2_step, (x << 3) + src2_offset);
        int dst_index  = mad24(y, dst_step,  (x << 3) + dst_offset);

        double data1 = *((__global double *)((__global char *)src1 + src1_index));
        double data2 = *((__global double *)((__global char *)src2 + src2_index));
        double tmp = fabs(data1-data2);

        *((__global double *)((__global char *)dst + dst_index)) = tmp;
    }
}
#endif

/**************************************absdiff with scalar**************************************/
__kernel void arithm_s_absdiff_C1_D0 (__global   uchar *src1, int src1_step, int src1_offset,
                                      __global   uchar *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{
    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 2;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align (dst_offset & 3)
        int src1_index = mad24(y, src1_step, x + src1_offset - dst_align);

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + x & (int)0xfffffffc);
        int src1_index_fix = src1_index < 0 ? 0 : src1_index;
        uchar4 src1_data = vload4(0, src1 + src1_index_fix);
        int4 src2_data = (int4)(src2.x, src2.x, src2.x, src2.x);
        if(src1_index < 0)
        {
            uchar4 tmp;
            tmp.xyzw = (src1_index == -2) ? src1_data.zwxy:src1_data.yzwx;
            src1_data.xyzw = (src1_index == -1) ? src1_data.wxyz:tmp.xyzw;
        }

        uchar4 data = *((__global uchar4 *)(dst + dst_index));
        uchar4 tmp_data = convert_uchar4_sat(abs_diff(convert_int4_sat(src1_data), src2_data));

        data.x = ((dst_index + 0 >= dst_start) && (dst_index + 0 < dst_end)) ? tmp_data.x : data.x;
        data.y = ((dst_index + 1 >= dst_start) && (dst_index + 1 < dst_end)) ? tmp_data.y : data.y;
        data.z = ((dst_index + 2 >= dst_start) && (dst_index + 2 < dst_end)) ? tmp_data.z : data.z;
        data.w = ((dst_index + 3 >= dst_start) && (dst_index + 3 < dst_end)) ? tmp_data.w : data.w;

        *((__global uchar4 *)(dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C1_D2 (__global   ushort *src1, int src1_step, int src1_offset,
                                      __global   ushort *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 1;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align ((dst_offset >> 1) & 1)
        int src1_index = mad24(y, src1_step, (x << 1) + src1_offset - (dst_align << 1));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x << 1) & (int)0xfffffffc);

        ushort2 src1_data = vload2(0, (__global ushort *)((__global char *)src1 + src1_index));
        int2 src2_data = (int2)(src2.x, src2.x);

        ushort2 data = *((__global ushort2 *)((__global uchar *)dst + dst_index));
        ushort2 tmp_data = convert_ushort2_sat(abs_diff(convert_int2_sat(src1_data), src2_data));

        data.x = (dst_index + 0 >= dst_start) ? tmp_data.x : data.x;
        data.y = (dst_index + 2 <  dst_end  ) ? tmp_data.y : data.y;

        *((__global ushort2 *)((__global uchar *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C1_D3 (__global   short *src1, int src1_step, int src1_offset,
                                      __global   short *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 1;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align ((dst_offset >> 1) & 1)
        int src1_index = mad24(y, src1_step, (x << 1) + src1_offset - (dst_align << 1));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x << 1) & (int)0xfffffffc);

        short2 src1_data = vload2(0, (__global short *)((__global char *)src1 + src1_index));
        int2 src2_data = (int2)(src2.x, src2.x);
        short2 data = *((__global short2 *)((__global uchar *)dst + dst_index));

        ushort2 tmp = convert_ushort2_sat(abs_diff(convert_int2_sat(src1_data), src2_data));
        short2 tmp_data = convert_short2_sat(tmp);

        data.x = (dst_index + 0 >= dst_start) ? tmp_data.x : data.x;
        data.y = (dst_index + 2 <  dst_end  ) ? tmp_data.y : data.y;

        *((__global short2 *)((__global uchar *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C1_D4 (__global   int *src1, int src1_step, int src1_offset,
                                      __global   int *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 2) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 2) + dst_offset);

        int src_data1 = *((__global int *)((__global char *)src1 + src1_index));
        int src_data2 = src2.x;
        int dst_data  = *((__global int *)((__global char *)dst  + dst_index));

        uint tmp_data = abs_diff(src_data1, src_data2);
        int  data = convert_int_sat(tmp_data);

        *((__global int *)((__global char *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C1_D5 (__global   float *src1, int src1_step, int src1_offset,
                                      __global   float *dst,  int dst_step,  int dst_offset,
                                      float4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 2) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 2) + dst_offset);

        float src_data1 = *((__global float *)((__global char *)src1 + src1_index));
        float src_data2 = src2.x;
        float dst_data  = *((__global float *)((__global char *)dst  + dst_index));

        float data = fabs(src_data1 - src_data2);

        *((__global float *)((__global char *)dst + dst_index)) = data;
    }
}

#if defined (DOUBLE_SUPPORT)
__kernel void arithm_s_absdiff_C1_D6 (__global   double *src1, int src1_step, int src1_offset,
                                      __global   double *dst,  int dst_step,  int dst_offset,
                                      double4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 3) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 3) + dst_offset);

        double src_data1 = *((__global double *)((__global char *)src1 + src1_index));
        double src2_data = src2.x;
        double dst_data  = *((__global double *)((__global char *)dst  + dst_index));

        double data = fabs(src_data1 - src2_data);

        *((__global double *)((__global char *)dst + dst_index)) = data;
    }
}
#endif

__kernel void arithm_s_absdiff_C2_D0 (__global   uchar *src1, int src1_step, int src1_offset,
                                      __global   uchar *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 1;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align ((dst_offset >> 1) & 1)
        int src1_index = mad24(y, src1_step, (x << 1) + src1_offset - (dst_align << 1));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x << 1) & (int)0xfffffffc);

        uchar4 src1_data = vload4(0, src1 + src1_index);
        int4 src2_data = (int4)(src2.x, src2.y, src2.x, src2.y);

        uchar4 data = *((__global uchar4 *)(dst + dst_index));
        uchar4 tmp_data = convert_uchar4_sat(abs_diff(convert_int4_sat(src1_data), src2_data));

        data.xy = (dst_index + 0 >= dst_start) ? tmp_data.xy : data.xy;
        data.zw = (dst_index + 2 <  dst_end  ) ? tmp_data.zw : data.zw;

        *((__global uchar4 *)(dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C2_D2 (__global   ushort *src1, int src1_step, int src1_offset,
                                      __global   ushort *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 2) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 2) + dst_offset);

        ushort2 src_data1 = *((__global ushort2 *)((__global char *)src1 + src1_index));
        int2 src_data2 = (int2)(src2.x, src2.y);
        ushort2 dst_data  = *((__global ushort2 *)((__global char *)dst  + dst_index));

        ushort2 data = convert_ushort2_sat( abs_diff(convert_int2_sat(src_data1), src_data2));

        *((__global ushort2 *)((__global char *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C2_D3 (__global   short *src1, int src1_step, int src1_offset,
                                      __global   short *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 2) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 2) + dst_offset);

        short2 src_data1 = *((__global short2 *)((__global char *)src1 + src1_index));
        int2 src_data2 = (int2)(src2.x, src2.y);
        short2 dst_data  = *((__global short2 *)((__global char *)dst  + dst_index));

        ushort2 tmp = convert_ushort2_sat(abs_diff(convert_int2_sat(src_data1), src_data2));
        short2 data = convert_short2_sat(tmp);

        *((__global short2 *)((__global char *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C2_D4 (__global   int *src1, int src1_step, int src1_offset,
                                      __global   int *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 3) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 3) + dst_offset);

        int2 src_data1 = *((__global int2 *)((__global char *)src1 + src1_index));
        int2 src_data2 = (int2)(src2.x, src2.y);
        int2 dst_data  = *((__global int2 *)((__global char *)dst  + dst_index));

        int2 data = convert_int2_sat(abs_diff(src_data1, src_data2));
        *((__global int2 *)((__global char *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C2_D5 (__global   float *src1, int src1_step, int src1_offset,
                                      __global   float *dst,  int dst_step,  int dst_offset,
                                      float4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 3) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 3) + dst_offset);

        float2 src_data1 = *((__global float2 *)((__global char *)src1 + src1_index));
        float2 src_data2 = (float2)(src2.x, src2.y);
        float2 dst_data  = *((__global float2 *)((__global char *)dst  + dst_index));

        float2 data = fabs(src_data1 - src_data2);
        *((__global float2 *)((__global char *)dst + dst_index)) = data;
    }
}
#if defined (DOUBLE_SUPPORT)
__kernel void arithm_s_absdiff_C2_D6 (__global   double *src1, int src1_step, int src1_offset,
                                      __global   double *dst,  int dst_step,  int dst_offset,
                                      double4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 4) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 4) + dst_offset);

        double2 src_data1 = *((__global double2 *)((__global char *)src1 + src1_index));
        double2 src_data2 = (double2)(src2.x, src2.y);
        double2 dst_data  = *((__global double2 *)((__global char *)dst  + dst_index));

        double2 data = fabs(src_data1 - src_data2);

        *((__global double2 *)((__global char *)dst + dst_index)) = data;
    }
}
#endif
__kernel void arithm_s_absdiff_C3_D0 (__global   uchar *src1, int src1_step, int src1_offset,
                                      __global   uchar *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 2;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align (((dst_offset % dst_step) / 3 ) & 3)
        int src1_index = mad24(y, src1_step, (x * 3) + src1_offset - (dst_align * 3));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x * 3) - (dst_align * 3));

        uchar4 src1_data_0 = vload4(0, src1 + src1_index + 0);
        uchar4 src1_data_1 = vload4(0, src1 + src1_index + 4);
        uchar4 src1_data_2 = vload4(0, src1 + src1_index + 8);

        int4 src2_data_0 = (int4)(src2.x, src2.y, src2.z, src2.x);
        int4 src2_data_1 = (int4)(src2.y, src2.z, src2.x, src2.y);
        int4 src2_data_2 = (int4)(src2.z, src2.x, src2.y, src2.z);

        uchar4 data_0 = *((__global uchar4 *)(dst + dst_index + 0));
        uchar4 data_1 = *((__global uchar4 *)(dst + dst_index + 4));
        uchar4 data_2 = *((__global uchar4 *)(dst + dst_index + 8));

        uchar4 tmp_data_0 = convert_uchar4_sat(abs_diff(convert_int4_sat(src1_data_0), src2_data_0));
        uchar4 tmp_data_1 = convert_uchar4_sat(abs_diff(convert_int4_sat(src1_data_1), src2_data_1));
        uchar4 tmp_data_2 = convert_uchar4_sat(abs_diff(convert_int4_sat(src1_data_2), src2_data_2));

        data_0.xyz = ((dst_index + 0 >= dst_start)) ? tmp_data_0.xyz : data_0.xyz;
        data_0.w   = ((dst_index + 3 >= dst_start) && (dst_index + 3 < dst_end))
                     ? tmp_data_0.w : data_0.w;

        data_1.xy  = ((dst_index + 3 >= dst_start) && (dst_index + 3 < dst_end))
                     ? tmp_data_1.xy : data_1.xy;
        data_1.zw  = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end))
                     ? tmp_data_1.zw : data_1.zw;

        data_2.x   = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end))
                     ? tmp_data_2.x : data_2.x;
        data_2.yzw = ((dst_index + 9 >= dst_start) && (dst_index + 9 < dst_end))
                     ? tmp_data_2.yzw : data_2.yzw;

        *((__global uchar4 *)(dst + dst_index + 0)) = data_0;
        *((__global uchar4 *)(dst + dst_index + 4)) = data_1;
        *((__global uchar4 *)(dst + dst_index + 8)) = data_2;
    }
}
__kernel void arithm_s_absdiff_C3_D2 (__global   ushort *src1, int src1_step, int src1_offset,
                                      __global   ushort *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 1;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align (((dst_offset % dst_step) / 6 ) & 1)
        int src1_index = mad24(y, src1_step, (x * 6) + src1_offset - (dst_align * 6));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x * 6) - (dst_align * 6));

        ushort2 src1_data_0 = vload2(0, (__global ushort *)((__global char *)src1 + src1_index + 0));
        ushort2 src1_data_1 = vload2(0, (__global ushort *)((__global char *)src1 + src1_index + 4));
        ushort2 src1_data_2 = vload2(0, (__global ushort *)((__global char *)src1 + src1_index + 8));

        int2 src2_data_0 = (int2)(src2.x, src2.y);
        int2 src2_data_1 = (int2)(src2.z, src2.x);
        int2 src2_data_2 = (int2)(src2.y, src2.z);

        ushort2 data_0 = *((__global ushort2 *)((__global char *)dst + dst_index + 0));
        ushort2 data_1 = *((__global ushort2 *)((__global char *)dst + dst_index + 4));
        ushort2 data_2 = *((__global ushort2 *)((__global char *)dst + dst_index + 8));

        ushort2 tmp_data_0 = convert_ushort2_sat(abs_diff(convert_int2_sat(src1_data_0), src2_data_0));
        ushort2 tmp_data_1 = convert_ushort2_sat(abs_diff(convert_int2_sat(src1_data_1), src2_data_1));
        ushort2 tmp_data_2 = convert_ushort2_sat(abs_diff(convert_int2_sat(src1_data_2), src2_data_2));

        data_0.xy = ((dst_index + 0 >= dst_start)) ? tmp_data_0.xy : data_0.xy;

        data_1.x  = ((dst_index + 0 >= dst_start) && (dst_index + 0 < dst_end))
                    ? tmp_data_1.x : data_1.x;
        data_1.y  = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end))
                    ? tmp_data_1.y : data_1.y;

        data_2.xy = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end))
                    ? tmp_data_2.xy : data_2.xy;

        *((__global ushort2 *)((__global char *)dst + dst_index + 0))= data_0;
        *((__global ushort2 *)((__global char *)dst + dst_index + 4))= data_1;
        *((__global ushort2 *)((__global char *)dst + dst_index + 8))= data_2;
    }
}
__kernel void arithm_s_absdiff_C3_D3 (__global   short *src1, int src1_step, int src1_offset,
                                      __global   short *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        x = x << 1;

#ifdef dst_align
#undef dst_align
#endif
#define dst_align (((dst_offset % dst_step) / 6 ) & 1)
        int src1_index = mad24(y, src1_step, (x * 6) + src1_offset - (dst_align * 6));

        int dst_start  = mad24(y, dst_step, dst_offset);
        int dst_end    = mad24(y, dst_step, dst_offset + dst_step1);
        int dst_index  = mad24(y, dst_step, dst_offset + (x * 6) - (dst_align * 6));

        short2 src1_data_0 = vload2(0, (__global short *)((__global char *)src1 + src1_index + 0));
        short2 src1_data_1 = vload2(0, (__global short *)((__global char *)src1 + src1_index + 4));
        short2 src1_data_2 = vload2(0, (__global short *)((__global char *)src1 + src1_index + 8));

        int2 src2_data_0 = (int2)(src2.x, src2.y);
        int2 src2_data_1 = (int2)(src2.z, src2.x);
        int2 src2_data_2 = (int2)(src2.y, src2.z);

        short2 data_0 = *((__global short2 *)((__global char *)dst + dst_index + 0));
        short2 data_1 = *((__global short2 *)((__global char *)dst + dst_index + 4));
        short2 data_2 = *((__global short2 *)((__global char *)dst + dst_index + 8));

        short2 tmp_data_0 = convert_short2_sat(abs_diff(convert_int2_sat(src1_data_0), src2_data_0));
        short2 tmp_data_1 = convert_short2_sat(abs_diff(convert_int2_sat(src1_data_1), src2_data_1));
        short2 tmp_data_2 = convert_short2_sat(abs_diff(convert_int2_sat(src1_data_2), src2_data_2));

        data_0.xy = ((dst_index + 0 >= dst_start)) ? tmp_data_0.xy : data_0.xy;

        data_1.x  = ((dst_index + 0 >= dst_start) && (dst_index + 0 < dst_end))
                    ? tmp_data_1.x : data_1.x;
        data_1.y  = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end))
                    ? tmp_data_1.y : data_1.y;

        data_2.xy = ((dst_index + 6 >= dst_start) && (dst_index + 6 < dst_end))
                    ? tmp_data_2.xy : data_2.xy;

        *((__global short2 *)((__global char *)dst + dst_index + 0))= data_0;
        *((__global short2 *)((__global char *)dst + dst_index + 4))= data_1;
        *((__global short2 *)((__global char *)dst + dst_index + 8))= data_2;
    }
}
__kernel void arithm_s_absdiff_C3_D4 (__global   int *src1, int src1_step, int src1_offset,
                                      __global   int *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x * 12) + src1_offset);
        int dst_index  = mad24(y, dst_step, dst_offset + (x * 12));

        int src1_data_0 = *((__global int *)((__global char *)src1 + src1_index + 0));
        int src1_data_1 = *((__global int *)((__global char *)src1 + src1_index + 4));
        int src1_data_2 = *((__global int *)((__global char *)src1 + src1_index + 8));

        int src2_data_0 = src2.x;
        int src2_data_1 = src2.y;
        int src2_data_2 = src2.z;

        int data_0 = *((__global int *)((__global char *)dst + dst_index + 0));
        int data_1 = *((__global int *)((__global char *)dst + dst_index + 4));
        int data_2 = *((__global int *)((__global char *)dst + dst_index + 8));

        int tmp_data_0 = convert_int_sat(abs_diff(src1_data_0, src2_data_0));
        int tmp_data_1 = convert_int_sat(abs_diff(src1_data_1, src2_data_1));
        int tmp_data_2 = convert_int_sat(abs_diff(src1_data_2, src2_data_2));

        *((__global int *)((__global char *)dst + dst_index + 0))= tmp_data_0;
        *((__global int *)((__global char *)dst + dst_index + 4))= tmp_data_1;
        *((__global int *)((__global char *)dst + dst_index + 8))= tmp_data_2;
    }
}
__kernel void arithm_s_absdiff_C3_D5 (__global   float *src1, int src1_step, int src1_offset,
                                      __global   float *dst,  int dst_step,  int dst_offset,
                                      float4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x * 12) + src1_offset);
        int dst_index  = mad24(y, dst_step, dst_offset + (x * 12));

        float src1_data_0 = *((__global float *)((__global char *)src1 + src1_index + 0));
        float src1_data_1 = *((__global float *)((__global char *)src1 + src1_index + 4));
        float src1_data_2 = *((__global float *)((__global char *)src1 + src1_index + 8));

        float src2_data_0 = src2.x;
        float src2_data_1 = src2.y;
        float src2_data_2 = src2.z;

        float data_0 = *((__global float *)((__global char *)dst + dst_index + 0));
        float data_1 = *((__global float *)((__global char *)dst + dst_index + 4));
        float data_2 = *((__global float *)((__global char *)dst + dst_index + 8));

        float tmp_data_0 = fabs(src1_data_0 - src2_data_0);
        float tmp_data_1 = fabs(src1_data_1 - src2_data_1);
        float tmp_data_2 = fabs(src1_data_2 - src2_data_2);

        *((__global float *)((__global char *)dst + dst_index + 0))= tmp_data_0;
        *((__global float *)((__global char *)dst + dst_index + 4))= tmp_data_1;
        *((__global float *)((__global char *)dst + dst_index + 8))= tmp_data_2;
    }
}

#if defined (DOUBLE_SUPPORT)
__kernel void arithm_s_absdiff_C3_D6 (__global   double *src1, int src1_step, int src1_offset,
                                      __global   double *dst,  int dst_step,  int dst_offset,
                                      double4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x * 24) + src1_offset);
        int dst_index  = mad24(y, dst_step, dst_offset + (x * 24));

        double src1_data_0 = *((__global double *)((__global char *)src1 + src1_index + 0 ));
        double src1_data_1 = *((__global double *)((__global char *)src1 + src1_index + 8 ));
        double src1_data_2 = *((__global double *)((__global char *)src1 + src1_index + 16));

        double src2_data_0 = src2.x;
        double src2_data_1 = src2.y;
        double src2_data_2 = src2.z;

        double data_0 = *((__global double *)((__global char *)dst + dst_index + 0 ));
        double data_1 = *((__global double *)((__global char *)dst + dst_index + 8 ));
        double data_2 = *((__global double *)((__global char *)dst + dst_index + 16));

        double tmp_data_0 = fabs(src1_data_0 - src2_data_0);
        double tmp_data_1 = fabs(src1_data_1 - src2_data_1);
        double tmp_data_2 = fabs(src1_data_2 - src2_data_2);

        *((__global double *)((__global char *)dst + dst_index + 0 ))= tmp_data_0;
        *((__global double *)((__global char *)dst + dst_index + 8 ))= tmp_data_1;
        *((__global double *)((__global char *)dst + dst_index + 16))= tmp_data_2;
    }
}
#endif
__kernel void arithm_s_absdiff_C4_D0 (__global   uchar *src1, int src1_step, int src1_offset,
                                      __global   uchar *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 2) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 2) + dst_offset);

        uchar4 src_data1 = *((__global uchar4 *)(src1 + src1_index));

        uchar4 data = convert_uchar4_sat(abs_diff(convert_int4_sat(src_data1), src2));

        *((__global uchar4 *)(dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C4_D2 (__global   ushort *src1, int src1_step, int src1_offset,
                                      __global   ushort *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 3) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 3) + dst_offset);

        ushort4 src_data1 = *((__global ushort4 *)((__global char *)src1 + src1_index));

        ushort4 data = convert_ushort4_sat(abs_diff(convert_int4_sat(src_data1), src2));

        *((__global ushort4 *)((__global char *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C4_D3 (__global   short *src1, int src1_step, int src1_offset,
                                      __global   short *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 3) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 3) + dst_offset);

        short4 src_data1 = *((__global short4 *)((__global char *)src1 + src1_index));

        short4 data = convert_short4_sat(abs_diff(convert_int4_sat(src_data1), src2));

        *((__global short4 *)((__global char *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C4_D4 (__global   int *src1, int src1_step, int src1_offset,
                                      __global   int *dst,  int dst_step,  int dst_offset,
                                      int4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 4) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 4) + dst_offset);

        int4 src_data1 = *((__global int4 *)((__global char *)src1 + src1_index));

        int4 data = convert_int4_sat(abs_diff(src_data1, src2));

        *((__global int4 *)((__global char *)dst + dst_index)) = data;
    }
}
__kernel void arithm_s_absdiff_C4_D5 (__global   float *src1, int src1_step, int src1_offset,
                                      __global   float *dst,  int dst_step,  int dst_offset,
                                      float4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 4) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 4) + dst_offset);

        float4 src_data1 = *((__global float4 *)((__global char *)src1 + src1_index));

        float4 data = fabs(src_data1 - src2);

        *((__global float4 *)((__global char *)dst + dst_index)) = data;
    }
}

#if defined (DOUBLE_SUPPORT)
__kernel void arithm_s_absdiff_C4_D6 (__global   double *src1, int src1_step, int src1_offset,
                                      __global   double *dst,  int dst_step,  int dst_offset,
                                      double4 src2, int rows, int cols, int dst_step1)
{

    int x = get_global_id(0);
    int y = get_global_id(1);

    if (x < cols && y < rows)
    {
        int src1_index = mad24(y, src1_step, (x << 5) + src1_offset);
        int dst_index  = mad24(y, dst_step,  (x << 5) + dst_offset);

        double4 src_data1 = *((__global double4 *)((__global char *)src1 + src1_index));

        double4 data = fabs(src_data1 - src2);

        *((__global double4 *)((__global char *)dst + dst_index)) = data;
    }
}
#endif