Complete CscGemm.cl

Complete CscGemm.cl

/*
 *
 *
 */

#pragma OPENCL EXTENSION cl_khr_int64_base_atomics : enable

__kernel void setNonZero(const cl_long beg,
                         const cl_long end,
                         __global const cl_long *idx,
                         const cl_long rows,
                         const cl_long cols,
                         __global const cl_long *colPtr,
                         __global const cl_long *rowIdx,
                         __global cl_long *out)
{
    uint gBase = get_global_id(0);
    uint gSize = get_global_size(0);

    // Clear buffer
    for(cl_long i = gBase; i < rows; i += gSize)
    {
        out[i] = 0;
    }

    // Each work item pick one cloumn
    for(cl_long i = beg + gBase; i < end + gBase; i += gSize)
    {
        cl_long col = idx[i];
        for(cl_long j = colPtr[col]; j < colPtr[col + 1]; ++j)
        {
            // Do not have to consider data race
            out[rowIdx[j]] = 1;
        }
    }
}

__kernel void calNonZero(const cl_long beg,
                         const cl_long end,
                         __global const cl_long *idx,
                         __global const cl_double *val,
                         const cl_long rows,
                         const cl_long cols,
                         __global const cl_long *colPtr,
                         __global const cl_long *rowIdx,
                         __global const cl_double *values,
                         __global cl_double *out)
{
    uint gBase = get_global_id(0);
    uint gSize = get_global_size(0);

    // Clear buffer
    for(cl_long i = gBase; i < rows; i += gSize)
    {
        out[i] = 0;
    }

    // Declare union for atom_cmpxchg
    union { ulong *ip; cl_double *fp; } tp, op;
    cl_double temp;
    tp.fp = &temp;
    op.fp = out;

    for(cl_long i = beg + gBase; i < end + gBase; i += gSize)
    {
        cl_long col = idx[i];
        for(cl_long j = colPtr[col]; j < colPtr[col + 1]; ++j)
        {
            // Compute & Save into output using FMA & Atomic operation
            temp = fma(val[i], values[j], out[rowIdx[j]]);
            atom_cmpxchg(op.ip + rowIdx[j], *(tp.ip), *(tp.ip));
        }
    }

}

__kernel void lCountZero(const cl_long n,
                        __global const cl_long *vec,
                        __local  cl_long *psum,
                        __global cl_long *count)
{
    uint gBase = get_global_id(0);
    uint gSize = get_global_size(0);

    for(cl_long i = gBase; i < n; i += gSize)
    {
        psum[gBase] += isequal(vec[i], 0.0);
    }

    uint j = log2(gSize) + 1;
    uint k = gSize - 1;
    for(uint i = gBase; i < j; ++i)
    {
        k = (k >> 1) + 1;
        if(i < k)
        {
            uint idx = i << 1;
            psum[idx] += psum[idx + 1];
        }
    }

    if(gBase == 0) *count = psum[0];
    return;
}

__kernel void dCountZero(const cl_long n,
                        __global const cl_double *vec,
                        __local  cl_long *psum,
                        __global cl_long *count)
{
    uint gBase = get_global_id(0);
    uint gSize = get_global_size(0);

    for(cl_long i = gBase; i < n; i += gSize)
    {
        psum[gBase] += isequal(vec[i], 0.0);
    }

    uint j = log2(gSize) + 1;
    uint k = gSize - 1;
    for(uint i = gBase; i < j; ++i)
    {
        k = (k >> 1) + 1;
        if(i < k)
        {
            uint idx = i << 1;
            psum[idx] += psum[idx + 1];
        }
    }

    if(gBase == 0) *count = psum[0];
    return;
}

__kernel
void accumulate(
    __global cl_long* buffer,
    __global cl_long size,
    __global cl_long val)
{

}


reduce v3

reduce v3

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable
#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable

#define ZERO 0
#define ONE 1
#define SMALLEST -1.0e37f

#define FAST_MATH

#ifdef FAST_MATH
#define exp_function native_exp
#define log_function native_log
#define log2_function native_log2
#define sqrt_function native_sqrt
#define rsqrt_function native_rsqrt
#define powr_function native_powr
#define divide_function native_divide
#define fmad_function(a,b,c) (mad ((a), (b), (c)))
#define logistic_function(x) (native_recip ( 1.0f + native_exp ( -(x) ) ))
#else
#define exp_function exp
#define log_function log
#define log2_function log2
#define sqrt_function sqrt
#define rsqrt_function rsqrt
#define powr_function powr
#define divide_function divide
#define fmad_function(a,b,c) ((c) + (a) * (b))
#define logistic_function(x) (1.0f / ( 1.0f + exp ( -(x) ) ))
#endif

//Function to perform the atomic max
inline void AtomicMax(volatile __global float *source, const float operand) {
	union {
		unsigned int intVal;
		float floatVal;
	} newVal;
	union {
		unsigned int intVal;
		float floatVal;
	} prevVal;
	do {
		prevVal.floatVal = *source;
		newVal.floatVal = max(prevVal.floatVal, operand);
	} while (atomic_cmpxchg((volatile __global unsigned int *)source, prevVal.intVal, newVal.intVal) != prevVal.intVal);
}

//TODO: test, try changing localsize1, localsize2
//local reduction + atomic global reduction
//larger REDUCTION_WORKGROUP_SIZE means fewer atomic ops
#define REDUCTION_BODY_v1(out) { \
						\
						float maxval = out[0]; \
						\
						while (id < n) { \
							float x = xgm[id]; \
							if (x >= maxval) { \
								maxval = x; \
							} \
							id += wsize * num_groups; \
						} \
						maxlm[lid] = maxval; \
						barrier(CLK_LOCAL_MEM_FENCE); \
					  	for (int s=wsize/2; s>0; s=s>>1) { \
					    	if (lid <= s) { \
					      		if (maxlm[lid + s] >= maxlm[lid]) { \
					        		maxlm[lid] = maxlm[lid + s]; \
					      		} \
					    	} \
					    	barrier(CLK_LOCAL_MEM_FENCE); \
					    	\
					  	} \
						if (lid == 0) { \
							AtomicMax(&out[0], maxlm[0]); \
						} \
			\
		}

// https://developer.apple.com/library/content/samplecode/OpenCL_Parallel_Reduction_Example/Listings/reduce_float_kernel_cl.html#//apple_ref/doc/uid/DTS40008188-reduce_float_kernel_cl-DontLinkElementID_7

//fixed 128 for now
//#define GROUP_SIZE 128
// manual unroll
#define REDUCTION_BODY_v2(out) { \
						\
						float maxval = out[0]; \
						\
						while (id < n) { \
							float x = xgm[id]; \
							if (x >= maxval) { \
								maxval = x; \
							} \
							id += wsize * num_groups; \
						} \
						maxlm[lid] = maxval; \
						barrier(CLK_LOCAL_MEM_FENCE); \
						if (lid < 64) { maxlm[lid] = fmax(maxlm[lid], maxlm[lid + 64]); } \
						barrier(CLK_LOCAL_MEM_FENCE); \
						if (lid < 32) { maxlm[lid] = fmax(maxlm[lid], maxlm[lid + 32]); } \
						barrier(CLK_LOCAL_MEM_FENCE); \
						if (lid < 16) { maxlm[lid] = fmax(maxlm[lid], maxlm[lid + 16]); } \
						barrier(CLK_LOCAL_MEM_FENCE); \
						if (lid < 8) { maxlm[lid] = fmax(maxlm[lid], maxlm[lid + 8]); } \
						barrier(CLK_LOCAL_MEM_FENCE); \
						if (lid < 4) { maxlm[lid] = fmax(maxlm[lid], maxlm[lid + 4]); } \
						barrier(CLK_LOCAL_MEM_FENCE); \
						if (lid < 2) { maxlm[lid] = fmax(maxlm[lid], maxlm[lid + 2]); } \
						barrier(CLK_LOCAL_MEM_FENCE); \
						if (lid < 1) { maxlm[lid] = fmax(maxlm[lid], maxlm[lid + 1]); } \
						barrier(CLK_LOCAL_MEM_FENCE); \
						\
						if (lid == 0) { \
							AtomicMax(&out[0], maxlm[0]); \
						} \
			\
		}

#define REDUCTION_BODY_v3(out) { \
						\
						float maxval = out[0]; \
						\
						while (id < n) { \
							float x = xgm[id]; \
							if (x >= maxval) { \
								maxval = x; \
							} \
							id += wsize * num_groups; \
						} \
						maxlm[lid] = maxval; \
						barrier(CLK_LOCAL_MEM_FENCE); \
						\
						_Pragma("unroll") \
					  	for (int s=wsize/2; s>0; s=s>>1) { \
					    	if (lid <= s) { \
					      		if (maxlm[lid + s] >= maxlm[lid]) { \
					        		maxlm[lid] = maxlm[lid + s]; \
					      		} \
					    	} \
					    	barrier(CLK_LOCAL_MEM_FENCE); \
					    	\
					  	} \
						if (lid == 0) { \
							AtomicMax(&out[0], maxlm[0]); \
						} \
			\
		}

__kernel void max_coeff (__global float * restrict y, __global float * restrict xgm, const unsigned int n, __local float *maxlm) {

	const unsigned int lid = get_local_id(0);
	const unsigned int wgid = get_group_id(0);
	const unsigned int wsize = get_local_size (0);
	unsigned int id = wgid * wsize + lid; //get_global_id(0);
	const unsigned int num_groups = get_num_groups(0);

	REDUCTION_BODY_v2(y)

}

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable
#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable

#define ZERO 0
#define ONE 1
#define SMALLEST -1.0e37f

#define FAST_MATH

#ifdef FAST_MATH
#define exp_function native_exp
#define log_function native_log
#define log2_function native_log2
#define sqrt_function native_sqrt
#define rsqrt_function native_rsqrt
#define powr_function native_powr
#define divide_function native_divide
#define fmad_function(a,b,c) (mad ((a), (b), (c)))
#define logistic_function(x) (native_recip ( 1.0f + native_exp ( -(x) ) ))
#else
#define exp_function exp
#define log_function log
#define log2_function log2
#define sqrt_function sqrt
#define rsqrt_function rsqrt
#define powr_function powr
#define divide_function divide
#define fmad_function(a,b,c) ((c) + (a) * (b))
#define logistic_function(x) (1.0f / ( 1.0f + exp ( -(x) ) ))
#endif

//Function to perform the atomic max
inline void AtomicMax(volatile __global float *source, const float operand) {
	union {
		unsigned int intVal;
		float floatVal;
	} newVal;
	union {
		unsigned int intVal;
		float floatVal;
	} prevVal;
	do {
		prevVal.floatVal = *source;
		newVal.floatVal = max(prevVal.floatVal, operand);
	} while (atomic_cmpxchg((volatile __global unsigned int *)source, prevVal.intVal, newVal.intVal) != prevVal.intVal);
}

//TODO: test, try changing localsize1, localsize2
//local reduction + atomic global reduction
//larger REDUCTION_WORKGROUP_SIZE means fewer atomic ops
#define REDUCTION_BODY_v1(out) { \
						\
						float maxval = out[0]; \
						\
						while (id < n) { \
							float x = xgm[id]; \
							if (x >= maxval) { \
								maxval = x; \
							} \
							id += wsize * num_groups; \
						} \
						maxlm[lid] = maxval; \
						barrier(CLK_LOCAL_MEM_FENCE); \
					  	for (int s=wsize/2; s>0; s=s>>1) { \
					    	if (lid <= s) { \
					      		if (maxlm[lid + s] >= maxlm[lid]) { \
					        		maxlm[lid] = maxlm[lid + s]; \
					      		} \
					    	} \
					    	barrier(CLK_LOCAL_MEM_FENCE); \
					    	\
					  	} \
						if (lid == 0) { \
							AtomicMax(&out[0], maxlm[0]); \
						} \
			\
		}

// https://developer.apple.com/library/content/samplecode/OpenCL_Parallel_Reduction_Example/Listings/reduce_float_kernel_cl.html#//apple_ref/doc/uid/DTS40008188-reduce_float_kernel_cl-DontLinkElementID_7
#define REDUCTION_BODY_v2(out) { \
						\
						float maxval = out[0]; \
						\
						while (id < n) { \
							float x = xgm[id]; \
							if (x >= maxval) { \
								maxval = x; \
							} \
							id += wsize * num_groups; \
						} \
						maxlm[lid] = maxval; \
						barrier(CLK_LOCAL_MEM_FENCE); \
						\
					  	for (int s=wsize/2; s>0; s=s>>1) { \
					    	if (lid <= s) { \
					      		if (maxlm[lid + s] >= maxlm[lid]) { \
					        		maxlm[lid] = maxlm[lid + s]; \
					      		} \
					    	} \
					    	barrier(CLK_LOCAL_MEM_FENCE); \
					    	\
					  	} \
						if (lid == 0) { \
							AtomicMax(&out[0], maxlm[0]); \
						} \
			\
		}

__kernel void max_coeff (__global float * restrict y, __global float * restrict xgm, const unsigned int n, __local float *maxlm) {

	const unsigned int lid = get_local_id(0);
	const unsigned int wgid = get_group_id(0);
	const unsigned int wsize = get_local_size (0);
	unsigned int id = wgid * wsize + lid; //get_global_id(0);
	const unsigned int num_groups = get_num_groups(0);

	REDUCTION_BODY_v2(y)

}


tweaking

tweaking

enum cell_type {
    FLUID = 0,
    NO_SLIP = 1,
    SRC = 2,
    COPY = 3
};

enum direction {
    NW = 0,
    N = 1,
    NE = 2,
    W = 3,
    C = 4,
    E = 5,
    SE = 6,
    S = 7,
    SW = 8
};

kernel void simulationStep(int width, int height,
                           global float* srcNW,
                           global float* srcN,
                           global float* srcNE,
                           global float* srcW,
                           global float* srcC,
                           global float* srcE,
                           global float* srcSW,
                           global float* srcS,
                           global float* srcSE,
                           global float* destNW,
                           global float* destN,
                           global float* destNE,
                           global float* destW,
                           global float* destC,
                           global float* destE,
                           global float* destSW,
                           global float* destS,
                           global float* destSE,
                           global int* flag_field) {

    global float* dest[9];
    global float* src[9];
    dest[NW] = destNW;
    dest[N] = destN;
    dest[NE] = destNE;
    dest[W] = destW;
    dest[C] = destC;
    dest[E] = destE;
    dest[SW] = destSW;
    dest[S] = destS;
    dest[SE] = destSE;

    src[NW] = srcNW;
    src[N] = srcN;
    src[NE] = srcNE;
    src[W] = srcW;
    src[C] = srcC;
    src[E] = srcE;
    src[SW] = srcSW;
    src[S] = srcS;
    src[SE] = srcSE;

    const int globalx = get_global_id(0);
    const int globaly = get_global_id(1);

    const int index = globaly*width + globalx;

    if( globalx < 0 || globalx >= width ||
        globaly < 0 || globaly >= height ) return;

    int dir_indices[9];
    dir_indices[NW] = index - width - 1;
    dir_indices[N]  = index - width;
    dir_indices[NE] = index - width + 1;
    dir_indices[W]  = index         - 1;
    dir_indices[C]  = index;
    dir_indices[E]  = index         + 1;
    dir_indices[SW] = index + width - 1;
    dir_indices[S]  = index + width;
    dir_indices[SE] = index + width + 1;

    if( flag_field[index] == NO_SLIP) return;

    int opposite[9];
    opposite[NW] = SE;
    opposite[N] = S;
    opposite[NE] = SW;
    opposite[W] = E;
    opposite[C] = C;
    opposite[E] = W;
    opposite[SW] = NE;
    opposite[S] = N;
    opposite[SE] = NW;

    float ftemp[9];

    if(flag_field[index] == COPY)
        return;

    if( flag_field[index] == FLUID) {
        //Collide
        float rho = 0;

        for( size_t i = 0; i < 9; i++) {
            rho += src[i][index];
        }

        float ux = ( src[NE][index] - src[NW][index] +
                     src[E][index] - src[W][index] +
                     src[SE][index] - src[SW][index] );

        float uy = ( src[SW][index] - src[NW][index] +
                     src[S][index] - src[N][index] +
                     src[SE][index] - src[NE][index] );

        ux /= rho;
        uy /= rho;

        float usquare = ux*ux+uy*uy;

        float f1 = 3.0f;
        float f2 = 9.0f/2.0f;
        float f3 = 3.0f/2.0f;
        float diag = 1.0f/36.0f;
        float axis = 1.0f/9.0f;
        float center = 4.0f/9.0f;

        float eq[9];

        eq[NW] =
            diag * rho * (1.0f + f1*(-ux-uy) + f2*(ux+uy)*(ux+uy) - f3* usquare);
        eq[N] =
            axis * rho * (1.0f + f1*(-uy) + f2*uy*uy - f3* usquare);
        eq[NE] =
            diag * rho * (1.0f + f1*(ux-uy) + f2*(ux-uy)*(ux-uy) - f3* usquare);
        eq[W] =
            axis * rho * (1.0f + f1*(-ux) + f2*ux*ux - f3*usquare);
        eq[C] =
            center * rho * (1.0f - f3*usquare);
        eq[E] =
            axis * rho * (1.0f + f1*(+ux) + f2*ux*ux - f3*usquare);
        eq[SW] =
            diag * rho * (1.0f + f1*(-ux+uy) + f2*(-ux+uy)*(-ux+uy) - f3* usquare);
        eq[S] =
            axis * rho * (1.0f + f1*(uy) + f2*uy*uy - f3* usquare);
        eq[SE] =
            diag * rho * (1.0f + f1*(ux+uy) + f2*(ux+uy)*(ux+uy) - f3* usquare);

        for( size_t i = 0; i < 9;i++) {
            ftemp[i] = src[i][index] - (src[i][index]-eq[i]) / 0.7f;
        }

    } else if( flag_field[index] == SRC) {
        for( size_t i = 0; i < 9; i++) {
            ftemp[i] = src[i][index];
        }
    }
    /*
    if( globalx == 2 && globaly == 10) {

        for( size_t i = 0; i < 3; i++) {
            printf("%f %f %f\n",
                   src[i*3][index], src[i*3+1][index], src[i*3+2][index]);
        }
        printf("\n");
        for( size_t i = 0; i < 3; i++) {
            printf("%f %f %f\n", ftemp[i*3], ftemp[i*3+1], ftemp[i*3+2]);
        }
        printf("\n\n");

    }*/

    if( flag_field[index] != NO_SLIP) {

        for( size_t i = 0; i < 9; i++) {
            if( flag_field[dir_indices[i]] == FLUID) {
                dest[i][dir_indices[i]] = ftemp[i];
            } else if( flag_field[dir_indices[i]] == NO_SLIP) {
                dest[opposite[i]][index] = ftemp[i];
            }
            if( flag_field[dir_indices[i]] == COPY) {
                dest[i][index] = ftemp[i];
            }
        }
    }

    return;
}

enum cell_type {
    FLUID = 0,
    NO_SLIP = 1,
    SRC = 2,
    COPY = 3
};

enum direction {
    NW = 0,
    N = 1,
    NE = 2,
    W = 3,
    C = 4,
    E = 5,
    SE = 6,
    S = 7,
    SW = 8
};

kernel void simulationStep(int width, int height,
                           global float* srcNW,
                           global float* srcN,
                           global float* srcNE,
                           global float* srcW,
                           global float* srcC,
                           global float* srcE,
                           global float* srcSW,
                           global float* srcS,
                           global float* srcSE,
                           global float* destNW,
                           global float* destN,
                           global float* destNE,
                           global float* destW,
                           global float* destC,
                           global float* destE,
                           global float* destSW,
                           global float* destS,
                           global float* destSE,
                           global int* flag_field) {

    global float* dest[9];
    global float* src[9];
    dest[NW] = destNW;
    dest[N] = destN;
    dest[NE] = destNE;
    dest[W] = destW;
    dest[C] = destC;
    dest[E] = destE;
    dest[SW] = destSW;
    dest[S] = destS;
    dest[SE] = destSE;

    src[NW] = srcNW;
    src[N] = srcN;
    src[NE] = srcNE;
    src[W] = srcW;
    src[C] = srcC;
    src[E] = srcE;
    src[SW] = srcSW;
    src[S] = srcS;
    src[SE] = srcSE;

    const int globalx = get_global_id(0);
    const int globaly = get_global_id(1);

    const int index = globaly*width + globalx;

    if( globalx < 0 || globalx >= width ||
        globaly < 0 || globaly >= height ) return;

    int dir_indices[9];
    dir_indices[NW] = index - width - 1;
    dir_indices[N]  = index - width;
    dir_indices[NE] = index - width + 1;
    dir_indices[W]  = index         - 1;
    dir_indices[C]  = index;
    dir_indices[E]  = index         + 1;
    dir_indices[SW] = index + width - 1;
    dir_indices[S]  = index + width;
    dir_indices[SE] = index + width + 1;

    if( flag_field[index] == NO_SLIP) return;

    int opposite[9];
    opposite[NW] = SE;
    opposite[N] = S;
    opposite[NE] = SW;
    opposite[W] = E;
    opposite[C] = C;
    opposite[E] = W;
    opposite[SW] = NE;
    opposite[S] = N;
    opposite[SE] = NW;

    float ftemp[9];

    if(flag_field[index] == COPY)
        return;

    if( flag_field[index] == FLUID) {
        //Collide
        float rho = 0;

        for( size_t i = 0; i < 9; i++) {
            rho += src[i][index];
        }

        float ux = ( src[NE][index] - src[NW][index] +
                     src[E][index] - src[W][index] +
                     src[SE][index] - src[SW][index] );

        float uy = ( src[SW][index] - src[NW][index] +
                     src[S][index] - src[N][index] +
                     src[SE][index] - src[NE][index] );

        ux /= rho;
        uy /= rho;

        float usquare = ux*ux+uy*uy;

        float f1 = 3.0f;
        float f2 = 9.0f/2.0f;
        float f3 = 3.0f/2.0f;
        float diag = 1.0f/36.0f;
        float axis = 1.0f/9.0f;
        float center = 4.0f/9.0f;

        float eq[9];

        eq[NW] =
            diag * rho * (1.0f + f1*(-ux-uy) + f2*(ux+uy)*(ux+uy) - f3* usquare);
        eq[N] =
            axis * rho * (1.0f + f1*(-uy) + f2*uy*uy - f3* usquare);
        eq[NE] =
            diag * rho * (1.0f + f1*(ux-uy) + f2*(ux-uy)*(ux-uy) - f3* usquare);
        eq[W] =
            axis * rho * (1.0f + f1*(-ux) + f2*ux*ux - f3*usquare);
        eq[C] =
            center * rho * (1.0f - f3*usquare);
        eq[E] =
            axis * rho * (1.0f + f1*(+ux) + f2*ux*ux - f3*usquare);
        eq[SW] =
            diag * rho * (1.0f + f1*(-ux+uy) + f2*(-ux+uy)*(-ux+uy) - f3* usquare);
        eq[S] =
            axis * rho * (1.0f + f1*(uy) + f2*uy*uy - f3* usquare);
        eq[SE] =
            diag * rho * (1.0f + f1*(ux+uy) + f2*(ux+uy)*(ux+uy) - f3* usquare);

        for( size_t i = 0; i < 9;i++) {
            ftemp[i] = src[i][index] - (src[i][index]-eq[i]) / 0.80f;
        }

    } else if( flag_field[index] == SRC) {
        for( size_t i = 0; i < 9; i++) {
            ftemp[i] = src[i][index];
        }
    }
    /*
    if( globalx == 2 && globaly == 10) {

        for( size_t i = 0; i < 3; i++) {
            printf("%f %f %f\n",
                   src[i*3][index], src[i*3+1][index], src[i*3+2][index]);
        }
        printf("\n");
        for( size_t i = 0; i < 3; i++) {
            printf("%f %f %f\n", ftemp[i*3], ftemp[i*3+1], ftemp[i*3+2]);
        }
        printf("\n\n");

    }*/

    if( flag_field[index] != NO_SLIP) {

        for( size_t i = 0; i < 9; i++) {
            if( flag_field[dir_indices[i]] == FLUID) {
                dest[i][dir_indices[i]] = ftemp[i];
            } else if( flag_field[dir_indices[i]] == NO_SLIP) {
                dest[opposite[i]][index] = ftemp[i];
            }
            if( flag_field[dir_indices[i]] == COPY) {
                dest[i][index] = ftemp[i];
            }
        }
    }

    return;
}


update backproperrorsv2cached.cl to use copyLocal function

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

void copyLocal( local float *target, global float const *source, int N ) {
    int numLoops = ( N + get_local_size(0) - 1 ) / get_local_size(0);
    for( int loop = 0; loop < numLoops; loop++ ) {
        int offset = loop * get_local_size(0) + get_local_id(0);
        if( offset < N ) {
            target[offset] = source[offset];
        }
    }
}

// as calcErrorsForUpstream, but with local cache
// convolve weights with errors to produce errorsForUpstream
// workgroupid: [n][inputPlane]
// localid: [upstreamrow][upstreamcol]
// per-thread aggregation: [outPlane][filterRow][filterCol]
// need to store locally:
// - _errorBoard. size = outputBoardSizeSquared
// - _filterBoard. size = filtersizesquared
// note: currently doesnt use bias as input.  thats probably an error?
// inputs: errors :convolve: filters => errorsForUpstream
//
// global:
// errors: [n][outPlane][outRow][outCol] 128 * 32 * 19 * 19 * 4
// weights: [filterId][upstreamplane][filterRow][filterCol] 32 * 32 * 5 * 5 * 4
// per workgroup:
// errors: [outPlane][outRow][outCol] 32 * 19 * 19 * 4 = 46KB
// weights: [filterId][filterRow][filterCol] 32 * 5 * 5 * 4 = 3.2KB
// errorsforupstream: [n][upstreamPlane][upstreamRow][upstreamCol]
void kernel calcErrorsForUpstreamCached(
        const int batchSize,
        global const float *errorsGlobal,
        global const float *filtersGlobal,
        global float *errorsForUpstream,
        local float *_errorBoard,
        local float *_filterBoard ) {

    const int globalId = get_global_id(0);
    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);

    const int n = workgroupId / gInputPlanes;
    const int upstreamPlane = workgroupId % gInputPlanes;

    const int upstreamRow = localId / gInputBoardSize;
    const int upstreamCol = localId % gInputBoardSize;

    const int minFilterRow = max( 0, upstreamRow + gMargin - (gOutputBoardSize - 1) );
    const int maxFilterRow = min( gFilterSize - 1, upstreamRow + gMargin );
    const int minFilterCol = max( 0, upstreamCol + gMargin - (gOutputBoardSize -1) );
    const int maxFilterCol = min( gFilterSize - 1, upstreamCol + gMargin );

    float sumWeightTimesOutError = 0;
    for( int outPlane = 0; outPlane < gNumFilters; outPlane++ ) {
        barrier(CLK_LOCAL_MEM_FENCE);
        copyLocal( _filterBoard, filtersGlobal + ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared, gFilterSizeSquared );
        copyLocal( _errorBoard, errorsGlobal + ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared, gOutputBoardSizeSquared );
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int filterRow = minFilterRow; filterRow <= maxFilterRow; filterRow++ ) {
            int outRow = upstreamRow + gMargin - filterRow;
            for( int filterCol = minFilterCol; filterCol <= maxFilterCol; filterCol++ ) {
                int outCol = upstreamCol + gMargin - filterCol;
                int resultIndex = outRow * gOutputBoardSize + outCol;
                float thisError = _errorBoard[resultIndex];
                int thisWeightIndex = filterRow * gFilterSize + filterCol;
                float thisWeight = _filterBoard[thisWeightIndex];
                float thisWeightTimesError = thisWeight * thisError;
                sumWeightTimesOutError += thisWeightTimesError;
            }
        }
    }
    const int upstreamBoardGlobalOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
    if( localId < gInputBoardSizeSquared ) {
        errorsForUpstream[upstreamBoardGlobalOffset + localId] = sumWeightTimesOutError;
    }
}

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

void copyLocal( local float *target, global float const *source, int N ) {
    int numLoops = ( N + get_local_size(0) - 1 ) / get_local_size(0);
    for( int loop = 0; loop < numLoops; loop++ ) {
        int offset = loop * get_local_size(0) + get_local_id(0);
        if( offset < N ) {
            target[offset] = source[offset];
        }
    }
}

// as calcErrorsForUpstream, but with local cache
// convolve weights with errors to produce errorsForUpstream
// workgroupid: [n][inputPlane]
// localid: [upstreamrow][upstreamcol]
// per-thread aggregation: [outPlane][filterRow][filterCol]
// need to store locally:
// - _errorBoard. size = outputBoardSizeSquared
// - _filterBoard. size = filtersizesquared
// note: currently doesnt use bias as input.  thats probably an error?
// inputs: errors :convolve: filters => errorsForUpstream
//
// global:
// errors: [n][outPlane][outRow][outCol] 128 * 32 * 19 * 19 * 4
// weights: [filterId][upstreamplane][filterRow][filterCol] 32 * 32 * 5 * 5 * 4
// per workgroup:
// errors: [outPlane][outRow][outCol] 32 * 19 * 19 * 4 = 46KB
// weights: [filterId][filterRow][filterCol] 32 * 5 * 5 * 4 = 3.2KB
// errorsforupstream: [n][upstreamPlane][upstreamRow][upstreamCol]
void kernel calcErrorsForUpstreamCached(
        const int batchSize,
        global const float *errorsGlobal,
        global const float *filtersGlobal,
        global float *errorsForUpstream,
        local float *_errorBoard,
        local float *_filterBoard ) {

    const int globalId = get_global_id(0);
    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);

    const int n = workgroupId / gInputPlanes;
    const int upstreamPlane = workgroupId % gInputPlanes;

    const int upstreamRow = localId / gInputBoardSize;
    const int upstreamCol = localId % gInputBoardSize;

    const int minFilterRow = max( 0, upstreamRow + gMargin - (gOutputBoardSize - 1) );
    const int maxFilterRow = min( gFilterSize - 1, upstreamRow + gMargin );
    const int minFilterCol = max( 0, upstreamCol + gMargin - (gOutputBoardSize -1) );
    const int maxFilterCol = min( gFilterSize - 1, upstreamCol + gMargin );

    const int filterPixelCopiesPerThread = ( gFilterSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int errorPixelCopiesPerThread = ( gOutputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int pixelCopiesPerThread = max( filterPixelCopiesPerThread, errorPixelCopiesPerThread );

    float sumWeightTimesOutError = 0;
    for( int outPlane = 0; outPlane < gNumFilters; outPlane++ ) {
        barrier(CLK_LOCAL_MEM_FENCE);
        copyLocal( _filterBoard, filtersGlobal + ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared, gFilterSizeSquared );
        copyLocal( _errorBoard, errorsGlobal + ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared, gOutputBoardSizeSquared );
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int filterRow = minFilterRow; filterRow <= maxFilterRow; filterRow++ ) {
            int outRow = upstreamRow + gMargin - filterRow;
            for( int filterCol = minFilterCol; filterCol <= maxFilterCol; filterCol++ ) {
                int outCol = upstreamCol + gMargin - filterCol;
                int resultIndex = outRow * gOutputBoardSize + outCol;
                float thisError = _errorBoard[resultIndex];
                int thisWeightIndex = filterRow * gFilterSize + filterCol;
                float thisWeight = _filterBoard[thisWeightIndex];
                float thisWeightTimesError = thisWeight * thisError;
                sumWeightTimesOutError += thisWeightTimesError;
            }
        }
    }
    const int upstreamBoardGlobalOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
    if( localId < gInputBoardSizeSquared ) {
        errorsForUpstream[upstreamBoardGlobalOffset + localId] = sumWeightTimesOutError;
    }
}