barvy meanshift

barvy meanshift

/*
 * Accelerated k-means and mean-shift algorithms via OpenCL
 * Authors: Martin Simon & Pavel Sirucek
 *
 * OpenCL kernels
 */

/* Implementace K-means
 * Prozatim v jednom kernelu (vyhoda je ze se nemusi sznchronizovat postup zpracovani kernelu,
                              ani si nejsem jisty jak to udelat, "nevyhodou" je uzke hrdlo pri pre
							  pocitavani stredu, kdy pracuje K vlaken a ostatni cekaji, tohle se
							  vetsim poctem kernelu nevyresi, protoze by stejne vlakna musely
							  pockat na prepocitani stredu)
 * Postup zpracovani (jak by mel byt, nejsem si jisty zda to tak je):
 *
 * Kazdy pixel se priradi do clusteru (k nejblizsimu stredu).
 * Pomoci barrier() se pocka na vsechna vlakna, pote se K vlaken
 * postara o prepocitani stredu. Pocka se na vsechna prepocitani a jede se znovu.
 */
 /* TODO - velke obrazky, zpracuje se cast..problem bude nekde u maximalniho poctu vlaken nebo tak
         - artefakty u nejakych obrazku, pripadne jednou to zpracuje dobre, podruhe podivne
		 - doba zpracovani (?)
		 - zadani parametru K z cmdl, omezeni na max napr 16.
		 - doladit (nekonceny cyklus napr, detekce zmeny clsuteru apod.) (?)
 */
__kernel void kmeans(__global uchar4* input, __global uchar4* output, __global uchar4* centroids, __global uint* pixels, uint width, uint height, uint K)
{
	uint gidX = get_global_id(0);
	uint gidY = get_global_id(1);
	bool center_change = true; // detekce zmeny stredu (prepocitani)

	uint pixel_index = gidX + width * gidY;
	float min_dist; // nejmensi vzdalenost

	uint last_cluster; // minuly cluster nebo stred
	float last_dist = 0.0f; // minula vzdalenost

	while (true)
	{
		for (uint i = 0; i < K; i++)
		{
			// spocteni vzdalenosti pixelu od stredu
			float dist = 0.0f;
			float3 distxyz;
			distxyz.x = convert_float(centroids[i].x) - convert_float(input[pixel_index].x);
			distxyz.x *= distxyz.x;
			distxyz.y = convert_float(centroids[i].y) - convert_float(input[pixel_index].y);
			distxyz.y *= distxyz.y;
			distxyz.z = convert_float(centroids[i].z) - convert_float(input[pixel_index].z);
			distxyz.z *= distxyz.z;
			dist = distxyz.x + distxyz.y + distxyz.z;

			if (i == 0)
			{
				min_dist = dist;
				pixels[pixel_index] = i;
			}
			else
			{
				if (dist < min_dist)
				{ // prirazeni pixelu do noveho clusteru
					min_dist = dist;
					pixels[pixel_index] = i;
				}
			}

		}

		// konec pokud se pixel nepresunul, vzdalenost zustala stejna (tedy zadna zmena) a pokud se neprepocitavaji stredy
		if (last_cluster == pixels[pixel_index] && last_dist == min_dist && !center_change)
		{
			output[pixel_index] = centroids[pixels[pixel_index]];
			output[pixel_index].w = 255;
			return;
		}

		last_cluster = pixels[pixel_index];
		last_dist = min_dist;
		// pockame na zpracovani vsech pixelu
		barrier(CLK_GLOBAL_MEM_FENCE);

		// poslani K vlaken na clustery -> prepocitani stredu
		if (gidX < K && center_change)
		{
			// prumer hodnot
			float4 sum = {0.0f, 0.0f, 0.0f, 0.0f};
			uint num = 0;

			for (uint i = 0; i < width * height; i++)
			{
				if (pixels[i] == gidX)
				{
					sum += convert_float4(input[i]);
					num++;
				}
			}
			// novy stred
			uchar4 newCenter = convert_uchar4(sum / convert_float(num));

			if (centroids[gidX].x != newCenter.x || centroids[gidX].y != newCenter.y || centroids[gidX].z != newCenter.z)
			{
				centroids[gidX] = newCenter;
				centroids[gidX].w = 255;
			}
			else // stred se nezmenil
			{
				center_change = false;
			}
		}
		else // toto vlakno neprepocitava stredy
		{
			center_change = false;
		}

		barrier(CLK_GLOBAL_MEM_FENCE);
	}
}

__kernel void meanshift(__global uchar4* input, __global uchar4* output, __global uint* peaks, __global uint* counts, uint width, uint height, uint winsize, __global int* peakcount, __global uint* uniquepeaks, __global uchar4* colors)
{
int x = get_global_id(0);
int y = get_global_id(1);

//reconstruct window by given param
float actx = convert_float(x);
float acty = convert_float(y);
int wymax = acty + ((winsize-1) / 2) + 1;
int wymin = acty - ((winsize-1) / 2);
int wxmax = actx + ((winsize-1) / 2) + 1;
int wxmin = actx - ((winsize-1) / 2);

float oldx = convert_float(x);
float oldy = convert_float(y);

//cycle control value
bool cont = true;
int iter = 0;
//mean structures
float newX, newY;

//begincycle - until the step is bigger than relevant
do {
    newX = newY = 0;
    for (int wy = wymin; wy < wymax; wy++)
    {
        for (int wx = wxmin; wx < wxmax; wx++)
        {
            if (wx < 0 || wy < 0 || wx >= width|| wy >= height)
            {
                //out of original window
                //set 0 to the position of the x in local cache
                continue;
            }
            else if (wx == convert_int(actx) && wy == convert_int(acty))
            {
                //set 1 to the position of the x
                //to [actx-wxmin,acty-wymin] set 1

                newX += (wx - actx);
                newY += (wy - acty);
            }
            else
            {
                //compute lengths for [wx,wy] - [x,y]
                //if the length is bigger the maxlength, set 0 there
                //otherwise set the length
                int gid = convert_int_rte(actx) + convert_int_rte(acty)*width;
                float length = 1/sqrt(convert_float(
                    (actx-wx)*(actx-wx) +
                    (acty-wy)*(acty-wy) +
                    (input[gid].s0 - input[wx + wy*width].s0)*(input[gid].s0 - input[wx + wy*width].s0) +
                    (input[gid].s1 - input[wx + wy*width].s1)*(input[gid].s1 - input[wx + wy*width].s1) +
                    (input[gid].s2 - input[wx + wy*width].s2)*(input[gid].s2 - input[wx + wy*width].s2)));

                newX += (wx - actx) * length;
                newY += (wy - acty) * length;
            }
        }
    }
//printf("[%d:%d]- old:[%f,%f], new diff:[%f,%f]\n", x,y,actx,acty,newX,newY);
    //recompute mean of the window
    oldx = actx;
    oldy = acty;

    actx = actx + newX;
    acty = acty + newY;

    //shift the window to the mean be in the center
    wymax = convert_int_rte(acty) + ((winsize-1) / 2) + 1;
    wymin = convert_int_rte(acty) - ((winsize-1) / 2);
    wxmax = convert_int_rte(actx) + ((winsize-1) / 2) + 1;
    wxmin = convert_int_rte(actx) - ((winsize-1) / 2);

    if((newX < 0 ? -1*newX : newX) <= 0.2 && (newY < 0 ? -1*newY : newY) <= 0.2)
    {
        //the step is lower the 0.5pix
        cont = false;
    }

    //endcycle
    iter++;
} while (cont && iter < 1000);

actx = convert_float(max(convert_int_rte(actx),0));
acty = convert_float(max(convert_int_rte(acty),0));

actx = convert_float(min(convert_int_rte(actx),convert_int_rte(width)-1));
acty = convert_float(min(convert_int_rte(acty),convert_int_rte(height)-1));

int gid = x + y*width;

//store the peak position to peaks array
peaks[gid] = convert_int_rte(actx) + convert_int_rte(acty)*width;

//increment value on peak position in counts array
counts[convert_int_rte(actx) + convert_int_rte(acty)*width] = counts[convert_int_rte(actx) + convert_int_rte(acty)*width] + 1;

barrier(CLK_GLOBAL_MEM_FENCE);

int peak = peaks[gid];

for (int i=0; i<100;i++)
{
    if(peaks[peaks[peak]] == peaks[peak])
    {
        break;
    }
    peaks[gid] = peaks[peak];
    peak = peaks[peaks[peak]];
}

barrier(CLK_GLOBAL_MEM_FENCE);

if (peakcount[0] == -1)
{
    peakcount[0] = 0; //lock the critical section

    //begin of critical section
    printf("[%d,%d] critic section - begin\n",x,y);
    for (int yy=0; yy<height; yy++)
    {
        for (int xx=0; xx<width; xx++)
        {
            int actpeak = peaks[xx + width*yy];

            /* Search the array for actpeak */
            int i;
            for (i=0; i<peakcount[0];i++)
            {
                if (uniquepeaks[i] == actpeak)
                {
                    break;
                }
            }

            /* If went through all the elements = haven't found */
            if (i == peakcount[0])
            {
                uniquepeaks[peakcount[0]++] = actpeak;
            }
        }
    }
    printf("pocet vrcholu: %d\n", peakcount[0]);
    //TODO rozlozim barvy a vytvorim mapovaci funkci
    int colorStep = convert_int_rte((256*256*256)/peakcount[0]);

    for (int n =0; n<peakcount[0]; n++)
    {
        colors[uniquepeaks[n]] = (n*colorStep);
    }

    printf("[%d,%d] critic section - end\n",x,y);
}

barrier(CLK_GLOBAL_MEM_FENCE);
//tady se podivam jestli globalni counter je 0, pak vyhledavam, jinak jdu k dalsi bariere
//vyhledavam postupne pokud jsem jeste nenasel

//TODO obarvim podle colors[peaks[gid]];
//output[gid].s0 = 255-peaks[gid];
//output[gid].s1 = 255-peaks[gid];
//output[gid].s2 = 255-peaks[gid];
//printf("[%d,%d,%d]\n",colors[gid].s0,colors[gid].s1,colors[gid].s2);
output[gid] = colors[peak];

output[gid].s3 = 255;
}

/*
 * Accelerated k-means and mean-shift algorithms via OpenCL
 * Authors: Martin Simon & Pavel Sirucek
 *
 * OpenCL kernels
 */

/* Implementace K-means
 * Prozatim v jednom kernelu (vyhoda je ze se nemusi sznchronizovat postup zpracovani kernelu,
                              ani si nejsem jisty jak to udelat, "nevyhodou" je uzke hrdlo pri pre
							  pocitavani stredu, kdy pracuje K vlaken a ostatni cekaji, tohle se
							  vetsim poctem kernelu nevyresi, protoze by stejne vlakna musely
							  pockat na prepocitani stredu)
 * Postup zpracovani (jak by mel byt, nejsem si jisty zda to tak je):
 *
 * Kazdy pixel se priradi do clusteru (k nejblizsimu stredu).
 * Pomoci barrier() se pocka na vsechna vlakna, pote se K vlaken
 * postara o prepocitani stredu. Pocka se na vsechna prepocitani a jede se znovu.
 */
 /* TODO - velke obrazky, zpracuje se cast..problem bude nekde u maximalniho poctu vlaken nebo tak
         - artefakty u nejakych obrazku, pripadne jednou to zpracuje dobre, podruhe podivne
		 - doba zpracovani (?)
		 - zadani parametru K z cmdl, omezeni na max napr 16.
		 - doladit (nekonceny cyklus napr, detekce zmeny clsuteru apod.) (?)
 */
__kernel void kmeans(__global uchar4* input, __global uchar4* output, __global uchar4* centroids, __global uint* pixels, uint width, uint height, uint K)
{
	uint gidX = get_global_id(0);
	uint gidY = get_global_id(1);
	bool center_change = true; // detekce zmeny stredu (prepocitani)

	uint pixel_index = gidX + width * gidY;
	float min_dist; // nejmensi vzdalenost

	uint last_cluster; // minuly cluster nebo stred
	float last_dist = 0.0f; // minula vzdalenost

	while (true)
	{
		for (uint i = 0; i < K; i++)
		{
			// spocteni vzdalenosti pixelu od stredu
			float dist = 0.0f;
			float3 distxyz;
			distxyz.x = convert_float(centroids[i].x) - convert_float(input[pixel_index].x);
			distxyz.x *= distxyz.x;
			distxyz.y = convert_float(centroids[i].y) - convert_float(input[pixel_index].y);
			distxyz.y *= distxyz.y;
			distxyz.z = convert_float(centroids[i].z) - convert_float(input[pixel_index].z);
			distxyz.z *= distxyz.z;
			dist = distxyz.x + distxyz.y + distxyz.z;

			if (i == 0)
			{
				min_dist = dist;
				pixels[pixel_index] = i;
			}
			else
			{
				if (dist < min_dist)
				{ // prirazeni pixelu do noveho clusteru
					min_dist = dist;
					pixels[pixel_index] = i;
				}
			}

		}

		// konec pokud se pixel nepresunul, vzdalenost zustala stejna (tedy zadna zmena) a pokud se neprepocitavaji stredy
		if (last_cluster == pixels[pixel_index] && last_dist == min_dist && !center_change)
		{
			output[pixel_index] = centroids[pixels[pixel_index]];
			output[pixel_index].w = 255;
			return;
		}

		last_cluster = pixels[pixel_index];
		last_dist = min_dist;
		// pockame na zpracovani vsech pixelu
		barrier(CLK_GLOBAL_MEM_FENCE);

		// poslani K vlaken na clustery -> prepocitani stredu
		if (gidX < K && center_change)
		{
			// prumer hodnot
			float4 sum = {0.0f, 0.0f, 0.0f, 0.0f};
			uint num = 0;

			for (uint i = 0; i < width * height; i++)
			{
				if (pixels[i] == gidX)
				{
					sum += convert_float4(input[i]);
					num++;
				}
			}
			// novy stred
			uchar4 newCenter = convert_uchar4(sum / convert_float(num));

			if (centroids[gidX].x != newCenter.x || centroids[gidX].y != newCenter.y || centroids[gidX].z != newCenter.z)
			{
				centroids[gidX] = newCenter;
				centroids[gidX].w = 255;
			}
			else // stred se nezmenil
			{
				center_change = false;
			}
		}
		else // toto vlakno neprepocitava stredy
		{
			center_change = false;
		}

		barrier(CLK_GLOBAL_MEM_FENCE);
	}
}

__kernel void meanshift(__global uchar4* input, __global uchar4* output, __global uint* peaks, __global uint* counts, uint width, uint height, uint winsize, __global int* peakcount, __global uint* uniquepeaks, __global uchar4* colors)
{
int x = get_global_id(0);
int y = get_global_id(1);

//reconstruct window by given param
float actx = convert_float(x);
float acty = convert_float(y);
int wymax = acty + ((winsize-1) / 2) + 1;
int wymin = acty - ((winsize-1) / 2);
int wxmax = actx + ((winsize-1) / 2) + 1;
int wxmin = actx - ((winsize-1) / 2);

float oldx = convert_float(x);
float oldy = convert_float(y);

//cycle control value
bool cont = true;
int iter = 0;
//mean structures
float newX, newY;

//begincycle - until the step is bigger than relevant
do {
    newX = newY = 0;
    for (int wy = wymin; wy < wymax; wy++)
    {
        for (int wx = wxmin; wx < wxmax; wx++)
        {
            if (wx < 0 || wy < 0 || wx >= width|| wy >= height)
            {
                //out of original window
                //set 0 to the position of the x in local cache
                continue;
            }
            else if (wx == convert_int(actx) && wy == convert_int(acty))
            {
                //set 1 to the position of the x
                //to [actx-wxmin,acty-wymin] set 1

                newX += (wx - actx);
                newY += (wy - acty);
            }
            else
            {
                //compute lengths for [wx,wy] - [x,y]
                //if the length is bigger the maxlength, set 0 there
                //otherwise set the length
                int gid = convert_int_rte(actx) + convert_int_rte(acty)*width;
                float length = 1/sqrt(convert_float(
                    (actx-wx)*(actx-wx) +
                    (acty-wy)*(acty-wy) +
                    (input[gid].s0 - input[wx + wy*width].s0)*(input[gid].s0 - input[wx + wy*width].s0) +
                    (input[gid].s1 - input[wx + wy*width].s1)*(input[gid].s1 - input[wx + wy*width].s1) +
                    (input[gid].s2 - input[wx + wy*width].s2)*(input[gid].s2 - input[wx + wy*width].s2)));

                newX += (wx - actx) * length;
                newY += (wy - acty) * length;
            }
        }
    }
//printf("[%d:%d]- old:[%f,%f], new diff:[%f,%f]\n", x,y,actx,acty,newX,newY);
    //recompute mean of the window
    oldx = actx;
    oldy = acty;

    actx = actx + newX;
    acty = acty + newY;

    //shift the window to the mean be in the center
    wymax = convert_int_rte(acty) + ((winsize-1) / 2) + 1;
    wymin = convert_int_rte(acty) - ((winsize-1) / 2);
    wxmax = convert_int_rte(actx) + ((winsize-1) / 2) + 1;
    wxmin = convert_int_rte(actx) - ((winsize-1) / 2);

    if((newX < 0 ? -1*newX : newX) <= 0.2 && (newY < 0 ? -1*newY : newY) <= 0.2)
    {
        //the step is lower the 0.5pix
        cont = false;
    }

    //endcycle
    iter++;
} while (cont && iter < 1000);

actx = convert_float(max(convert_int_rte(actx),0));
acty = convert_float(max(convert_int_rte(acty),0));

actx = convert_float(min(convert_int_rte(actx),convert_int_rte(width)-1));
acty = convert_float(min(convert_int_rte(acty),convert_int_rte(height)-1));

int gid = x + y*width;

//store the peak position to peaks array
peaks[gid] = convert_int_rte(actx) + convert_int_rte(acty)*width;

//increment value on peak position in counts array
counts[convert_int_rte(actx) + convert_int_rte(acty)*width] = counts[convert_int_rte(actx) + convert_int_rte(acty)*width] + 1;

barrier(CLK_GLOBAL_MEM_FENCE);

int peak = peaks[gid];

for (int i=0; i<100;i++)
{
    if(peaks[peaks[peak]] == peaks[peak])
    {
        break;
    }
    peaks[gid] = peaks[peak];
    peak = peaks[peaks[peak]];
}

barrier(CLK_GLOBAL_MEM_FENCE);

if (peakcount[0] == -1)
{
    peakcount[0] = 0; //lock the critical section

    //begin of critical section
    printf("[%d,%d] critic section - begin\n",x,y);
    for (int yy=0; yy<height; yy++)
    {
        for (int xx=0; xx<width; xx++)
        {
            int actpeak = peaks[xx + width*yy];

            /* Search the array for actpeak */
            int i;
            for (i=0; i<peakcount[0];i++)
            {
                if (uniquepeaks[i] == actpeak)
                {
                    break;
                }
            }

            /* If went through all the elements = haven't found */
            if (i == peakcount[0])
            {
                uniquepeaks[peakcount[0]++] = actpeak;
            }
        }
    }
    printf("pocet vrcholu: %d\n", peakcount[0]);
    //TODO rozlozim barvy a vytvorim mapovaci funkci

    printf("[%d,%d] critic section - end\n",x,y);
}

barrier(CLK_GLOBAL_MEM_FENCE);
//tady se podivam jestli globalni counter je 0, pak vyhledavam, jinak jdu k dalsi bariere
//vyhledavam postupne pokud jsem jeste nenasel

//TODO obarvim podle colors[peaks[gid]];
//output[gid].s0 = 255-peaks[gid];
//output[gid].s1 = 255-peaks[gid];
//output[gid].s2 = 255-peaks[gid];
printf("%d\n",counts[gid]);
output[gid] = peak;

output[gid].s3 = 255;
}

__kernel void mean_shift_begin(__global uchar4* input, __global uint* peaks, __global uint* counts,
__local float* cache, uint width, uint height, uint winsize, float maxlength)
{
int x = get_global_id(0);
int y = get_global_id(1);

//reconstruct window by given param
float actx = convert_float(x);
float acty = convert_float(y);
int wymax = acty + ((winsize-1) / 2) + 1;
int wymin = acty - ((winsize-1) / 2);
int wxmax = actx + ((winsize-1) / 2) + 1;
int wxmin = actx - ((winsize-1) / 2);

float oldx = convert_float(x);
float oldy = convert_float(y);

//cycle control value
bool cont = true;
int iter = 0;
//mean structures
float newX, newY;

//begincycle - until the step is bigger than relevant
do {
    newX = newY = 0;
    for (int wy = wymin; wy < wymax; wy++)
    {
        for (int wx = wxmin; wx < wxmax; wx++)
        {
            if (wx < 0 || wy < 0 || wx > width-1|| wy > height-1)
            {
                //out of original window
                //set 0 to the position of the x in local cache
                continue;
            }
            else if (wx == convert_int(actx) && wy == convert_int(acty))
            {
                //set 1 to the position of the x
                //to [actx-wxmin,acty-wymin] set 1

                newX += (wx - actx);
                newY += (wy - acty);
            }
            else
            {
                //compute lengths for [wx,wy] - [x,y]
                //if the length is bigger the maxlength, set 0 there
                //otherwise set the length
                int gid = convert_int_rte(actx) + convert_int_rte(acty)*width;
                float length = 1/sqrt(convert_float(
                    (actx-wx)*(actx-wx) +
                    (acty-wy)*(acty-wy) +
                    (input[gid].s0 - input[wx + wy*width].s0)*(input[gid].s0 - input[wx + wy*width].s0) +
                    (input[gid].s1 - input[wx + wy*width].s1)*(input[gid].s1 - input[wx + wy*width].s1) +
                    (input[gid].s2 - input[wx + wy*width].s2)*(input[gid].s2 - input[wx + wy*width].s2)));

                newX += (wx - actx) * length;
                newY += (wy - acty) * length;
            }
        }
    }
//printf("[%d:%d]- old:[%f,%f], new diff:[%f,%f]\n", x,y,actx,acty,newX,newY);
    //recompute mean of the window
    oldx = actx;
    oldy = acty;

    actx = actx + newX;
    acty = acty + newY;

    //shift the window to the mean be in the center
    wymax = convert_int_rte(acty) + ((winsize-1) / 2) + 1;
    wymin = convert_int_rte(acty) - ((winsize-1) / 2);
    wxmax = convert_int_rte(actx) + ((winsize-1) / 2) + 1;
    wxmin = convert_int_rte(actx) - ((winsize-1) / 2);

    if((newX < 0 ? -1*newX : newX) <= 0.2 && (newY < 0 ? -1*newY : newY) <= 0.2)
    {
        //the step is lower the 0.5pix
        cont = false;
    }

    //endcycle
    iter++;
} while (cont);

actx = convert_float(max(convert_int_rte(actx),0));
acty = convert_float(max(convert_int_rte(acty),0));

actx = convert_float(min(convert_int_rte(actx),convert_int_rte(width)-1));
acty = convert_float(min(convert_int_rte(acty),convert_int_rte(height)-1));

//printf("[%d,%d]/%d reached: [%d, %d]\n",x,y,iter,convert_int_rte(actx),convert_int_rte(acty));

//store the peak position to peaks array
peaks[x+y*width] = convert_int_rte(actx) + convert_int_rte(acty)*width;

//increment value on peak position in counts array
counts[convert_int_rte(actx) + convert_int_rte(acty)*width] = counts[convert_int_rte(actx) + convert_int_rte(acty)*width] + 1;

}

__kernel void mean_shift_peaks(__global uint* peaks, uint width, uint height)
{
    int x = get_global_id(0);
	int y = get_global_id(1);

//    if (x < 0 || y < 0 || x >= width || y >= height)
//    {
//        return;
//    }

    printf("[%d,%d]\n",x,y);

    int gid = x + width*y;
    int peak = peaks[gid];
    int i = 0;

    while (peaks[peaks[peak]] != peaks[peak] && i < 10)
    {
        i++;
        peaks[gid] = peaks[peak];
        peak = peaks[peaks[peak]];
    }
}

__kernel void mean_shift_result(__global uint* counts, __global uint* peaks, __global uchar4* output, uint width, uint height)
{
	int x = get_global_id(0);
	int y = get_global_id(1);

	if(x < width && y < height)
	{
		int gid = x + y*width;
//printf(peaks[gid]);
		//output[gid] = add_sat(sobel_x[gid], sobel_y[gid]);
        //if(peaks[gid] != 0)
        //    output[gid].s0 = 255;
        //else
        output[gid].s0 = 255-peaks[gid];
        output[gid].s1 = 255-peaks[gid];
        output[gid].s2 = 255-peaks[gid];

        output[gid].s3 = 255;
    }
}


running almost

running almost

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	_FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS* MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	int m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	int p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	int m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	int p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;
//	gbl_out_off += p4 * MLO_READ_UNIT;
// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);
// inside input range
	bool inside_map_range_input = ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = (p4 < MLO_MAP_WK_SZ &&  inside_map_range_input);

	// inside output range
	bool inside_range_output = (p4 < MLO_MAP_WK_SZ);

	for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		int bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0

			if (last_pixel)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range

					bool inside_range_input2 = inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS);

					bot_off = (inside_range_input2) ? bot_off : 0;

					for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
					}
					for (int i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
			else
#endif
			{
				// check
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{
					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range

					bool inside_range_input2 = inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS);

					bot_off = (inside_range_input2) ? bot_off : 0;

					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
					}

				}

			} // if (last_pixel)

//			int top_off = (inside_range_output) ? gbl_out_off : 0;

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				int m = iDiv(p, MLO_MAP_WK_SZ);
				int pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				int m = ((uint)p / MLO_MAP_WK_SZ);
				int pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				int top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif
#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{
					for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df[top_off1 + i];
					}
					for (int i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df[top_off1 + i];

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

// final summation optimized.

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

#if 1 // MLO_LG2_REDUC_ROUNDS > 1
// first round
// transpose and split into sub-group for logar summation
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			int split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			int split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			int split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			int split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (int j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= (MLO_REDUC_LOOP_STEP << rd) && p4 < (MLO_REDUC_LOOP_STEP << (rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP << rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

#undef MLO_N_MAPS_PER_GROUP
#undef MLO_TOP_DAT_SZ
#undef MLO_BOT_DAT_SZ
#undef MLO_ACCUM_SZ

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	_FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (int pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		int b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		int p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		int b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		int p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		gbl_in_off = b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		gbl_out_off = b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
				}

			}
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df[top_off + i];
				}
			}

		}
		else
#endif
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
				}

			}
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df[top_off + i];
				}
			}
		}

		// processing
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];
				}
			}
		}

	}

	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// final summation optimized.

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

#if 0

	// first round
	// transpose and split into sub-group for logar summation
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			int split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			int split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			int split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			int split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (int j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= (MLO_REDUC_LOOP_STEP << rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << rd))
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

#elif 0
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (c_idx + c < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	_FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS* MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	int m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	int p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	int m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	int p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;
//	gbl_out_off += p4 * MLO_READ_UNIT;
// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);
// inside input range
	bool inside_map_range_input = ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = (p4 < MLO_MAP_WK_SZ &&  inside_map_range_input);

	// inside output range
	bool inside_range_output = (p4 < MLO_MAP_WK_SZ);

	for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		int bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0

			if (last_pixel)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range

					bool inside_range_input2 = inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS);

					bot_off = (inside_range_input2) ? bot_off : 0;

					for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
					}
					for (int i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
			else
#endif
			{
				// check
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{
					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range

					bool inside_range_input2 = inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS);

					bot_off = (inside_range_input2) ? bot_off : 0;

					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
					}

				}

			} // if (last_pixel)

//			int top_off = (inside_range_output) ? gbl_out_off : 0;

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				int m = iDiv(p, MLO_MAP_WK_SZ);
				int pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				int m = ((uint)p / MLO_MAP_WK_SZ);
				int pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				int top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif
#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{
					for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df[top_off1 + i];
					}
					for (int i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df[top_off1 + i];

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

// final summation optimized.

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

#if 1 // MLO_LG2_REDUC_ROUNDS > 1
// first round
// transpose and split into sub-group for logar summation
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			int split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			int split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			int split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			int split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (int j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= (MLO_REDUC_LOOP_STEP << rd) && p4 < (MLO_REDUC_LOOP_STEP << (rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP << rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

#undef MLO_N_MAPS_PER_GROUP
#undef MLO_TOP_DAT_SZ
#undef MLO_BOT_DAT_SZ
#undef MLO_ACCUM_SZ

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	_FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;

	int lcl_id = get_local_id(0);

	int k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (int pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		int b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		int p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		int b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		int p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		gbl_in_off = b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		gbl_out_off = b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
				}

			}
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df[top_off + i];
				}
			}

		}
		else
#endif
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				int bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot[bot_off + i];
				}

			}
			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df[top_off + i];
				}
			}
		}

		// processing
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];
				}
			}
		}

	}

	// write out
	// inputs are outputs
	int wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// final summation optimized.

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			int split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			int split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			int split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			int split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (int j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= (MLO_REDUC_LOOP_STEP << rd) && lcl_id < (MLO_REDUC_LOOP_STEP << (rd + 1)))
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << rd))
		{
			for (int rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (c_idx + c < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}