sixtracklib/opencl: streamline the handling of the success_status flag inside the OpenCL kernel

#ifndef SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__
#define SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__

#if defined( _GPUCODE )

void __kernel Track_remap_serialized_blocks_buffer(
     __global unsigned char* restrict  particles_data_buffer,
     __global unsigned char* restrict beam_elements_data_buffer,
     __global unsigned char* restrict elem_by_elem_data_buffer,
     __global long int*      restrict ptr_success_flag )
{
    size_t const global_id   = get_global_id( 0 );
    size_t const global_size = get_global_size( 0 );

    size_t const gid_to_remap_particles = 0;

    size_t const gid_to_remap_beam_elements = ( global_size > 1u )
        ? 1u : gid_to_remap_particles;

    size_t const gid_to_remap_elem_by_elem = ( global_size > 2u )
        ? 2u : gid_to_remap_beam_elements;

    long int  success_flag = 0;

    if( global_id <= gid_to_remap_elem_by_elem )
    {
        if( global_id == gid_to_remap_particles )
        {
            NS(Blocks) particles_buffer;
            NS(Blocks_preset)( &particles_buffer );

            if( 0 != NS(Blocks_unserialize)( &particles_buffer,
                        particles_data_buffer ) )
            {
                success_flag |= -1;
            }
        }

        if( ( success_flag == 0 ) &&
            ( global_id == gid_to_remap_beam_elements ) )
        {
            NS(Blocks) beam_elements;
            NS(Blocks_preset)( &beam_elements );

            if( 0 != NS(Blocks_unserialize)( &beam_elements,
                        beam_elements_data_buffer ) )
            {
                success_flag = -2;
            }
        }

        if( ( success_flag == 0 ) &&
            ( global_id == gid_to_remap_elem_by_elem ) )
        {
            __global unsigned long const* header =
                    ( __global unsigned long const* )elem_by_elem_data_buffer;

            if( ( header != 0 ) && ( header[ 0 ] != 0u ) )
            {
                NS(Blocks) elem_by_elem_buffer;
                NS(Blocks_preset)( &elem_by_elem_buffer );

                if( 0 != NS(Blocks_unserialize)( &elem_by_elem_buffer ) )
                {
                    success_flag = -4;
                }
            }
        }

        if( ( success_flag != 0 ) && ( ptr_success_flag  != 0 ) )
        {
            *ptr_success_flag |= success_flag;
        }
    }

    return;
}

void __kernel Track_particles_kernel_opencl(
    unsigned long const num_of_turns,
    __global unsigned char* restrict particles_data_buffer,
    __global unsigned char* restrict beam_elements_data_buffer,
    __global unsigned char* restrict elem_by_elem_data_buffer,
    __global long int*      restrict ptr_success_flag )
{
    typedef __global unsigned long const* g_ulong_cptr_t;
    typedef __global NS(BlockInfo)*       g_info_ptr_t;
    typedef __global NS(Particles)*       g_particles_ptr_t;

    size_t const global_id   = get_global_id( 0 );
    size_t const global_size = get_global_size( 0 );

    NS(block_size_t) num_particle_blocks              = 0u;
    NS(block_size_t) num_beam_elements                = 0u;
    NS(block_size_t) num_elem_by_elem_blocks          = 0u;
    NS(block_size_t) num_elem_by_elem_blocks_per_turn = 0u;
    NS(block_size_t) num_required_elem_by_elem_blocks = 0u;
    NS(block_size_t) num_of_particles                 = 0u;

    NS(Blocks) particles_buffer;
    NS(Blocks) beam_elements;
    NS(Blocks) elem_by_elem_buffer;

    NS(Particles) particles;

    long int success_flag = 0;
    bool use_elem_by_elem_buffer = false;

    /* --------------------------------------------------------------------- */
    /* *****  SECTION FOR DEFUSING THE HEISENBUG *** */
    /* This printf section seems to defuse the Heisenbug on the AMDGPU
     * implementation available to the author -> YMMV */

    /*
    if( global_id == 0u )
    {
        g_particles_ptr_t particles_header =
            ( g_particles_ptr_t )particles_data_buffer;

        printf( "before unserialization: \r\n" );
        printf( "global_id     = %u\r\n", global_id );

        printf( "particles_header[ 0 ]   = %16x at %18x\r\n",
                 particles_header[ 0 ],
                 ( uintptr_t )( particles_data_buffer +  0 ) );

        printf( "particles_header[ 1 ]   = %16x at %18x\r\n",
                 particles_header[ 1 ],
                 ( uintptr_t )( particles_data_buffer +  8 ) );

        printf( "particles_header[ 2 ]   = %16x at %18x\r\n",
                 particles_header[ 2 ],
                ( uintptr_t )( particles_data_buffer + 16 ) );

        printf( "particles_header[ 3 ]   = %16x at %18x\r\n",
                 particles_header[ 3 ],
                ( uintptr_t )( particles_data_buffer + 32 ) );
    }

    barrier( CLK_GLOBAL_MEM_FENCE );
    */

    /* *****  END OF SECTION FOR DEFUSING THE HEISENBUG *** */
    /* --------------------------------------------------------------------- */

    NS(Blocks_preset)( &particles_buffer );

    if( 0 == NS(Blocks_unserialize_without_remapping)(
                &particles_buffer, particles_data_buffer ) )
    {
        num_particle_blocks = NS(Blocks_get_num_of_blocks)( &particles_buffer );

        if( num_particle_blocks == 1u )
        {
            g_particles_ptr_t ptr_particles  = 0;
            g_info_ptr_t  ptr_particles_info =
                NS(Blocks_get_block_infos_begin)( &particles_buffer );

            NS(BlockInfo) particles_info;

            if( ptr_particles_info != 0 )
            {
                particles_info = *ptr_particles_info;
            }
            else
            {
                NS(BlockInfo_preset)( &particles_info );
            }

            ptr_particles = NS(Blocks_get_particles)( &particles_info );

            if( ptr_particles != 0 )
            {
                particles = *ptr_particles;
            }
            else
            {
                NS(Particles_preset)( &particles );
            }

            num_of_particles = NS(Particles_get_num_particles)( &particles );
        }
    }

    if( num_of_particles == 0u )
    {
        NS(Blocks_preset)( &particles_buffer );
        NS(Particles_preset)( &particles );
        success_flag |= -1;
    }

    NS(Blocks_preset)( &beam_elements );

    if( 0 == NS(Blocks_unserialize_without_remapping)(
                &beam_elements, beam_elements_data_buffer ) )
    {
        num_beam_elements = NS(Blocks_get_num_of_blocks)( &beam_elements );
    }

    if( num_beam_elements == 0u )
    {
        NS(Blocks_preset)( &beam_elements );
        success_flag |= -2;
    }

    if( elem_by_elem_data_buffer != 0 )
    {
        g_ulong_cptr_t elem_by_elem_header =
            ( g_ulong_cptr_t )elem_by_elem_data_buffer;

        num_elem_by_elem_blocks_per_turn =
            num_beam_elements * num_particle_blocks;

        num_required_elem_by_elem_blocks =
            num_of_turns * num_elem_by_elem_blocks_per_turn;

        NS(Blocks_preset)( &elem_by_elem_buffer );

        if( 0 == NS(Blocks_unserialize_without_remapping)(
                    &elem_by_elem_buffer, elem_by_elem_data_buffer ) )
        {
            num_elem_by_elem_blocks =
                NS(Blocks_get_num_of_blocks)( &elem_by_elem_buffer );

            if( ( required_num_elem_by_elem > 0u ) &&
                ( num_elem_by_elem_blocks >= required_num_elem_by_elem ) )
            {
                use_elem_by_elem_buffer = true;
            }
        }
        else if( elem_by_elem_header[ 0 ] != ( unsigned long )0u )
        {
            NS(Blocks_preset)( &elem_by_elem_buffer );
            success_flag |= -4;
        }
    }

    if( ( success_flag        == 0  ) && ( num_of_turns     != 0u ) &&
        ( num_beam_elements   != 0u ) && ( num_of_particles != 0u ) &&
        ( num_particle_blocks == 1u ) && ( global_id < num_of_particles ) )
    {
        int ret = 0;
        unsigned long ii = 0u;

        if( !use_elem_by_elem_buffer )
        {
            for( ; ii < num_of_turns ; ++ii )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, 0 );
            }

            if( ret != 0 ) success_flag |= -8;
        }
        else
        {
            SIXTRL_GLOBAL_DEC NS(BlockInfo)* io_info_it =
                NS(Blocks_get_block_infos_begin)( &elem_by_elem_buffer );

            for( ; ii < num_of_turns ; ++ii,
                    io_info_it = io_info_it + num_beam_elements )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, io_info_it );

                if( io_info_it != 0 )
                {
                    io_info_it = io_info_it + num_elem_by_elem_blocks_per_turn;
                }
            }

            if( ret != 0 ) success_flag |= -16;
        }
    }
    else
    {
        success_flag |= -32;
    }

    if( ( success_flag != 0 ) && ( ptr_success_flag != 0 ) )
    {
        *ptr_success_flag |= success_flag;
    }

    return;
}

#endif /* defined( _GPUCODE ) */

#endif /* SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__ */

/* end: sixtracklib/opencl/track_particles_kernel.cl */

#ifndef SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__
#define SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__

#if defined( _GPUCODE )

void __kernel Track_remap_serialized_blocks_buffer(
     __global unsigned char* restrict  particles_data_buffer,
     __global unsigned char* restrict beam_elements_data_buffer,
     __global unsigned char* restrict elem_by_elem_data_buffer,
     __global long int*      restrict ptr_success_flag )
{
    size_t const global_id   = get_global_id( 0 );
    size_t const global_size = get_global_size( 0 );

    size_t const gid_to_remap_particles = 0;

    size_t const gid_to_remap_beam_elements = ( global_size > 1u )
        ? 1u : gid_to_remap_particles;

    size_t const gid_to_remap_elem_by_elem = ( global_size > 2u )
        ? 2u : gid_to_remap_beam_elements;

    long int  success_flag = 0;

    if( global_id <= gid_to_remap_elem_by_elem )
    {
        int ret = 0;

        NS(Blocks) elem_by_elem_buffer;
        NS(Blocks_preset)( &elem_by_elem_buffer );

        if( global_id == gid_to_remap_particles )
        {
            NS(Blocks) particles_buffer;
            NS(Blocks_preset)( &particles_buffer );

            if( 0 != NS(Blocks_unserialize)( &particles_buffer,
                        particles_data_buffer ) )
            {
                ret = -1;
                success_flag |= -1;
            }
        }

        if( ( ret == 0 ) && ( global_id == gid_to_remap_beam_elements ) )
        {
            NS(Blocks) beam_elements;
            NS(Blocks_preset)( &beam_elements );

            if( 0 != NS(Blocks_unserialize)( &beam_elements,
                        beam_elements_data_buffer ) )
            {
                ret = -1;
                success_flag = -2;
            }
        }

        if( ( ret == 0 ) && ( global_id == gid_to_remap_elem_by_elem ) )
        {
            __global unsigned long const* header =
                    ( __global unsigned long const* )elem_by_elem_data_buffer;

            if( header != 0 )
            {
                NS(Blocks) elem_by_elem_buffer;
                NS(Blocks_preset)( &elem_by_elem_buffer );

                if( ( 0 != NS(Blocks_unserialize)( &elem_by_elem_buffer ) ) &&
                    ( *header != ( unsigned long )0 ) )
                {
                    ret = -1;
                    success_flag = -4;
                }
            }
        }

        if( ( success_flag != 0 ) && ( ptr_success_flag  != 0 ) )
        {
            *ptr_success_flag |= success_flag;
        }
    }

    return;
}

void __kernel Track_particles_kernel_opencl(
    unsigned long const num_of_turns,
    __global unsigned char* restrict particles_data_buffer,
    __global unsigned char* restrict beam_elements_data_buffer,
    __global unsigned char* restrict elem_by_elem_data_buffer,
    __global long int*      restrict ptr_success_flag )
{
    typedef __global unsigned long const* g_ulong_cptr_t;
    typedef __global NS(BlockInfo)*       g_info_ptr_t;
    typedef __global NS(Particles)*       g_particles_ptr_t;

    size_t const global_id   = get_global_id( 0 );
    size_t const global_size = get_global_size( 0 );

    NS(block_size_t) num_particle_blocks              = 0u;
    NS(block_size_t) num_beam_elements                = 0u;
    NS(block_size_t) num_elem_by_elem_blocks          = 0u;
    NS(block_size_t) num_elem_by_elem_blocks_per_turn = 0u;
    NS(block_size_t) num_required_elem_by_elem_blocks = 0u;
    NS(block_size_t) num_of_particles                 = 0u;

    NS(Blocks) particles_buffer;
    NS(Blocks) beam_elements;
    NS(Blocks) elem_by_elem_buffer;

    NS(Particles) particles;

    long int success_flag = 0;
    bool use_elem_by_elem_buffer = false;

    /* --------------------------------------------------------------------- */
    /* *****  SECTION FOR DEFUSING THE HEISENBUG *** */
    /* This printf section seems to defuse the Heisenbug on the AMDGPU
     * implementation available to the author -> YMMV */

    /*
    if( global_id == 0u )
    {
        g_particles_ptr_t particles_header =
            ( g_particles_ptr_t )particles_data_buffer;

        printf( "before unserialization: \r\n" );
        printf( "global_id     = %u\r\n", global_id );

        printf( "particles_header[ 0 ]   = %16x at %18x\r\n",
                 particles_header[ 0 ],
                 ( uintptr_t )( particles_data_buffer +  0 ) );

        printf( "particles_header[ 1 ]   = %16x at %18x\r\n",
                 particles_header[ 1 ],
                 ( uintptr_t )( particles_data_buffer +  8 ) );

        printf( "particles_header[ 2 ]   = %16x at %18x\r\n",
                 particles_header[ 2 ],
                ( uintptr_t )( particles_data_buffer + 16 ) );

        printf( "particles_header[ 3 ]   = %16x at %18x\r\n",
                 particles_header[ 3 ],
                ( uintptr_t )( particles_data_buffer + 32 ) );
    }

    barrier( CLK_GLOBAL_MEM_FENCE );
    */

    /* *****  END OF SECTION FOR DEFUSING THE HEISENBUG *** */
    /* --------------------------------------------------------------------- */

    NS(Blocks_preset)( &particles_buffer );
    NS(Blocks_preset)( &beam_elements );

    if( 0 == NS(Blocks_unserialize_without_remapping)(
                &particles_buffer, particles_data_buffer ) )
    {
        num_particle_blocks = NS(Blocks_get_num_of_blocks)( &particles_buffer );

        if( num_particle_blocks == 1u )
        {
            g_particles_ptr_t ptr_particles  = 0;
            g_info_ptr_t  ptr_particles_info =
                NS(Blocks_get_block_infos_begin)( &particles_buffer );

            NS(BlockInfo) particles_info;

            if( ptr_particles_info != 0 )
            {
                particles_info = *ptr_particles_info;
            }
            else
            {
                NS(BlockInfo_preset)( &particles_info );
            }

            ptr_particles = NS(Blocks_get_particles)( &particles_info );

            if( ptr_particles != 0 )
            {
                particles = *ptr_particles;
            }
            else
            {
                NS(Particles_preset)( &particles );
            }

            num_of_particles = NS(Particles_get_num_particles)( &particles );
        }
    }
    else
    {
        NS(Blocks_preset)( &particles_buffer );
        NS(Particles_preset)( &particles );
        success_flag |= -1;
    }

    if( 0 == NS(Blocks_unserialize_without_remapping)(
                &beam_elements, beam_elements_data_buffer ) )
    {
        num_beam_elements = NS(Blocks_get_num_of_blocks)( &beam_elements );
    }
    else
    {
        NS(Blocks_preset)( &beam_elements );
        success_flag |= -2;
    }

    if( elem_by_elem_data_buffer != 0 )
    {
        g_ulong_cptr_t elem_by_elem_header =
            ( g_ulong_cptr_t )elem_by_elem_data_buffer;

        num_elem_by_elem_blocks_per_turn =
            num_beam_elements * num_particle_blocks;

        num_required_elem_by_elem_blocks =
            num_of_turns * num_elem_by_elem_blocks_per_turn;

        NS(Blocks_preset)( &elem_by_elem_buffer );

        if( 0 == NS(Blocks_unserialize_without_remapping)(
                    &elem_by_elem_buffer, elem_by_elem_data_buffer ) )
        {
            num_elem_by_elem_blocks =
                NS(Blocks_get_num_of_blocks)( &elem_by_elem_buffer );

            if( ( required_num_elem_by_elem > 0u ) &&
                ( num_elem_by_elem_blocks >= required_num_elem_by_elem ) )
            {
                use_elem_by_elem_buffer = true;
            }
        }
        else if( elem_by_elem_header[ 0 ] != ( unsigned long )0u )
        {
            success_flag |= -4;
        }
    }

    if( ( success_flag        == 0  ) && ( num_of_turns     != 0u ) &&
        ( num_beam_elements   != 0u ) && ( num_of_particles != 0u ) &&
        ( num_particle_blocks == 1u ) && ( global_id < num_of_particles ) )
    {
        int ret = 0;
        unsigned long ii = 0u;

        if( !use_elem_by_elem_buffer )
        {
            for( ; ii < num_of_turns ; ++ii )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, 0 );
            }

            if( ret != 0 ) success_flag |= -8;
        }
        else
        {
            SIXTRL_GLOBAL_DEC NS(BlockInfo)* io_info_it =
                NS(Blocks_get_block_infos_begin)( &elem_by_elem_buffer );

            for( ; ii < num_of_turns ; ++ii,
                    io_info_it = io_info_it + num_beam_elements )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, io_info_it );

                if( io_info_it != 0 )
                {
                    io_info_it = io_info_it + num_elem_by_elem_blocks_per_turn;
                }
            }

            if( ret != 0 ) success_flag |= -16;
        }
    }

    if( ( success_flag != 0 ) && ( ptr_success_flag != 0 ) )
    {
        *ptr_success_flag |= success_flag;
    }

    return;
}

#endif /* defined( _GPUCODE ) */

#endif /* SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__ */

/* end: sixtracklib/opencl/track_particles_kernel.cl */


Well I got something coming up, performance is worse than I was hoping, but there's a lot of optimization to go


open cl compiler error fix

open cl compiler error fix

__kernel void MinSquareVal(__global float* pA, __global float* pC, int square_size, int offset)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);
	const int width = get_global_size(0);
	const int height = get_global_size(1);

	int minH = offset > y ? 0 : y - offset;
	int maxH = min(y + square_size - offset, height);

	int minW = offset > x ? 0 : x - offset;
	int maxW = min(x + square_size - offset, width);

	float minValue = pA[minH * width + minW];

	for (int j = minH; j < maxH; j++)
	{
		for (int i = minW; i < maxW; i++)
		{
			float tmp = pA[j * width + i];
			if (tmp < minValue) minValue = tmp;
		}
	}

	pC[y * width + x] = minValue;
}

float calcWeight(__global float* multipliers, int len)
{
	float weight_no_border = 0;
	for (int j = 0; j < len; j++)
	{
		weight_no_border += multipliers[j];
	}
	return weight_no_border;
}

__kernel void Convolution(__global float* multipliers, __global float* inp, __global float* result,
							int xsize, int xstep, int len, int offset, float border_ratio)
{
	const int ox = get_global_id(0);
	const int y = get_global_id(1);

	const int oxsize = get_global_size(0);
	const int ysize = get_global_size(1);

	const int x = ox * xstep;
/*
	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}
*/
	float weight_no_border = calcWeight(multipliers, len);

	int minx = x < offset ? 0 : x - offset;
	int maxx = min(xsize, x + len - offset);

	float weight = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		weight += multipliers[j - x + offset];
	}

	weight = (1.0 - border_ratio) * weight + border_ratio * weight_no_border;
	float scale = 1.0 / weight;

	float sum = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		sum += inp[y * xsize + j] * multipliers[j - x + offset];
	}

	result[ox * ysize + y] = sum * scale;
}
/*
__kernel void ConvolutionX(__global float* multipliers, __global float* inp, __global float* result,
	int len, int offset, float border_ratio)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}

	int minx = x < offset ? 0 : x - offset;
	int maxx = min(xsize, x + len - offset);

	int miny = y < offset ? 0 : y - offset;
	int maxy = min(ysize, y + len - offset);

	float weightX = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		weightX += multipliers[j - x + offset];
	}

	weightX = (1.0 - border_ratio) * weightX + border_ratio * weight_no_border;

	float weightY = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		weightY += multipliers[j - y + offset];
	}

	weightY = (1.0 - border_ratio) * weightY + border_ratio * weight_no_border;

	float sum = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		float sumx = 0.0;
		for (int i = minx; i < maxx; i++)
		{
			sumx += inp[j * xsize + i] * multipliers[i - x + offset];
		}

		sum += sumx * multipliers[j - y + offset];
	}

	result[y * xsize + x] = sum / weightY / weightX;
}
*/

__kernel void ConvolutionX(__global float* multipliers, __global float* inp, __global float* result,
	int len, int offset, float border_ratio)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}

	int minx = x < offset ? 0 : x - offset;
	int maxx = min(xsize, x + len - offset);

	float weight = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		weight += multipliers[j - x + offset];
	}

	weight = (1.0 - border_ratio) * weight + border_ratio * weight_no_border;
	float scale = 1.0 / weight;

	float sum = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		sum += inp[y * xsize + j] * multipliers[j - x + offset];
	}

	result[x * ysize + y] = sum * scale;
}

__kernel void ConvolutionY(__global float* multipliers, __global float* inp, __global float* result,
	int len, int offset, float border_ratio)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}

	int miny = y < offset ? 0 : y - offset;
	int maxy = min(ysize, y + len - offset);

	float weight = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		weight += multipliers[j - y + offset];
	}

	weight = (1.0 - border_ratio) * weight + border_ratio * weight_no_border;
	float scale = 1.0 / weight;

	float sum = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		sum += inp[j * xsize + x] * multipliers[j - y + offset];
	}

	result[y * xsize + x] = sum * scale;
}

__kernel void DownSample(__global float* pA, __global float* pC, int square)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	const int oxsize = xsize / square;

	const int sample_x = x / square;
	const int sample_y = y / square;

	pC[y * xsize + x] = pA[sample_y * oxsize + sample_x];
}

void OpsinAbsorbance(const double in[3], double out[3])
{
	const float mix[12] = {
	0.348036746003,
	0.577814843137,
	0.0544556093735,
	0.774145581713,
	0.26922717275,
	0.767247733938,
	0.0366922708552,
	0.920130265014,
	0.0882062883536,
	0.158581714673,
	0.712857943858,
	10.6524069248,
	};

	out[0] = mix[0] * in[0] + mix[1] * in[1] + mix[2] * in[2] + mix[3];
	out[1] = mix[4] * in[0] + mix[5] * in[1] + mix[6] * in[2] + mix[7];
	out[2] = mix[8] * in[0] + mix[9] * in[1] + mix[10] * in[2] + mix[11];
}

double EvaluatePolynomial(const double x, const double *coefficients, int n)
{
	double b1 = 0.0;
	double b2 = 0.0;

	for (int i = n - 1; i >= 0; i--)
	{
		if (i == 0) {
			const double x_b1 = x * b1;
			b1 = x_b1 - b2 + coefficients[0];
			break;
		}
		const double x_b1 = x * b1;
		const double t = (x_b1 + x_b1) - b2 + coefficients[i];
		b2 = b1;
		b1 = t;
	}

	return b1;
}

double Gamma(double v)
{
	double min_value = 0.770000000000000;
	double max_value = 274.579999999999984;

	const double p[5 + 1] = {
		881.979476556478289, 1496.058452015812463, 908.662212739659481,
		373.566100223287378, 85.840860336314364, 6.683258861509244,
	};
	const double q[5 + 1] = {
		12.262350348616792, 20.557285797683576, 12.161463238367844,
		4.711532733641639, 0.899112889751053, 0.035662329617191,
	};

	const double x01 = (v - min_value) / (max_value - min_value);
	const double xc = 2.0 * x01 - 1.0;

	const double yp = EvaluatePolynomial(xc, p, 6);
	const double yq = EvaluatePolynomial(xc, q, 6);
	if (yq == 0.0) return 0.0;
	return (float)(yp / yq);
}

void RgbToXyb(double r, double g, double b, double *valx, double *valy, double *valz)
{
	const double a0 = 1.01611726948;
	const double a1 = 0.982482243696;
	const double a2 = 1.43571362627;
	const double a3 = 0.896039849412;
	*valx = a0 * r - a1 * g;
	*valy = a2 * r + a3 * g;
	*valz = b;
}

__kernel void OpsinDynamicsImage(__global float *r, __global float *g, __global float *b, __global float *r_blurred, __global float *g_blurred, __global float *b_blurred, int size)
{
	const int i = get_global_id(0);
	double pre[3] = { r_blurred[i], g_blurred[i],  b_blurred[i] };
	double pre_mixed[3];
	OpsinAbsorbance(pre, pre_mixed);
	double sensitivity[3];
	sensitivity[0] = Gamma(pre_mixed[0]) / pre_mixed[0];
	sensitivity[1] = Gamma(pre_mixed[1]) / pre_mixed[1];
	sensitivity[2] = Gamma(pre_mixed[2]) / pre_mixed[2];

	double cur_rgb[3] = { r_blurred[i], g_blurred[i],  b_blurred[i] };
	double cur_mixed[3];
    OpsinAbsorbance(cur_rgb, cur_mixed);
    cur_mixed[0] *= sensitivity[0];
    cur_mixed[1] *= sensitivity[1];
    cur_mixed[2] *= sensitivity[2];
    double x, y, z;
	RgbToXyb(cur_mixed[0], cur_mixed[1], cur_mixed[2], &x, &y, &z);
    r[i] = x;
    g[i] = y;
    b[i] = z;
}

__kernel void MinSquareVal(__global float* pA, __global float* pC, int square_size, int offset)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);
	const int width = get_global_size(0);
	const int height = get_global_size(1);

	int minH = offset > y ? 0 : y - offset;
	int maxH = min(y + square_size - offset, height);

	int minW = offset > x ? 0 : x - offset;
	int maxW = min(x + square_size - offset, width);

	float minValue = pA[minH * width + minW];

	for (int j = minH; j < maxH; j++)
	{
		for (int i = minW; i < maxW; i++)
		{
			float tmp = pA[j * width + i];
			if (tmp < minValue) minValue = tmp;
		}
	}

	pC[y * width + x] = minValue;
}

float calcWeight(__global float* multipliers, int len)
{
	float weight_no_border = 0;
	for (int j = 0; j < len; j++)
	{
		weight_no_border += multipliers[j];
	}
	return weight_no_border;
}

__kernel void Convolution(__global float* multipliers, __global float* inp, __global float* result,
							int xsize, int xstep, int len, int offset, float border_ratio)
{
	const int ox = get_global_id(0);
	const int y = get_global_id(1);

	const int oxsize = get_global_size(0);
	const int ysize = get_global_size(1);

	const int x = ox * xstep;
/*
	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}
*/
	float weight_no_border = calcWeight(multipliers, len);

	int minx = x < offset ? 0 : x - offset;
	int maxx = min(xsize, x + len - offset);

	float weight = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		weight += multipliers[j - x + offset];
	}

	weight = (1.0 - border_ratio) * weight + border_ratio * weight_no_border;
	float scale = 1.0 / weight;

	float sum = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		sum += inp[y * xsize + j] * multipliers[j - x + offset];
	}

	result[ox * ysize + y] = sum * scale;
}
/*
__kernel void ConvolutionX(__global float* multipliers, __global float* inp, __global float* result,
	int len, int offset, float border_ratio)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}

	int minx = x < offset ? 0 : x - offset;
	int maxx = min(xsize, x + len - offset);

	int miny = y < offset ? 0 : y - offset;
	int maxy = min(ysize, y + len - offset);

	float weightX = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		weightX += multipliers[j - x + offset];
	}

	weightX = (1.0 - border_ratio) * weightX + border_ratio * weight_no_border;

	float weightY = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		weightY += multipliers[j - y + offset];
	}

	weightY = (1.0 - border_ratio) * weightY + border_ratio * weight_no_border;

	float sum = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		float sumx = 0.0;
		for (int i = minx; i < maxx; i++)
		{
			sumx += inp[j * xsize + i] * multipliers[i - x + offset];
		}

		sum += sumx * multipliers[j - y + offset];
	}

	result[y * xsize + x] = sum / weightY / weightX;
}
*/

__kernel void ConvolutionX(__global float* multipliers, __global float* inp, __global float* result,
	int len, int offset, float border_ratio)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}

	int minx = x < offset ? 0 : x - offset;
	int maxx = min(xsize, x + len - offset);

	float weight = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		weight += multipliers[j - x + offset];
	}

	weight = (1.0 - border_ratio) * weight + border_ratio * weight_no_border;
	float scale = 1.0 / weight;

	float sum = 0.0;
	for (int j = minx; j < maxx; j++)
	{
		sum += inp[y * xsize + j] * multipliers[j - x + offset];
	}

	result[x * ysize + y] = sum * scale;
}

__kernel void ConvolutionY(__global float* multipliers, __global float* inp, __global float* result,
	int len, int offset, float border_ratio)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	float weight_no_border = 0;
	for (int j = 0; j <= 2 * offset; j++)
	{
		weight_no_border += multipliers[j];
	}

	int miny = y < offset ? 0 : y - offset;
	int maxy = min(ysize, y + len - offset);

	float weight = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		weight += multipliers[j - y + offset];
	}

	weight = (1.0 - border_ratio) * weight + border_ratio * weight_no_border;
	float scale = 1.0 / weight;

	float sum = 0.0;
	for (int j = miny; j < maxy; j++)
	{
		sum += inp[j * xsize + x] * multipliers[j - y + offset];
	}

	result[y * xsize + x] = sum * scale;
}

__kernel void DownSample(__global float* pA, __global float* pC, int square)
{
	const int x = get_global_id(0);
	const int y = get_global_id(1);

	const int xsize = get_global_size(0);
	const int ysize = get_global_size(1);

	const int oxsize = xsize / square;

	const int sample_x = x / square;
	const int sample_y = y / square;

	pC[y * xsize + x] = pA[sample_y * oxsize + sample_x];
}

void OpsinAbsorbance(const double in[3], double out[3])
{
	const float mix[12] = {
	0.348036746003,
	0.577814843137,
	0.0544556093735,
	0.774145581713,
	0.26922717275,
	0.767247733938,
	0.0366922708552,
	0.920130265014,
	0.0882062883536,
	0.158581714673,
	0.712857943858,
	10.6524069248,
	};

	out[0] = mix[0] * in[0] + mix[1] * in[1] + mix[2] * in[2] + mix[3];
	out[1] = mix[4] * in[0] + mix[5] * in[1] + mix[6] * in[2] + mix[7];
	out[2] = mix[8] * in[0] + mix[9] * in[1] + mix[10] * in[2] + mix[11];
}

double EvaluatePolynomial(const double x, const double *coefficients, int n)
{
	double b1 = 0.0;
	double b2 = 0.0;

	for (int i = n - 1; i >= 0; i--)
	{
		if (i == 0) {
			const double x_b1 = x * b1;
			b1 = x_b1 - b2 + coefficients[0];
			break;
		}
		const double x_b1 = x * b1;
		const double t = (x_b1 + x_b1) - b2 + coefficients[i];
		b2 = b1;
		b1 = t;
	}

	return b1;
}

float Gamma(double v)
{
	double min_value = 0.770000000000000;
	double max_value = 274.579999999999984;

	/*static*/ const double p[5 + 1] = {
		881.979476556478289, 1496.058452015812463, 908.662212739659481,
		373.566100223287378, 85.840860336314364, 6.683258861509244,
	};
	/*static*/ const double q[5 + 1] = {
		12.262350348616792, 20.557285797683576, 12.161463238367844,
		4.711532733641639, 0.899112889751053, 0.035662329617191,
	};

	const double x01 = (v - min_value) / (max_value - min_value);
	const double xc = 2.0 * x01 - 1.0;

	const double yp = EvaluatePolynomial(xc, p, 6);
	const double yq = EvaluatePolynomial(xc, q, 6);
	if (yq == 0.0) return 0.0;
	return (float)(yp / yq);
}

void RgbToXyb(double r, double g, double b, double *valx, double *valy, double *valz)
{
	/*static*/ const double a0 = 1.01611726948;
	/*static*/ const double a1 = 0.982482243696;
	/*static*/ const double a2 = 1.43571362627;
	/*static*/ const double a3 = 0.896039849412;
	*valx = a0 * r - a1 * g;
	*valy = a2 * r + a3 * g;
	*valz = b;
}

__kernel void OpsinDynamicsImage(__global float *r, __global float *g, __global float *b, __global float *r_blurred, __global float *g_blurred, __global float *b_blurred, int size)
{
	const int i = get_global_id(0);
	double pre[3] = { r_blurred[i], g_blurred[i],  b_blurred[i] };
	double pre_mixed[3];
	OpsinAbsorbance(pre, pre_mixed);
	double sensitivity[3];
	sensitivity[0] = Gamma(pre_mixed[0]) / pre_mixed[0];
	sensitivity[1] = Gamma(pre_mixed[1]) / pre_mixed[1];
	sensitivity[2] = Gamma(pre_mixed[2]) / pre_mixed[2];

	double cur_rgb[3] = { r_blurred[i], g_blurred[i],  b_blurred[i] };
	double cur_mixed[3];
    OpsinAbsorbance(cur_rgb, cur_mixed);
    cur_mixed[0] *= sensitivity[0];
    cur_mixed[1] *= sensitivity[1];
    cur_mixed[2] *= sensitivity[2];
    double x, y, z;
	RgbToXyb(cur_mixed[0], cur_mixed[1], cur_mixed[2], &x, &y, &z);
    r[i] = x;
    g[i] = y;
    b[i] = z;
}


Use the correct definition

Use the correct definition

ENESIM_OPENCL_KERNEL(
__kernel void background(read_write image2d_t out, int rop, uchar4 c)
{
	int x = get_global_id(0);
	int y = get_global_id(1);

	write_imageui(out, (int2)(x, y), (uint4)(c.x, c.y, c.z, c.w));
}
);

ENESIM_OPENCL_KERNEL(
__kernel void background(read_write image2d_t out, cl_int rop, uchar4 c)
{
	int x = get_global_id(0);
	int y = get_global_id(1);

	write_imageui(out, (int2)(x, y), (uint4)(c.x, c.y, c.z, c.w));
}
);


fixed 1x1 wrw PF

fixed 1x1 wrw PF

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24(u, d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	uint m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	uint p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	uint m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	uint p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS);
#endif

#ifdef __AMDGCN__
#pragma unroll 2
#endif

	for (uint b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{
		if (m_id < MLO_N_MAPS_PER_GROUP)
		{

			// read all inputs into registers
			uint bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
			bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

			if (last_pixel)
			{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					const __global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
			else
#endif
			{
				// check
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{
					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif
					const __global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif
					}

				}

			} // if (last_pixel)
		}

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (uint kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (uint p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				uint m = iDiv(p, MLO_MAP_WK_SZ);
				uint pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				uint m = ((uint)p / MLO_MAP_WK_SZ);
				uint pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				uint top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif

				const __global _FLOAT * top_df1 = &top_df[top_off1];

#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (uint k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	uint wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			uint wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	uint gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	uint gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (uint pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		uint p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		uint b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		uint p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		uint gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		uint gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif

				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;

#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

		// processing
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	uint wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))
#define DBG_OUT_OF_RNGE 0

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v *(1.f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24(u, d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS output maps into LDS
// read MLO_N_LCL_IN_MAPS per wk_item input maps

// alg

// convolve
// reduce with transform

// write out

**********************************************************************************************************/

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)
#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)
#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS)

/*

	Small  maps

*/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrWSmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS * MLO_N_MAPS_PER_GROUP); // input map index based

	uint ib = get_group_id(2); // batch id

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

// map id inside the group, super-pixel inside the map
#if (MLO_MAP_WK_SZ &  (MLO_MAP_WK_SZ - 1))

	uint m_id = iDiv(lcl_id, MLO_MAP_WK_SZ);  // map
	uint p4 = iMod(lcl_id, m_id, MLO_MAP_WK_SZ); // pixel
#else
	uint m_id = ((uint)lcl_id / MLO_MAP_WK_SZ);  // map
	uint p4 = ((uint)lcl_id & (MLO_MAP_WK_SZ - 1)); // pixel

#endif

	gbl_in_off += p4 * MLO_READ_UNIT;

// input is kept in registers at the start
	gbl_in_off += m_id * MLO_IN_CHANNEL_STRIDE;

// inside input range

#if MLO_N_IN_MAPS_ALIGNED == 0
	bool inside_map_range = (p4 < MLO_MAP_WK_SZ  && m_id < MLO_N_MAPS_PER_GROUP);
	bool inside_range_input = inside_map_range & ((c_idx + m_id) < MLO_N_INPUTS && m_id < MLO_N_MAPS_PER_GROUP);
#endif

#ifdef __AMDGCN__
#pragma unroll 2
#endif

	for (uint b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)
	{

		// read all inputs into registers
		uint bot_off = gbl_in_off;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
				{

					// reading in order per group and jump over maps been read
					// read arbitrary data but inside the range
#if MLO_N_IN_MAPS_ALIGNED == 0
					bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif

					const __global _FLOAT * bot1 = &bot[bot_off];

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						bot_dat[c*MLO_READ_UNIT + i] = 0;
					}

				}

			}
		else
#endif
		{
			// check
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c, bot_off += MLO_N_MAPS_PER_GROUP*MLO_IN_CHANNEL_STRIDE)
			{
				// reading in order per group and jump over maps been read
				// read arbitrary data but inside the range

#if MLO_N_IN_MAPS_ALIGNED == 0
				bot_off = (inside_range_input && ((c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS)) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}

		} // if (last_pixel)

		int top_off = gbl_out_off;

		// read all outputs
		// assum division by MLO_N_LCL_OUT
		for (uint kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

			for (uint p = lcl_id; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)
			{
#if (MLO_MAP_WK_SZ & (MLO_MAP_WK_SZ - 1))
				uint m = iDiv(p, MLO_MAP_WK_SZ);
				uint pm = iMod(p, m, MLO_MAP_WK_SZ);
#else
				uint m = ((uint)p / MLO_MAP_WK_SZ);
				uint pm = ((uint)p & (MLO_MAP_WK_SZ - 1));
#endif

				uint top_off1 = top_off + m * MLO_OUT_CHANNEL_STRIDE + pm*MLO_READ_UNIT;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off1 = (k_idx + m < MLO_N_OUTPUTS) ? top_off1 : 0;
#endif

				const __global _FLOAT * top_df1 = &top_df[top_off1];

#if MLO_N_PIXS_OFF > 0
				if (pm == MLO_MAP_WK_SZ - 1)
				{

					for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
					for (uint i = MLO_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = 0;
					}

				}
				else

#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_mem[p*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
						if (top_off1 + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:out-off-range\n");
						}
#endif

					}
				}

			} // for(int p = 0; p < MLO_OUT_LCL_BLK * MLO_MAP_WK_SZ; p += MLO_GRP_SZ)

			barrier(CLK_LOCAL_MEM_FENCE);

			for (uint k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				// processing
				for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						_FLOAT bot_val = bot_dat[c*MLO_READ_UNIT + i];
						_FLOAT top_val = lcl_mem[((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
						pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += top_val * bot_val;
#if 0
						if (top_val * bot_val !=0  &&  k == 0 && c == 0 && m_id == 0 && get_group_id(0) == 0 && get_group_id(1) == 1 && lcl_id ==0 )
						{

							printf("K:c: %d %d %d %d %f %f %f %f\n",
								c_idx + m_id + c*MLO_N_MAPS_PER_GROUP,
								lcl_id,
								c*MLO_READ_UNIT + i,
								((k - kb*MLO_OUT_LCL_BLK) * MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i,
								pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
								top_val * bot_val,
								bot_val,
								top_val

							);
						}

#endif

					}
				}
			}

		} // for(int kb = 0; kb < MLO_N_LCL_OUT; kb++, top_off += MLO_OUT_LCL_BLK * MLO_OUT_CHANNEL_STRIDE)
	} // for (int b = 0; b < MLO_BATCH_SZ; ++b, gbl_in_off += MLO_IN_BATCH_STRIDE, gbl_out_off += MLO_OUT_BATCH_STRIDE)

// FINAL REDUCTION

	// write out
	// inputs are outputs
	uint wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + (c_idx + m_id) * MLO_WEI_CHANNEL_STRIDE;

#define MLO_N_FIRST_SPLITS  (1 << (MLO_LG2_REDUC_ROUNDS - 1))
// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1
// first round
// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
// write out only valid pixels
#if MLO_MAP_WK_SZ < MLO_GRP_SZ
		if (lcl_id < MLO_N_MAPS_PER_GROUP*MLO_MAP_WK_SZ)
#endif

		{

			for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < (MLO_REDUC_LOOP_STEP<< (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(p4, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(p4, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)p4 / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)p4 & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_MAP_WK_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

#if MLO_LG2_REDUC_ROUNDS > 1
// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				lcl_mem[base_off + p4] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && m_id < MLO_N_MAPS_PER_GROUP)
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = (rr*MLO_N_MAPS_PER_GROUP + m_id)*MLO_MAP_WK_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + p4];
			}
		}
	}

#endif

	if (p4 < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			uint wei_idx = r* (MLO_REDUC_LOOP_STEP)+p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (m_id < MLO_N_MAPS_PER_GROUP
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive reduction
#elif 1
//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (p4 < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_MAP_WK_SZ; j++)
				{
					final_sum[r] += lcl_mem[(m_id*MLO_MAP_WK_SZ + j)*MLO_REDUC_LOOP_STEP + p4];
				}

			}
		}

		if (p4 < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* ( MLO_REDUC_LOOP_STEP) + p4;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS-1));
#endif

				if (m_id < MLO_N_MAPS_PER_GROUP && (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)

// verification
#else

	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (p4 == 0 && inside_range_input)
		{
			for (int j = 1; j < MLO_MAP_WK_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (p4 == 0)
	{
		for (int kb = 0; kb < MLO_N_LCL_OUT; kb++)
		{
			for (int k = kb*MLO_OUT_LCL_BLK; k < (kb + 1)*MLO_OUT_LCL_BLK; ++k)
			{
				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					if ( (c_idx + m_id + c*MLO_N_MAPS_PER_GROUP) < MLO_N_INPUTS
#if MLO_N_OUT_MAPS_ALIGNED == 0
						  && k_idx + k < MLO_N_OUTPUTS
#endif
						)
					{
						int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_N_MAPS_PER_GROUP*MLO_WEI_CHANNEL_STRIDE;
						weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
					}

				}

			}
		}

	}

#endif

}

/*

  Large maps

*/

#undef MLO_N_MAPS_PER_GROUP

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrWLmap(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
	UNUSED _FLOAT padding_val
)
{
	// reduction memory.

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];

	uint lcl_id = get_local_id(0);

	uint k_idx = get_group_id(0) * (MLO_N_LCL_OUT_MAPS); // output map index base

	uint c_idx = get_group_id(1) * (MLO_N_LCL_IN_MAPS); // input map index based

	uint gbl_in_off0 = c_idx * MLO_IN_CHANNEL_STRIDE;
	uint gbl_out_off0 = k_idx * MLO_OUT_CHANNEL_STRIDE;

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (uint i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}

	for (uint pix4 = lcl_id; pix4 < MLO_MAP_WK_SZ * MLO_BATCH_SZ; pix4 += MLO_GRP_SZ)
	{

#if (MLO_MAP_WK_SZ) &  (MLO_MAP_WK_SZ - 1)

		uint b = iDiv(pix4, MLO_MAP_WK_SZ);  // batch
		uint p4 = iMod(pix4, b, MLO_MAP_WK_SZ); // pixel block
#else
		uint b = ((uint)pix4 / MLO_MAP_WK_SZ);  // batch
		uint p4 = ((uint)pix4 & (MLO_MAP_WK_SZ - 1)); // pixel block

#endif
		uint gbl_in_off = gbl_in_off0 + b * MLO_IN_BATCH_STRIDE + p4*MLO_READ_UNIT;
		uint gbl_out_off = gbl_out_off0 + b * MLO_OUT_BATCH_STRIDE + p4*MLO_READ_UNIT;

#if MLO_N_PIXS_OFF > 0
		bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);

		if (last_pixel)
		{
			for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
			{
				top_dat[i] = 0;
			}
			for (uint i = 0; i < MLO_BOT_DAT_SZ; ++i)
			{
				bot_dat[i] = 0;
			}
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif

				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif

				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;

#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_N_PIXS_OFF; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif

				}
			}

		}
		else
#endif
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				uint bot_off = gbl_in_off + c*MLO_IN_CHANNEL_STRIDE;
#if MLO_N_IN_MAPS_ALIGNED == 0
				// reading garbage, will be thrown away on the way output
				bot_off = (c_idx + c < MLO_N_INPUTS) ? bot_off : 0;
#endif
				const __global _FLOAT * bot1 = &bot[bot_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					bot_dat[c*MLO_READ_UNIT + i] = bot1[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-off-range\n");
					}
#endif
				}

			}
			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_off = gbl_out_off + k*MLO_OUT_CHANNEL_STRIDE;
#if MLO_N_OUT_MAPS_ALIGNED == 0
				top_off = (k_idx + k < MLO_N_OUTPUTS) ? top_off : 0;
#endif
				const __global _FLOAT * top_df1 = &top_df[top_off];

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					top_dat[k*MLO_READ_UNIT + i] = top_df1[i];
#if DBG_OUT_OF_RNGE
					if (top_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:out-off-range\n");
					}
#endif
				}
			}
		}

		// processing
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_accum[k * MLO_N_LCL_IN_MAPS + c] += bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i];

#if 0
					if (get_group_id(0) == 1 && lcl_id == 0 && k == 0 )
					{
						printf("K:c: %f %f %f %f\n",
							pvt_accum[k * MLO_N_LCL_IN_MAPS + c],
							bot_dat[c*MLO_READ_UNIT + i] * top_dat[k*MLO_READ_UNIT + i],
							bot_dat[c*MLO_READ_UNIT + i],
							top_dat[k*MLO_READ_UNIT + i]
							);
					}
#endif
				}
			}
		}

	}

// FINAL REDUCTION
	// write out
	// inputs are outputs
	uint wei_df_off = k_idx * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	// transpose data using MLO_REDUC_LOOP_STEP wk-items from each small map
	__private _FLOAT final_sum[(MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP)];

// final log reduction with the initail input not pow2
#if 1

	// first round
	// transpose and split into sub-group for logar summation
	for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		final_sum[r] = 0;

		barrier(CLK_LOCAL_MEM_FENCE);
		// write out only valid pixels

		for (uint rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < (MLO_REDUC_LOOP_STEP << (MLO_LG2_REDUC_ROUNDS - 1)))
		{
			// what split the pix belong to
#if (MLO_REDUC_LOOP_STEP & (MLO_REDUC_LOOP_STEP -1))
			uint split = iDiv(lcl_id, MLO_REDUC_LOOP_STEP);
			uint split_pix = iMod(lcl_id, split, MLO_REDUC_LOOP_STEP);
#else
			uint split = ((uint)lcl_id / MLO_REDUC_LOOP_STEP);
			uint split_pix = ((uint)lcl_id & (MLO_REDUC_LOOP_STEP - 1));
#endif

			for (uint j = 0; j < MLO_FIRST_ROUND; j++)
			{
#if MLO_FIRST_CAN_DIVIDE ==0
				if (split*MLO_FIRST_ROUND + j < MLO_GRP_SZ)
#endif
				{
					final_sum[r] += lcl_mem[(split*MLO_FIRST_ROUND + j)*MLO_REDUC_LOOP_STEP + split_pix];
				}
			}

		}
	}

	// log summation
	for (int rd = (MLO_LG2_REDUC_ROUNDS - 2); rd >= 0; --rd)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id >= ((uint)MLO_REDUC_LOOP_STEP << (uint)rd) && lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)(rd + 1)))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				lcl_mem[base_off + lcl_id] = final_sum[rr];

			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id < ((uint)MLO_REDUC_LOOP_STEP << (uint)rd))
		{
			for (uint rr = 0; rr < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++rr)
			{
				int base_off = rr*MLO_GRP_SZ;
				final_sum[rr] += lcl_mem[base_off + (MLO_REDUC_LOOP_STEP << rd) + lcl_id];
			}
		}
	}

	if (lcl_id < MLO_REDUC_LOOP_STEP)
	{
		for (uint r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			int wei_idx = r* (MLO_REDUC_LOOP_STEP)+ lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
			uint k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
			uint c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
			uint k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
			uint c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

			if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
				&& k_idx + k < MLO_N_OUTPUTS
#endif
				)
			{
				uint wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
				weights_df[wei_off] = final_sum[r];
			}

		} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	} // if (p4 < MLO_REDUC_LOOP_STEP)

// naive
#elif 1
	//	if (inside_range_input)
	{
		for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		{
			final_sum[r] = 0;

			barrier(CLK_LOCAL_MEM_FENCE);

			for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
			{
				lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
			}
			barrier(CLK_LOCAL_MEM_FENCE);

			if (lcl_id < MLO_REDUC_LOOP_STEP)
			{
				for (int j = 0; j < MLO_GRP_SZ; j++)
				{
					final_sum[r] += lcl_mem[j*MLO_REDUC_LOOP_STEP + lcl_id];
				}

			}
		}

		if (lcl_id < MLO_REDUC_LOOP_STEP)
		{
			for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
			{
				int wei_idx = r* (MLO_REDUC_LOOP_STEP)+lcl_id;

#if (MLO_N_LCL_IN_MAPS & (MLO_N_LCL_IN_MAPS - 1))
				int k = iDiv(wei_idx, MLO_N_LCL_IN_MAPS);
				int c = iMod(wei_idx, k, MLO_N_LCL_IN_MAPS);
#else
				int k = ((uint)wei_idx / MLO_N_LCL_IN_MAPS);
				int c = ((uint)wei_idx & (MLO_N_LCL_IN_MAPS - 1));
#endif

				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
					&& c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = final_sum[r];
				}

			} // for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
		} // if (p4 < MLO_REDUC_LOOP_STEP)
	} // if (inside_range_input)
#else
	for (int r = 0; r < (MLO_ACCUM_SZ / MLO_REDUC_LOOP_STEP); ++r)
	{
		barrier(CLK_LOCAL_MEM_FENCE);

		for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
		{
			lcl_mem[lcl_id*MLO_REDUC_LOOP_STEP + rr] = pvt_accum[r*MLO_REDUC_LOOP_STEP + rr];
		}
		barrier(CLK_LOCAL_MEM_FENCE);

		if (lcl_id == 0)
		{
			for (int j = 1; j < MLO_GRP_SZ; j++)
			{
				for (int rr = 0; rr < MLO_REDUC_LOOP_STEP; ++rr)
				{
					pvt_accum[r*MLO_REDUC_LOOP_STEP + rr] += lcl_mem[(lcl_id + j)*MLO_REDUC_LOOP_STEP + rr];
				}
			}
		}
	}

	if (lcl_id == 0)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				if (true
#if MLO_N_IN_MAPS_ALIGNED == 0
				 && c_idx + c < MLO_N_INPUTS
#endif
#if MLO_N_OUT_MAPS_ALIGNED == 0
					&& k_idx + k < MLO_N_OUTPUTS
#endif
					)
				{
					int wei_off = wei_df_off + k*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE;
					weights_df[wei_off] = pvt_accum[k*MLO_N_LCL_IN_MAPS + c];
				}

			}

		}

	}

#endif

}


moved gid_off outside of while and comments

/* Only works for NCHW
 * bitmap tracks which dims are the same betwen 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

__kernel void AddTensor(global float *a,
        const int a_c, const int a_h, const int a_w,
        const int a_nstride, a_cstride,
        global float *c,
        const int c_c, const int c_h, const int c_w,
        const int c_nstride, c_cstride,
        const unsigned int bitmap, const int work_per_wg)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    float add = a[gid];

    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    int o_w_gid_off = gid % a_w;
    int o_h_gid_off = (gid / a_w) % a_h;
    int o_c_gid_off = (gid / a_cstride) % a_c;
    int o_n_gid_off = gid / a_nstride;

    while(lid < work_per_wg) {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

        c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w] += add;

        lid += get_local_size(0);
    }
}

/* Only works for NCHW
 */

__kernel void AddTensor(global float *a,
        const int a_c, const int a_h, const int a_w,
        const int a_nstride, a_cstride,
        global float *c,
        const int c_c, const int c_h, const int c_w,
        const int c_nstride, c_cstride,
        const unsigned int bitmap, const int work_per_wg)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    float add = a[gid];

    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    while(lid < work_per_wg) {
        int o_w = (bitmap & (1 << 0)) ? gid % a_w               : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? (gid / a_w) % a_h       : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? (gid / a_cstride) % a_c : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? gid / a_nstride         : lid / o_n_div;

        c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w] += add;

        lid += get_local_size(0);
    }
}