fixed bug with wrong component being stored in texture

#pragma OPENCL EXTENSION cl_khr_3d_image_writes : enable

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVF3DInit(__read_only image3d_t volume, __write_only image3d_t init_vector_field, __write_only image3d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    float f100 = read_imagef(volume, sampler, pos + (int4)(1,0,0,0)).x;
    float f_100 = read_imagef(volume, sampler, pos - (int4)(1,0,0,0)).x;
    float f010 = read_imagef(volume, sampler, pos + (int4)(0,1,0,0)).x;
    float f0_10 = read_imagef(volume, sampler, pos - (int4)(0,1,0,0)).x;
    float f001 = read_imagef(volume, sampler, pos + (int4)(0,0,1,0)).x;
    float f00_1 = read_imagef(volume, sampler, pos - (int4)(0,0,1,0)).x;

    float4 gradient = {
        0.5f*(f100-f_100),
        0.5f*(f010-f0_10),
        0.5f*(f001-f00_1),
        0.5f*(f001-f00_1)
    };

    write_imagef(init_vector_field, pos, gradient);
    write_imagef(vector_field, pos, gradient);
}

#define LA3D(x,y,z) (x + (y<<3) + (z<<6))
__kernel __attribute__((reqd_work_group_size(8,8,4))) void GVF3DIteration(__read_only image3d_t init_vector_field, __read_only image3d_t read_vector_field, __write_only image3d_t write_vector_field, __private float mu) {

    int4 writePos = {
        get_global_id(0)-(get_group_id(0)*2+1),
        get_global_id(1)-(get_group_id(1)*2+1),
        get_global_id(2)-(get_group_id(2)*2+1),
        0
    };
    if(writePos.x > 255 || writePos.y > 255 || writePos.z > 255
            || writePos.x < 0 || writePos.y < 0 || writePos.z < 0)
        writePos = (int4)(50, 50, 50, 0);

    int3 localPos = {get_local_id(0), get_local_id(1), get_local_id(2)};

    // Enforce mirror boundary conditions
    //int4 size = {get_global_size(0), get_global_size(1), get_global_size(2), 0};
    //int4 pos = writePos;
    //pos = select(pos, (int4)(2,2,2,0), pos == (int4)(0,0,0,0));
    //pos = select(pos, size-3, pos == size-1);

    // Allocate shared memory
    __local float2 sharedMemory[256];
	__local float sharedMemorySingle[256];

    // Read into shared memory
    float4 v = read_imagef(read_vector_field, sampler, writePos);
    sharedMemory[LA3D(localPos.x,localPos.y,localPos.z)]= v.xy;
    sharedMemorySingle[LA3D(localPos.x,localPos.y,localPos.z)] = v.z;

    /*
    int x = localPos.x;
    int y = localPos.y;
    int z = localPos.z;
    int w = 0;
    int rowID2 = (LA(x,y,z) >> 4) & 0x1F;
    int bankID2 = ((LA(x,y,z)) & 0xF) << 1;
    int rowID = (LA(x,y,z) >> 5) & 0x1F;
	int bankID = ((LA(x,y,z)) & 0x1F);
    printf("Float2s: %d %d %d - %d - %d\n", x,y,z, bankID2, rowID2);
	printf("Floats: %d %d %d - %d - %d\n", x,y,z, bankID, rowID);
    */
    int3 comp = (localPos == (int3)(0,0,0)) +
        (localPos == (int3)(7,7,3));

    // Synchronize the threads in the group
    barrier(CLK_LOCAL_MEM_FENCE);

    if(comp.x+comp.y+comp.z ==  0) {
        // Load data from shared memory and do calculations
        float2 init_vector = read_imagef(init_vector_field, sampler, writePos).xy; // should read from pos

        float3 fx1, fx_1, fy1, fy_1, fz1, fz_1;
        fx1.xy = sharedMemory[LA3D(localPos.x+1,localPos.y,localPos.z)];
        fx1.z = sharedMemorySingle[LA3D(localPos.x+1,localPos.y,localPos.z)];
        fx_1.xy = sharedMemory[LA3D(localPos.x-1,localPos.y,localPos.z)];
        fx_1.z = sharedMemorySingle[LA3D(localPos.x-1,localPos.y,localPos.z)];
        fy1.xy = sharedMemory[LA3D(localPos.x,localPos.y+1,localPos.z)];
        fy1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y+1,localPos.z)];
        fy_1.xy = sharedMemory[LA3D(localPos.x,localPos.y-1,localPos.z)];
        fy_1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y-1,localPos.z)];
        fz1.xy = sharedMemory[LA3D(localPos.x,localPos.y,localPos.z+1)];
        fz1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y,localPos.z+1)];
        fz_1.xy = sharedMemory[LA3D(localPos.x,localPos.y,localPos.z-1)];
        fz_1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y,localPos.z-1)];

        // Update the vector field: Calculate Laplacian using a 3D central difference scheme
        float3 laplacian = -6*v.xyz + fx1 + fx_1 + fy1 + fy_1 + fz1 + fz_1;

        v.xyz += mu * laplacian - (v.xyz - (float3)(init_vector.x, init_vector.y, v.w))*
            (init_vector.x*init_vector.x+init_vector.y*init_vector.y+v.w*v.w);

        write_imagef(write_vector_field, writePos, v);
    }
}

__kernel void GVF3DResult(__write_only image3d_t result, __read_only image3d_t vectorField) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.w = 0;
    //if(1 < length(vector))
    //    printf("%f %f %f\n", vector.x, vector.y, vector.z);
    write_imagef(result, pos, length(vector));
}

#pragma OPENCL EXTENSION cl_khr_3d_image_writes : enable

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;

__kernel void GVF3DInit(__read_only image3d_t volume, __write_only image3d_t init_vector_field, __write_only image3d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};

    float f100 = read_imagef(volume, sampler, pos + (int4)(1,0,0,0)).x;
    float f_100 = read_imagef(volume, sampler, pos - (int4)(1,0,0,0)).x;
    float f010 = read_imagef(volume, sampler, pos + (int4)(0,1,0,0)).x;
    float f0_10 = read_imagef(volume, sampler, pos - (int4)(0,1,0,0)).x;
    float f001 = read_imagef(volume, sampler, pos + (int4)(0,0,1,0)).x;
    float f00_1 = read_imagef(volume, sampler, pos - (int4)(0,0,1,0)).x;

    float4 gradient = {
        0.5f*(f100-f_100),
        0.5f*(f010-f0_10),
        0.5f*(f001-f00_1),
        0.5f*(f100-f_100)
    };

    write_imagef(init_vector_field, pos, gradient);
    write_imagef(vector_field, pos, gradient);
}

#define LA3D(x,y,z) (x + (y<<3) + (z<<6))
__kernel __attribute__((reqd_work_group_size(8,8,4))) void GVF3DIteration(__read_only image3d_t init_vector_field, __read_only image3d_t read_vector_field, __write_only image3d_t write_vector_field, __private float mu) {

    int4 writePos = {
        get_global_id(0)-(get_group_id(0)*2+1),
        get_global_id(1)-(get_group_id(1)*2+1),
        get_global_id(2)-(get_group_id(2)*2+1),
        0
    };
    if(writePos.x > 255 || writePos.y > 255 || writePos.z > 255
            || writePos.x < 0 || writePos.y < 0 || writePos.z < 0)
        writePos = (int4)(50, 50, 50, 0);

    int3 localPos = {get_local_id(0), get_local_id(1), get_local_id(2)};

    // Enforce mirror boundary conditions
    //int4 size = {get_global_size(0), get_global_size(1), get_global_size(2), 0};
    //int4 pos = writePos;
    //pos = select(pos, (int4)(2,2,2,0), pos == (int4)(0,0,0,0));
    //pos = select(pos, size-3, pos == size-1);

    // Allocate shared memory
    __local float2 sharedMemory[256];
	__local float sharedMemorySingle[256];

    // Read into shared memory
    float4 v = read_imagef(read_vector_field, sampler, writePos);
    sharedMemory[LA3D(localPos.x,localPos.y,localPos.z)]= v.xy;
    sharedMemorySingle[LA3D(localPos.x,localPos.y,localPos.z)] = v.z;

    /*
    int x = localPos.x;
    int y = localPos.y;
    int z = localPos.z;
    int w = 0;
    int rowID2 = (LA(x,y,z) >> 4) & 0x1F;
    int bankID2 = ((LA(x,y,z)) & 0xF) << 1;
    int rowID = (LA(x,y,z) >> 5) & 0x1F;
	int bankID = ((LA(x,y,z)) & 0x1F);
    printf("Float2s: %d %d %d - %d - %d\n", x,y,z, bankID2, rowID2);
	printf("Floats: %d %d %d - %d - %d\n", x,y,z, bankID, rowID);
    */
    int3 comp = (localPos == (int3)(0,0,0)) +
        (localPos == (int3)(7,7,3));

    // Synchronize the threads in the group
    barrier(CLK_LOCAL_MEM_FENCE);

    if(comp.x+comp.y+comp.z ==  0) {
        // Load data from shared memory and do calculations
        float2 init_vector = read_imagef(init_vector_field, sampler, writePos).xy; // should read from pos

        float3 fx1, fx_1, fy1, fy_1, fz1, fz_1;
        fx1.xy = sharedMemory[LA3D(localPos.x+1,localPos.y,localPos.z)];
        fx1.z = sharedMemorySingle[LA3D(localPos.x+1,localPos.y,localPos.z)];
        fx_1.xy = sharedMemory[LA3D(localPos.x-1,localPos.y,localPos.z)];
        fx_1.z = sharedMemorySingle[LA3D(localPos.x-1,localPos.y,localPos.z)];
        fy1.xy = sharedMemory[LA3D(localPos.x,localPos.y+1,localPos.z)];
        fy1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y+1,localPos.z)];
        fy_1.xy = sharedMemory[LA3D(localPos.x,localPos.y-1,localPos.z)];
        fy_1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y-1,localPos.z)];
        fz1.xy = sharedMemory[LA3D(localPos.x,localPos.y,localPos.z+1)];
        fz1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y,localPos.z+1)];
        fz_1.xy = sharedMemory[LA3D(localPos.x,localPos.y,localPos.z-1)];
        fz_1.z = sharedMemorySingle[LA3D(localPos.x,localPos.y,localPos.z-1)];

        // Update the vector field: Calculate Laplacian using a 3D central difference scheme
        float3 laplacian = -6*v.xyz + fx1 + fx_1 + fy1 + fy_1 + fz1 + fz_1;

        v.xyz += mu * laplacian - (v.xyz - (float3)(init_vector.x, init_vector.y, v.w))*
            (init_vector.x*init_vector.x+init_vector.y*init_vector.y+v.w*v.w);

        write_imagef(write_vector_field, writePos, v);
    }
}

__kernel void GVF3DResult(__write_only image3d_t result, __read_only image3d_t vectorField) {

    int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.w = 0;
    //if(1 < length(vector))
    //    printf("%f %f %f\n", vector.x, vector.y, vector.z);
    write_imagef(result, pos, length(vector));
}


revised the variables names in building the grid

/*
    Build the grid of vectors
    Animate vector field
    Copyright (c) 2013-2014, Randall Maas
    Created by Randall Maas on 12/22/13.

    Translated from  https://github.com/cambecc/earth
    Copyright (c) 2013 Cameron Beccario
    The MIT License - http://opensource.org/licenses/MIT

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
*//*
    Vectors are represented as unit direction
    Points are 2D

    The data is provided as a  vector field arranged by lat/lon
    The vectors are in the units of m/s

    Problem #1
    The problem is that lat/lon doesn't directly map to meter distances.
    The fix is that Cameron uses a finite differences method.  At the poles
    these distances between two points is close.  He seems to distort the vecto
    magnitudes at these.

    A more sophisticate method might accomodate the wobbly shape of the earth.

    Problem #2
    When the particles going over the poles or off the wrap around, the point is lost.
    This singularity hopefully won't be noticed.  But it isn't handled elegantly.

    Problem #3
    We need a bigger field.  The field is not sufficiently large for a large sphere.
 */

#define M_PI   3.1415926535897932384626433832795f

/** Bilinear interpolation
    @param x
    @param y
    @param g00
    @param g10
    @param g01
    @param g11
    @returns {float}
*/
float2 bilinear(float x, float y, float2 g00, float2 g10, float2 g01, float2 g11)
{
    float a = (1.0 - x) * (1.0 - y);
    float b = x * (1.0 - y);
    float c = (1.0 - x) * y;
    float d = x * y;
    float u = g00.x * a + g10.x * b + g01.x * c + g11.x * d;
    float v = g00.y * a + g10.y * b + g01.y * c + g11.y * d;
    return (float2)(u, v);
}

/** Interpolates
    @param coord  The coordinate of the point of discussion
    @param origin The origin of the coordinate space
    @param delta  The distance between grid points (e.g., 2.5 deg lon, 2.5 deg lat)
    @returns vector
*/
float2 interpolate(float2 coord
                   , uint2 srcSize  // The number of uv vectors wide and high
                   , __global float2* srcField)
{
    //         1      2           After converting λ and φ to fractional grid indexes i and j, we find the
    //        fi  i   ci          four points "G" that enclose point (i, j). These points are at the four
    //         | =1.4 |           corners specified by the floor and ceiling of i and j. For example, given
    //      ---G--|---G--- fj 8   i = 1.4 and j = 8.3, the four surrounding grid points are (1, 8), (2, 8),
    //    j ___|_ .   |           (1, 9) and (2, 9).
    //  =8.3   |      |
    //      ---G------G--- cj 9   Note that for wrapped grids, the first column is duplicated as the last
    //         |      |           column, so the index ci can be used without taking a modulo.

    // Calculate the longitude (x) index within the source field
    int fi = floor(coord.x);
    // calculate the latitude (y) index withing the source field
    int fj = floor(coord.y);

    uint ci = min(fi+1u, srcSize.x-1u);
    uint cj = min(fj+1u, srcSize.y-1u);

    // The starting spot of the row
    int rowOfs = fj*srcSize.x;

    float2 g00 = srcField[rowOfs+fi];
    float2 g10 = srcField[rowOfs+ci];

    rowOfs = cj*srcSize.x;
    float2 g01 = srcField[rowOfs+fi];
    float2 g11 = srcField[rowOfs+ci];
    // All four points found, so use bilinear interpolation to calculate the wind vector.
    return bilinear(coord.x - fi, coord.y - fj, g00, g10, g01, g11);

}

/**
 * Returns the distortion introduced by the specified projection at the given point.
 *
 * This method uses finite difference estimates to calculate warping by adding a very small amount (h) to
 * both the longitude and latitude to create two lines. These lines are then projected to pixel space, where
 * they become diagonals of triangles that represent how much the projection warps longitude and latitude at
 * that location.
 *
 * <pre>
 *        (u, v+h)                  (xu, yu)
 *           .                         .
 *           |               ==>        \
 *           |                           \   __. (xv, yv)
 *    (u, v) .____. (u+h, v)       (x, y) .--
 * </pre>
 *
 * See:
 *     Map Projections: A Working Manual, Snyder, John P: pubs.er.usgs.gov/publication/pp1395
 *     gis.stackexchange.com/questions/5068/how-to-create-an-accurate-tissot-indicatrix
 *     www.jasondavies.com/maps/tissot
 *
 * @returns {float4} float4 of scaled derivatives [dx/du, dy/du, dx/dv, dy/dv]
 */
float4 distortion(float2 latlon)
{
    float H = 0.0000630957344f; // pow(10, -5.2);
    float hu = latlon.x < 0 ? H : -H;
    float hv = latlon.y < 0 ? H : -H;
    float2 pu = (float2)(latlon.x + hu, latlon.y);
    float2 pv = (float2)(latlon.x,      latlon.y + hv);

    // Meridian scale factor (see Snyder, equation 4-3), where R = 1. This handles issue where length of 1º λ
    // changes depending on v. Without this, there is a pinching effect at the poles.
    float k = cos(latlon.y * M_PI *2.0 / 360.0);

    return (float4)(
                    (pu.x - latlon.x) / hu / k,
                    (pu.y - latlon.y) / hu / k,
                    (pv.x - latlon.x) / hv,
                    (pv.y - latlon.y) / hv
                    );
}

/** Calculate distortion of the wind vector caused by the shape of the projection at point
     (x, y). The wind vector is modified in place and returned by this function.
    @param latlon  The coordinate of the point of discussion
    @returns distorted vector
 */
float2 distort(float2 latlon, float scale, float2 wind)
{
    float2 uv = wind * scale;
    float4 d = distortion(latlon);

    // Scale distortion vectors by u and v, then add.
    wind.x = d.x * uv.x + d.z * uv.y;
    wind.y = d.y * uv.x + d.w * uv.y;

    return wind;
}

/** Build the internal form of the grid
    create the data grid by interpolating these, and handling the metric distance for the
    projection

    @param srcSize    The number of elements in the original grid
    @param srcField   The internal representation of the u-v grid
    @param origin     The origin of the coordinate space (e.g., 0.0E, 90.0N)
    @param delta      The distance between grid points (e.g., 2.5 deg lon, 2.5 deg lat)
    @param velocityScale The ?
    @param tgtSize    The number of bins wide and high the tgtField is (in the part that corresponds to the srcField)
    @param tgtStride     The number of entries per row (there may be more than in tgtSize)
    @param tgtField   The field modified for animation

    tgtField is indexed as
       x + y * stride
    stride == the number of internal grid columns.
    stride is >= the number of ideal columns
    The extra columns on the right (or left) of the main one are duplicated, to ease the math
*/
kernel void gridInterpolate(
                             uint2  srcSize
                           , __global float2* srcField
                           , float2 origin
                           , float2 delta
                           , float velocityScale
                           , uint2 tgtSize
                           , uint  tgtStride
                           , __global float2* tgtField
                           )
{
    // Step thru each of the degree points
    // Get each work-items unique row-column
    // Make this into a point (vector)
    uint2 tgtPoint = (uint2)(get_global_id(0),get_global_id(1));
    if (tgtSize.x == 0 || tgtSize.y == 0)
        return;

    // First, calculate the point within the source field
    // We use the srcSize (which may be longer than the original) so that the right hand size gets wrap around info
    float2 srcPoint= (float2)( clamp((float)(tgtPoint.x*srcSize.x)/(float)tgtSize.x, 0.0f, (float)srcSize.x-0.1f)
                             , clamp((float)(tgtPoint.y*srcSize.y)/(float)tgtSize.y, 0.0f, (float)srcSize.y-0.1f) );
    float2 wind = interpolate(srcPoint, srcSize, srcField);

    // Second, calculate the lat/lon of the wind vector
    float2 latlon;
    latlon . x = (srcPoint.x -   origin.x);
    latlon . y  =  (origin.y - srcPoint.y);

    // If the wind vector is valid, use the grid distoration
    // Calculate the distortion from the particular projection onto the grid
    wind = distort(latlon, velocityScale, wind);

    // The source data often starts at Longitude 0, while the land maps start at -180.
    // Shift this to match those maps
    tgtField[tgtPoint.x +     tgtPoint.y*tgtStride] = wind;
}

/** Build the internal form of the grid
    @param uData       The grid of the u component's of the vector
    @param vData       The grid of the v component's of the vector
    @param srcGridSize The number of elements in the original grid
    @param delta       The distance between grid points (e.g., 2.5 deg lon, 2.5 deg lat)
    @param srcGrid     The internal representation of the u-v grid

    srcgrid data both go from x,y coords to index as
       index = x + stride * y
    stride == the number of internal grid columns.
       stride is >= the number of ideal columns
       The extra columns on the right (or left) of the main one are duplicated, to ease the math

 */
kernel void gridBuild(
                     __constant float* uData, __constant float* vData
                     // The number of source grid points W-E and N-S (e.g., 144 x 73)
                     , uint2 srcGridSize
                     , float2 delta
                     , bool isContinuous
                     , __global  float2* srcGrid
                      )
{
    // Get each work-items unique row
    int j = get_global_id(0);
    int p = j * srcGridSize.x;

    int stride = srcGridSize.x + (isContinuous?1:0);

    // Scan mode 0 assumed. Longitude increases from λ0, and latitude decreases from φ0.
    // http://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_table3-4.shtml
    // Continuous srcGrids have an extra column
    int rowOfs =  stride * j;

    // HACK, I am shift the x origin by 180degrees; this assumes the origin, spacing, and size
    for (int i = 0; i < srcGridSize.x; i++, p++)
    {
        // Note: because the latitude decreases, the direction of the y component is flipped
        // I fix it here
        srcGrid[rowOfs + (i+180)%srcGridSize.x] = (float2)(uData[p], (j<360?-1:0)*vData[p]);
    }
    if (isContinuous)
    {
        // For wrapped grids, duplicate first column as last column to simplify interpolation logic
        srcGrid[rowOfs + srcGridSize.x] = srcGrid[rowOfs];
    }
}

/*
    Build the grid of vectors
    Animate vector field
    Copyright (c) 2013-2014, Randall Maas
    Created by Randall Maas on 12/22/13.

    Translated from  https://github.com/cambecc/earth
    Copyright (c) 2013 Cameron Beccario
    The MIT License - http://opensource.org/licenses/MIT

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
*//*
    Vectors are represented as unit direction
    Points are 2D

    The data is provided as a  vector field arranged by lat/lon
    The vectors are in the units of m/s

    Problem #1
    The problem is that lat/lon doesn't directly map to meter distances.
    The fix is that Cameron uses a finite differences method.  At the poles
    these distances between two points is close.  He seems to distort the vecto
    magnitudes at these.

    A more sophisticate method might accomodate the wobbly shape of the earth.

    Problem #2
    When the particles going over the poles or off the wrap around, the point is lost.
    This singularity hopefully won't be noticed.  But it isn't handled elegantly.

    Problem #3
    We need a bigger field.  The field is not sufficiently large for a large sphere.
 */

#define M_PI   3.1415926535897932384626433832795f

/** Bilinear interpolation
    @param x
    @param y
    @param g00
    @param g10
    @param g01
    @param g11
    @returns {float}
*/
float2 bilinear(float x, float y, float2 g00, float2 g10, float2 g01, float2 g11)
{
    float a = (1.0 - x) * (1.0 - y);
    float b = x * (1.0 - y);
    float c = (1.0 - x) * y;
    float d = x * y;
    float u = g00.x * a + g10.x * b + g01.x * c + g11.x * d;
    float v = g00.y * a + g10.y * b + g01.y * c + g11.y * d;
    return (float2)(u, v);
}

/** Interpolates
    @param coord  The coordinate of the point of discussion
    @param origin The origin of the coordinate space
    @param delta  The distance between grid points (e.g., 2.5 deg lon, 2.5 deg lat)
    @returns vector
*/
float2 interpolate(float2 coord
                   , uint2 srcSize  // The number of uv vectors wide and high
                   , __global float2* srcField)
{
    //         1      2           After converting λ and φ to fractional grid indexes i and j, we find the
    //        fi  i   ci          four points "G" that enclose point (i, j). These points are at the four
    //         | =1.4 |           corners specified by the floor and ceiling of i and j. For example, given
    //      ---G--|---G--- fj 8   i = 1.4 and j = 8.3, the four surrounding grid points are (1, 8), (2, 8),
    //    j ___|_ .   |           (1, 9) and (2, 9).
    //  =8.3   |      |
    //      ---G------G--- cj 9   Note that for wrapped grids, the first column is duplicated as the last
    //         |      |           column, so the index ci can be used without taking a modulo.

    // Calculate the longitude (x) index within the source field
    int fi = floor(coord.x);
    // calculate the latitude (y) index withing the source field
    int fj = floor(coord.y);
    uint ci = min(fi+1u, srcSize.x-1u);
    uint cj = min(fj+1u, srcSize.y-1u);

    // The starting spot of the row
    int rowOfs = fj*srcSize.x;

    float2 g00 = srcField[rowOfs+fi];
    float2 g10 = srcField[rowOfs+ci];

    rowOfs += srcSize.x;
    float2 g01 = srcField[rowOfs+fi];
    float2 g11 = srcField[rowOfs+ci];
    // All four points found, so use bilinear interpolation to calculate the wind vector.
    return bilinear(coord.x - fi, coord.y - fj, g00, g10, g01, g11);

}

/**
 * Returns the distortion introduced by the specified projection at the given point.
 *
 * This method uses finite difference estimates to calculate warping by adding a very small amount (h) to
 * both the longitude and latitude to create two lines. These lines are then projected to pixel space, where
 * they become diagonals of triangles that represent how much the projection warps longitude and latitude at
 * that location.
 *
 * <pre>
 *        (u, v+h)                  (xu, yu)
 *           .                         .
 *           |               ==>        \
 *           |                           \   __. (xv, yv)
 *    (u, v) .____. (u+h, v)       (x, y) .--
 * </pre>
 *
 * See:
 *     Map Projections: A Working Manual, Snyder, John P: pubs.er.usgs.gov/publication/pp1395
 *     gis.stackexchange.com/questions/5068/how-to-create-an-accurate-tissot-indicatrix
 *     www.jasondavies.com/maps/tissot
 *
 * @returns {float4} float4 of scaled derivatives [dx/du, dy/du, dx/dv, dy/dv]
 */
float4 distortion(float2 latlon)
{
    float H = 0.0000630957344f; // pow(10, -5.2);
    float hu = latlon.x < 0 ? H : -H;
    float hv = latlon.y < 0 ? H : -H;
    float2 pu = (float2)(latlon.x + hu, latlon.y);
    float2 pv = (float2)(latlon.x,      latlon.y + hv);

    // Meridian scale factor (see Snyder, equation 4-3), where R = 1. This handles issue where length of 1º λ
    // changes depending on v. Without this, there is a pinching effect at the poles.
    float k = cos(latlon.y / 360.0 * M_PI *2.0);

    return (float4)(
                    (pu.x - latlon.x) / hu / k,
                    (pu.y - latlon.y) / hu / k,
                    (pv.x - latlon.x) / hv,
                    (pv.y - latlon.y) / hv
                    );
}

/** Calculate distortion of the wind vector caused by the shape of the projection at point
     (x, y). The wind vector is modified in place and returned by this function.
    @param latlon  The coordinate of the point of discussion
    @returns distorted vector
 */
float2 distort(float2 latlon, float scale, float2 wind)
{
    float2 uv = wind * scale;
    float4 d = distortion(latlon);

    // Scale distortion vectors by u and v, then add.
    wind.x = d.x * uv.x + d.z * uv.y;
    wind.y = d.y * uv.x + d.w * uv.y;

    return wind;
}

/** Build the internal form of the grid
    create the data grid by interpolating these, and handling the metric distance for the
    projection

    @param gridSize   The number of elements in the original grid
    @param srcGrid    The internal representation of the u-v grid
    @param origin     The origin of the coordinate space (e.g., 0.0E, 90.0N)
    @param delta      The distance between grid points (e.g., 2.5 deg lon, 2.5 deg lat)
    @param velocityScale The ?
    @param numXBins   The number of bins wide the out grid is
    @param vectorField The grid modified for animation
*/
kernel void gridInterpolate(
                             uint2  srcSize
                           , __global float2* srcField
                           , float2 origin
                           , float2 delta
                           , float velocityScale
                           , uint2 tgtSize
                           , __global float2* tgtField
                           )
{
    // Step thru each of the degree points
    // Get each work-items unique row-column
    // Make this into a point (vector)
    uint2 tgtPoint = (uint2)(get_global_id(0),get_global_id(1));
    if (tgtSize.x == 0 || tgtSize.y == 0)
        return;

    // First, calculate the point within the source field
    // We use the srcSize (which may be longer than the original) so that the right hand size gets wrap around info
    float2 srcPoint= (float2)( clamp((float)(tgtPoint.x*srcSize.x)/(float)tgtSize.x, 0.0f, (float)srcSize.x-0.1f)
                             , clamp((float)(tgtPoint.y*srcSize.y)/(float)tgtSize.y, 0.0f, (float)srcSize.y-0.1f) );
    float2 wind = interpolate(srcPoint, srcSize, srcField);

    // Second, calculate the lat/lon of the wind vector
    float2 latlon;
    latlon . x = srcPoint.x - origin.x;
    latlon . y  =  origin.y - srcPoint.y;

    // If the wind vector is valid, use the grid distoration
    // Calculate the distortion from the particular projection onto the grid
    wind = distort(latlon, velocityScale, wind);

    // The source data often starts at Longitude 0, while the land maps start at -180.
    // Shift this to match those maps
    tgtField[tgtPoint.x +     tgtPoint.y*tgtSize.x] = wind;
}

/** Build the internal form of the grid

    @param uData      The grid of the u component's of the vector
    @param vData      The grid of the v component's of the vector
    @param gridSize   The number of elements in the original grid
    @param delta      The distance between grid points (e.g., 2.5 deg lon, 2.5 deg lat)
    @param srcGrid    The internal representation of the u-v grid
 */
kernel void gridBuild(
                     __constant float* uData, __constant float* vData
                     // The number of source grid points W-E and N-S (e.g., 144 x 73)
                     , uint2 gridSize
                     , float2 delta
                     , bool isContinuous
                     , __global  float2* srcGrid
                      )
{
    // Get each work-items unique row
    int j = get_global_id(0);
    int p = j * gridSize.x;

    // Scan mode 0 assumed. Longitude increases from λ0, and latitude decreases from φ0.
    // http://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_table3-4.shtml
    // Continuous srcGrids have an extra column
    int rowOfs = (gridSize.x + (isContinuous?1:0)) * j;

    // HACK, I am shift the x origin by 180degrees; this assumes the origin, spacing, and size
    for (int i = 0; i < gridSize.x; i++, p++)
    {
        // Note: because the latitude decreases, the direction of the y component is flipped
        // I fix it here
        srcGrid[rowOfs + (i+180)%gridSize.x] = (float2)(uData[p], -vData[p]);
    }
    if (isContinuous)
    {
        // For wrapped grids, duplicate first column as last column to simplify interpolation logic
        srcGrid[rowOfs + gridSize.x] = srcGrid[rowOfs];
    }
}


tici camerart 3.1 (#20143)

tici camerart 3.1 (#20143)

Co-authored-by: Comma Device <f3a929b3364b471a481f4f7cda0b4559ecde9aba@comma.ai>

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const half black_level = 42.0;

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.25985206, -0.378923, -0.21356857),
  (half3)(-0.11117607, 1.3962182, -0.46342976),
  (half3)(-0.21523926, -0.13449348, 1.47665819),
};

const __constant half3 alt_ccm[3] = {
  (half3)(0.84166382, -0.25025687, 0.1593848),
  (half3)(0.25124881, 1.13196952, 0.16597516),
  (half3)(-0.09291263, 0.11828735, 0.67464003),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.01;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 8.6 - 66*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb, int ggain) {
  half3 ret = (0,0,0);
  half cpx = clamp(0.03h, 0.1h, cpxb + cpxk * min(10, ggain));
  rgb.x = mf(rgb.x, cpx);
  rgb.y = mf(rgb.y, cpx);
  rgb.z = mf(rgb.z, cpx);
  half alp = 1.0 - 0.07*ggain;
  ret += (half)rgb.x * (alp*color_correction[0]+(1-alp)*alt_ccm[0]);
  ret += (half)rgb.y * (alp*color_correction[1]+(1-alp)*alt_ccm[1]);
  ret += (half)rgb.z * (alp*color_correction[2]+(1-alp)*alt_ccm[2]);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

half val_from_10(const uchar * source, int gx, int gy) {
  // parse 10bit
  int start = gy * FRAME_STRIDE + (5 * (gx / 4));
  int offset = gx % 4;
  uint major = (uint)source[start + offset] << 2;
  uint minor = (source[start + 4] >> (2 * offset)) & 3;
  half pv = (half)(major + minor);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv *= 0.00101833h; // /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        uint ggain
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global);
  cached[localOffset] = pv;

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global);
  } else if (x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global);
  }

  if (y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1);
    }
  } else if (y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb, (int)ggain);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);

}

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const half black_level = 42.0;

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.25985206, -0.378923, -0.21356857),
  (half3)(-0.11117607, 1.3962182, -0.46342976),
  (half3)(-0.21523926, -0.13449348, 1.47665819),
};

const __constant half3 base_ccm[3] = {
  (half3)(1,0,0),
  (half3)(0,1,0),
  (half3)(0,0,1),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.01;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 8.6 - 66*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb, int ggain) {
  half3 ret = (0,0,0);
  half cpx = clamp(0.03h, 0.1h, cpxb + cpxk * min(10, ggain));
  rgb.x = mf(rgb.x, cpx);
  rgb.y = mf(rgb.y, cpx);
  rgb.z = mf(rgb.z, cpx);
  ret += (half)rgb.x * (color_correction[0]);
  ret += (half)rgb.y * (color_correction[1]);
  ret += (half)rgb.z * (color_correction[2]);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

half val_from_10(const uchar * source, int gx, int gy) {
  // parse 10bit
  int start = gy * FRAME_STRIDE + (5 * (gx / 4));
  int offset = gx % 4;
  uint major = (uint)source[start + offset] << 2;
  uint minor = (source[start + 4] >> (2 * offset)) & 3;
  half pv = (half)(major + minor);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv *= 0.00101833h; // /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        uint ggain
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global);
  cached[localOffset] = pv;

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global);
  } else if (x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global);
  }

  if (y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1);
    }
  } else if (y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb, (int)ggain);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);

}


Tuning of MIOpenConvBwdWrW_LxG_P53 on LC

Added one pragma unroll to mitigate register pressure.

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
#if MLO_FILTER_PAD1 > 0

#if  MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS - 2)
#define MLO_IN_VERT_READS (MLO_IN_EXTENT1 + MLO_FILTER_PAD1)
#else
#define MLO_N_GENERIC_LOOPS 0
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

#else
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS)
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24((uint)u, (uint)d);
	return(r);
}

__attribute__((always_inline))
void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, uint lcl_id, uint scan_lcl, uint sum_stride, uint unit_len, UNUSED bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (uint i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (uint j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (uint i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
__attribute__((always_inline))
void readInput(uint lcl_id, uint gbl_in_scan_off, uint n_v_reads, const __global _FLOAT * __restrict bot, __local _FLOAT * __restrict lcl_bot)
{
	for (uint p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];
// TODO : more than 1 input
		uint c = 0;
		uint t_p4 = p4;
#if MLO_N_LCL_IN_MAPS > 1
		c = iDiv(p4, (MLO_N_IN_HORIZ_READS * n_v_reads));
		t_p4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS * n_v_reads));
#endif

#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
		uint c_scan = iDiv(t_p4, (MLO_N_IN_HORIZ_READS));
		uint c_pix4 = iMod(t_p4, c_scan, (MLO_N_IN_HORIZ_READS));
#else
		uint c_scan = t_p4 / (MLO_N_IN_HORIZ_READS);
		uint c_pix4 = t_p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

//		if (c < MLO_N_INPUTS)

		{
			uint bot_off = gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
			const __global _FLOAT * bot_p = &bot[bot_off];
#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (uint i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}

				for (uint i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}
			}

// MLO_N_LCL_IN_MAPS inputs
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan* MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}

/*
	core processing loop
	bot - input, from local (1 span)
	top - output diff, from global (array of spans, filters vertical size)

	loop over filter vertical size

*/
__attribute__((always_inline))
void Processing(UNUSED uint sc, uint sc_lcl_off, uint top_lim, int bot_lim, __private _FLOAT * __restrict pvt_accum, __local _FLOAT * __restrict lcl_bot, __private _FLOAT * __restrict top_dat)
{
	for (int l = top_lim; l >= bot_lim; --l)
	{
		for (uint m = 0; m < MLO_IN_TILE0; ++m)
		{
			for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					int bot_off = sc_lcl_off + c*MLO_IN_LCL_SZ + n + m;
					_FLOAT bot_val = lcl_bot[bot_off];
					for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
					{
						uint pvt_top_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
						uint pvt_accum_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;

						_FLOAT top_val = top_dat[pvt_top_off];

						pvt_accum[pvt_accum_off]
						// each wk-it process an input
							+= bot_val*top_val;
#if 0
						if (/*bot_val * top_val != 0 && */l == 0 && n == 1 && get_local_id(0) == 0 && get_local_id(1) == 0 && get_local_id(2) == 0 && k == 0)
						{
							printf("K: %d %d %d %f %f %f %f\n",
								sc,
								bot_off,
								pvt_top_off,
								pvt_accum[pvt_accum_off],
								bot_val * top_val,
								bot_val,
								top_val
							);
						}
#endif

					}
				}
			}

		}

	}

}

__attribute__((always_inline))
void moveOutputUp(__private _FLOAT * __restrict top_dat)
{
	// move up output to reduce overfetch
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
		{
			for (uint i = 0; i < MLO_IN_TILE0; ++i)
			{
				uint pvt_off_n = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0 + i;
				uint pvt_off_o = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) *MLO_IN_TILE0 + i;
				top_dat[pvt_off_n] = top_dat[pvt_off_o];
			}
		}
	}
}

__attribute__((always_inline))

__attribute__((always_inline))
void spanReadingOutput(int spn, int k, int j, int top_df_off, _FLOAT mask,
	__private _FLOAT * __restrict top_dat, const __global _FLOAT * __restrict top_df)
{
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
	const __global _FLOAT * top_df_p = &top_df[top_df_off];
#if MLO_OUT_N_PIXS_OFF > 0
	if (spn == MLO_N_SPANS_PER_SCAN - 1)
	{
		uint i = 0;
		for (; i < MLO_OUT_N_PIXS_OFF; ++i)
		{
			top_dat[pvt_off + i] = top_df_p[i] * mask;

#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif
		}
		for (; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = 0;
		}
	}
	else
#else
	(void)spn;
#endif
	{
		for (uint i = 0; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = top_df_p[i] * mask;
#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * __restrict bias_df,
#endif
	UNUSED _FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ) + 1];
	__local _FLOAT * lcl_bot = lcl;

	uint lcl_id = get_local_id(0);

	uint c_idx_base = get_group_id(0); // input map index base

	uint o_idx_base = get_group_id(1);// output map index base

	uint ib_base = get_group_id(2);

	uint ib = ib_base*(MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);

	uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	uint o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
	// 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
	// TODO: more than 1 input
#if MLO_N_SPANS_PER_SCAN & (MLO_N_SPANS_PER_SCAN - 1)
	uint o = iDiv(lcl_id, MLO_N_SPANS_PER_SCAN);
	uint spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);
#else
	uint o = lcl_id / MLO_N_SPANS_PER_SCAN;
	uint spn = lcl_id & (MLO_N_SPANS_PER_SCAN - 1);
#endif
	//	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);

	uint lcl_bot_off = spn * MLO_IN_TILE0;
	uint out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
	gbl_out_off += out_wk_item_off;
// no output out of range
	gbl_out_off = (o_idx + o < MLO_N_OUTPUTS && o < MLO_OUT_STACKS) ? gbl_out_off : 0;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_N_LCL_IN_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	// over all batches
	for (uint b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// top border input block
		uint gbl_in_scan_off = gbl_in_off;
		uint gbl_out_scan_off = gbl_out_off;

		// read input map
		readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);
		// move input pointer
		gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

		for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
		{
			top_dat[i] = 0;
		}

		// prefetch output

		int gbl_out_scan_off1 = gbl_out_scan_off;
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
		{
			for (uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				// loop around all output maps
				uint top_df_off = gbl_out_scan_off1 + j*MLO_OUT_STRIDE;
				_FLOAT mask = 1;
#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

					spanReadingOutput(spn, k, j, top_df_off, mask,
						top_dat, top_df);

			}
		}

		gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

		uint sc = 0;
		uint sc_lcl_off = lcl_bot_off;

		// prolog
		// handling padding

		// top padding
		for (; sc < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
		}

#ifdef __AMDGCN__
#pragma unroll 2
#endif
		for (; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
			- MLO_FILTER_PAD1
#endif
			; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
						top_dat, top_df);

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		}

// non-border input blocks
		for (uint i_loop = 0;i_loop < MLO_N_GENERIC_LOOPS; ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);

			readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < (i_loop + 2) * MLO_IN_EXTENT1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// bottom border block

		for (int i_loop = 0; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1); ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read 1 scan line less
// padding processing takes care of the bottom border.

#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))

			readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

			// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

		// epilog
		// handling padding

		for (; sc < MLO_IN_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, (MLO_FILTER_PAD1 + 1 - (MLO_IN_HEIGHT - sc)), pvt_accum, lcl_bot, top_dat);
			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

	// final summation over all output maps and each filter row
	// this coudl be done with log but it negligeble anyway
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
		{

			for (uint l = 0; l < MLO_FILTER_SIZE1; ++l)
			{

				barrier(CLK_LOCAL_MEM_FENCE);
				for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
				{
					uint pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
					lcl[lcl_id * MLO_FILTER_SIZE0 + n] =
						pvt_accum[pvt_off];

				}

				barrier(CLK_LOCAL_MEM_FENCE);

				if (spn == 0)
				{
					for (uint s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
					{

						for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
						{
							uint pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
							pvt_accum[pvt_off]
								+= lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
						}

					}
				}

			}
		}
	}

// output
// inputs are outputs
// TODO : for more than 1 input

	uint wei_df_off = (((ib/MLO_N_BATCH_LOOPS) * MLO_N_OUTPUTS + o_idx + o) * (uint)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		if (spn == 0 && o_idx + o + k*MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
				{
					weights_df[wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE + i] = pvt_accum[(k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + i];
				}
			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * __restrict weight_df_tmp,
	__global _FLOAT * __restrict weights_df
)
{
	uint gbl_id = get_global_id(0);
	uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
	uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	uint wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
	uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
	uint wei_idx = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (uint i = 0; i < batch_loop; ++i)
	{
		for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
#if MLO_FILTER_PAD1 > 0

#if  MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS - 2)
#define MLO_IN_VERT_READS (MLO_IN_EXTENT1 + MLO_FILTER_PAD1)
#else
#define MLO_N_GENERIC_LOOPS 0
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

#else
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS)
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24((uint)u, (uint)d);
	return(r);
}

__attribute__((always_inline))
void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, uint lcl_id, uint scan_lcl, uint sum_stride, uint unit_len, UNUSED bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (uint i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (uint j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (uint i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
__attribute__((always_inline))
void readInput(uint lcl_id, uint gbl_in_scan_off, uint n_v_reads, const __global _FLOAT * __restrict bot, __local _FLOAT * __restrict lcl_bot)
{
	for (uint p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];
// TODO : more than 1 input
		uint c = 0;
		uint t_p4 = p4;
#if MLO_N_LCL_IN_MAPS > 1
		c = iDiv(p4, (MLO_N_IN_HORIZ_READS * n_v_reads));
		t_p4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS * n_v_reads));
#endif

#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
		uint c_scan = iDiv(t_p4, (MLO_N_IN_HORIZ_READS));
		uint c_pix4 = iMod(t_p4, c_scan, (MLO_N_IN_HORIZ_READS));
#else
		uint c_scan = t_p4 / (MLO_N_IN_HORIZ_READS);
		uint c_pix4 = t_p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

//		if (c < MLO_N_INPUTS)

		{
			uint bot_off = gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
			const __global _FLOAT * bot_p = &bot[bot_off];
#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (uint i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}

				for (uint i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}
			}

// MLO_N_LCL_IN_MAPS inputs
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan* MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}

/*
	core processing loop
	bot - input, from local (1 span)
	top - output diff, from global (array of spans, filters vertical size)

	loop over filter vertical size

*/
__attribute__((always_inline))
void Processing(UNUSED uint sc, uint sc_lcl_off, uint top_lim, int bot_lim, __private _FLOAT * __restrict pvt_accum, __local _FLOAT * __restrict lcl_bot, __private _FLOAT * __restrict top_dat)
{
	for (int l = top_lim; l >= bot_lim; --l)
	{
		for (uint m = 0; m < MLO_IN_TILE0; ++m)
		{
			for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					int bot_off = sc_lcl_off + c*MLO_IN_LCL_SZ + n + m;
					_FLOAT bot_val = lcl_bot[bot_off];
					for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
					{
						uint pvt_top_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
						uint pvt_accum_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;

						_FLOAT top_val = top_dat[pvt_top_off];

						pvt_accum[pvt_accum_off]
						// each wk-it process an input
							+= bot_val*top_val;
#if 0
						if (/*bot_val * top_val != 0 && */l == 0 && n == 1 && get_local_id(0) == 0 && get_local_id(1) == 0 && get_local_id(2) == 0 && k == 0)
						{
							printf("K: %d %d %d %f %f %f %f\n",
								sc,
								bot_off,
								pvt_top_off,
								pvt_accum[pvt_accum_off],
								bot_val * top_val,
								bot_val,
								top_val
							);
						}
#endif

					}
				}
			}

		}

	}

}

__attribute__((always_inline))
void moveOutputUp(__private _FLOAT * __restrict top_dat)
{
	// move up output to reduce overfetch
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
		{
			for (uint i = 0; i < MLO_IN_TILE0; ++i)
			{
				uint pvt_off_n = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0 + i;
				uint pvt_off_o = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) *MLO_IN_TILE0 + i;
				top_dat[pvt_off_n] = top_dat[pvt_off_o];
			}
		}
	}
}

__attribute__((always_inline))

__attribute__((always_inline))
void spanReadingOutput(int spn, int k, int j, int top_df_off, _FLOAT mask,
	__private _FLOAT * __restrict top_dat, const __global _FLOAT * __restrict top_df)
{
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
	const __global _FLOAT * top_df_p = &top_df[top_df_off];
#if MLO_OUT_N_PIXS_OFF > 0
	if (spn == MLO_N_SPANS_PER_SCAN - 1)
	{
		uint i = 0;
		for (; i < MLO_OUT_N_PIXS_OFF; ++i)
		{
			top_dat[pvt_off + i] = top_df_p[i] * mask;

#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif
		}
		for (; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = 0;
		}
	}
	else
#else
	(void)spn;
#endif
	{
		for (uint i = 0; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = top_df_p[i] * mask;
#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * __restrict bias_df,
#endif
	UNUSED _FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ) + 1];
	__local _FLOAT * lcl_bot = lcl;

	uint lcl_id = get_local_id(0);

	uint c_idx_base = get_group_id(0); // input map index base

	uint o_idx_base = get_group_id(1);// output map index base

	uint ib_base = get_group_id(2);

	uint ib = ib_base*(MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);

	uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	uint o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
	// 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
	// TODO: more than 1 input
#if MLO_N_SPANS_PER_SCAN & (MLO_N_SPANS_PER_SCAN - 1)
	uint o = iDiv(lcl_id, MLO_N_SPANS_PER_SCAN);
	uint spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);
#else
	uint o = lcl_id / MLO_N_SPANS_PER_SCAN;
	uint spn = lcl_id & (MLO_N_SPANS_PER_SCAN - 1);
#endif
	//	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);

	uint lcl_bot_off = spn * MLO_IN_TILE0;
	uint out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
	gbl_out_off += out_wk_item_off;
// no output out of range
	gbl_out_off = (o_idx + o < MLO_N_OUTPUTS && o < MLO_OUT_STACKS) ? gbl_out_off : 0;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_N_LCL_IN_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	// over all batches
	for (uint b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// top border input block
		uint gbl_in_scan_off = gbl_in_off;
		uint gbl_out_scan_off = gbl_out_off;

		// read input map
		readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);
		// move input pointer
		gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

		for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
		{
			top_dat[i] = 0;
		}

		// prefetch output

		int gbl_out_scan_off1 = gbl_out_scan_off;
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
		{
			for (uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				// loop around all output maps
				uint top_df_off = gbl_out_scan_off1 + j*MLO_OUT_STRIDE;
				_FLOAT mask = 1;
#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

					spanReadingOutput(spn, k, j, top_df_off, mask,
						top_dat, top_df);

			}
		}

		gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

		uint sc = 0;
		uint sc_lcl_off = lcl_bot_off;

		// prolog
		// handling padding

		// top padding
		for (; sc < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
		}

		for (; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
			- MLO_FILTER_PAD1
#endif
			; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
						top_dat, top_df);

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		}

// non-border input blocks
		for (uint i_loop = 0;i_loop < MLO_N_GENERIC_LOOPS; ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);

			readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < (i_loop + 2) * MLO_IN_EXTENT1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// bottom border block

//#if (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1) != 0
		for (int i_loop = 0; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1); ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
//#endif
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read 1 scan line less
// padding processing takes care of the bottom border.

#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))

			readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

			// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

		// epilog
		// handling padding

		for (; sc < MLO_IN_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, (MLO_FILTER_PAD1 + 1 - (MLO_IN_HEIGHT - sc)), pvt_accum, lcl_bot, top_dat);
			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

	// final summation over all output maps and each filter row
	// this coudl be done with log but it negligeble anyway
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
		{

			for (uint l = 0; l < MLO_FILTER_SIZE1; ++l)
			{

				barrier(CLK_LOCAL_MEM_FENCE);
				for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
				{
					uint pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
					lcl[lcl_id * MLO_FILTER_SIZE0 + n] =
						pvt_accum[pvt_off];

				}

				barrier(CLK_LOCAL_MEM_FENCE);

				if (spn == 0)
				{
					for (uint s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
					{

						for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
						{
							uint pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
							pvt_accum[pvt_off]
								+= lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
						}

					}
				}

			}
		}
	}

// output
// inputs are outputs
// TODO : for more than 1 input

	uint wei_df_off = (((ib/MLO_N_BATCH_LOOPS) * MLO_N_OUTPUTS + o_idx + o) * (uint)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		if (spn == 0 && o_idx + o + k*MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
				{
					weights_df[wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE + i] = pvt_accum[(k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + i];
				}
			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * __restrict weight_df_tmp,
	__global _FLOAT * __restrict weights_df
)
{
	uint gbl_id = get_global_id(0);
	uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
	uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	uint wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
	uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
	uint wei_idx = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (uint i = 0; i < batch_loop; ++i)
	{
		for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}