Made transpose kernel struct init proper according to the C standard

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the common kernels shared among different BLAS functions. This file contains
// a kernel to transpose matrices. This is a 'fast' version with restrictions, see the
// 'padtranspose.opencl' file for a general transpose kernel.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Data-widths
#if TRA_WPT == 1
  typedef real realT;
#elif TRA_WPT == 2
  typedef real2 realT;
#elif TRA_WPT == 4
  typedef real4 realT;
#elif TRA_WPT == 8
  typedef real8 realT;
#elif TRA_WPT == 16
  typedef real16 realT;
#endif

// =================================================================================================

// Transposes and copies a matrix. Requires both matrices to be of the same dimensions and without
// offset. A more general version is available in 'padtranspose.opencl'.
__kernel __attribute__((reqd_work_group_size(TRA_DIM, TRA_DIM, 1)))
void TransposeMatrixFast(const int ld,
                         __global const realT* restrict src,
                         __global realT* dest,
                         const real_arg arg_alpha) {
  const real alpha = GetRealArg(arg_alpha);

  // Sets the group identifiers. They might be 'shuffled' around to distribute work in a different
  // way over workgroups, breaking memory-bank dependencies.
  const int gid0 = get_group_id(0);
  #if TRA_SHUFFLE == 1
    const int gid1 = (get_group_id(0) + get_group_id(1)) % get_num_groups(0);
  #else
    const int gid1 = get_group_id(1);
  #endif

  // Local memory to store a tile of the matrix (for coalescing)
  __local realT tile[TRA_WPT*TRA_DIM][TRA_DIM + TRA_PAD];

  // Loops over the work per thread
  #pragma unroll
  for (int w_one=0; w_one<TRA_WPT; ++w_one) {

    // Computes the identifiers for the source matrix. Note that the local and global dimensions
    // do not correspond to each other!
    const int id_one = gid1 * TRA_DIM + get_local_id(0);
    const int id_two = (gid0 * TRA_DIM + get_local_id(1))*TRA_WPT + w_one;

    // Loads data into the local memory
    realT value = src[id_two*(ld/TRA_WPT) + id_one];
    tile[get_local_id(0)*TRA_WPT + w_one][get_local_id(1)] = value;
  }

  // Synchronizes all threads in a workgroup
  barrier(CLK_LOCAL_MEM_FENCE);

  // Loads transposed data from the local memory
  realT v[TRA_WPT];
  #pragma unroll
  for (int w_one=0; w_one<TRA_WPT; ++w_one) {
    v[w_one] = tile[get_local_id(1)*TRA_WPT + w_one][get_local_id(0)];
  }

  // Performs the register-level transpose of the vectorized data
  realT results[TRA_WPT];
  #if TRA_WPT == 1
    results[0] = v[0];
  #elif TRA_WPT == 2
    results[0].x = v[0].x; results[0].y = v[1].x;
    results[1].x = v[0].y; results[1].y = v[1].y;
  #elif TRA_WPT == 4
    results[0].x = v[0].x; results[0].y; = v[1].x; results[0].z = v[2].x; results[0].w = v[3].x;
    results[1].x = v[0].y; results[1].y; = v[1].y; results[1].z = v[2].y; results[1].w = v[3].y;
    results[2].x = v[0].z; results[2].y; = v[1].z; results[2].z = v[2].z; results[2].w = v[3].z;
    results[3].x = v[0].w; results[3].y; = v[1].w; results[3].z = v[2].w; results[3].w = v[3].w;
  #elif TRA_WPT == 8
    results[0].s0 = v[0].s0; results[0].s1 = v[1].s0; results[0].s2 = v[2].s0; results[0].s3 = v[3].s0; results[0].s4 = v[4].s0; results[0].s5 = v[5].s0; results[0].s6 = v[6].s0; results[0].s7 = v[7].s0;
    results[1].s0 = v[0].s1; results[1].s1 = v[1].s1; results[1].s2 = v[2].s1; results[1].s3 = v[3].s1; results[1].s4 = v[4].s1; results[1].s5 = v[5].s1; results[1].s6 = v[6].s1; results[1].s7 = v[7].s1;
    results[2].s0 = v[0].s2; results[2].s1 = v[1].s2; results[2].s2 = v[2].s2; results[2].s3 = v[3].s2; results[2].s4 = v[4].s2; results[2].s5 = v[5].s2; results[2].s6 = v[6].s2; results[2].s7 = v[7].s2;
    results[3].s0 = v[0].s3; results[3].s1 = v[1].s3; results[3].s2 = v[2].s3; results[3].s3 = v[3].s3; results[3].s4 = v[4].s3; results[3].s5 = v[5].s3; results[3].s6 = v[6].s3; results[3].s7 = v[7].s3;
    results[4].s0 = v[0].s4; results[4].s1 = v[1].s4; results[4].s2 = v[2].s4; results[4].s3 = v[3].s4; results[4].s4 = v[4].s4; results[4].s5 = v[5].s4; results[4].s6 = v[6].s4; results[4].s7 = v[7].s4;
    results[5].s0 = v[0].s5; results[5].s1 = v[1].s5; results[5].s2 = v[2].s5; results[5].s3 = v[3].s5; results[5].s4 = v[4].s5; results[5].s5 = v[5].s5; results[5].s6 = v[6].s5; results[5].s7 = v[7].s5;
    results[6].s0 = v[0].s6; results[6].s1 = v[1].s6; results[6].s2 = v[2].s6; results[6].s3 = v[3].s6; results[6].s4 = v[4].s6; results[6].s5 = v[5].s6; results[6].s6 = v[6].s6; results[6].s7 = v[7].s6;
    results[7].s0 = v[0].s7; results[7].s1 = v[1].s7; results[7].s2 = v[2].s7; results[7].s3 = v[3].s7; results[7].s4 = v[4].s7; results[7].s5 = v[5].s7; results[7].s6 = v[6].s7; results[7].s7 = v[7].s7;
  #elif TRA_WPT == 16
    results[ 0].s0 = v[0].s0; results[ 0].s1 = v[1].s0; results[ 0].s2 = v[2].s0; results[ 0].s3 = v[3].s0; results[ 0].s4 = v[4].s0; results[ 0].s5 = v[5].s0; results[ 0].s6 = v[6].s0; results[ 0].s7 = v[7].s0; results[ 0].s8 = v[8].s0; results[ 0].s9 = v[9].s0; results[ 0].sA = v[10].s0;, results[ 0].sB = v[11].s0;, results[ 0].sC = v[12].s0;, results[ 0].sD = v[13].s0;, results[ 0].sE = v[14].s0;, results[ 0].sF = v[15].s0;
    results[ 1].s0 = v[0].s1; results[ 1].s1 = v[1].s1; results[ 1].s2 = v[2].s1; results[ 1].s3 = v[3].s1; results[ 1].s4 = v[4].s1; results[ 1].s5 = v[5].s1; results[ 1].s6 = v[6].s1; results[ 1].s7 = v[7].s1; results[ 1].s8 = v[8].s1; results[ 1].s9 = v[9].s1; results[ 1].sA = v[10].s1;, results[ 1].sB = v[11].s1;, results[ 1].sC = v[12].s1;, results[ 1].sD = v[13].s1;, results[ 1].sE = v[14].s1;, results[ 1].sF = v[15].s1;
    results[ 2].s0 = v[0].s2; results[ 2].s1 = v[1].s2; results[ 2].s2 = v[2].s2; results[ 2].s3 = v[3].s2; results[ 2].s4 = v[4].s2; results[ 2].s5 = v[5].s2; results[ 2].s6 = v[6].s2; results[ 2].s7 = v[7].s2; results[ 2].s8 = v[8].s2; results[ 2].s9 = v[9].s2; results[ 2].sA = v[10].s2;, results[ 2].sB = v[11].s2;, results[ 2].sC = v[12].s2;, results[ 2].sD = v[13].s2;, results[ 2].sE = v[14].s2;, results[ 2].sF = v[15].s2;
    results[ 3].s0 = v[0].s3; results[ 3].s1 = v[1].s3; results[ 3].s2 = v[2].s3; results[ 3].s3 = v[3].s3; results[ 3].s4 = v[4].s3; results[ 3].s5 = v[5].s3; results[ 3].s6 = v[6].s3; results[ 3].s7 = v[7].s3; results[ 3].s8 = v[8].s3; results[ 3].s9 = v[9].s3; results[ 3].sA = v[10].s3;, results[ 3].sB = v[11].s3;, results[ 3].sC = v[12].s3;, results[ 3].sD = v[13].s3;, results[ 3].sE = v[14].s3;, results[ 3].sF = v[15].s3;
    results[ 4].s0 = v[0].s4; results[ 4].s1 = v[1].s4; results[ 4].s2 = v[2].s4; results[ 4].s3 = v[3].s4; results[ 4].s4 = v[4].s4; results[ 4].s5 = v[5].s4; results[ 4].s6 = v[6].s4; results[ 4].s7 = v[7].s4; results[ 4].s8 = v[8].s4; results[ 4].s9 = v[9].s4; results[ 4].sA = v[10].s4;, results[ 4].sB = v[11].s4;, results[ 4].sC = v[12].s4;, results[ 4].sD = v[13].s4;, results[ 4].sE = v[14].s4;, results[ 4].sF = v[15].s4;
    results[ 5].s0 = v[0].s5; results[ 5].s1 = v[1].s5; results[ 5].s2 = v[2].s5; results[ 5].s3 = v[3].s5; results[ 5].s4 = v[4].s5; results[ 5].s5 = v[5].s5; results[ 5].s6 = v[6].s5; results[ 5].s7 = v[7].s5; results[ 5].s8 = v[8].s5; results[ 5].s9 = v[9].s5; results[ 5].sA = v[10].s5;, results[ 5].sB = v[11].s5;, results[ 5].sC = v[12].s5;, results[ 5].sD = v[13].s5;, results[ 5].sE = v[14].s5;, results[ 5].sF = v[15].s5;
    results[ 6].s0 = v[0].s6; results[ 6].s1 = v[1].s6; results[ 6].s2 = v[2].s6; results[ 6].s3 = v[3].s6; results[ 6].s4 = v[4].s6; results[ 6].s5 = v[5].s6; results[ 6].s6 = v[6].s6; results[ 6].s7 = v[7].s6; results[ 6].s8 = v[8].s6; results[ 6].s9 = v[9].s6; results[ 6].sA = v[10].s6;, results[ 6].sB = v[11].s6;, results[ 6].sC = v[12].s6;, results[ 6].sD = v[13].s6;, results[ 6].sE = v[14].s6;, results[ 6].sF = v[15].s6;
    results[ 7].s0 = v[0].s7; results[ 7].s1 = v[1].s7; results[ 7].s2 = v[2].s7; results[ 7].s3 = v[3].s7; results[ 7].s4 = v[4].s7; results[ 7].s5 = v[5].s7; results[ 7].s6 = v[6].s7; results[ 7].s7 = v[7].s7; results[ 7].s8 = v[8].s7; results[ 7].s9 = v[9].s7; results[ 7].sA = v[10].s7;, results[ 7].sB = v[11].s7;, results[ 7].sC = v[12].s7;, results[ 7].sD = v[13].s7;, results[ 7].sE = v[14].s7;, results[ 7].sF = v[15].s7;
    results[ 8].s0 = v[0].s8; results[ 8].s1 = v[1].s8; results[ 8].s2 = v[2].s8; results[ 8].s3 = v[3].s8; results[ 8].s4 = v[4].s8; results[ 8].s5 = v[5].s8; results[ 8].s6 = v[6].s8; results[ 8].s7 = v[7].s8; results[ 8].s8 = v[8].s8; results[ 8].s9 = v[9].s8; results[ 8].sA = v[10].s8;, results[ 8].sB = v[11].s8;, results[ 8].sC = v[12].s8;, results[ 8].sD = v[13].s8;, results[ 8].sE = v[14].s8;, results[ 8].sF = v[15].s8;
    results[ 9].s0 = v[0].s9; results[ 9].s1 = v[1].s9; results[ 9].s2 = v[2].s9; results[ 9].s3 = v[3].s9; results[ 9].s4 = v[4].s9; results[ 9].s5 = v[5].s9; results[ 9].s6 = v[6].s9; results[ 9].s7 = v[7].s9; results[ 9].s8 = v[8].s9; results[ 9].s9 = v[9].s9; results[ 9].sA = v[10].s9;, results[ 9].sB = v[11].s9;, results[ 9].sC = v[12].s9;, results[ 9].sD = v[13].s9;, results[ 9].sE = v[14].s9;, results[ 9].sF = v[15].s9;
    results[10].s0 = v[0].sA; results[10].s1 = v[1].sA; results[10].s2 = v[2].sA; results[10].s3 = v[3].sA; results[10].s4 = v[4].sA; results[10].s5 = v[5].sA; results[10].s6 = v[6].sA; results[10].s7 = v[7].sA; results[10].s8 = v[8].sA; results[10].s9 = v[9].sA; results[10].sA = v[10].sA;, results[10].sB = v[11].sA;, results[10].sC = v[12].sA;, results[10].sD = v[13].sA;, results[10].sE = v[14].sA;, results[10].sF = v[15].sA;
    results[11].s0 = v[0].sB; results[11].s1 = v[1].sB; results[11].s2 = v[2].sB; results[11].s3 = v[3].sB; results[11].s4 = v[4].sB; results[11].s5 = v[5].sB; results[11].s6 = v[6].sB; results[11].s7 = v[7].sB; results[11].s8 = v[8].sB; results[11].s9 = v[9].sB; results[11].sA = v[10].sB;, results[11].sB = v[11].sB;, results[11].sC = v[12].sB;, results[11].sD = v[13].sB;, results[11].sE = v[14].sB;, results[11].sF = v[15].sB;
    results[12].s0 = v[0].sC; results[12].s1 = v[1].sC; results[12].s2 = v[2].sC; results[12].s3 = v[3].sC; results[12].s4 = v[4].sC; results[12].s5 = v[5].sC; results[12].s6 = v[6].sC; results[12].s7 = v[7].sC; results[12].s8 = v[8].sC; results[12].s9 = v[9].sC; results[12].sA = v[10].sC;, results[12].sB = v[11].sC;, results[12].sC = v[12].sC;, results[12].sD = v[13].sC;, results[12].sE = v[14].sC;, results[12].sF = v[15].sC;
    results[13].s0 = v[0].sD; results[13].s1 = v[1].sD; results[13].s2 = v[2].sD; results[13].s3 = v[3].sD; results[13].s4 = v[4].sD; results[13].s5 = v[5].sD; results[13].s6 = v[6].sD; results[13].s7 = v[7].sD; results[13].s8 = v[8].sD; results[13].s9 = v[9].sD; results[13].sA = v[10].sD;, results[13].sB = v[11].sD;, results[13].sC = v[12].sD;, results[13].sD = v[13].sD;, results[13].sE = v[14].sD;, results[13].sF = v[15].sD;
    results[14].s0 = v[0].sE; results[14].s1 = v[1].sE; results[14].s2 = v[2].sE; results[14].s3 = v[3].sE; results[14].s4 = v[4].sE; results[14].s5 = v[5].sE; results[14].s6 = v[6].sE; results[14].s7 = v[7].sE; results[14].s8 = v[8].sE; results[14].s9 = v[9].sE; results[14].sA = v[10].sE;, results[14].sB = v[11].sE;, results[14].sC = v[12].sE;, results[14].sD = v[13].sE;, results[14].sE = v[14].sE;, results[14].sF = v[15].sE;
    results[15].s0 = v[0].sF; results[15].s1 = v[1].sF; results[15].s2 = v[2].sF; results[15].s3 = v[3].sF; results[15].s4 = v[4].sF; results[15].s5 = v[5].sF; results[15].s6 = v[6].sF; results[15].s7 = v[7].sF; results[15].s8 = v[8].sF; results[15].s9 = v[9].sF; results[15].sA = v[10].sF;, results[15].sB = v[11].sF;, results[15].sC = v[12].sF;, results[15].sD = v[13].sF;, results[15].sE = v[14].sF;, results[15].sF = v[15].sF;
  #endif

  // Multiplies by alpha and then stores the results into the destination matrix
  #pragma unroll
  for (int w_two=0; w_two<TRA_WPT; ++w_two) {
    realT result;
    #if TRA_WPT == 1
      Multiply(result, alpha, results[w_two]);
    #elif TRA_WPT == 2
      Multiply(result.x, alpha, results[w_two].x);
      Multiply(result.y, alpha, results[w_two].y);
    #elif TRA_WPT == 4
      Multiply(result.x, alpha, results[w_two].x);
      Multiply(result.y, alpha, results[w_two].y);
      Multiply(result.z, alpha, results[w_two].z);
      Multiply(result.w, alpha, results[w_two].w);
    #elif TRA_WPT == 8
      Multiply(result.s0, alpha, results[w_two].s0);
      Multiply(result.s1, alpha, results[w_two].s1);
      Multiply(result.s2, alpha, results[w_two].s2);
      Multiply(result.s3, alpha, results[w_two].s3);
      Multiply(result.s4, alpha, results[w_two].s4);
      Multiply(result.s5, alpha, results[w_two].s5);
      Multiply(result.s6, alpha, results[w_two].s6);
      Multiply(result.s7, alpha, results[w_two].s7);
    #elif TRA_WPT == 16
      Multiply(result.s0, alpha, results[w_two].s0);
      Multiply(result.s1, alpha, results[w_two].s1);
      Multiply(result.s2, alpha, results[w_two].s2);
      Multiply(result.s3, alpha, results[w_two].s3);
      Multiply(result.s4, alpha, results[w_two].s4);
      Multiply(result.s5, alpha, results[w_two].s5);
      Multiply(result.s6, alpha, results[w_two].s6);
      Multiply(result.s7, alpha, results[w_two].s7);
      Multiply(result.s8, alpha, results[w_two].s8);
      Multiply(result.s9, alpha, results[w_two].s9);
      Multiply(result.sA, alpha, results[w_two].sA);
      Multiply(result.sB, alpha, results[w_two].sB);
      Multiply(result.sC, alpha, results[w_two].sC);
      Multiply(result.sD, alpha, results[w_two].sD);
      Multiply(result.sE, alpha, results[w_two].sE);
      Multiply(result.sF, alpha, results[w_two].sF);
    #endif
    const int id_one = gid0*TRA_DIM + get_local_id(0);
    const int id_two = (gid1*TRA_DIM + get_local_id(1))*TRA_WPT + w_two;
    dest[id_two*(ld/TRA_WPT) + id_one] = result;
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the common kernels shared among different BLAS functions. This file contains
// a kernel to transpose matrices. This is a 'fast' version with restrictions, see the
// 'padtranspose.opencl' file for a general transpose kernel.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Data-widths
#if TRA_WPT == 1
  typedef real realT;
#elif TRA_WPT == 2
  typedef real2 realT;
#elif TRA_WPT == 4
  typedef real4 realT;
#elif TRA_WPT == 8
  typedef real8 realT;
#elif TRA_WPT == 16
  typedef real16 realT;
#endif

// =================================================================================================

// Transposes and copies a matrix. Requires both matrices to be of the same dimensions and without
// offset. A more general version is available in 'padtranspose.opencl'.
__kernel __attribute__((reqd_work_group_size(TRA_DIM, TRA_DIM, 1)))
void TransposeMatrixFast(const int ld,
                         __global const realT* restrict src,
                         __global realT* dest,
                         const real_arg arg_alpha) {
  const real alpha = GetRealArg(arg_alpha);

  // Sets the group identifiers. They might be 'shuffled' around to distribute work in a different
  // way over workgroups, breaking memory-bank dependencies.
  const int gid0 = get_group_id(0);
  #if TRA_SHUFFLE == 1
    const int gid1 = (get_group_id(0) + get_group_id(1)) % get_num_groups(0);
  #else
    const int gid1 = get_group_id(1);
  #endif

  // Local memory to store a tile of the matrix (for coalescing)
  __local realT tile[TRA_WPT*TRA_DIM][TRA_DIM + TRA_PAD];

  // Loops over the work per thread
  #pragma unroll
  for (int w_one=0; w_one<TRA_WPT; ++w_one) {

    // Computes the identifiers for the source matrix. Note that the local and global dimensions
    // do not correspond to each other!
    const int id_one = gid1 * TRA_DIM + get_local_id(0);
    const int id_two = (gid0 * TRA_DIM + get_local_id(1))*TRA_WPT + w_one;

    // Loads data into the local memory
    realT value = src[id_two*(ld/TRA_WPT) + id_one];
    tile[get_local_id(0)*TRA_WPT + w_one][get_local_id(1)] = value;
  }

  // Synchronizes all threads in a workgroup
  barrier(CLK_LOCAL_MEM_FENCE);

  // Loads transposed data from the local memory
  realT v[TRA_WPT];
  #pragma unroll
  for (int w_one=0; w_one<TRA_WPT; ++w_one) {
    v[w_one] = tile[get_local_id(1)*TRA_WPT + w_one][get_local_id(0)];
  }

  // Performs the register-level transpose of the vectorized data
  realT results[TRA_WPT];
  #if TRA_WPT == 1
    results[0] = v[0];
  #elif TRA_WPT == 2
    results[0] = (realT) {v[0].x, v[1].x};
    results[1] = (realT) {v[0].y, v[1].y};
  #elif TRA_WPT == 4
    results[0] = (realT) {v[0].x, v[1].x, v[2].x, v[3].x};
    results[1] = (realT) {v[0].y, v[1].y, v[2].y, v[3].y};
    results[2] = (realT) {v[0].z, v[1].z, v[2].z, v[3].z};
    results[3] = (realT) {v[0].w, v[1].w, v[2].w, v[3].w};
  #elif TRA_WPT == 8
    results[0] = (realT) {v[0].s0, v[1].s0, v[2].s0, v[3].s0, v[4].s0, v[5].s0, v[6].s0, v[7].s0};
    results[1] = (realT) {v[0].s1, v[1].s1, v[2].s1, v[3].s1, v[4].s1, v[5].s1, v[6].s1, v[7].s1};
    results[2] = (realT) {v[0].s2, v[1].s2, v[2].s2, v[3].s2, v[4].s2, v[5].s2, v[6].s2, v[7].s2};
    results[3] = (realT) {v[0].s3, v[1].s3, v[2].s3, v[3].s3, v[4].s3, v[5].s3, v[6].s3, v[7].s3};
    results[4] = (realT) {v[0].s4, v[1].s4, v[2].s4, v[3].s4, v[4].s4, v[5].s4, v[6].s4, v[7].s4};
    results[5] = (realT) {v[0].s5, v[1].s5, v[2].s5, v[3].s5, v[4].s5, v[5].s5, v[6].s5, v[7].s5};
    results[6] = (realT) {v[0].s6, v[1].s6, v[2].s6, v[3].s6, v[4].s6, v[5].s6, v[6].s6, v[7].s6};
    results[7] = (realT) {v[0].s7, v[1].s7, v[2].s7, v[3].s7, v[4].s7, v[5].s7, v[6].s7, v[7].s7};
  #elif TRA_WPT == 16
    results[ 0] = (realT) {v[0].s0, v[1].s0, v[2].s0, v[3].s0, v[4].s0, v[5].s0, v[6].s0, v[7].s0, v[8].s0, v[9].s0, v[10].s0, v[11].s0, v[12].s0, v[13].s0, v[14].s0, v[15].s0};
    results[ 1] = (realT) {v[0].s1, v[1].s1, v[2].s1, v[3].s1, v[4].s1, v[5].s1, v[6].s1, v[7].s1, v[8].s1, v[9].s1, v[10].s1, v[11].s1, v[12].s1, v[13].s1, v[14].s1, v[15].s1};
    results[ 2] = (realT) {v[0].s2, v[1].s2, v[2].s2, v[3].s2, v[4].s2, v[5].s2, v[6].s2, v[7].s2, v[8].s2, v[9].s2, v[10].s2, v[11].s2, v[12].s2, v[13].s2, v[14].s2, v[15].s2};
    results[ 3] = (realT) {v[0].s3, v[1].s3, v[2].s3, v[3].s3, v[4].s3, v[5].s3, v[6].s3, v[7].s3, v[8].s3, v[9].s3, v[10].s3, v[11].s3, v[12].s3, v[13].s3, v[14].s3, v[15].s3};
    results[ 4] = (realT) {v[0].s4, v[1].s4, v[2].s4, v[3].s4, v[4].s4, v[5].s4, v[6].s4, v[7].s4, v[8].s4, v[9].s4, v[10].s4, v[11].s4, v[12].s4, v[13].s4, v[14].s4, v[15].s4};
    results[ 5] = (realT) {v[0].s5, v[1].s5, v[2].s5, v[3].s5, v[4].s5, v[5].s5, v[6].s5, v[7].s5, v[8].s5, v[9].s5, v[10].s5, v[11].s5, v[12].s5, v[13].s5, v[14].s5, v[15].s5};
    results[ 6] = (realT) {v[0].s6, v[1].s6, v[2].s6, v[3].s6, v[4].s6, v[5].s6, v[6].s6, v[7].s6, v[8].s6, v[9].s6, v[10].s6, v[11].s6, v[12].s6, v[13].s6, v[14].s6, v[15].s6};
    results[ 7] = (realT) {v[0].s7, v[1].s7, v[2].s7, v[3].s7, v[4].s7, v[5].s7, v[6].s7, v[7].s7, v[8].s7, v[9].s7, v[10].s7, v[11].s7, v[12].s7, v[13].s7, v[14].s7, v[15].s7};
    results[ 8] = (realT) {v[0].s8, v[1].s8, v[2].s8, v[3].s8, v[4].s8, v[5].s8, v[6].s8, v[7].s8, v[8].s8, v[9].s8, v[10].s8, v[11].s8, v[12].s8, v[13].s8, v[14].s8, v[15].s8};
    results[ 9] = (realT) {v[0].s9, v[1].s9, v[2].s9, v[3].s9, v[4].s9, v[5].s9, v[6].s9, v[7].s9, v[8].s9, v[9].s9, v[10].s9, v[11].s9, v[12].s9, v[13].s9, v[14].s9, v[15].s9};
    results[10] = (realT) {v[0].sA, v[1].sA, v[2].sA, v[3].sA, v[4].sA, v[5].sA, v[6].sA, v[7].sA, v[8].sA, v[9].sA, v[10].sA, v[11].sA, v[12].sA, v[13].sA, v[14].sA, v[15].sA};
    results[11] = (realT) {v[0].sB, v[1].sB, v[2].sB, v[3].sB, v[4].sB, v[5].sB, v[6].sB, v[7].sB, v[8].sB, v[9].sB, v[10].sB, v[11].sB, v[12].sB, v[13].sB, v[14].sB, v[15].sB};
    results[12] = (realT) {v[0].sC, v[1].sC, v[2].sC, v[3].sC, v[4].sC, v[5].sC, v[6].sC, v[7].sC, v[8].sC, v[9].sC, v[10].sC, v[11].sC, v[12].sC, v[13].sC, v[14].sC, v[15].sC};
    results[13] = (realT) {v[0].sD, v[1].sD, v[2].sD, v[3].sD, v[4].sD, v[5].sD, v[6].sD, v[7].sD, v[8].sD, v[9].sD, v[10].sD, v[11].sD, v[12].sD, v[13].sD, v[14].sD, v[15].sD};
    results[14] = (realT) {v[0].sE, v[1].sE, v[2].sE, v[3].sE, v[4].sE, v[5].sE, v[6].sE, v[7].sE, v[8].sE, v[9].sE, v[10].sE, v[11].sE, v[12].sE, v[13].sE, v[14].sE, v[15].sE};
    results[15] = (realT) {v[0].sF, v[1].sF, v[2].sF, v[3].sF, v[4].sF, v[5].sF, v[6].sF, v[7].sF, v[8].sF, v[9].sF, v[10].sF, v[11].sF, v[12].sF, v[13].sF, v[14].sF, v[15].sF};
  #endif

  // Multiplies by alpha and then stores the results into the destination matrix
  #pragma unroll
  for (int w_two=0; w_two<TRA_WPT; ++w_two) {
    realT result;
    #if TRA_WPT == 1
      Multiply(result, alpha, results[w_two]);
    #elif TRA_WPT == 2
      Multiply(result.x, alpha, results[w_two].x);
      Multiply(result.y, alpha, results[w_two].y);
    #elif TRA_WPT == 4
      Multiply(result.x, alpha, results[w_two].x);
      Multiply(result.y, alpha, results[w_two].y);
      Multiply(result.z, alpha, results[w_two].z);
      Multiply(result.w, alpha, results[w_two].w);
    #elif TRA_WPT == 8
      Multiply(result.s0, alpha, results[w_two].s0);
      Multiply(result.s1, alpha, results[w_two].s1);
      Multiply(result.s2, alpha, results[w_two].s2);
      Multiply(result.s3, alpha, results[w_two].s3);
      Multiply(result.s4, alpha, results[w_two].s4);
      Multiply(result.s5, alpha, results[w_two].s5);
      Multiply(result.s6, alpha, results[w_two].s6);
      Multiply(result.s7, alpha, results[w_two].s7);
    #elif TRA_WPT == 16
      Multiply(result.s0, alpha, results[w_two].s0);
      Multiply(result.s1, alpha, results[w_two].s1);
      Multiply(result.s2, alpha, results[w_two].s2);
      Multiply(result.s3, alpha, results[w_two].s3);
      Multiply(result.s4, alpha, results[w_two].s4);
      Multiply(result.s5, alpha, results[w_two].s5);
      Multiply(result.s6, alpha, results[w_two].s6);
      Multiply(result.s7, alpha, results[w_two].s7);
      Multiply(result.s8, alpha, results[w_two].s8);
      Multiply(result.s9, alpha, results[w_two].s9);
      Multiply(result.sA, alpha, results[w_two].sA);
      Multiply(result.sB, alpha, results[w_two].sB);
      Multiply(result.sC, alpha, results[w_two].sC);
      Multiply(result.sD, alpha, results[w_two].sD);
      Multiply(result.sE, alpha, results[w_two].sE);
      Multiply(result.sF, alpha, results[w_two].sF);
    #endif
    const int id_one = gid0*TRA_DIM + get_local_id(0);
    const int id_two = (gid1*TRA_DIM + get_local_id(1))*TRA_WPT + w_two;
    dest[id_two*(ld/TRA_WPT) + id_one] = result;
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================


removed extra paranthesis

removed extra paranthesis

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0
// number of filter taps in the processing wk_item
//#define MLO_WEI_WKITEM 5

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_WIDTH + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_OUT_WEI_SCAN_BLK (MLO_OUT_WEI_EXT_SCAN_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_WIDTH + MLO_OUT_WEI_SCAN_BLK - 1)/MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ *MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

#if MLO_OUT_HORIZ_PIX_SZ <  (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#else
#define MLO_OUT_HORIZ_PIX_EXT_SZ MLO_OUT_HORIZ_PIX_SZ
#endif
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)
#if ((MLO_OUT_HEIGHT/MLO_N_ALIGNED_OUT_SCAN_BLK)*MLO_N_ALIGNED_OUT_SCAN_BLK == MLO_OUT_HEIGHT)
#define MLO_BLK_ALIGNED 1
#else
#define MLO_BLK_ALIGNED 0
#endif

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH) //((MLO_OUT_WIDTH-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0 - 2 * MLO_FILTER_PAD0)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#if (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24((uint)u, (uint)d);
	return(r);
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split filter taps into sub-tiles along x and y axis with number of tap groups muliples of stride or 1
// for example
// the 5x10 filter has been split into 10 sub-tiles 1x5 each, 1 tap in y direction and 5 taps in x direction.
// those horizontal taps are 0, 2, 4, 6, 8 and 1, 3, 5, 7, 9
// a single vertical tap is 0 or 1 or 2 or 3 or 4.
// one may say sub-tiles are indexed by a vertical tap.
// the partial sum has been calculated into those 10 sub-tiles in parallel.
// the full filter has been calulated by reducing all sub-tiles into a single filter per group.
// teh accumulation has been done over all pixels of several outputs being shared with a single input.
// the accuulation has been done per batch.
//
// the total reduction over all batches has been doesn a separete kerenel.
//
// alg
//
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for j in output scans
//				for i in output scan interval
//                  accumulate the weights into sub-tiles
//
//		reduce sub-tiles into a single filter for each output
//		write accululated weights
//
// group layout
// 0 - n waves * wave size (n_waves has been defined by host)
// 1 - input channel index
// 2 - output channel/batch index
//
//
// for each batch
//	 accumulate all weights per input/output pair

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * __restrict bias_df,
#endif
	UNUSED _FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	uint c_idx_base = get_group_id(0); // input map index base

	uint o_idx_base = get_group_id(1); // output map index base
	uint ib_base = get_group_id(2); // batch index base

	uint ib = ib_base*MLO_N_LCL_BATCHS;

	uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	uint o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	uint lcl_id = get_local_id(0);

// weight tile id
#if MLO_WEI_BLK_SZ & (MLO_WEI_BLK_SZ - 1)
	uint w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	uint w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
#else
	uint w_blk_idx = lcl_id / MLO_WEI_BLK_SZ;
	uint w_idx = lcl_id & (MLO_WEI_BLK_SZ - 1);
#endif
#if MLO_WEI_BLK_SZ0 & (MLO_WEI_BLK_SZ0 - 1)
// weight y
	uint w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
// weight starting x
	uint w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);
#else
	uint w_y = w_idx / MLO_WEI_BLK_SZ0;
	uint w_x0 = w_idx & (MLO_WEI_BLK_SZ0 - 1);
#endif

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	for (uint i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	// over all batches

	for (uint b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		uint in_y = 0;
		uint out_y = 0;

// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		uint gbl_in_scan_off = gbl_in_off;
		uint gbl_out_scan_off = gbl_out_off;
		uint c_scan = 0;

		for (uint p4 = lcl_id;  p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1);
			p4 += MLO_GRP_SZ)
		{
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
			c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS);
			uint c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);
#else
			c_scan = p4 / MLO_N_IN_HORIZ_READS;
			uint c_pix4 = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				in_rd_data[i] = 0;
			}
			if (in_y + c_scan < MLO_IN_HEIGHT)
			{
				uint bot_off = gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
				const __global _FLOAT * bot_p = &bot[bot_off];
				// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
				if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
				{
					uint i = 0;
					for (; i < MLO_IN_N_PIXS_OFF; ++i)
					{
						in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-of-range\n");
						}
#endif
					}

				}
				else
#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-of-range\n");
						}
#endif
					}

				}
			}
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
			}

		}

		in_y += MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;

// TO DO: HANDLE PADDING
// over all out blocks
// processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

		for (uint ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
// TODO:: HANDLE multiple INPUTS
// an overshoot has to be handled by zero outing output
			gbl_in_scan_off = gbl_in_off + mul24(in_y, ( uint )MLO_IN_STRIDE);
			uint c_scan2 = 0;
			for (uint p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS * (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1);
				p4 += MLO_GRP_SZ)
			{
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
				c_scan2 = iDiv(p4, MLO_N_IN_HORIZ_READS);
				uint c_pix4 = iMod(p4, c_scan2, MLO_N_IN_HORIZ_READS);
#else
				c_scan2 = p4 / MLO_N_IN_HORIZ_READS;
				uint c_pix4 = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

				if (in_y + c_scan2 < MLO_IN_HEIGHT)
				{
					uint bot_off = gbl_in_scan_off + c_scan2 * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
					const __global _FLOAT * bot_p = &bot[bot_off];

#if MLO_IN_N_PIXS_OFF > 0
					if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
					{

						uint i = 0;
						for (; i < MLO_IN_N_PIXS_OFF; ++i)
						{
							in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
							if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}

					}
					else
#endif
					{

						for (uint i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
							if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}
					}

				} // if (in_y + c_scan2 < MLO_IN_HEIGHT)

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					uint lcl_off = (c_scan2 + MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
					lcl_bot[lcl_off + i] = in_rd_data[i];
				}

			}

			gbl_out_scan_off = gbl_out_off + mul24(out_y, ( uint )MLO_OUT_STRIDE);

// over all outputs groups
// MLO_N_OUT_BLK_GRP outputs reuse the same input
// each output blk is MLO_N_LCL_OUT_MAPS outputs
// MLO_N_LCL_OUT_MAPS nuber is restricted by LDS size

			uint gbl_out_scan_off1 = gbl_out_scan_off;
			for (uint og = 0; og < MLO_N_OUT_BLK_GRP
				; ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS*MLO_OUT_CHANNEL_STRIDE)
			{
				barrier(CLK_LOCAL_MEM_FENCE);

// fetch output. MLO_N_ALIGNED_OUT_SCAN_BLK output scans, each of size MLO_N_OUT_HORIZ_READS

				__private _FLOAT out_rd_data[MLO_READ_UNIT];

				for (uint oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS);
																											oo_p4 += MLO_GRP_SZ)
				{
#if (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS) & ((MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS) - 1)
					uint o = iDiv(oo_p4, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
					uint o_pX4 = iMod(oo_p4, o, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
#else
					uint o = oo_p4 / (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS);
					uint o_pX4 = oo_p4 & ((MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS) - 1);
#endif
#if MLO_N_OUT_HORIZ_READS & (MLO_N_OUT_HORIZ_READS - 1)
					uint o_scan = iDiv(o_pX4, MLO_N_OUT_HORIZ_READS);
					uint o_pix4 = iMod(o_pX4, o_scan, MLO_N_OUT_HORIZ_READS);
#else
					uint o_scan = o_pX4 / MLO_N_OUT_HORIZ_READS;
					uint o_pix4 = o_pX4 & (MLO_N_OUT_HORIZ_READS - 1);
#endif
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						out_rd_data[i] = 0;
					}
					uint top_df_off = gbl_out_scan_off1 + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT;
					const __global _FLOAT * top_df_p = &top_df[top_df_off];

					if (out_y + o_scan < MLO_OUT_HEIGHT
						&& o_idx + og *MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS
						)
					{
// scan has been fetch by 4
// here the non-multiple of 4 scan has been handled
// also makes sure the input garbage hs been multipled by 0
#if MLO_OUT_N_PIXS_OFF > 0
						if (o_pix4 == (MLO_N_OUT_HORIZ_READS - 1))
						{
							uint i = 0;
							for (; i < MLO_OUT_N_PIXS_OFF; ++i)
							{
								out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
								if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
								{
									printf("k:err:out-of-range\n");
								}
#endif
							}
						}

						else
#endif

						{
							for (uint i = 0; i < MLO_READ_UNIT; ++i)
							{
								out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
								if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
								{
									printf("k:err:out-of-range\n");
								}
#endif
							}
						}

					} // if (out_y + o_scan < MLO_OUT_HEIGHT && o_idx + og *MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS)

// write into LDS with MLO_OUT_HORIZ_PIX_EXT_SZ stride to zero out weights block overshoot

					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						uint lcl_off = o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT;
						lcl_top[lcl_off + i] = out_rd_data[i];
					}

				} //	for (uint oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS); oo_p4 += MLO_GRP_SZ)

				barrier(CLK_LOCAL_MEM_FENCE);

// process
// algorithm

					// over all input scans in LDS
					for (uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
					{

						// prefetch proper inputs pixels.
						// they are MLO_WEI_BLK_SZ0 apart taps of the filter

						_FLOAT i_vals[MLO_WEI_WKITEM];

						for (uint w = 0; w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)); ++w)
						{
							uint w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
							uint i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[w] = i_val;

						}

						// if we overshoot the scanline
						// out data will be 0 by initial setting
						for (uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
						{

							// read the current input pixel
							for (uint w = (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)); w < MLO_WEI_WKITEM; ++w)
							{
								uint w_x = w_x0 + w* MLO_WEI_BLK_SZ0;
								uint i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
								_FLOAT i_val = lcl_bot[i_off];

								i_vals[w] = i_val;
							}
							// for each output accumulate a proper filter tap
							for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
							{

								// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
								_FLOAT o_val
									= lcl_top[(o)*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

								uint w = 0;
								_FLOAT i_val;
								for (/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)
								{

									i_val = i_vals[w];

									pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;

								} // for (/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)

							} // for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)

							for (uint w = 0; w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)); ++w)
							{
								i_vals[w] = i_vals[w + (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)];

							}
						} // for (uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
					} // for (uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

// move the input data tail inside LDS to reduce mem bandwidth
			for (uint c_scan3 = 0; c_scan3 < (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1); ++c_scan3)
			{
				barrier(CLK_LOCAL_MEM_FENCE);

				for (uint p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS;
												p4 += MLO_GRP_SZ)
				{

					uint c_pix4 = p4;
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_bot[c_scan3*(MLO_IN_LCL_WIDTH)+MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i]
							= lcl_bot[(c_scan3 + (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1))*(MLO_IN_LCL_WIDTH)+MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i];
					}

				}
			}

		} // for (uint ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (uint b = 0;

// send it out

	uint wei_lcl_off = 0;
	_FLOAT final_sum = 0;

// save in lcl and orgnize in a proper order
	// outputs
	//	filter size1
	//	  filter size0

// TO DO:: DEPENDING ON THE GROUP SIZE
	for (uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if ( w_blk_idx < MLO_MAX_WEI_BLK_LOOP)
		{
			for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
			{
				uint w = 0;
				for (; w < MLO_WEI_WKITEM; ++w)
				{
					// save "virtual" filter table
					uint w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
					wei_lcl_off = ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1 + w_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + w_x;
					lcl[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];

				}

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		// read into real filter table

		for(uint l = lcl_id; l < (MLO_N_LCL_OUT_MAPS *MLO_WEI_CHANNEL_STRIDE); l+=MLO_GRP_SZ)
		{
#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
			uint oo = iDiv(l, MLO_WEI_CHANNEL_STRIDE);
			uint wei_i = iMod(l, oo, MLO_WEI_CHANNEL_STRIDE);
#else
			uint oo = l / MLO_WEI_CHANNEL_STRIDE;
			uint wei_i = l & MLO_WEI_CHANNEL_STRIDE - 1;
#endif
#if (MLO_FILTER_SIZE0) & ((MLO_FILTER_SIZE0) - 1)
			uint wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
			uint wei_i_x = iMod(wei_i, wei_i_y, (MLO_FILTER_SIZE0));
#else
			uint wei_i_y = wei_i / (MLO_FILTER_SIZE0);
			uint wei_i_x = wei_i & ((MLO_FILTER_SIZE0) - 1);
#endif
			// send it out
			// inputs are outputs
			uint wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (uint)MLO_WEI_BATCH_STRIDE)
				// this input channel
				+ mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);

			final_sum = 0;
			for (uint i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
			{
				final_sum += lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x];
			}

			weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //
		}

	} // for(uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * __restrict weight_df_tmp,
	__global _FLOAT * __restrict weights_df
)
{
	uint gbl_id = get_global_id(0);
	uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
	uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	uint wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
	uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
	uint wei_idx = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	uint batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (uint i = 0; i < batch_loop; ++i)
	{
		for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0
// number of filter taps in the processing wk_item
//#define MLO_WEI_WKITEM 5

#define MLO_N_OUT_HORIZ_READS (MLO_ALIGNED_OUT_SCAN_LN)
#define MLO_OUT_HORIZ_PIX_SZ (MLO_N_OUT_HORIZ_READS * MLO_READ_UNIT)

// n of of filter blks
#define MLO_WEI_BLK_SZ0 ((MLO_FILTER_SIZE0 + MLO_WEI_WKITEM -1)/MLO_WEI_WKITEM)
#define MLO_WEI_BLK_SZ (MLO_FILTER_SIZE1*MLO_WEI_BLK_SZ0)
// n of filter tiles in the group grid
#define MLO_N_WEI_BLK (MLO_GRP_SZ/ MLO_WEI_BLK_SZ)
// n of steps per scan line to be made in the inner loop
// extended scan to deal with overshot in the inner loop
#define MLO_OUT_WEI_EXT_SCAN_BLK ((MLO_OUT_WIDTH + MLO_N_WEI_BLK - 1) / MLO_N_WEI_BLK)

#define MLO_OUT_WEI_SCAN_BLK (MLO_OUT_WEI_EXT_SCAN_BLK)

// max loop over virtual blocks for small images
#define MLO_MAX_WEI_BLK_LOOP_TMP ((MLO_OUT_WIDTH + MLO_OUT_WEI_SCAN_BLK - 1)/MLO_OUT_WEI_SCAN_BLK)
#if MLO_MAX_WEI_BLK_LOOP_TMP < MLO_N_WEI_BLK
#define MLO_MAX_WEI_BLK_LOOP MLO_MAX_WEI_BLK_LOOP_TMP
#else
#define MLO_MAX_WEI_BLK_LOOP MLO_N_WEI_BLK
#endif

#define MLO_WEI_BLKS_SZ (MLO_MAX_WEI_BLK_LOOP * MLO_WEI_BLK_SZ *MLO_WEI_WKITEM)
#define MLO_WEI_BLKS_LCL_SZ (MLO_WEI_BLKS_SZ * MLO_N_LCL_OUT_MAPS)

#define MLO_OUT_BLK_GRP_PIX_SZ (MLO_OUT_HORIZ_PIX_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_BLK_GRP_WK_SZ (MLO_OUT_BLK_GRP_PIX_SZ / MLO_READ_UNIT)

#if MLO_OUT_HORIZ_PIX_SZ <  (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#define MLO_OUT_HORIZ_PIX_EXT_SZ (MLO_OUT_WEI_EXT_SCAN_BLK * MLO_MAX_WEI_BLK_LOOP)
#else
#define MLO_OUT_HORIZ_PIX_EXT_SZ MLO_OUT_HORIZ_PIX_SZ
#endif
#define MLO_OUT_BLK_GRP_EXT_PIX_SZ (MLO_OUT_HORIZ_PIX_EXT_SZ * MLO_N_ALIGNED_OUT_SCAN_BLK)
#define MLO_OUT_LCL_SZ (MLO_OUT_BLK_GRP_EXT_PIX_SZ)
// LDS OUT SIZE
#define MLO_TOTAL_OUT_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_OUT_MAPS*MLO_OUT_LCL_SZ)
#if ((MLO_OUT_HEIGHT/MLO_N_ALIGNED_OUT_SCAN_BLK)*MLO_N_ALIGNED_OUT_SCAN_BLK == MLO_OUT_HEIGHT)
#define MLO_BLK_ALIGNED 1
#else
#define MLO_BLK_ALIGNED 0
#endif

// input size depends on output scan length and
// number of output scans
// this number is constrained by amount or LDS and size of register file.
// TO DO:: CHECK PADDING!!!
#define MLO_IN_LCL_HEIGHT ((MLO_N_ALIGNED_OUT_SCAN_BLK-1)*MLO_FILTER_STRIDE1 + MLO_FILTER_SIZE1)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH) //((MLO_OUT_WIDTH-1)*MLO_FILTER_STRIDE0 + MLO_FILTER_SIZE0 - 2 * MLO_FILTER_PAD0)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS  / MLO_READ_UNIT)*MLO_READ_UNIT)

#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2 * MLO_FILTER_PAD0)
#define MLO_IN_BLK_GRP_PIX_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_IN_BLK_GRP_WK_SZ (MLO_IN_BLK_GRP_PIX_SZ/MLO_READ_UNIT)
#define MLO_IN_LCL_SZ (MLO_IN_BLK_GRP_PIX_SZ)
// LDS IN SIZE
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)
#define MLO_IN_VERT_READS (MLO_GRP_SZ/MLO_N_IN_HORIZ_READS)

#if (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ) > (MLO_WEI_BLKS_LCL_SZ)
#define MLO_LCL_SZ (MLO_TOTAL_OUT_LCL_SZ + MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_BLKS_LCL_SZ)
#endif

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f/ (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24((uint)u, (uint)d);
	return(r);
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split filter taps into sub-tiles along x and y axis with number of tap groups muliples of stride or 1
// for example
// the 5x10 filter has been split into 10 sub-tiles 1x5 each, 1 tap in y direction and 5 taps in x direction.
// those horizontal taps are 0, 2, 4, 6, 8 and 1, 3, 5, 7, 9
// a single vertical tap is 0 or 1 or 2 or 3 or 4.
// one may say sub-tiles are indexed by a vertical tap.
// the partial sum has been calculated into those 10 sub-tiles in parallel.
// the full filter has been calulated by reducing all sub-tiles into a single filter per group.
// teh accumulation has been done over all pixels of several outputs being shared with a single input.
// the accuulation has been done per batch.
//
// the total reduction over all batches has been doesn a separete kerenel.
//
// alg
//
//		until end of output map (MLO_N_OUT_BLK)
//			load input map block in LDS
//			load output maps in LDS
//          for j in output scans
//				for i in output scan interval
//                  accumulate the weights into sub-tiles
//
//		reduce sub-tiles into a single filter for each output
//		write accululated weights
//
// group layout
// 0 - n waves * wave size (n_waves has been defined by host)
// 1 - input channel index
// 2 - output channel/batch index
//
//
// for each batch
//	 accumulate all weights per input/output pair

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * __restrict bias_df,
#endif
	UNUSED _FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ)];
	__local _FLOAT * lcl_bot = lcl;
	__local _FLOAT * lcl_top = lcl + MLO_TOTAL_IN_LCL_SZ;

	uint c_idx_base = get_group_id(0); // input map index base

	uint o_idx_base = get_group_id(1); // output map index base
	uint ib_base = get_group_id(2); // batch index base

	uint ib = ib_base*MLO_N_LCL_BATCHS;

	uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	uint o_idx = o_idx_base * (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS); // output map index

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	uint lcl_id = get_local_id(0);

// weight tile id
#if MLO_WEI_BLK_SZ & (MLO_WEI_BLK_SZ - 1)
	uint w_blk_idx = iDiv(lcl_id, MLO_WEI_BLK_SZ);
	uint w_idx = iMod(lcl_id, w_blk_idx, MLO_WEI_BLK_SZ);
#else
	uint w_blk_idx = lcl_id / MLO_WEI_BLK_SZ;
	uint w_idx = lcl_id & (MLO_WEI_BLK_SZ - 1);
#endif
#if MLO_WEI_BLK_SZ0 & (MLO_WEI_BLK_SZ0 - 1)
// weight y
	uint w_y = iDiv(w_idx, MLO_WEI_BLK_SZ0);
// weight starting x
	uint w_x0 = iMod(w_idx, w_y, MLO_WEI_BLK_SZ0);
#else
	uint w_y = w_idx / MLO_WEI_BLK_SZ0;
	uint w_x0 = w_idx & (MLO_WEI_BLK_SZ0 - 1);
#endif

	__private _FLOAT pvt_accum[(MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM)];

	for (uint i = 0; i < (MLO_N_OUT_BLK_GRP * MLO_N_LCL_OUT_MAPS * MLO_WEI_WKITEM); ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	// over all batches

	for (uint b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		barrier(CLK_LOCAL_MEM_FENCE);

		uint in_y = 0;
		uint out_y = 0;

// prefetch MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1 input scans
		__private _FLOAT in_rd_data[MLO_READ_UNIT];

		uint gbl_in_scan_off = gbl_in_off;
		uint gbl_out_scan_off = gbl_out_off;
		uint c_scan = 0;

		for (uint p4 = lcl_id;  p4 < MLO_N_IN_HORIZ_READS * (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1);
			p4 += MLO_GRP_SZ)
		{
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
			c_scan = iDiv(p4, MLO_N_IN_HORIZ_READS);
			uint c_pix4 = iMod(p4, c_scan, MLO_N_IN_HORIZ_READS);
#else
			c_scan = p4 / MLO_N_IN_HORIZ_READS;
			uint c_pix4 = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				in_rd_data[i] = 0;
			}
			if (in_y + c_scan < MLO_IN_HEIGHT)
			{
				uint bot_off = gbl_in_scan_off + c_scan * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
				const __global _FLOAT * bot_p = &bot[bot_off];
				// still problems with unaligned LDS access
#if MLO_IN_N_PIXS_OFF > 0
				if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
				{
					uint i = 0;
					for (; i < MLO_IN_N_PIXS_OFF; ++i)
					{
						in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-of-range\n");
						}
#endif
					}

				}
				else
#endif
				{
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
						if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
						{
							printf("k:err:in-of-range\n");
						}
#endif
					}

				}
			}
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				lcl_bot[(c_scan + MLO_FILTER_PAD1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i] = in_rd_data[i];
			}

		}

		in_y += MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1 - MLO_FILTER_PAD1;

// TO DO: HANDLE PADDING
// over all out blocks
// processing per MLO_N_ALIGNED_OUT_SCAN_BLK output scans

		for (uint ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
		{

			barrier(CLK_LOCAL_MEM_FENCE);

// fetch input: (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1)
// TODO:: HANDLE multiple INPUTS
// an overshoot has to be handled by zero outing output
			gbl_in_scan_off = gbl_in_off + mul24(in_y, ( uint )MLO_IN_STRIDE);
			uint c_scan2 = 0;
			for (uint p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS * (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1);
				p4 += MLO_GRP_SZ)
			{
#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
				c_scan2 = iDiv(p4, MLO_N_IN_HORIZ_READS);
				uint c_pix4 = iMod(p4, c_scan2, MLO_N_IN_HORIZ_READS);
#else
				c_scan2 = p4 / MLO_N_IN_HORIZ_READS;
				uint c_pix4 = p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

				if (in_y + c_scan2 < MLO_IN_HEIGHT)
				{
					uint bot_off = gbl_in_scan_off + c_scan2 * MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
					const __global _FLOAT * bot_p = &bot[bot_off];

#if MLO_IN_N_PIXS_OFF > 0
					if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
					{

						uint i = 0;
						for (; i < MLO_IN_N_PIXS_OFF; ++i)
						{
							in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
							if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}

					}
					else
#endif
					{

						for (uint i = 0; i < MLO_READ_UNIT; ++i)
						{
							in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
							if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
							{
								printf("k:err:in-of-range\n");
							}
#endif
						}
					}

				} // if (in_y + c_scan2 < MLO_IN_HEIGHT)

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					uint lcl_off = (c_scan2 + MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1)*MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT;
					lcl_bot[lcl_off + i] = in_rd_data[i];
				}

			}

			gbl_out_scan_off = gbl_out_off + mul24(out_y, ( uint )MLO_OUT_STRIDE);

// over all outputs groups
// MLO_N_OUT_BLK_GRP outputs reuse the same input
// each output blk is MLO_N_LCL_OUT_MAPS outputs
// MLO_N_LCL_OUT_MAPS nuber is restricted by LDS size

			uint gbl_out_scan_off1 = gbl_out_scan_off;
			for (uint og = 0; og < MLO_N_OUT_BLK_GRP
				; ++og, gbl_out_scan_off1 += MLO_N_LCL_OUT_MAPS*MLO_OUT_CHANNEL_STRIDE)
			{
				barrier(CLK_LOCAL_MEM_FENCE);

// fetch output. MLO_N_ALIGNED_OUT_SCAN_BLK output scans, each of size MLO_N_OUT_HORIZ_READS

				__private _FLOAT out_rd_data[MLO_READ_UNIT];

				for (uint oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS);
																											oo_p4 += MLO_GRP_SZ)
				{
#if (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS) & ((MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS) - 1)
					uint o = iDiv(oo_p4, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
					uint o_pX4 = iMod(oo_p4, o, (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS));
#else
					uint o = oo_p4 / (MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS);
					uint o_pX4 = oo_p4 & ((MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS) - 1);
#endif
#if MLO_N_OUT_HORIZ_READS & (MLO_N_OUT_HORIZ_READS - 1)
					uint o_scan = iDiv(o_pX4, MLO_N_OUT_HORIZ_READS);
					uint o_pix4 = iMod(o_pX4, o_scan, MLO_N_OUT_HORIZ_READS);
#else
					uint o_scan = o_pX4 / MLO_N_OUT_HORIZ_READS;
					uint o_pix4 = o_pX4 & (MLO_N_OUT_HORIZ_READS - 1);
#endif
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						out_rd_data[i] = 0;
					}
					uint top_df_off = gbl_out_scan_off1 + o*MLO_OUT_CHANNEL_STRIDE + o_scan * MLO_OUT_STRIDE + o_pix4*MLO_READ_UNIT;
					const __global _FLOAT * top_df_p = &top_df[top_df_off];

					if (out_y + o_scan < MLO_OUT_HEIGHT
						&& o_idx + og *MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS
						)
					{
// scan has been fetch by 4
// here the non-multiple of 4 scan has been handled
// also makes sure the input garbage hs been multipled by 0
#if MLO_OUT_N_PIXS_OFF > 0
						if (o_pix4 == (MLO_N_OUT_HORIZ_READS - 1))
						{
							uint i = 0;
							for (; i < MLO_OUT_N_PIXS_OFF; ++i)
							{
								out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
								if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
								{
									printf("k:err:out-of-range\n");
								}
#endif
							}
						}

						else
#endif

						{
							for (uint i = 0; i < MLO_READ_UNIT; ++i)
							{
								out_rd_data[i] = top_df_p[i];
#if DBG_OUT_OF_RNGE
								if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
								{
									printf("k:err:out-of-range\n");
								}
#endif
							}
						}

					} // if (out_y + o_scan < MLO_OUT_HEIGHT && o_idx + og *MLO_N_LCL_OUT_MAPS + o < MLO_N_OUTPUTS)

// write into LDS with MLO_OUT_HORIZ_PIX_EXT_SZ stride to zero out weights block overshoot

					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						uint lcl_off = o * MLO_OUT_LCL_SZ + o_scan * MLO_OUT_HORIZ_PIX_EXT_SZ + o_pix4*MLO_READ_UNIT;
						lcl_top[lcl_off + i] = out_rd_data[i];
					}

				} //	for (uint oo_p4 = lcl_id; oo_p4 < (MLO_N_LCL_OUT_MAPS*MLO_N_ALIGNED_OUT_SCAN_BLK*MLO_N_OUT_HORIZ_READS); oo_p4 += MLO_GRP_SZ)

				barrier(CLK_LOCAL_MEM_FENCE);

// process
// algorithm

					// over all input scans in LDS
					for (uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)
					{

						// prefetch proper inputs pixels.
						// they are MLO_WEI_BLK_SZ0 apart taps of the filter

						_FLOAT i_vals[MLO_WEI_WKITEM];

						for (uint w = 0; w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)); ++w)
						{
							uint w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
							uint i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + 0) * MLO_FILTER_STRIDE0 + w_x;
							_FLOAT i_val = lcl_bot[i_off];

							i_vals[w] = i_val;

						}

						// if we overshoot the scanline
						// out data will be 0 by initial setting
						for (uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
						{

							// read the current input pixel
							for (uint w = (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)); w < MLO_WEI_WKITEM; ++w)
							{
								uint w_x = w_x0 + w* MLO_WEI_BLK_SZ0;
								uint i_off = (j*MLO_FILTER_STRIDE1 + w_y) * MLO_IN_LCL_WIDTH + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i) * MLO_FILTER_STRIDE0 + w_x;
								_FLOAT i_val = lcl_bot[i_off];

								i_vals[w] = i_val;
							}
							// for each output accumulate a proper filter tap
							for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
							{

								// read with MLO_OUT_HORIX_PIX_EXT_SZ stride
								_FLOAT o_val
									= lcl_top[(o)*MLO_OUT_LCL_SZ + j * MLO_OUT_HORIZ_PIX_EXT_SZ + (w_blk_idx*MLO_OUT_WEI_SCAN_BLK + i)];

								uint w = 0;
								_FLOAT i_val;
								for (/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)
								{

									i_val = i_vals[w];

									pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w] += i_val * o_val;

								} // for (/*uint w = 0*/; w < MLO_WEI_WKITEM; ++w)

							} // for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)

							for (uint w = 0; w < (MLO_WEI_WKITEM - (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)); ++w)
							{
								i_vals[w] = i_vals[w + (MLO_FILTER_STRIDE0 / MLO_WEI_BLK_SZ0)];

							}
						} // for (uint i = 0; i < MLO_OUT_WEI_SCAN_BLK; ++i)
					} // for (uint j = 0; j < MLO_N_ALIGNED_OUT_SCAN_BLK; ++j)

			} // for(; og < (MLO_N_OUT_BLK_GRP; ++og )

// move the input data tail inside LDS to reduce mem bandwidth
			for (uint c_scan3 = 0; c_scan3 < (MLO_FILTER_SIZE1 - MLO_FILTER_STRIDE1); ++c_scan3)
			{
				barrier(CLK_LOCAL_MEM_FENCE);

				for (uint p4 = lcl_id; p4 < MLO_N_IN_HORIZ_READS;
												p4 += MLO_GRP_SZ)
				{

					uint c_pix4 = p4;
					for (uint i = 0; i < MLO_READ_UNIT; ++i)
					{
						lcl_bot[c_scan3*(MLO_IN_LCL_WIDTH)+MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i]
							= lcl_bot[(c_scan3 + (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + MLO_FILTER_STRIDE1))*(MLO_IN_LCL_WIDTH)+MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i];
					}

				}
			}

		} // for (uint ob = 0; ob < MLO_N_OUT_BLK; ++ob, in_y += (MLO_IN_LCL_HEIGHT - MLO_FILTER_SIZE1 + 1), out_y += MLO_N_ALIGNED_OUT_SCAN_BLK)
	} // for (uint b = 0;

// send it out

	uint wei_lcl_off = 0;
	_FLOAT final_sum = 0;

// save in lcl and orgnize in a proper order
	// outputs
	//	filter size1
	//	  filter size0

// TO DO:: DEPENDING ON THE GROUP SIZE
	for (uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if ( w_blk_idx < MLO_MAX_WEI_BLK_LOOP)
		{
			for (uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
			{
				uint w = 0;
				for (; w < MLO_WEI_WKITEM; ++w)
				{
					// save "virtual" filter table
					uint w_x = w_x0 + w*MLO_WEI_BLK_SZ0;
					wei_lcl_off = ((o * MLO_MAX_WEI_BLK_LOOP + w_blk_idx) * MLO_FILTER_SIZE1 + w_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + w_x;
					lcl[wei_lcl_off] = pvt_accum[(og * MLO_N_LCL_OUT_MAPS + o) * MLO_WEI_WKITEM + w];

				}

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		// read into real filter table

		for(uint l = lcl_id; l < (MLO_N_LCL_OUT_MAPS *MLO_WEI_CHANNEL_STRIDE); l+=MLO_GRP_SZ)
		{
#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
			uint oo = iDiv(l, MLO_WEI_CHANNEL_STRIDE);
			uint wei_i = iMod(l, oo, MLO_WEI_CHANNEL_STRIDE);
#else
			uint oo = l / MLO_WEI_CHANNEL_STRIDE;
			uint wei_i = l & MLO_WEI_CHANNEL_STRIDE - 1);
#endif
#if (MLO_FILTER_SIZE0) & ((MLO_FILTER_SIZE0) - 1)
			uint wei_i_y = iDiv(wei_i, (MLO_FILTER_SIZE0));
			uint wei_i_x = iMod(wei_i, wei_i_y, (MLO_FILTER_SIZE0));
#else
			uint wei_i_y = wei_i / (MLO_FILTER_SIZE0);
			uint wei_i_x = wei_i & ((MLO_FILTER_SIZE0) - 1);
#endif
			// send it out
			// inputs are outputs
			uint wei_df_off = ((ib * MLO_N_OUTPUTS + o_idx) * (uint)MLO_WEI_BATCH_STRIDE)
				// this input channel
				+ mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);

			final_sum = 0;
			for (uint i = 0; i < MLO_MAX_WEI_BLK_LOOP; ++i)
			{
				final_sum += lcl_bot[((oo * MLO_MAX_WEI_BLK_LOOP + i) * MLO_FILTER_SIZE1 + wei_i_y) * (MLO_WEI_BLK_SZ0 *MLO_WEI_WKITEM) + wei_i_x];
			}

			weights_df[wei_df_off + (og *  MLO_N_LCL_OUT_MAPS + oo) * MLO_WEI_BATCH_STRIDE + wei_i] = final_sum; //lcl_bot[lcl_id]; //
		}

	} // for(uint og = 0; og < MLO_N_OUT_BLK_GRP; ++og)

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * __restrict weight_df_tmp,
	__global _FLOAT * __restrict weights_df
)
{
	uint gbl_id = get_global_id(0);
	uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
	uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	uint wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
	uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
	uint wei_idx = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	uint batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (uint i = 0; i < batch_loop; ++i)
	{
		for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}


Bug fix: ImageResizer causing edge artifacts

__constant sampler_t samplerLinear = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_LINEAR;
__constant sampler_t samplerNearest = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

void writeToImage(__write_only image2d_t input, int2 position, float4 value) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		write_imagef(input, position, value);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
		write_imageui(input, position, convert_uint4(value));
	} else {
		write_imagei(input, position, convert_int4(value));
    }
}

float4 readFromImage(__read_only image2d_t input, sampler_t sampler, float2 position) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		return read_imagef(input, sampler, position);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
        return convert_float4(read_imageui(input, sampler, position));
	} else {
        return convert_float4(read_imagei(input, sampler, position));
    }
}

__kernel void resize2DpreserveAspect(
	__read_only image2d_t input,
	__write_only image2d_t output,
    __private int newHeight,
    __private char useInterpolation
	) {
	const int2 position = {get_global_id(0), get_global_id(1)};
    const float scale = (float)newHeight / get_image_height(input);
	const float2 readPosition = {(float)position.x / get_global_size(0), (float)position.y / (scale*get_image_height(input))};
	float4 value;
    if(useInterpolation == 1) {
        value = readFromImage(input, samplerLinear, readPosition);
    } else {
        value = readFromImage(input, samplerNearest, readPosition);
    }
    if(position.y >= newHeight) {
        writeToImage(output, position, (float4)(0,0,0,0));
    } else {
        writeToImage(output, position, value);
    }
}

__kernel void resize2D(
		__read_only image2d_t input,
		__write_only image2d_t output,
        __private char useInterpolation
    ) {
	const int2 position = {get_global_id(0), get_global_id(1)};
	const float2 normalizedPosition = {(float)position.x / get_global_size(0), (float)position.y / get_global_size(1)};
	int dataType = get_image_channel_data_type(input);
	float4 value;
    if(useInterpolation == 1) {
        value = readFromImage(input, samplerLinear, normalizedPosition);
    } else {
        value = readFromImage(input, samplerNearest, normalizedPosition);
    }
    writeToImage(output, position, value);
}

// ================ 3D
#ifdef fast_3d_image_writes
void writeToImage3D(__write_only image3d_t input, int4 position, float4 value) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		write_imagef(input, position, value);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
		write_imageui(input, position, convert_uint4(value));
	} else {
		write_imagei(input, position, convert_int4(value));
    }
}
#endif

float4 readFromImage3D(__read_only image3d_t input, sampler_t sampler, float4 position) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		return read_imagef(input, sampler, position);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
        return convert_float4(read_imageui(input, sampler, position));
	} else {
        return convert_float4(read_imagei(input, sampler, position));
    }
}

__kernel void resize3D(
		__read_only image3d_t input,
#ifdef fast_3d_image_writes
		__write_only image3d_t output,
#else
        __global void* output,
#endif
        __private char useInterpolation
    ) {
	const int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
	const float4 normalizedPosition = {
	        (float)pos.x / get_global_size(0),
            (float)pos.y / get_global_size(1),
            (float)pos.z / get_global_size(2),
            0
	};
	int dataType = get_image_channel_data_type(input);
	float4 value;
    if(useInterpolation == 1) {
        value = readFromImage3D(input, samplerLinear, normalizedPosition);
    } else {
        value = readFromImage3D(input, samplerNearest, normalizedPosition);
    }

#ifdef fast_3d_image_writes
    writeToImage3D(output, pos, value);
#else
    // TODO only one channel supported atm
    if(dataType == CLK_UNSIGNED_INT8) {
        ((__global uchar*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_SIGNED_INT8) {
        ((__global char*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_UNSIGNED_INT16) {
        ((__global ushort*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_SIGNED_INT16) {
        ((__global short*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_FLOAT) {
        ((__global float*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    }
#endif
}

__constant sampler_t samplerLinear = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_NONE | CLK_FILTER_LINEAR;
__constant sampler_t samplerNearest = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST;

void writeToImage(__write_only image2d_t input, int2 position, float4 value) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		write_imagef(input, position, value);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
		write_imageui(input, position, convert_uint4(value));
	} else {
		write_imagei(input, position, convert_int4(value));
    }
}

float4 readFromImage(__read_only image2d_t input, sampler_t sampler, float2 position) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		return read_imagef(input, sampler, position);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
        return convert_float4(read_imageui(input, sampler, position));
	} else {
        return convert_float4(read_imagei(input, sampler, position));
    }
}

__kernel void resize2DpreserveAspect(
	__read_only image2d_t input,
	__write_only image2d_t output,
    __private int newHeight,
    __private char useInterpolation
	) {
	const int2 position = {get_global_id(0), get_global_id(1)};
    const float scale = (float)newHeight / get_image_height(input);
	const float2 readPosition = {(float)position.x / get_global_size(0), (float)position.y / (scale*get_image_height(input))};
	float4 value;
    if(useInterpolation == 1) {
        value = readFromImage(input, samplerLinear, readPosition);
    } else {
        value = readFromImage(input, samplerNearest, readPosition);
    }
    if(position.y >= newHeight) {
        writeToImage(output, position, (float4)(0,0,0,0));
    } else {
        writeToImage(output, position, value);
    }
}

__kernel void resize2D(
		__read_only image2d_t input,
		__write_only image2d_t output,
        __private char useInterpolation
    ) {
	const int2 position = {get_global_id(0), get_global_id(1)};
	const float2 normalizedPosition = {(float)position.x / get_global_size(0), (float)position.y / get_global_size(1)};
	int dataType = get_image_channel_data_type(input);
	float4 value;
    if(useInterpolation == 1) {
        value = readFromImage(input, samplerLinear, normalizedPosition);
    } else {
        value = readFromImage(input, samplerNearest, normalizedPosition);
    }
    writeToImage(output, position, value);
}

// ================ 3D
#ifdef fast_3d_image_writes
void writeToImage3D(__write_only image3d_t input, int4 position, float4 value) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		write_imagef(input, position, value);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
		write_imageui(input, position, convert_uint4(value));
	} else {
		write_imagei(input, position, convert_int4(value));
    }
}
#endif

float4 readFromImage3D(__read_only image3d_t input, sampler_t sampler, float4 position) {
    int dataType = get_image_channel_data_type(input);
	if(dataType == CLK_FLOAT) {
		return read_imagef(input, sampler, position);
	} else if(dataType == CLK_UNSIGNED_INT8 || dataType == CLK_UNSIGNED_INT16) {
        return convert_float4(read_imageui(input, sampler, position));
	} else {
        return convert_float4(read_imagei(input, sampler, position));
    }
}

__kernel void resize3D(
		__read_only image3d_t input,
#ifdef fast_3d_image_writes
		__write_only image3d_t output,
#else
        __global void* output,
#endif
        __private char useInterpolation
    ) {
	const int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
	const float4 normalizedPosition = {
	        (float)pos.x / get_global_size(0),
            (float)pos.y / get_global_size(1),
            (float)pos.z / get_global_size(2),
            0
	};
	int dataType = get_image_channel_data_type(input);
	float4 value;
    if(useInterpolation == 1) {
        value = readFromImage3D(input, samplerLinear, normalizedPosition);
    } else {
        value = readFromImage3D(input, samplerNearest, normalizedPosition);
    }

#ifdef fast_3d_image_writes
    writeToImage3D(output, pos, value);
#else
    // TODO only one channel supported atm
    if(dataType == CLK_UNSIGNED_INT8) {
        ((__global uchar*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_SIGNED_INT8) {
        ((__global char*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_UNSIGNED_INT16) {
        ((__global ushort*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_SIGNED_INT16) {
        ((__global short*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    } else if(dataType == CLK_FLOAT) {
        ((__global float*)output)[pos.x+pos.y*get_global_size(0)+pos.z*get_global_size(0)*get_global_size(1)] = value.x;
    }
#endif
}


added comments

added comments

/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

__kernel void AddTensor(global float *a,
        const int a_c, const int a_h, const int a_w,
        const int a_nstride, const int a_cstride,
        global float *c,
        const int c_n, const int c_c, const int c_h, const int c_w,
        const int c_nstride, const int c_cstride,
        const unsigned int bitmap, const int work_per_wg)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

#if FWD_CONV_BIAS == 1 && INCR_WG == 1 // case when num_wg = c_n*c_c;
    int o_n = gid / a_c;
    int o_c = gid % a_c;
    float add = a[o_c];

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + lid] += add;

        lid += get_local_size(0);
    }

#elif FWD_CONV_BIAS == 1 && INCR_WG == 0 // case when num_wg = c_c (or a_c)
    float add = a[gid];
    int work_off = work_per_wg / c_n;

    while(lid < work_per_wg) {
        int o_hw = lid % work_off;
        int o_n =  lid / work_off;
        c[o_n*c_nstride + gid*c_cstride + o_hw] += add;

        lid += get_local_size(0);
    }

#else // generic add tensor
        float add = a[gid];
        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

        int o_w_gid_off = gid % a_w;
        int o_h_gid_off = (gid / a_w) % a_h;
        int o_c_gid_off = (gid / a_cstride) % a_c;
        int o_n_gid_off = gid / a_nstride;

        while(lid < work_per_wg) {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

            c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w] += add;
            // c = op(a, b)

            lid += get_local_size(0);
        }
#endif // FWD_CONV_BIAS
}

#define OP_0(a, b) (a)+(b)
#define OP_1(a, b) (a)*(b)
#define OP_2(a ,b) (((a)<(b))?(a):(b))
#define OP_3(a, b) (((a)>(b))?(a):(b))
#define OP(op, a, b) (op == 0 ? OP_0(a, b) : \
        (op == 1 ? OP_1(a, b) : \
         (op == 2 ? OP_2(a, b) : OP_3(a, b))))

__kernel void OpTensor(global float *a, global float *b,
        const int b_c, const int b_h, const int b_w,
        const int b_nstride, const int b_cstride,
        global float *c,
        const int c_n, const int c_c, const int c_h, const int c_w,
        const int c_nstride, const int c_cstride,
        const unsigned int bitmap, const int work_per_wg,
        int op)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

    // each workgroup computes H*W for each C (bias-term)
    // number of workgroups = c_n*c_c;
#if FWD_CONV_BIAS == 1 && INCR_WG == 1
    int o_n = gid / b_c;
    int o_c = gid % b_c;
    float operand = b[o_c];

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + lid] = OP(op, a[o_n*c_nstride + o_c*c_cstride + lid], operand);

        lid += get_local_size(0);
    }

    // each workgroup computes N*H*W for each C (bias-term)
    // number of workgroups = c_c (b_c)
#elif FWD_CONV_BIAS == 1 && INCR_WG == 0
    float operand = b[gid];
    int work_off = work_per_wg / c_n;

    while(lid < work_per_wg) {
        int o_hw = lid % work_off;
        int o_n =  lid / work_off;
        c[o_n*c_nstride + gid*c_cstride + o_hw] = OP(op, a[o_n*c_nstride + gid*c_cstride + o_hw], operand);

        lid += get_local_size(0);
    }

#elif LEADING_ONES == 1 && FIRST_NOT_ONE == 2 // bitmap = 1,1,1,0
    float operand = b[gid];

    int o_h = gid % c_h;
    int o_c = (gid / c_h) % c_c;
    int o_n = gid / (c_c*c_h);

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + lid] = OP(op, a[o_n*c_nstride + o_c*c_cstride + o_h*c_w + lid], operand);

        lid += get_local_size(0);
    }
#elif LEADING_ONES == 1 && FIRST_NOT_ONE == 3 // bitmap = 1,1,0,0
    float operand = b[gid];

    int o_c = gid % c_c;
    int o_n = gid / c_c;

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + lid] = OP(op, a[o_n*c_nstride + o_c*c_cstride + lid], operand);

        lid += get_local_size(0);
    }

#elif LEADING_ONES == 1 && FIRST_NOT_ONE == 4 // bitmap = 1,0,0,0
    float operand = b[gid];

    while(lid < work_per_wg) {
        c[gid*c_nstride + lid] = OP(op, a[gid*c_nstride + lid], operand);

        lid += get_local_size(0);
    }

#else // generic op tensor
    float operand = b[gid];
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_c_gid_off = (gid / b_cstride) % b_c;
    int o_n_gid_off = gid / b_nstride;

    while(lid < work_per_wg) {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

        c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w] = OP(op, a[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w], operand);

        lid += get_local_size(0);
    }
#endif
}

/* Only works for NCHW
 * bitmap tracks which dims are the same between 'a' and 'c'.
 * Example: 0, 1, 1, 0 means that C and H dims are the same and the rest are ones
 * bitmap dims with 0 contribute to the work_per_wg,
 * whereas dims with 1 contribute to the #workgroups (gid)
 * work_per_wg = product of dims with 0s (dims of 'c tensor') and
 * num_wg = product of dims with 1s (dims of 'a')
 * Bitmap for fwd_bias looks like 0, 1, 0, 0
 */

__kernel void AddTensor(global float *a,
        const int a_c, const int a_h, const int a_w,
        const int a_nstride, const int a_cstride,
        global float *c,
        const int c_n, const int c_c, const int c_h, const int c_w,
        const int c_nstride, const int c_cstride,
        const unsigned int bitmap, const int work_per_wg)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

#if FWD_CONV_BIAS == 1 && INCR_WG == 1 // case when num_wg = c_n*c_c;
    int o_n = gid / a_c;
    int o_c = gid % a_c;
    float add = a[o_c];

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + lid] += add;

        lid += get_local_size(0);
    }

#elif FWD_CONV_BIAS == 1 && INCR_WG == 0 // case when num_wg = c_c (or a_c)
    float add = a[gid];
    int work_off = work_per_wg / c_n;

    while(lid < work_per_wg) {
        int o_hw = lid % work_off;
        int o_n =  lid / work_off;
        c[o_n*c_nstride + gid*c_cstride + o_hw] += add;

        lid += get_local_size(0);
    }

#else // generic add tensor
        float add = a[gid];
        int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
        int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
        int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

        int o_w_gid_off = gid % a_w;
        int o_h_gid_off = (gid / a_w) % a_h;
        int o_c_gid_off = (gid / a_cstride) % a_c;
        int o_n_gid_off = gid / a_nstride;

        while(lid < work_per_wg) {
            int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
            int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
            int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
            int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

            c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w] += add;
            // c = op(a, b)

            lid += get_local_size(0);
        }
#endif // FWD_CONV_BIAS
}

#define OP_0(a, b) (a)+(b)
#define OP_1(a, b) (a)*(b)
#define OP_2(a ,b) (((a)<(b))?(a):(b))
#define OP_3(a, b) (((a)>(b))?(a):(b))
#define OP(op, a, b) (op == 0 ? OP_0(a, b) : \
        (op == 1 ? OP_1(a, b) : \
         (op == 2 ? OP_2(a, b) : OP_3(a, b))))

__kernel void OpTensor(global float *a, global float *b,
        const int b_c, const int b_h, const int b_w,
        const int b_nstride, const int b_cstride,
        global float *c,
        const int c_n, const int c_c, const int c_h, const int c_w,
        const int c_nstride, const int c_cstride,
        const unsigned int bitmap, const int work_per_wg,
        int op)
{
    int gid = get_group_id(0);
    int lid = get_local_id(0);

#if FWD_CONV_BIAS == 1 && INCR_WG == 1 // case when num_wg = c_n*c_c;
    int o_n = gid / b_c;
    int o_c = gid % b_c;
    float operand = b[o_c];

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + lid] = OP(op, a[o_n*c_nstride + o_c*c_cstride + lid], operand);

        lid += get_local_size(0);
    }

#elif FWD_CONV_BIAS == 1 && INCR_WG == 0 // case when num_wg = c_c (or a_c)
    float operand = b[gid];
    int work_off = work_per_wg / c_n;

    while(lid < work_per_wg) {
        int o_hw = lid % work_off;
        int o_n =  lid / work_off;
        c[o_n*c_nstride + gid*c_cstride + o_hw] = OP(op, a[o_n*c_nstride + gid*c_cstride + o_hw], operand);

        lid += get_local_size(0);
    }

#elif LEADING_ONES == 1 && FIRST_NOT_ONE == 2
    float operand = b[gid];

    int o_h = gid % c_h;
    int o_c = (gid / c_h) % c_c;
    int o_n = gid / (c_c*c_h);

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + lid] = OP(op, a[o_n*c_nstride + o_c*c_cstride + o_h*c_w + lid], operand);

        lid += get_local_size(0);
    }
#elif LEADING_ONES == 1 && FIRST_NOT_ONE == 3
    float operand = b[gid];

    int o_c = gid % c_c;
    int o_n = gid / c_c;

    while(lid < work_per_wg) {
        c[o_n*c_nstride + o_c*c_cstride + lid] = OP(op, a[o_n*c_nstride + o_c*c_cstride + lid], operand);

        lid += get_local_size(0);
    }

#elif LEADING_ONES == 1 && FIRST_NOT_ONE == 4
    float operand = b[gid];

    while(lid < work_per_wg) {
        c[gid*c_nstride + lid] = OP(op, a[gid*c_nstride + lid], operand);

        lid += get_local_size(0);
    }

#else // generic op tensor
    float operand = b[gid];
    int o_h_div = bitmap & (1 << 0) ? 1 : c_w;
    int o_c_div = o_h_div * (bitmap & (1 << 1) ? 1 : c_h);
    int o_n_div = o_c_div * (bitmap & (1 << 2) ? 1 : c_c);

    int o_w_gid_off = gid % b_w;
    int o_h_gid_off = (gid / b_w) % b_h;
    int o_c_gid_off = (gid / b_cstride) % b_c;
    int o_n_gid_off = gid / b_nstride;

    while(lid < work_per_wg) {
        int o_w = (bitmap & (1 << 0)) ? o_w_gid_off : lid % c_w;
        int o_h = (bitmap & (1 << 1)) ? o_h_gid_off : (lid / o_h_div) % c_h;
        int o_c = (bitmap & (1 << 2)) ? o_c_gid_off : (lid / o_c_div) % c_c;
        int o_n = (bitmap & (1 << 3)) ? o_n_gid_off : lid / o_n_div;

        c[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w] = OP(op, a[o_n*c_nstride + o_c*c_cstride + o_h*c_w + o_w], operand);

        lid += get_local_size(0);
    }
#endif // FWD_CONV_BIAS
}