covariance - simplify blocks termination when using SYMMETRIC parameter

// *************************************************************************************************
// The following kernel is a slight modification of the GEMM tunable example in CLTune project, so
// it can be used in Covariance problem. The original notice follows.
// *************************************************************************************************
//
// =================================================================================================
// This file is part of the CLTune project, which loosely follows the Google C++ styleguide and uses
// a tab-size of two spaces and a max-width of 100 characters per line.
//
// Author: cedric.nugteren@surfsara.nl (Cedric Nugteren)
//
// This file contains an example OpenCL kernel as part of the gemm.cc example. It is an optimized
// matrix-multiplication kernel according to the paper by Matsumoto et al. and the tutorial on
// http://www.cedricnugteren.nl/tutorial.php. It is fully configurable (and tunable!) using more or
// less the same parameters/naming conventions as in the paper. It supports single and double
// precision (SGEMM/DGEMM) through a pre-processor define.
//
// Note: this kernel requires a compiler compliant to OpenCL 1.1 or higher.
//
// -------------------------------------------------------------------------------------------------
//
// Copyright 2014 SURFsara
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// =================================================================================================
//
// Matrices are accessed as follows:
// A: [k*M + m], with 'k' ranging from 0:K and 'm' from 0:M (m,k,m)
// B: [k*N + n], with 'k' ranging from 0:K and 'n' from 0:N (n,k,n)
// C: [n*M + m], with 'n' ranging from 0:N and 'm' from 0:M (m,n,m)
//
// Or as an image (assuming column-major)
//       K
//    o-------o
//    |       |
//  N | [B^T] |
//    |       |
//    o-------o
//        K               N
//    o-------o        o-----o
//  M |  [A]  |      M | [C] |
//    |       |        |     |
//    o-------o        o-----o
//
//
// Parameters determined by the tuner
// MWG       : Tile-size in dimension M (e.g. 64, 128)
// NWG       : Tile-size in dimension N (e.g. 64, 128)
// KWG       : Tile-size in dimension K (e.g. 8, 16)
// MDIMC     : Threads per workgroup in M-dimension (e.g. 8, 16, 32)
// NDIMC     : Threads per workgroup in N-dimension (e.g. 8, 16, 32)
// MDIMA     : Re-shaped tile dimension of matrix A: KDIMA * MDIMA
// NDIMB     : Re-shaped tile dimension of matrix B: KDIMB * NDIMB
// KWI       : Unroll factor of the KWG loop (smaller or equal than KWG)
// VWM       : Vector width of matrices A and C (supported 1, 2, 4, and 8)
// VWN       : Vector width of matrix B (supported 1, 2, 4, and 8)
// STRM      : Use strided access within a thread in the M-dimension (1) or not (0)
// STRN      : Use strided access within a thread in the N-dimension (1) or not (0)
// SA        : Use local/shared memory to cache matrix A (1) or not (0)
// SB        : Use local/shared memory to cache matrix B (1) or not (0)
// PRECISION : Whether to use single (32) or double (64) precision data-types
//
// =================================================================================================

// Helper parameters based on the above tuning parameters
#define MWI (MWG / MDIMC)                  // Work per work-item (M-dimension)
#define NWI (NWG / NDIMC)                  // Work per work-item (N-dimension)
#define KDIMA ((MDIMC * NDIMC) / (MDIMA))  // Re-shaped tile dimension of matrix A: KDIMA * MDIMA
#define KDIMB ((MDIMC * NDIMC) / (NDIMB))  // Re-shaped tile dimension of matrix B: KDIMB * NDIMB
#define MWA (MWG / MDIMA)                  // Amount of loads-per-thread for matrix A (M-dimension)
#define KWA (KWG / KDIMA)                  // Amount of loads-per-thread for matrix A (K-dimension)
#define KWB (KWG / KDIMB)                  // Amount of loads-per-thread for matrix B (K-dimension)
#define NWB (NWG / NDIMB)                  // Amount of loads-per-thread for matrix B (N-dimension)

// Settings
#define USE_VECTOR_MAD 1  // Don't unroll the vector MAD computation
#define USE_CL_MAD 0      // Uses the non-IEEE754 compliant OpenCL mad() (if above is 0)

// =================================================================================================

// Data-type: single or double precision
#if PRECISION == 32
typedef float real;
typedef float2 real2;
typedef float4 real4;
typedef float8 real8;
typedef float16 real16;
#  define ZERO 0.0f
#elif PRECISION == 64
#  if __OPENCL_VERSION__ <= CL_VERSION_1_1  // This the default on OpenCL 1.2 or higher
#    pragma OPENCL EXTENSION cl_khr_fp64 : enable
#  endif
typedef double real;
typedef double2 real2;
typedef double4 real4;
typedef double8 real8;
typedef double16 real16;
#  define ZERO 0.0
#endif

// =================================================================================================

// Data-widths in dimension M
#if VWM == 1
typedef real realM;
#elif VWM == 2
typedef real2 realM;
#elif VWM == 4
typedef real4 realM;
#elif VWM == 8
typedef real8 realM;
#elif VWM == 16
typedef real16 realM;
#endif

// Data-widths in dimension N
#if VWN == 1
typedef real realN;
#elif VWN == 2
typedef real2 realN;
#elif VWN == 4
typedef real4 realN;
#elif VWN == 8
typedef real8 realN;
#elif VWN == 16
typedef real16 realN;
#endif

// =================================================================================================

// Caches global off-chip memory into local (shared) memory on-chip. This function is specific for
// caching the A input matrix.
#if SA == 1
inline void GlobalToLocalA(const __global realM* restrict agm, __local realM* alm, const int kSizeM,
    const int tid, const int kwg) {
  const int la0 = tid % MDIMA;
  const int la1 = tid / MDIMA;
#  pragma unroll
  for (int mia = 0; mia < MWA / VWM; ++mia) {
#  pragma unroll
    for (int kia = 0; kia < KWA; ++kia) {
// Computes the indices based on strided/non-strided access
#  if STRM == 0
      int mg = mia + la0 * (MWA / VWM);
#  elif STRM == 1
      int mg = la0 + mia * MDIMA;
#  endif

      // Computes the indices for the global memory
      int kg = kia + la1 * KWA;
      int idm = mg + get_group_id(0) * (MWG / VWM);
      int idk = kg + kwg;

      // Loads the data from global memory (not transposed) into the local memory
      alm[kg * (MWG / VWM) + mg] = agm[idk * (kSizeM / VWM) + idm];
    }
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
inline void GlobalToLocalB(const __global realN* restrict bgm, __local realN* blm, const int kSizeN,
    const int tid, const int kwg) {
  const int lb0 = tid % NDIMB;
  const int lb1 = tid / NDIMB;
#  pragma unroll
  for (int kib = 0; kib < KWB; ++kib) {
#  pragma unroll
    for (int nib = 0; nib < NWB / VWN; ++nib) {
// Computes the indices based on strided/non-strided access
#  if STRN == 0
      int ng = nib + lb0 * (NWB / VWN);
#  elif STRN == 1
      int ng = lb0 + nib * NDIMB;
#  endif

      // Computes the indices for the global memory
      int kg = kib + lb1 * KWB;
      int idn = ng + get_group_id(1) * (NWG / VWN);
      int idk = kg + kwg;

      // Loads the data from global memory (transposed) into the local memory
      blm[kg * (NWG / VWN) + ng] = bgm[idk * (kSizeN / VWN) + idn];
    }
  }
}
#endif

// =================================================================================================

// Caches global off-chip memory directly into per-thread private memory (registers). This function
// is specific for caching the A input matrix.
#if SA == 0
inline void GlobalToPrivateA(const __global realM* restrict agm, realM apm[MWI / VWM],
    const int kSizeM, const int idk, const int kwg) {
#  pragma unroll
  for (int mi = 0; mi < MWI / VWM; ++mi) {
// Computes the indices based on strided/non-strided access
#  if STRM == 0
    int mg = mi + get_local_id(0) * (MWI / VWM);
#  elif STRM == 1
    int mg = get_local_id(0) + mi * MDIMC;
#  endif

    // Computes the indices for the global memory
    int idm = mg + get_group_id(0) * (MWG / VWM);

    // Loads the data from global memory (not transposed) and stores into registers
    apm[mi] = agm[idk * (kSizeM / VWM) + idm];
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 0
inline void GlobalToPrivateB(
    const __global realN* restrict bgm, realN bpm[NWI / VWN], const int kSizeN, const int idk) {
#  pragma unroll
  for (int ni = 0; ni < NWI / VWN; ++ni) {
// Computes the indices based on strided/non-strided access
#  if STRN == 0
    int ng = ni + get_local_id(1) * (NWI / VWN);
#  elif STRN == 1
    int ng = get_local_id(1) + ni * NDIMC;
#  endif

    // Computes the indices for the global memory
    int idn = ng + get_group_id(1) * (NWG / VWN);

    // Loads the data from global memory (transposed) and stores into registers
    bpm[ni] = bgm[idk * (kSizeN / VWN) + idn];
  }
}
#endif

// =================================================================================================

// Caches on-chip local memory into per-thread private memory (registers). This function is specific
// for caching the A input matrix.
#if SA == 1
inline void LocalToPrivateA(__local realM* alm, realM apm[MWI / VWM], const int kg) {
#  pragma unroll
  for (int mi = 0; mi < MWI / VWM; ++mi) {
#  if STRM == 0
    int mg = mi + get_local_id(0) * (MWI / VWM);
#  elif STRM == 1
    int mg = get_local_id(0) + mi * MDIMC;
#  endif
    apm[mi] = alm[kg * (MWG / VWM) + mg];
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
inline void LocalToPrivateB(__local realN* blm, realN bpm[NWI / VWN], const int kg) {
#  pragma unroll
  for (int ni = 0; ni < NWI / VWN; ++ni) {
#  if STRN == 0
    int ng = ni + get_local_id(1) * (NWI / VWN);
#  elif STRN == 1
    int ng = get_local_id(1) + ni * NDIMC;
#  endif
    bpm[ni] = blm[kg * (NWG / VWN) + ng];
  }
}
#endif

// =================================================================================================

// Merges the results in Cpm with the global array in Cgm
inline void StoreResults(__global realM* cgm, realM cpm[NWI][MWI / VWM], const int kSizeM) {
#pragma unroll
  for (int ni = 0; ni < NWI; ++ni) {
#pragma unroll
    for (int mi = 0; mi < MWI / VWM; ++mi) {
#if STRM == 0
      int mg = mi + get_local_id(0) * (MWI / VWM);
#elif STRM == 1
      int mg = get_local_id(0) + mi * MDIMC;
#endif
#if STRN == 0
      int ng = ni + get_local_id(1) * NWI;
#elif STRN == 1
      int ng = ni % VWN + get_local_id(1) * VWN + (ni / VWN) * VWN * NDIMC;
#endif
      int idm = mg + get_group_id(0) * (MWG / VWM);
      int idn = ng + get_group_id(1) * NWG;
      int index = idn * (kSizeM / VWM) + idm;
      cgm[index] = cpm[ni][mi];
#if SYM_STORE == 1
      cgm[idm * (kSizeM / VWM) + idn] = cpm[ni][mi];
#endif
    }
  }
}

// =================================================================================================

// The basic scalar multiply-add function
#if USE_CL_MAD == 1
#  define MultiplyAdd(cval, aval, bval) (cval = mad(aval, bval, cval))
#else
#  define MultiplyAdd(cval, aval, bval) (cval += (aval) * (bval))
#endif

// The vectorised multiply-add function
inline realM MultiplyAddVector(realM cvec, const realM avec, const real bval) {
#if USE_VECTOR_MAD == 1
  cvec += avec * bval;
#else
#  if VWM == 1
  MultiplyAdd(cvec, avec, bval);
#  elif VWM == 2
  MultiplyAdd(cvec.x, avec.x, bval);
  MultiplyAdd(cvec.y, avec.y, bval);
#  elif VWM == 4
  MultiplyAdd(cvec.x, avec.x, bval);
  MultiplyAdd(cvec.y, avec.y, bval);
  MultiplyAdd(cvec.z, avec.z, bval);
  MultiplyAdd(cvec.w, avec.w, bval);
#  elif VWM == 8
  MultiplyAdd(cvec.s0, avec.s0, bval);
  MultiplyAdd(cvec.s1, avec.s1, bval);
  MultiplyAdd(cvec.s2, avec.s2, bval);
  MultiplyAdd(cvec.s3, avec.s3, bval);
  MultiplyAdd(cvec.s4, avec.s4, bval);
  MultiplyAdd(cvec.s5, avec.s5, bval);
  MultiplyAdd(cvec.s6, avec.s6, bval);
  MultiplyAdd(cvec.s7, avec.s7, bval);
#  elif VWM == 16
  MultiplyAdd(cvec.s0, avec.s0, bval);
  MultiplyAdd(cvec.s1, avec.s1, bval);
  MultiplyAdd(cvec.s2, avec.s2, bval);
  MultiplyAdd(cvec.s3, avec.s3, bval);
  MultiplyAdd(cvec.s4, avec.s4, bval);
  MultiplyAdd(cvec.s5, avec.s5, bval);
  MultiplyAdd(cvec.s6, avec.s6, bval);
  MultiplyAdd(cvec.s7, avec.s7, bval);
  MultiplyAdd(cvec.s8, avec.s8, bval);
  MultiplyAdd(cvec.s9, avec.s9, bval);
  MultiplyAdd(cvec.sA, avec.sA, bval);
  MultiplyAdd(cvec.sB, avec.sB, bval);
  MultiplyAdd(cvec.sC, avec.sC, bval);
  MultiplyAdd(cvec.sD, avec.sD, bval);
  MultiplyAdd(cvec.sE, avec.sE, bval);
  MultiplyAdd(cvec.sF, avec.sF, bval);
#  endif
#endif
  return cvec;
}

// Performs the actual computation: Cpm += Apm * Bpm
inline void MultiplyAccumulate(
    realM cpm[NWI][MWI / VWM], realM apm[MWI / VWM], realN bpm[NWI / VWN]) {
#pragma unroll
  for (int ni = 0; ni < NWI / VWN; ++ni) {
#pragma unroll
    for (int mi = 0; mi < MWI / VWM; ++mi) {
#if VWN == 1
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni]);
#elif VWN == 2
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].x);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].y);
#elif VWN == 4
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].x);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].y);
      cpm[ni * VWN + 2][mi] = MultiplyAddVector(cpm[ni * VWN + 2][mi], apm[mi], bpm[ni].z);
      cpm[ni * VWN + 3][mi] = MultiplyAddVector(cpm[ni * VWN + 3][mi], apm[mi], bpm[ni].w);
#elif VWN == 8
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].s0);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].s1);
      cpm[ni * VWN + 2][mi] = MultiplyAddVector(cpm[ni * VWN + 2][mi], apm[mi], bpm[ni].s2);
      cpm[ni * VWN + 3][mi] = MultiplyAddVector(cpm[ni * VWN + 3][mi], apm[mi], bpm[ni].s3);
      cpm[ni * VWN + 4][mi] = MultiplyAddVector(cpm[ni * VWN + 4][mi], apm[mi], bpm[ni].s4);
      cpm[ni * VWN + 5][mi] = MultiplyAddVector(cpm[ni * VWN + 5][mi], apm[mi], bpm[ni].s5);
      cpm[ni * VWN + 6][mi] = MultiplyAddVector(cpm[ni * VWN + 6][mi], apm[mi], bpm[ni].s6);
      cpm[ni * VWN + 7][mi] = MultiplyAddVector(cpm[ni * VWN + 7][mi], apm[mi], bpm[ni].s7);
#elif VWN == 16
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].s0);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].s1);
      cpm[ni * VWN + 2][mi] = MultiplyAddVector(cpm[ni * VWN + 2][mi], apm[mi], bpm[ni].s2);
      cpm[ni * VWN + 3][mi] = MultiplyAddVector(cpm[ni * VWN + 3][mi], apm[mi], bpm[ni].s3);
      cpm[ni * VWN + 4][mi] = MultiplyAddVector(cpm[ni * VWN + 4][mi], apm[mi], bpm[ni].s4);
      cpm[ni * VWN + 5][mi] = MultiplyAddVector(cpm[ni * VWN + 5][mi], apm[mi], bpm[ni].s5);
      cpm[ni * VWN + 6][mi] = MultiplyAddVector(cpm[ni * VWN + 6][mi], apm[mi], bpm[ni].s6);
      cpm[ni * VWN + 7][mi] = MultiplyAddVector(cpm[ni * VWN + 7][mi], apm[mi], bpm[ni].s7);
      cpm[ni * VWN + 8][mi] = MultiplyAddVector(cpm[ni * VWN + 8][mi], apm[mi], bpm[ni].s8);
      cpm[ni * VWN + 9][mi] = MultiplyAddVector(cpm[ni * VWN + 9][mi], apm[mi], bpm[ni].s9);
      cpm[ni * VWN + 10][mi] = MultiplyAddVector(cpm[ni * VWN + 10][mi], apm[mi], bpm[ni].sA);
      cpm[ni * VWN + 11][mi] = MultiplyAddVector(cpm[ni * VWN + 11][mi], apm[mi], bpm[ni].sB);
      cpm[ni * VWN + 12][mi] = MultiplyAddVector(cpm[ni * VWN + 12][mi], apm[mi], bpm[ni].sC);
      cpm[ni * VWN + 13][mi] = MultiplyAddVector(cpm[ni * VWN + 13][mi], apm[mi], bpm[ni].sD);
      cpm[ni * VWN + 14][mi] = MultiplyAddVector(cpm[ni * VWN + 14][mi], apm[mi], bpm[ni].sE);
      cpm[ni * VWN + 15][mi] = MultiplyAddVector(cpm[ni * VWN + 15][mi], apm[mi], bpm[ni].sF);
#endif
    }
  }
}

// =================================================================================================

// Main entry of the kernel. This function contains the basic skeleton, the functionality is
// provided by the inlined functions above.
__attribute__((reqd_work_group_size(MDIMC, NDIMC, 1))) __kernel void gemm_fast(
    const int kSizeM, const int kSizeK, const __global realM* restrict agm, __global realM* cgm) {
#if SYMMETRIC == 1
  const int bx = get_group_id(0);
  const int by = get_group_id(1);
  if (by * NWG > bx * MWG + MWG - 1) return;
#endif
// Combined thread identifier
#if SA == 1 || SB == 1
  volatile int tid = get_local_id(0) + MDIMC * get_local_id(1);
#endif

// Allocates workgroup-private memory (local memory)
#if SA == 1
  __local realM alm[KWG * MWG / VWM];
#endif
#if SB == 1
  __local realN blm[KWG * NWG / VWN];
#endif

  // Allocates workitem-private memory (registers)
  realM apm[MWI / VWM];
  realN bpm[NWI / VWN];
  realM cpm[NWI][MWI / VWM];

// Initializes the accumulation registers
#pragma unroll
  for (int mi = 0; mi < MWI / VWM; ++mi) {
#pragma unroll
    for (int ni = 0; ni < NWI; ++ni) {
      cpm[ni][mi] = (realM)ZERO;
    }
  }

  // Loops over all workgroup tiles
  for (int kwg = 0; kwg < kSizeK; kwg += KWG) {
// Loads data: off-chip --> local (matrix A)
#if SA == 1
    GlobalToLocalA(agm, alm, kSizeM, tid, kwg);
#endif
// Loads data: off-chip --> local (matrix B)
#if SB == 1
    GlobalToLocalB(agm, blm, kSizeM, tid, kwg);
#endif

// Synchronizes all threads in a workgroup
#if SA == 1 || SB == 1
    barrier(CLK_LOCAL_MEM_FENCE);
#endif

    // Loops over all workitem tiles, unrolled by a factor KWI
    for (int pwi = 0; pwi < KWG; pwi += KWI) {
#pragma unroll
      for (int pit = 0; pit < KWI; ++pit) {
#if SA == 0 || SB == 0
        int idk = kwg + pwi + pit;
#endif
#if SA == 1 || SB == 1
        int kg = pwi + pit;
#endif

// Loads data: local --> private (matrix A)
#if SA == 1
        LocalToPrivateA(alm, apm, kg);
// Loads data: off-chip --> private (matrix A)
#else
        GlobalToPrivateA(agm, apm, kSizeM, idk, kwg);
#endif

// Loads data: local --> private (matrix B)
#if SB == 1
        LocalToPrivateB(blm, bpm, kg);
// Loads data: off-chip --> private (matrix B)
#else
        GlobalToPrivateB(agm, bpm, kSizeM, idk);
#endif

        // Performs the accumulation (Cpm += Apm * Bpm)
        MultiplyAccumulate(cpm, apm, bpm);
      }
    }

// Synchronizes all threads in a workgroup
#if SA == 1 || SB == 1
    barrier(CLK_LOCAL_MEM_FENCE);
#endif
  }

  // Stores an MWG * NWG tile of results
  StoreResults(cgm, cpm, kSizeM);
}
// =================================================================================================

// *************************************************************************************************
// The following kernel is a slight modification of the GEMM tunable example in CLTune project, so
// it can be used in Covariance problem. The original notice follows.
// *************************************************************************************************
//
// =================================================================================================
// This file is part of the CLTune project, which loosely follows the Google C++ styleguide and uses
// a tab-size of two spaces and a max-width of 100 characters per line.
//
// Author: cedric.nugteren@surfsara.nl (Cedric Nugteren)
//
// This file contains an example OpenCL kernel as part of the gemm.cc example. It is an optimized
// matrix-multiplication kernel according to the paper by Matsumoto et al. and the tutorial on
// http://www.cedricnugteren.nl/tutorial.php. It is fully configurable (and tunable!) using more or
// less the same parameters/naming conventions as in the paper. It supports single and double
// precision (SGEMM/DGEMM) through a pre-processor define.
//
// Note: this kernel requires a compiler compliant to OpenCL 1.1 or higher.
//
// -------------------------------------------------------------------------------------------------
//
// Copyright 2014 SURFsara
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//  http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// =================================================================================================
//
// Matrices are accessed as follows:
// A: [k*M + m], with 'k' ranging from 0:K and 'm' from 0:M (m,k,m)
// B: [k*N + n], with 'k' ranging from 0:K and 'n' from 0:N (n,k,n)
// C: [n*M + m], with 'n' ranging from 0:N and 'm' from 0:M (m,n,m)
//
// Or as an image (assuming column-major)
//       K
//    o-------o
//    |       |
//  N | [B^T] |
//    |       |
//    o-------o
//        K               N
//    o-------o        o-----o
//  M |  [A]  |      M | [C] |
//    |       |        |     |
//    o-------o        o-----o
//
//
// Parameters determined by the tuner
// MWG       : Tile-size in dimension M (e.g. 64, 128)
// NWG       : Tile-size in dimension N (e.g. 64, 128)
// KWG       : Tile-size in dimension K (e.g. 8, 16)
// MDIMC     : Threads per workgroup in M-dimension (e.g. 8, 16, 32)
// NDIMC     : Threads per workgroup in N-dimension (e.g. 8, 16, 32)
// MDIMA     : Re-shaped tile dimension of matrix A: KDIMA * MDIMA
// NDIMB     : Re-shaped tile dimension of matrix B: KDIMB * NDIMB
// KWI       : Unroll factor of the KWG loop (smaller or equal than KWG)
// VWM       : Vector width of matrices A and C (supported 1, 2, 4, and 8)
// VWN       : Vector width of matrix B (supported 1, 2, 4, and 8)
// STRM      : Use strided access within a thread in the M-dimension (1) or not (0)
// STRN      : Use strided access within a thread in the N-dimension (1) or not (0)
// SA        : Use local/shared memory to cache matrix A (1) or not (0)
// SB        : Use local/shared memory to cache matrix B (1) or not (0)
// PRECISION : Whether to use single (32) or double (64) precision data-types
//
// =================================================================================================

// Helper parameters based on the above tuning parameters
#define MWI (MWG / MDIMC)                  // Work per work-item (M-dimension)
#define NWI (NWG / NDIMC)                  // Work per work-item (N-dimension)
#define KDIMA ((MDIMC * NDIMC) / (MDIMA))  // Re-shaped tile dimension of matrix A: KDIMA * MDIMA
#define KDIMB ((MDIMC * NDIMC) / (NDIMB))  // Re-shaped tile dimension of matrix B: KDIMB * NDIMB
#define MWA (MWG / MDIMA)                  // Amount of loads-per-thread for matrix A (M-dimension)
#define KWA (KWG / KDIMA)                  // Amount of loads-per-thread for matrix A (K-dimension)
#define KWB (KWG / KDIMB)                  // Amount of loads-per-thread for matrix B (K-dimension)
#define NWB (NWG / NDIMB)                  // Amount of loads-per-thread for matrix B (N-dimension)

// Settings
#define USE_VECTOR_MAD 1  // Don't unroll the vector MAD computation
#define USE_CL_MAD 0      // Uses the non-IEEE754 compliant OpenCL mad() (if above is 0)

// =================================================================================================

// Data-type: single or double precision
#if PRECISION == 32
typedef float real;
typedef float2 real2;
typedef float4 real4;
typedef float8 real8;
typedef float16 real16;
#  define ZERO 0.0f
#elif PRECISION == 64
#  if __OPENCL_VERSION__ <= CL_VERSION_1_1  // This the default on OpenCL 1.2 or higher
#    pragma OPENCL EXTENSION cl_khr_fp64 : enable
#  endif
typedef double real;
typedef double2 real2;
typedef double4 real4;
typedef double8 real8;
typedef double16 real16;
#  define ZERO 0.0
#endif

// =================================================================================================

// Data-widths in dimension M
#if VWM == 1
typedef real realM;
#elif VWM == 2
typedef real2 realM;
#elif VWM == 4
typedef real4 realM;
#elif VWM == 8
typedef real8 realM;
#elif VWM == 16
typedef real16 realM;
#endif

// Data-widths in dimension N
#if VWN == 1
typedef real realN;
#elif VWN == 2
typedef real2 realN;
#elif VWN == 4
typedef real4 realN;
#elif VWN == 8
typedef real8 realN;
#elif VWN == 16
typedef real16 realN;
#endif

// =================================================================================================

// Caches global off-chip memory into local (shared) memory on-chip. This function is specific for
// caching the A input matrix.
#if SA == 1
inline void GlobalToLocalA(const __global realM* restrict agm, __local realM* alm, const int kSizeM,
    const int tid, const int kwg) {
  const int la0 = tid % MDIMA;
  const int la1 = tid / MDIMA;
#  pragma unroll
  for (int mia = 0; mia < MWA / VWM; ++mia) {
#  pragma unroll
    for (int kia = 0; kia < KWA; ++kia) {
// Computes the indices based on strided/non-strided access
#  if STRM == 0
      int mg = mia + la0 * (MWA / VWM);
#  elif STRM == 1
      int mg = la0 + mia * MDIMA;
#  endif

      // Computes the indices for the global memory
      int kg = kia + la1 * KWA;
      int idm = mg + get_group_id(0) * (MWG / VWM);
      int idk = kg + kwg;

      // Loads the data from global memory (not transposed) into the local memory
      alm[kg * (MWG / VWM) + mg] = agm[idk * (kSizeM / VWM) + idm];
    }
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
inline void GlobalToLocalB(const __global realN* restrict bgm, __local realN* blm, const int kSizeN,
    const int tid, const int kwg) {
  const int lb0 = tid % NDIMB;
  const int lb1 = tid / NDIMB;
#  pragma unroll
  for (int kib = 0; kib < KWB; ++kib) {
#  pragma unroll
    for (int nib = 0; nib < NWB / VWN; ++nib) {
// Computes the indices based on strided/non-strided access
#  if STRN == 0
      int ng = nib + lb0 * (NWB / VWN);
#  elif STRN == 1
      int ng = lb0 + nib * NDIMB;
#  endif

      // Computes the indices for the global memory
      int kg = kib + lb1 * KWB;
      int idn = ng + get_group_id(1) * (NWG / VWN);
      int idk = kg + kwg;

      // Loads the data from global memory (transposed) into the local memory
      blm[kg * (NWG / VWN) + ng] = bgm[idk * (kSizeN / VWN) + idn];
    }
  }
}
#endif

// =================================================================================================

// Caches global off-chip memory directly into per-thread private memory (registers). This function
// is specific for caching the A input matrix.
#if SA == 0
inline void GlobalToPrivateA(const __global realM* restrict agm, realM apm[MWI / VWM],
    const int kSizeM, const int idk, const int kwg) {
#  pragma unroll
  for (int mi = 0; mi < MWI / VWM; ++mi) {
// Computes the indices based on strided/non-strided access
#  if STRM == 0
    int mg = mi + get_local_id(0) * (MWI / VWM);
#  elif STRM == 1
    int mg = get_local_id(0) + mi * MDIMC;
#  endif

    // Computes the indices for the global memory
    int idm = mg + get_group_id(0) * (MWG / VWM);

    // Loads the data from global memory (not transposed) and stores into registers
    apm[mi] = agm[idk * (kSizeM / VWM) + idm];
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 0
inline void GlobalToPrivateB(
    const __global realN* restrict bgm, realN bpm[NWI / VWN], const int kSizeN, const int idk) {
#  pragma unroll
  for (int ni = 0; ni < NWI / VWN; ++ni) {
// Computes the indices based on strided/non-strided access
#  if STRN == 0
    int ng = ni + get_local_id(1) * (NWI / VWN);
#  elif STRN == 1
    int ng = get_local_id(1) + ni * NDIMC;
#  endif

    // Computes the indices for the global memory
    int idn = ng + get_group_id(1) * (NWG / VWN);

    // Loads the data from global memory (transposed) and stores into registers
    bpm[ni] = bgm[idk * (kSizeN / VWN) + idn];
  }
}
#endif

// =================================================================================================

// Caches on-chip local memory into per-thread private memory (registers). This function is specific
// for caching the A input matrix.
#if SA == 1
inline void LocalToPrivateA(__local realM* alm, realM apm[MWI / VWM], const int kg) {
#  pragma unroll
  for (int mi = 0; mi < MWI / VWM; ++mi) {
#  if STRM == 0
    int mg = mi + get_local_id(0) * (MWI / VWM);
#  elif STRM == 1
    int mg = get_local_id(0) + mi * MDIMC;
#  endif
    apm[mi] = alm[kg * (MWG / VWM) + mg];
  }
}
#endif

// Same as above, but now for the B input matrix
#if SB == 1
inline void LocalToPrivateB(__local realN* blm, realN bpm[NWI / VWN], const int kg) {
#  pragma unroll
  for (int ni = 0; ni < NWI / VWN; ++ni) {
#  if STRN == 0
    int ng = ni + get_local_id(1) * (NWI / VWN);
#  elif STRN == 1
    int ng = get_local_id(1) + ni * NDIMC;
#  endif
    bpm[ni] = blm[kg * (NWG / VWN) + ng];
  }
}
#endif

// =================================================================================================

// Merges the results in Cpm with the global array in Cgm
inline void StoreResults(__global realM* cgm, realM cpm[NWI][MWI / VWM], const int kSizeM) {
#pragma unroll
  for (int ni = 0; ni < NWI; ++ni) {
#pragma unroll
    for (int mi = 0; mi < MWI / VWM; ++mi) {
#if STRM == 0
      int mg = mi + get_local_id(0) * (MWI / VWM);
#elif STRM == 1
      int mg = get_local_id(0) + mi * MDIMC;
#endif
#if STRN == 0
      int ng = ni + get_local_id(1) * NWI;
#elif STRN == 1
      int ng = ni % VWN + get_local_id(1) * VWN + (ni / VWN) * VWN * NDIMC;
#endif
      int idm = mg + get_group_id(0) * (MWG / VWM);
      int idn = ng + get_group_id(1) * NWG;
      int index = idn * (kSizeM / VWM) + idm;
      cgm[index] = cpm[ni][mi];
#if SYM_STORE == 1
      cgm[idm * (kSizeM / VWM) + idn] = cpm[ni][mi];
#endif
    }
  }
}

// =================================================================================================

// The basic scalar multiply-add function
#if USE_CL_MAD == 1
#  define MultiplyAdd(cval, aval, bval) (cval = mad(aval, bval, cval))
#else
#  define MultiplyAdd(cval, aval, bval) (cval += (aval) * (bval))
#endif

// The vectorised multiply-add function
inline realM MultiplyAddVector(realM cvec, const realM avec, const real bval) {
#if USE_VECTOR_MAD == 1
  cvec += avec * bval;
#else
#  if VWM == 1
  MultiplyAdd(cvec, avec, bval);
#  elif VWM == 2
  MultiplyAdd(cvec.x, avec.x, bval);
  MultiplyAdd(cvec.y, avec.y, bval);
#  elif VWM == 4
  MultiplyAdd(cvec.x, avec.x, bval);
  MultiplyAdd(cvec.y, avec.y, bval);
  MultiplyAdd(cvec.z, avec.z, bval);
  MultiplyAdd(cvec.w, avec.w, bval);
#  elif VWM == 8
  MultiplyAdd(cvec.s0, avec.s0, bval);
  MultiplyAdd(cvec.s1, avec.s1, bval);
  MultiplyAdd(cvec.s2, avec.s2, bval);
  MultiplyAdd(cvec.s3, avec.s3, bval);
  MultiplyAdd(cvec.s4, avec.s4, bval);
  MultiplyAdd(cvec.s5, avec.s5, bval);
  MultiplyAdd(cvec.s6, avec.s6, bval);
  MultiplyAdd(cvec.s7, avec.s7, bval);
#  elif VWM == 16
  MultiplyAdd(cvec.s0, avec.s0, bval);
  MultiplyAdd(cvec.s1, avec.s1, bval);
  MultiplyAdd(cvec.s2, avec.s2, bval);
  MultiplyAdd(cvec.s3, avec.s3, bval);
  MultiplyAdd(cvec.s4, avec.s4, bval);
  MultiplyAdd(cvec.s5, avec.s5, bval);
  MultiplyAdd(cvec.s6, avec.s6, bval);
  MultiplyAdd(cvec.s7, avec.s7, bval);
  MultiplyAdd(cvec.s8, avec.s8, bval);
  MultiplyAdd(cvec.s9, avec.s9, bval);
  MultiplyAdd(cvec.sA, avec.sA, bval);
  MultiplyAdd(cvec.sB, avec.sB, bval);
  MultiplyAdd(cvec.sC, avec.sC, bval);
  MultiplyAdd(cvec.sD, avec.sD, bval);
  MultiplyAdd(cvec.sE, avec.sE, bval);
  MultiplyAdd(cvec.sF, avec.sF, bval);
#  endif
#endif
  return cvec;
}

// Performs the actual computation: Cpm += Apm * Bpm
inline void MultiplyAccumulate(
    realM cpm[NWI][MWI / VWM], realM apm[MWI / VWM], realN bpm[NWI / VWN]) {
#pragma unroll
  for (int ni = 0; ni < NWI / VWN; ++ni) {
#pragma unroll
    for (int mi = 0; mi < MWI / VWM; ++mi) {
#if VWN == 1
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni]);
#elif VWN == 2
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].x);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].y);
#elif VWN == 4
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].x);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].y);
      cpm[ni * VWN + 2][mi] = MultiplyAddVector(cpm[ni * VWN + 2][mi], apm[mi], bpm[ni].z);
      cpm[ni * VWN + 3][mi] = MultiplyAddVector(cpm[ni * VWN + 3][mi], apm[mi], bpm[ni].w);
#elif VWN == 8
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].s0);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].s1);
      cpm[ni * VWN + 2][mi] = MultiplyAddVector(cpm[ni * VWN + 2][mi], apm[mi], bpm[ni].s2);
      cpm[ni * VWN + 3][mi] = MultiplyAddVector(cpm[ni * VWN + 3][mi], apm[mi], bpm[ni].s3);
      cpm[ni * VWN + 4][mi] = MultiplyAddVector(cpm[ni * VWN + 4][mi], apm[mi], bpm[ni].s4);
      cpm[ni * VWN + 5][mi] = MultiplyAddVector(cpm[ni * VWN + 5][mi], apm[mi], bpm[ni].s5);
      cpm[ni * VWN + 6][mi] = MultiplyAddVector(cpm[ni * VWN + 6][mi], apm[mi], bpm[ni].s6);
      cpm[ni * VWN + 7][mi] = MultiplyAddVector(cpm[ni * VWN + 7][mi], apm[mi], bpm[ni].s7);
#elif VWN == 16
      cpm[ni * VWN + 0][mi] = MultiplyAddVector(cpm[ni * VWN + 0][mi], apm[mi], bpm[ni].s0);
      cpm[ni * VWN + 1][mi] = MultiplyAddVector(cpm[ni * VWN + 1][mi], apm[mi], bpm[ni].s1);
      cpm[ni * VWN + 2][mi] = MultiplyAddVector(cpm[ni * VWN + 2][mi], apm[mi], bpm[ni].s2);
      cpm[ni * VWN + 3][mi] = MultiplyAddVector(cpm[ni * VWN + 3][mi], apm[mi], bpm[ni].s3);
      cpm[ni * VWN + 4][mi] = MultiplyAddVector(cpm[ni * VWN + 4][mi], apm[mi], bpm[ni].s4);
      cpm[ni * VWN + 5][mi] = MultiplyAddVector(cpm[ni * VWN + 5][mi], apm[mi], bpm[ni].s5);
      cpm[ni * VWN + 6][mi] = MultiplyAddVector(cpm[ni * VWN + 6][mi], apm[mi], bpm[ni].s6);
      cpm[ni * VWN + 7][mi] = MultiplyAddVector(cpm[ni * VWN + 7][mi], apm[mi], bpm[ni].s7);
      cpm[ni * VWN + 8][mi] = MultiplyAddVector(cpm[ni * VWN + 8][mi], apm[mi], bpm[ni].s8);
      cpm[ni * VWN + 9][mi] = MultiplyAddVector(cpm[ni * VWN + 9][mi], apm[mi], bpm[ni].s9);
      cpm[ni * VWN + 10][mi] = MultiplyAddVector(cpm[ni * VWN + 10][mi], apm[mi], bpm[ni].sA);
      cpm[ni * VWN + 11][mi] = MultiplyAddVector(cpm[ni * VWN + 11][mi], apm[mi], bpm[ni].sB);
      cpm[ni * VWN + 12][mi] = MultiplyAddVector(cpm[ni * VWN + 12][mi], apm[mi], bpm[ni].sC);
      cpm[ni * VWN + 13][mi] = MultiplyAddVector(cpm[ni * VWN + 13][mi], apm[mi], bpm[ni].sD);
      cpm[ni * VWN + 14][mi] = MultiplyAddVector(cpm[ni * VWN + 14][mi], apm[mi], bpm[ni].sE);
      cpm[ni * VWN + 15][mi] = MultiplyAddVector(cpm[ni * VWN + 15][mi], apm[mi], bpm[ni].sF);
#endif
    }
  }
}

// =================================================================================================

// Main entry of the kernel. This function contains the basic skeleton, the functionality is
// provided by the inlined functions above.
__attribute__((reqd_work_group_size(MDIMC, NDIMC, 1))) __kernel void gemm_fast(const int kSizeM,
    const int kSizeK, const __global realM* restrict agm, __global realM* cgm) {
#if SYMMETRIC == 1
  const int bx = get_group_id(0);
  const int by = get_group_id(1);
#  if MWG == NWG
  if (by > bx) return;
#  elif MWG > NWG
  if (by > bx * MWG / NWG + MWG / NWG - 1) return;
#  else
  if (by * NWG / MWG > bx) return;
#  endif
#endif  // SYMMETRIC == 1
// Combined thread identifier
#if SA == 1 || SB == 1
  volatile int tid = get_local_id(0) + MDIMC * get_local_id(1);
#endif

// Allocates workgroup-private memory (local memory)
#if SA == 1
  __local realM alm[KWG * MWG / VWM];
#endif
#if SB == 1
  __local realN blm[KWG * NWG / VWN];
#endif

  // Allocates workitem-private memory (registers)
  realM apm[MWI / VWM];
  realN bpm[NWI / VWN];
  realM cpm[NWI][MWI / VWM];

// Initializes the accumulation registers
#pragma unroll
  for (int mi = 0; mi < MWI / VWM; ++mi) {
#pragma unroll
    for (int ni = 0; ni < NWI; ++ni) {
      cpm[ni][mi] = (realM)ZERO;
    }
  }

  // Loops over all workgroup tiles
  for (int kwg = 0; kwg < kSizeK; kwg += KWG) {
// Loads data: off-chip --> local (matrix A)
#if SA == 1
    GlobalToLocalA(agm, alm, kSizeM, tid, kwg);
#endif
// Loads data: off-chip --> local (matrix B)
#if SB == 1
    GlobalToLocalB(agm, blm, kSizeM, tid, kwg);
#endif

// Synchronizes all threads in a workgroup
#if SA == 1 || SB == 1
    barrier(CLK_LOCAL_MEM_FENCE);
#endif

    // Loops over all workitem tiles, unrolled by a factor KWI
    for (int pwi = 0; pwi < KWG; pwi += KWI) {
#pragma unroll
      for (int pit = 0; pit < KWI; ++pit) {
#if SA == 0 || SB == 0
        int idk = kwg + pwi + pit;
#endif
#if SA == 1 || SB == 1
        int kg = pwi + pit;
#endif

// Loads data: local --> private (matrix A)
#if SA == 1
        LocalToPrivateA(alm, apm, kg);
// Loads data: off-chip --> private (matrix A)
#else
        GlobalToPrivateA(agm, apm, kSizeM, idk, kwg);
#endif

// Loads data: local --> private (matrix B)
#if SB == 1
        LocalToPrivateB(blm, bpm, kg);
// Loads data: off-chip --> private (matrix B)
#else
        GlobalToPrivateB(agm, bpm, kSizeM, idk);
#endif

        // Performs the accumulation (Cpm += Apm * Bpm)
        MultiplyAccumulate(cpm, apm, bpm);
      }
    }

// Synchronizes all threads in a workgroup
#if SA == 1 || SB == 1
    barrier(CLK_LOCAL_MEM_FENCE);
#endif
  }

  // Stores an MWG * NWG tile of results
  StoreResults(cgm, cpm, kSizeM);
}
// =================================================================================================


HSAIL kernel lib: move mul_hi() to new macros

/* OpenCL built-in library: mul_hi()

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "hsail_templates.h"

#undef mul_hi
DEFINE_LLVM_INTRIN_SU_INT32_ONLY(_cl_mul_hi, V_VV, hsail.smulhi, hsail.umulhi)

/* OpenCL built-in library: mul_hi()

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "hsail_templates.h"

DEFINE_BUILTIN_V_VV_SU_INT32_ONLY(mul_hi, hsail.smulhi, hsail.umulhi)


Fixed a missing synchronization barrier in the invert kernel; fixes TRSM tests

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains kernels to invert squared diagonal blocks of a matrix. These kernels are based
// on the TRSM implementation in the CUDA version of Magma version 2.2.0 and the poster "Triangular
// Linear System Solver for GPU with CUDA and OpenCL" by Peng Du, Stanimire Tomov, Piotr Luszczek,
// and Jack Dongarra.
//
// =================================================================================================
//
//  Let A be an block_size*block_size lower triangular matrix, and B its inverse.
//  Then the block decomposition
//
//      [ A11   0  ] * [ B11   0  ] = [ I 0 ]
//      [ A21  A22 ]   [ B21  B22 ]   [ 0 I ]
//
//  yields
//
//      A11*B11 = I            ==>  B11 =  A11^{-1},
//      A22*B22 = I            ==>  B22 =  A22^{-1},
//      A21*B11 + A22*B21 = 0  ==>  B21 = -A22^{-1}*A21*B11 = -B22*A21*B11.
//
//  The InvertDiagonalBlock kernel inverts A11 and A22.
//  The TripleMatMul routines multiply:
//  part 1:  B21 =  A21 * B11,
//  part 2:  B21 = -B22 * B21.
//
//  At this level, inner block is current_size=16, with one 4 x 4 work-group per inner block. Each
//  submatrix Aij and Bij is current_size x current_size. The submatrix dimension is multiplied by 2
//  at each level, so the next level is current_size*2 = 32. A 'page' is the next bigger block,
//  here current_size*2=32,
//                 [ B11   0  ]
//  which contains [ B21  B22 ].
//  Outer blocks are block_size x block_size.
//
//  A21 may have < current_size rows, but is guaranteed to have current_size cols since A22 is on
//  the right. This makes a single check easy to do.
//
//  B is stored in workspace that is a full multiple of block_size x block_size; no checks needed.
//
//  We split this into part1 & part2 to synchronize all blocks and make sure
//  that writes to B12 are observed by all blocks.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================
#if defined(ROUTINE_INVERT)

#define LOCALX 17 // 16 + 1 to avoid bank conflicts
#define LOCALY 16

// =================================================================================================

// Inverts a diagonal block of INTERNAL_BLOCK_SIZE by INTERNAL_BLOCK_SIZE elements in a larger matrix
__kernel __attribute__((reqd_work_group_size(INTERNAL_BLOCK_SIZE, 1, 1)))
void InvertDiagonalBlock(int n, __global const real* restrict src, const int src_offset, const int src_ld,
                         __global real* restrict dest, const int outer_block_size,
                         const int unit_diagonal, const int is_upper)
{
  const int thread_index = get_local_id(0);
  const int block_index = get_group_id(0);

  // Sets the offset for this particular block in the source and destination matrices
  const int src_block_offset = block_index * (INTERNAL_BLOCK_SIZE + src_ld * INTERNAL_BLOCK_SIZE) + src_offset;
  const int num_inner_blocks = outer_block_size / INTERNAL_BLOCK_SIZE;
  const int dest_block_offset = (block_index / num_inner_blocks) * outer_block_size * outer_block_size + // go to the (block_index / num_inner_blocks) outer outer_block_size*outer_block_size block,
                                (block_index % num_inner_blocks) * (outer_block_size*INTERNAL_BLOCK_SIZE + INTERNAL_BLOCK_SIZE); // then to the (block_index % num_inner_blocks) inner INTERNAL_BLOCK_SIZE*INTERNAL_BLOCK_SIZE block inside that

  // Local memory to store the inverted block of INTERNAL_BLOCK_SIZE by INTERNAL_BLOCK_SIZE
  __local real lm[INTERNAL_BLOCK_SIZE][INTERNAL_BLOCK_SIZE];

  // Loads the source lower triangle into local memory. Any values in the upper triangle or
  // outside of the matrix are set to zero
  #pragma unroll
  for (int j = 0; j < INTERNAL_BLOCK_SIZE; ++j) {
    const bool condition = (is_upper) ? (thread_index <= j && block_index*INTERNAL_BLOCK_SIZE + j < n) :
                                        (thread_index >= j && block_index*INTERNAL_BLOCK_SIZE + thread_index < n);
    if (condition) {
      lm[thread_index][j] = src[j*src_ld + thread_index + src_block_offset];
    }
    else {
      SetToZero(lm[thread_index][j]);
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);

  // Inverts the diagonal
  real inverted_diagonal;
  SetToOne(inverted_diagonal);
  if (unit_diagonal == 0) {
    const real diagonal_value = lm[thread_index][thread_index];
    if (!IsZero(diagonal_value)) { // Only for non-singular values and values inside the matrix
      real constant_one;
      SetToOne(constant_one);
      DivideFull(inverted_diagonal, constant_one, diagonal_value);
    }
  }
  lm[thread_index][thread_index] = inverted_diagonal;
  barrier(CLK_LOCAL_MEM_FENCE);

  // Upper-triangular
  if (is_upper) {

    // Computes the elements 0:j-1 of the j-th column
    for (int j = 1; j < INTERNAL_BLOCK_SIZE; ++j) {
      real sum;
      if (thread_index < j) {
        SetToZero(sum);
        #pragma unroll
        for (int k = 0; k < j; ++k) {
          MultiplyAdd(sum, lm[thread_index][k], lm[k][j]);
        }
      }
      barrier(CLK_LOCAL_MEM_FENCE);
      if (thread_index < j) {
        real diagonal_value = lm[j][j];
        Negate(diagonal_value);
        Multiply(lm[thread_index][j], diagonal_value, sum);
      }
      barrier(CLK_LOCAL_MEM_FENCE);
    }
  }

  // Lower triangular
  else {

    // Computes the elements j+1:INTERNAL_BLOCK_SIZE-1 of the j-th column
    for (int j = INTERNAL_BLOCK_SIZE - 2; j >= 0; --j) {
      real sum;
      if (thread_index > j) {
        SetToZero(sum);
        #pragma unroll
        for (int k = j + 1; k < INTERNAL_BLOCK_SIZE; ++k) {
          MultiplyAdd(sum, lm[thread_index][k], lm[k][j]);
        }
      }
      barrier(CLK_LOCAL_MEM_FENCE);
      if (thread_index > j) {
        real diagonal_value = lm[j][j];
        Negate(diagonal_value);
        Multiply(lm[thread_index][j], diagonal_value, sum);
      }
      barrier(CLK_LOCAL_MEM_FENCE);
    }
  }

  // Writes the result to global memory
  #pragma unroll
  for (int j = 0; j < INTERNAL_BLOCK_SIZE; ++j) {
    dest[j*outer_block_size + thread_index + dest_block_offset] = lm[thread_index][j];
  }
}

// =================================================================================================

// Triple matrix-multiplication kernel: C = A * B
inline void TripleMatMul(const int size, const bool upper, const int part, __local real* blm, int n,
                         __global const real* agm, __global const real* bgm, __global real* cgm,
                         const int lda, const int ldb, const int ldc,
                         int current_size, int num_pages, const int block_size) {

  // Emulates a 3D grid: NX * (NY * num_pages)
  const int by   = get_group_id(1) / num_pages;
  const int page = get_group_id(1) % num_pages;
  const int lidx = get_local_id(0);
  const int lidy = get_local_id(1);
  const int ibx  = get_group_id(0) * (get_local_size(0)*get_local_size(1));
  const int iby  = by*16;
  const int id   = lidx + lidy*get_local_size(0);
  const int row  = page*current_size*2 + current_size + ibx + id;
  int col        = page*current_size*2 + current_size;

  // Sets the offsets for this specific thread
  agm += ibx + id;
  bgm += lidx + (iby + lidy)*ldb;
  cgm += ibx + id + iby*ldc;

  // Initializes the result registers
  real cpm[16];
  #pragma unroll
  for (int j = 0; j < 16; ++j) {
    SetToZero(cpm[j]);
  }

  // Computes NT x 16 block of C, each thread computes one 1 x 16 row
  for (int k = 0; k < current_size; k += 16) {

    // Loads a 16 x 16 block of B into local memory using NX x 4 threads
    #pragma unroll
    for( int i=0; i < 16; i += (size/4) ) {  // += get_local_size(0)
      #pragma unroll
      for( int j=0; j < 16; j += 4 ) {  // += get_local_size(1)
        blm[(lidx + i) * LOCALX + (lidy + j)] = bgm[k + i + j*ldb];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // Upper triangular
    if (upper) {

      // Performs 16 x 16 multiply-add operations
      #pragma unroll
      for (int i = 0; i < 16; ++i) {
        if (part == 2 || col++ < n) {
          #pragma unroll
          for (int j = 0; j < 16; ++j) {
            MultiplyAdd(cpm[j], agm[(i + k) * lda], blm[i * LOCALX + j]);
          }
        }
      }
    }

    // Lower triangular
    else {
      if (row < n) {

        // Performs 16 x 16 multiply-add operations
        #pragma unroll
        for (int i = 0; i < 16; ++i) {
          #pragma unroll
          for (int j = 0; j < 16; ++j) {
            MultiplyAdd(cpm[j], agm[(i + k) * lda], blm[i * LOCALX + j]);
          }
        }
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
  }

  // Stores NT x 16 results: each thread writes one 16 x 1 row
  #pragma unroll
  for (int i = 0; i < 16; ++i) {
    if (part == 2) { Negate(cpm[i]); }
    cgm[0] = cpm[i];
    cgm += ldc;
  }
}

// =================================================================================================

// Triple matrix-multiplication kernel part 1: B12 = A12 * B22 (upper) or B21 = A21 * B11 (lower)
inline void TripleMatMulPart1(const int size, const bool upper, __local real* blm, int n,
                              __global const real* src, const int a_offset, const int lda,
                              __global real* dest, int current_size, int num_pages, const int block_size) {

  // Emulates a 3D grid: NX * (NY * num_pages)
  const int page = get_group_id(1) % num_pages;

  // Computes the destination block offset:
  // - go to the (page / pages_per_block) outer block_size * block_size block
  // - then the (page % pages_per_block) inner (current_size*2) * (current_size*2) page inside that
  const int pages_per_block = block_size / (current_size*2);
  dest += (page / pages_per_block) * block_size * block_size +
          (page % pages_per_block) * (current_size*2*block_size + current_size*2);

  // Using the GEMM notation: C = A*B
  __global const real* agm;
  __global const real* bgm;
  __global real* cgm;
  if (upper) { // upper triangular: B12 = A12 * B22
    agm = src + a_offset + page*current_size*2*lda + page*current_size*2 + current_size*lda;  // A12
    bgm = dest + current_size*block_size + current_size;                                      // B22
    cgm = dest + current_size*block_size;                                                     // B12
  }
  else { // lower triangular: B21 = A21 * B11
    agm = src + a_offset + page*current_size*2*lda + page*current_size*2 + current_size;  // A21
    bgm = dest;                                                                           // B11
    cgm = dest + current_size;                                                            // B21
  }

  // Runs the generic C = A * B matrix multiplication
  const int ldb = block_size;
  const int ldc = block_size;
  TripleMatMul(size, upper, 1, blm, n, agm, bgm, cgm, lda, ldb, ldc, current_size, num_pages, block_size);
}

// Triple matrix-multiplication kernel part 1: B12 = -B11 * B12 (upper) or B21 = -B22 * B21 (lower)
inline void TripleMatMulPart2(const int size, const bool upper, __local real* blm, const int n,
                              __global real* dest, int current_size, int num_pages, const int block_size) {

  // Emulates a 3D grid: NX * (NY * num_pages)
  const int page = get_group_id(1) % num_pages;

  // Computes the destination block offset:
  // - go to the (page / pages_per_block) outer block_size * block_size block
  // - then the (page % pages_per_block) inner (current_size*2) * (current_size*2) page inside that
  const int pages_per_block = block_size / (current_size*2);
  dest += (page / pages_per_block) * block_size * block_size +
          (page % pages_per_block) * (current_size*2*block_size + current_size*2);

  // Using the GEMM notation: C = A*B
  __global const real* agm;
  __global const real* bgm;
  __global real* cgm;
  if (upper) { // upper triangular: B12 = -B11 * B12
    agm = dest;                            // B11
    cgm = dest + current_size*block_size;  // B12
    bgm = cgm;                             // B12, okay to overwrite
  }

  else { // lower triangular: B21 = -B22 * B21
    agm = dest + current_size*block_size + current_size;  // B22
    cgm = dest + current_size;                            // B21
    bgm = cgm;                                            // B21, okay to overwrite
  }

  // Runs the generic C = A * B matrix multiplication
  const int lda = block_size;
  const int ldb = block_size;
  const int ldc = block_size;
  TripleMatMul(size, upper, 2, blm, n, agm, bgm, cgm, lda, ldb, ldc, current_size, num_pages, block_size);
}

// =================================================================================================

// B21 = A21 * B11
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part1Lower(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(16, false, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B21 = -B22 * B21
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part2Lower(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(16, false, lm, n, dest, current_size, num_pages, block_size);
}

// B21 = A21 * B11
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part1Lower(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(32, false, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B21 = -B22 * B21
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part2Lower(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(32, false, lm, n, dest, current_size, num_pages, block_size);
}

// B21 = A21 * B11
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part1Lower(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(64, false, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B21 = -B22 * B21
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part2Lower(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(64, false, lm, n, dest, current_size, num_pages, block_size);
}

// =================================================================================================

// B12 =  A12 * B22
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part1Upper(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(16, true, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B12 = -B11 * B12
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part2Upper(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(16, true, lm, n, dest, current_size, num_pages, block_size);
}

// B12 =  A12 * B22
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part1Upper(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(32, true, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B12 = -B11 * B12
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part2Upper(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(32, true, lm, n, dest, current_size, num_pages, block_size);
}

// B12 =  A12 * B22
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part1Upper(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(64, true, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B12 = -B11 * B12
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part2Upper(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(64, true, lm, n, dest, current_size, num_pages, block_size);
}

#endif
// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains kernels to invert squared diagonal blocks of a matrix. These kernels are based
// on the TRSM implementation in the CUDA version of Magma version 2.2.0 and the poster "Triangular
// Linear System Solver for GPU with CUDA and OpenCL" by Peng Du, Stanimire Tomov, Piotr Luszczek,
// and Jack Dongarra.
//
// =================================================================================================
//
//  Let A be an block_size*block_size lower triangular matrix, and B its inverse.
//  Then the block decomposition
//
//      [ A11   0  ] * [ B11   0  ] = [ I 0 ]
//      [ A21  A22 ]   [ B21  B22 ]   [ 0 I ]
//
//  yields
//
//      A11*B11 = I            ==>  B11 =  A11^{-1},
//      A22*B22 = I            ==>  B22 =  A22^{-1},
//      A21*B11 + A22*B21 = 0  ==>  B21 = -A22^{-1}*A21*B11 = -B22*A21*B11.
//
//  The InvertDiagonalBlock kernel inverts A11 and A22.
//  The TripleMatMul routines multiply:
//  part 1:  B21 =  A21 * B11,
//  part 2:  B21 = -B22 * B21.
//
//  At this level, inner block is current_size=16, with one 4 x 4 work-group per inner block. Each
//  submatrix Aij and Bij is current_size x current_size. The submatrix dimension is multiplied by 2
//  at each level, so the next level is current_size*2 = 32. A 'page' is the next bigger block,
//  here current_size*2=32,
//                 [ B11   0  ]
//  which contains [ B21  B22 ].
//  Outer blocks are block_size x block_size.
//
//  A21 may have < current_size rows, but is guaranteed to have current_size cols since A22 is on
//  the right. This makes a single check easy to do.
//
//  B is stored in workspace that is a full multiple of block_size x block_size; no checks needed.
//
//  We split this into part1 & part2 to synchronize all blocks and make sure
//  that writes to B12 are observed by all blocks.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================
#if defined(ROUTINE_INVERT)

#define LOCALX 17 // 16 + 1 to avoid bank conflicts
#define LOCALY 16

// =================================================================================================

// Inverts a diagonal block of INTERNAL_BLOCK_SIZE by INTERNAL_BLOCK_SIZE elements in a larger matrix
__kernel __attribute__((reqd_work_group_size(INTERNAL_BLOCK_SIZE, 1, 1)))
void InvertDiagonalBlock(int n, __global const real* restrict src, const int src_offset, const int src_ld,
                         __global real* restrict dest, const int outer_block_size,
                         const int unit_diagonal, const int is_upper)
{
  const int thread_index = get_local_id(0);
  const int block_index = get_group_id(0);

  // Sets the offset for this particular block in the source and destination matrices
  const int src_block_offset = block_index * (INTERNAL_BLOCK_SIZE + src_ld * INTERNAL_BLOCK_SIZE) + src_offset;
  const int num_inner_blocks = outer_block_size / INTERNAL_BLOCK_SIZE;
  const int dest_block_offset = (block_index / num_inner_blocks) * outer_block_size * outer_block_size + // go to the (block_index / num_inner_blocks) outer outer_block_size*outer_block_size block,
                                (block_index % num_inner_blocks) * (outer_block_size*INTERNAL_BLOCK_SIZE + INTERNAL_BLOCK_SIZE); // then to the (block_index % num_inner_blocks) inner INTERNAL_BLOCK_SIZE*INTERNAL_BLOCK_SIZE block inside that

  // Local memory to store the inverted block of INTERNAL_BLOCK_SIZE by INTERNAL_BLOCK_SIZE
  __local real lm[INTERNAL_BLOCK_SIZE][INTERNAL_BLOCK_SIZE];

  // Loads the source lower triangle into local memory. Any values in the upper triangle or
  // outside of the matrix are set to zero
  #pragma unroll
  for (int j = 0; j < INTERNAL_BLOCK_SIZE; ++j) {
    const bool condition = (is_upper) ? (thread_index <= j && block_index*INTERNAL_BLOCK_SIZE + j < n) :
                                        (thread_index >= j && block_index*INTERNAL_BLOCK_SIZE + thread_index < n);
    if (condition) {
      lm[thread_index][j] = src[j*src_ld + thread_index + src_block_offset];
    }
    else {
      SetToZero(lm[thread_index][j]);
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);

  // Inverts the diagonal
  real inverted_diagonal;
  SetToOne(inverted_diagonal);
  if (unit_diagonal == 0) {
    const real diagonal_value = lm[thread_index][thread_index];
    if (!IsZero(diagonal_value)) { // Only for non-singular values and values inside the matrix
      real constant_one;
      SetToOne(constant_one);
      DivideFull(inverted_diagonal, constant_one, diagonal_value);
    }
  }
  lm[thread_index][thread_index] = inverted_diagonal;
  barrier(CLK_LOCAL_MEM_FENCE);

  // Upper-triangular
  if (is_upper) {

    // Computes the elements 0:j-1 of the j-th column
    for (int j = 1; j < INTERNAL_BLOCK_SIZE; ++j) {
      if (thread_index < j) {
        real sum;
        SetToZero(sum);
        #pragma unroll
        for (int k = 0; k < j; ++k) {
          MultiplyAdd(sum, lm[thread_index][k], lm[k][j]);
        }
        real diagonal_value = lm[j][j];
        Negate(diagonal_value);
        Multiply(lm[thread_index][j], diagonal_value, sum);
      }
      barrier(CLK_LOCAL_MEM_FENCE);
    }
  }

  // Lower triangular
  else {

    // Computes the elements j+1:INTERNAL_BLOCK_SIZE-1 of the j-th column
    for (int j = INTERNAL_BLOCK_SIZE - 2; j >= 0; --j) {
      if (thread_index > j) {
        real sum;
        SetToZero(sum);
        #pragma unroll
        for (int k = j + 1; k < INTERNAL_BLOCK_SIZE; ++k) {
          MultiplyAdd(sum, lm[thread_index][k], lm[k][j]);
        }
        real diagonal_value = lm[j][j];
        Negate(diagonal_value);
        Multiply(lm[thread_index][j], diagonal_value, sum);
      }
      barrier(CLK_LOCAL_MEM_FENCE);
    }
  }

  // Writes the result to global memory
  #pragma unroll
  for (int j = 0; j < INTERNAL_BLOCK_SIZE; ++j) {
    dest[j*outer_block_size + thread_index + dest_block_offset] = lm[thread_index][j];
  }
}

// =================================================================================================

// Triple matrix-multiplication kernel: C = A * B
inline void TripleMatMul(const int size, const bool upper, const int part, __local real* blm, int n,
                         __global const real* agm, __global const real* bgm, __global real* cgm,
                         const int lda, const int ldb, const int ldc,
                         int current_size, int num_pages, const int block_size) {

  // Emulates a 3D grid: NX * (NY * num_pages)
  const int by   = get_group_id(1) / num_pages;
  const int page = get_group_id(1) % num_pages;
  const int lidx = get_local_id(0);
  const int lidy = get_local_id(1);
  const int ibx  = get_group_id(0) * (get_local_size(0)*get_local_size(1));
  const int iby  = by*16;
  const int id   = lidx + lidy*get_local_size(0);
  const int row  = page*current_size*2 + current_size + ibx + id;
  int col        = page*current_size*2 + current_size;

  // Sets the offsets for this specific thread
  agm += ibx + id;
  bgm += lidx + (iby + lidy)*ldb;
  cgm += ibx + id + iby*ldc;

  // Initializes the result registers
  real cpm[16];
  #pragma unroll
  for (int j = 0; j < 16; ++j) {
    SetToZero(cpm[j]);
  }

  // Computes NT x 16 block of C, each thread computes one 1 x 16 row
  for (int k = 0; k < current_size; k += 16) {

    // Loads a 16 x 16 block of B into local memory using NX x 4 threads
    #pragma unroll
    for( int i=0; i < 16; i += (size/4) ) {  // += get_local_size(0)
      #pragma unroll
      for( int j=0; j < 16; j += 4 ) {  // += get_local_size(1)
        blm[(lidx + i) * LOCALX + (lidy + j)] = bgm[k + i + j*ldb];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // Upper triangular
    if (upper) {

      // Performs 16 x 16 multiply-add operations
      #pragma unroll
      for (int i = 0; i < 16; ++i) {
        if (part == 2 || col++ < n) {
          #pragma unroll
          for (int j = 0; j < 16; ++j) {
            MultiplyAdd(cpm[j], agm[(i + k) * lda], blm[i * LOCALX + j]);
          }
        }
      }
    }

    // Lower triangular
    else {
      if (row < n) {

        // Performs 16 x 16 multiply-add operations
        #pragma unroll
        for (int i = 0; i < 16; ++i) {
          #pragma unroll
          for (int j = 0; j < 16; ++j) {
            MultiplyAdd(cpm[j], agm[(i + k) * lda], blm[i * LOCALX + j]);
          }
        }
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);
  }

  // Stores NT x 16 results: each thread writes one 16 x 1 row
  #pragma unroll
  for (int i = 0; i < 16; ++i) {
    if (part == 2) { Negate(cpm[i]); }
    cgm[0] = cpm[i];
    cgm += ldc;
  }
}

// =================================================================================================

// Triple matrix-multiplication kernel part 1: B12 = A12 * B22 (upper) or B21 = A21 * B11 (lower)
inline void TripleMatMulPart1(const int size, const bool upper, __local real* blm, int n,
                              __global const real* src, const int a_offset, const int lda,
                              __global real* dest, int current_size, int num_pages, const int block_size) {

  // Emulates a 3D grid: NX * (NY * num_pages)
  const int page = get_group_id(1) % num_pages;

  // Computes the destination block offset:
  // - go to the (page / pages_per_block) outer block_size * block_size block
  // - then the (page % pages_per_block) inner (current_size*2) * (current_size*2) page inside that
  const int pages_per_block = block_size / (current_size*2);
  dest += (page / pages_per_block) * block_size * block_size +
          (page % pages_per_block) * (current_size*2*block_size + current_size*2);

  // Using the GEMM notation: C = A*B
  __global const real* agm;
  __global const real* bgm;
  __global real* cgm;
  if (upper) { // upper triangular: B12 = A12 * B22
    agm = src + a_offset + page*current_size*2*lda + page*current_size*2 + current_size*lda;  // A12
    bgm = dest + current_size*block_size + current_size;                                      // B22
    cgm = dest + current_size*block_size;                                                     // B12
  }
  else { // lower triangular: B21 = A21 * B11
    agm = src + a_offset + page*current_size*2*lda + page*current_size*2 + current_size;  // A21
    bgm = dest;                                                                           // B11
    cgm = dest + current_size;                                                            // B21
  }

  // Runs the generic C = A * B matrix multiplication
  const int ldb = block_size;
  const int ldc = block_size;
  TripleMatMul(size, upper, 1, blm, n, agm, bgm, cgm, lda, ldb, ldc, current_size, num_pages, block_size);
}

// Triple matrix-multiplication kernel part 1: B12 = -B11 * B12 (upper) or B21 = -B22 * B21 (lower)
inline void TripleMatMulPart2(const int size, const bool upper, __local real* blm, const int n,
                              __global real* dest, int current_size, int num_pages, const int block_size) {

  // Emulates a 3D grid: NX * (NY * num_pages)
  const int page = get_group_id(1) % num_pages;

  // Computes the destination block offset:
  // - go to the (page / pages_per_block) outer block_size * block_size block
  // - then the (page % pages_per_block) inner (current_size*2) * (current_size*2) page inside that
  const int pages_per_block = block_size / (current_size*2);
  dest += (page / pages_per_block) * block_size * block_size +
          (page % pages_per_block) * (current_size*2*block_size + current_size*2);

  // Using the GEMM notation: C = A*B
  __global const real* agm;
  __global const real* bgm;
  __global real* cgm;
  if (upper) { // upper triangular: B12 = -B11 * B12
    agm = dest;                            // B11
    cgm = dest + current_size*block_size;  // B12
    bgm = cgm;                             // B12, okay to overwrite
  }

  else { // lower triangular: B21 = -B22 * B21
    agm = dest + current_size*block_size + current_size;  // B22
    cgm = dest + current_size;                            // B21
    bgm = cgm;                                            // B21, okay to overwrite
  }

  // Runs the generic C = A * B matrix multiplication
  const int lda = block_size;
  const int ldb = block_size;
  const int ldc = block_size;
  TripleMatMul(size, upper, 2, blm, n, agm, bgm, cgm, lda, ldb, ldc, current_size, num_pages, block_size);
}

// =================================================================================================

// B21 = A21 * B11
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part1Lower(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(16, false, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B21 = -B22 * B21
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part2Lower(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(16, false, lm, n, dest, current_size, num_pages, block_size);
}

// B21 = A21 * B11
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part1Lower(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(32, false, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B21 = -B22 * B21
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part2Lower(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(32, false, lm, n, dest, current_size, num_pages, block_size);
}

// B21 = A21 * B11
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part1Lower(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(64, false, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B21 = -B22 * B21
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part2Lower(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(64, false, lm, n, dest, current_size, num_pages, block_size);
}

// =================================================================================================

// B12 =  A12 * B22
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part1Upper(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(16, true, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B12 = -B11 * B12
__kernel __attribute__((reqd_work_group_size(4, 4, 1)))
void TripleMatMul16Part2Upper(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(16, true, lm, n, dest, current_size, num_pages, block_size);
}

// B12 =  A12 * B22
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part1Upper(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(32, true, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B12 = -B11 * B12
__kernel __attribute__((reqd_work_group_size(8, 4, 1)))
void TripleMatMul32Part2Upper(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(32, true, lm, n, dest, current_size, num_pages, block_size);
}

// B12 =  A12 * B22
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part1Upper(int n, __global const real* restrict src, const int a_offset, const int lda,
                              __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart1(64, true, lm, n, src, a_offset, lda, dest, current_size, num_pages, block_size);
}

// B12 = -B11 * B12
__kernel __attribute__((reqd_work_group_size(16, 4, 1)))
void TripleMatMul64Part2Upper(int n, __global real* restrict dest, int current_size, int num_pages, const int block_size)
{
  __local real lm[LOCALY * LOCALX];
  TripleMatMulPart2(64, true, lm, n, dest, current_size, num_pages, block_size);
}

#endif
// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================