gamma last and only one ccm

gamma last and only one ccm

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const half black_level = 42.0;

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.25985206, -0.378923, -0.21356857),
  (half3)(-0.11117607, 1.3962182, -0.46342976),
  (half3)(-0.21523926, -0.13449348, 1.47665819),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.01;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 8.6 - 66*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb, int ggain) {
  half3 ret = (0,0,0);
  half cpx = clamp(0.03h, 0.1h, cpxb + cpxk * min(10, ggain));
  ret += (half)rgb.x * color_correction[0];
  ret += (half)rgb.y * color_correction[1];
  ret += (half)rgb.z * color_correction[2];
  ret.x = mf(ret.x, cpx);
  ret.y = mf(ret.y, cpx);
  ret.z = mf(ret.z, cpx);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

half val_from_10(const uchar * source, int gx, int gy) {
  // parse 10bit
  int start = gy * FRAME_STRIDE + (5 * (gx / 4));
  int offset = gx % 4;
  uint major = (uint)source[start + offset] << 2;
  uint minor = (source[start + 4] >> (2 * offset)) & 3;
  half pv = (half)(major + minor);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv *= 0.00101833h; // /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        uint ggain
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global);
  cached[localOffset] = pv;

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global);
  } else if (x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global);
  }

  if (y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1);
    }
  } else if (y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb, (int)ggain);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);

}

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

const half black_level = 42.0;

const __constant half3 color_correction[3] = {
  // post wb CCM
  (half3)(1.25985206, -0.378923, -0.21356857),
  (half3)(-0.11117607, 1.3962182, -0.46342976),
  (half3)(-0.21523926, -0.13449348, 1.47665819),
};

const __constant half3 alt_ccm[3] = {
  (half3)(0.84166382, -0.25025687, 0.1593848),
  (half3)(0.25124881, 1.13196952, 0.16597516),
  (half3)(-0.09291263, 0.11828735, 0.67464003),
};

// tone mapping params
const half cpk = 0.75;
const half cpb = 0.125;
const half cpxk = 0.01;
const half cpxb = 0.01;

half mf(half x, half cp) {
  half rk = 8.6 - 66*cp;
  if (x > cp) {
    return (rk * (x-cp) * (1-(cpk*cp+cpb)) * (1+1/(rk*(1-cp))) / (1+rk*(x-cp))) + cpk*cp + cpb;
  } else if (x < cp) {
    return (rk * (x-cp) * (cpk*cp+cpb) * (1+1/(rk*cp)) / (1-rk*(x-cp))) + cpk*cp + cpb;
  } else {
    return x;
  }
}

half3 color_correct(half3 rgb, int ggain) {
  half3 ret = (0,0,0);
  half cpx = clamp(0.03h, 0.1h, cpxb + cpxk * min(10, ggain));
  rgb.x = mf(rgb.x, cpx);
  rgb.y = mf(rgb.y, cpx);
  rgb.z = mf(rgb.z, cpx);
  half alp = 1.0 - 0.07*ggain;
  ret += (half)rgb.x * (alp*color_correction[0]+(1-alp)*alt_ccm[0]);
  ret += (half)rgb.y * (alp*color_correction[1]+(1-alp)*alt_ccm[1]);
  ret += (half)rgb.z * (alp*color_correction[2]+(1-alp)*alt_ccm[2]);
  ret = clamp(0.0h, 255.0h, ret*255.0h);
  return ret;
}

half val_from_10(const uchar * source, int gx, int gy) {
  // parse 10bit
  int start = gy * FRAME_STRIDE + (5 * (gx / 4));
  int offset = gx % 4;
  uint major = (uint)source[start + offset] << 2;
  uint minor = (source[start + 4] >> (2 * offset)) & 3;
  half pv = (half)(major + minor);

  // normalize
  pv = max(0.0h, pv - black_level);
  pv *= 0.00101833h; // /= (1024.0f - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    float r = gx*gx + gy*gy;
    half s;
    if (r < 62500) {
      s = (half)(1.0f + 0.0000008f*r);
    } else if (r < 490000) {
      s = (half)(0.9625f + 0.0000014f*r);
    } else if (r < 1102500) {
      s = (half)(1.26434f + 0.0000000000016f*r*r);
    } else {
      s = (half)(0.53503625f + 0.0000000000022f*r*r);
    }
    pv = s * pv;
  }

  pv = clamp(0.0h, 1.0h, pv);
  return pv;
}

half fabs_diff(half x, half y) {
  return fabs(x-y);
}

half phi(half x) {
  // detection funtion
  return 2 - x;
  // if (x > 1) {
  //   return 1 / x;
  // } else {
  //   return 2 - x;
  // }
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        uint ggain
                       )
{
  const int x_global = get_global_id(0);
  const int y_global = get_global_id(1);

  const int localRowLen = 2 + get_local_size(0); // 2 padding
  const int x_local = get_local_id(0); // 0-15
  const int y_local = get_local_id(1); // 0-15
  const int localOffset = (y_local + 1) * localRowLen + x_local + 1; // max 18x18-1

  int out_idx = 3 * x_global + 3 * y_global * RGB_WIDTH;

  half pv = val_from_10(in, x_global, y_global);
  cached[localOffset] = pv;

  // don't care
  if (x_global < 1 || x_global >= RGB_WIDTH - 1 || y_global < 1 || y_global >= RGB_HEIGHT - 1) {
    return;
  }

  // cache padding
  int localColOffset = -1;
  int globalColOffset = -1;

  // cache padding
  if (x_local < 1) {
    localColOffset = x_local;
    globalColOffset = -1;
    cached[(y_local + 1) * localRowLen + x_local] = val_from_10(in, x_global-1, y_global);
  } else if (x_local >= get_local_size(0) - 1) {
    localColOffset = x_local + 2;
    globalColOffset = 1;
    cached[localOffset + 1] = val_from_10(in, x_global+1, y_global);
  }

  if (y_local < 1) {
    cached[y_local * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global-1);
    if (localColOffset != -1) {
      cached[y_local * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global-1);
    }
  } else if (y_local >= get_local_size(1) - 1) {
    cached[(y_local + 2) * localRowLen + x_local + 1] = val_from_10(in, x_global, y_global+1);
    if (localColOffset != -1) {
      cached[(y_local + 2) * localRowLen + localColOffset] = val_from_10(in, x_global+globalColOffset, y_global+1);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half d1 = cached[localOffset - localRowLen - 1];
  half d2 = cached[localOffset - localRowLen + 1];
  half d3 = cached[localOffset + localRowLen - 1];
  half d4 = cached[localOffset + localRowLen + 1];
  half n1 = cached[localOffset - localRowLen];
  half n2 = cached[localOffset + 1];
  half n3 = cached[localOffset + localRowLen];
  half n4 = cached[localOffset - 1];

  half3 rgb;

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  if (x_global % 2 == 0) {
    if (y_global % 2 == 0) {
      rgb.y = pv; // G1(R)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G1
      rgb.x = (k2*n2+k4*n4)/(k2+k4);
      // B_G1
      rgb.z = (k1*n1+k3*n3)/(k1+k3);
    } else {
      rgb.z = pv; // B
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_B
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // R_B
      rgb.x = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    }
  } else {
    if (y_global % 2 == 0) {
      rgb.x = pv; // R
      half k1 = phi(fabs_diff(d1, d3) + fabs_diff(d2, d4));
      half k2 = phi(fabs_diff(n1, n4) + fabs_diff(n2, n3));
      half k3 = phi(fabs_diff(d1, d2) + fabs_diff(d3, d4));
      half k4 = phi(fabs_diff(n1, n2) + fabs_diff(n3, n4));
      // G_R
      rgb.y = (k1*(n1+n3)*0.5+k3*(n2+n4)*0.5)/(k1+k3);
      // B_R
      rgb.z = (k2*(d2+d3)*0.5+k4*(d1+d4)*0.5)/(k2+k4);
    } else {
      rgb.y = pv; // G2(B)
      half k1 = phi(fabs_diff(d1, pv) + fabs_diff(d2, pv));
      half k2 = phi(fabs_diff(d2, pv) + fabs_diff(d4, pv));
      half k3 = phi(fabs_diff(d3, pv) + fabs_diff(d4, pv));
      half k4 = phi(fabs_diff(d1, pv) + fabs_diff(d3, pv));
      // R_G2
      rgb.x = (k1*n1+k3*n3)/(k1+k3);
      // B_G2
      rgb.z = (k2*n2+k4*n4)/(k2+k4);
    }
  }

  rgb = clamp(0.0h, 1.0h, rgb);
  rgb = color_correct(rgb, (int)ggain);

  out[out_idx + 0] = (uchar)(rgb.z);
  out[out_idx + 1] = (uchar)(rgb.y);
  out[out_idx + 2] = (uchar)(rgb.x);

}


[OpenCL] In test/Driver/opencl.cl, don't require name of Clang binary to contain "clang"

The test currently fails if the name of the Clang binary doesn't contain "clang".

This patch removes that requirement, as some environments may choose to run the test with a differently named binary. This shouldn't make the test any less strict -- the only place where the flags we're searching for can really occur is the Clang command line.

Patch by Martin Böhme!

Differential Revision: https://reviews.llvm.org/D22359

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@275428 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang -S -### -cl-std=CL %s 2>&1 | FileCheck --check-prefix=CHECK-CL %s
// RUN: %clang -S -### -cl-std=CL1.1 %s 2>&1 | FileCheck --check-prefix=CHECK-CL11 %s
// RUN: %clang -S -### -cl-std=CL1.2 %s 2>&1 | FileCheck --check-prefix=CHECK-CL12 %s
// RUN: %clang -S -### -cl-std=CL2.0 %s 2>&1 | FileCheck --check-prefix=CHECK-CL20 %s
// RUN: %clang -S -### -cl-opt-disable %s 2>&1 | FileCheck --check-prefix=CHECK-OPT-DISABLE %s
// RUN: %clang -S -### -cl-strict-aliasing %s 2>&1 | FileCheck --check-prefix=CHECK-STRICT-ALIASING %s
// RUN: %clang -S -### -cl-single-precision-constant %s 2>&1 | FileCheck --check-prefix=CHECK-SINGLE-PRECISION-CONST %s
// RUN: %clang -S -### -cl-finite-math-only %s 2>&1 | FileCheck --check-prefix=CHECK-FINITE-MATH-ONLY %s
// RUN: %clang -S -### -cl-kernel-arg-info %s 2>&1 | FileCheck --check-prefix=CHECK-KERNEL-ARG-INFO %s
// RUN: %clang -S -### -cl-unsafe-math-optimizations %s 2>&1 | FileCheck --check-prefix=CHECK-UNSAFE-MATH-OPT %s
// RUN: %clang -S -### -cl-fast-relaxed-math %s 2>&1 | FileCheck --check-prefix=CHECK-FAST-RELAXED-MATH %s
// RUN: %clang -S -### -cl-mad-enable %s 2>&1 | FileCheck --check-prefix=CHECK-MAD-ENABLE %s
// RUN: %clang -S -### -cl-no-signed-zeros %s 2>&1 | FileCheck --check-prefix=CHECK-NO-SIGNED-ZEROS %s
// RUN: %clang -S -### -cl-denorms-are-zero %s 2>&1 | FileCheck --check-prefix=CHECK-DENORMS-ARE-ZERO %s
// RUN: not %clang -cl-std=c99 -DOPENCL %s 2>&1 | FileCheck --check-prefix=CHECK-C99 %s
// RUN: not %clang -cl-std=invalid -DOPENCL %s 2>&1 | FileCheck --check-prefix=CHECK-INVALID %s

// CHECK-CL: "-cc1" {{.*}} "-cl-std=CL"
// CHECK-CL11: "-cc1" {{.*}} "-cl-std=CL1.1"
// CHECK-CL12: "-cc1" {{.*}} "-cl-std=CL1.2"
// CHECK-CL20: "-cc1" {{.*}} "-cl-std=CL2.0"
// CHECK-OPT-DISABLE: "-cc1" {{.*}} "-cl-opt-disable"
// CHECK-STRICT-ALIASING: "-cc1" {{.*}} "-cl-strict-aliasing"
// CHECK-SINGLE-PRECISION-CONST: "-cc1" {{.*}} "-cl-single-precision-constant"
// CHECK-FINITE-MATH-ONLY: "-cc1" {{.*}} "-cl-finite-math-only"
// CHECK-KERNEL-ARG-INFO: "-cc1" {{.*}} "-cl-kernel-arg-info"
// CHECK-UNSAFE-MATH-OPT: "-cc1" {{.*}} "-cl-unsafe-math-optimizations"
// CHECK-FAST-RELAXED-MATH: "-cc1" {{.*}} "-cl-fast-relaxed-math"
// CHECK-MAD-ENABLE: "-cc1" {{.*}} "-cl-mad-enable"
// CHECK-NO-SIGNED-ZEROS: "-cc1" {{.*}} "-cl-no-signed-zeros"
// CHECK-DENORMS-ARE-ZERO: "-cc1" {{.*}} "-cl-denorms-are-zero"
// CHECK-C99: error: invalid value 'c99' in '-cl-std=c99'
// CHECK-INVALID: error: invalid value 'invalid' in '-cl-std=invalid'

kernel void func(void);

// RUN: %clang -S -### -cl-std=CL %s 2>&1 | FileCheck --check-prefix=CHECK-CL %s
// RUN: %clang -S -### -cl-std=CL1.1 %s 2>&1 | FileCheck --check-prefix=CHECK-CL11 %s
// RUN: %clang -S -### -cl-std=CL1.2 %s 2>&1 | FileCheck --check-prefix=CHECK-CL12 %s
// RUN: %clang -S -### -cl-std=CL2.0 %s 2>&1 | FileCheck --check-prefix=CHECK-CL20 %s
// RUN: %clang -S -### -cl-opt-disable %s 2>&1 | FileCheck --check-prefix=CHECK-OPT-DISABLE %s
// RUN: %clang -S -### -cl-strict-aliasing %s 2>&1 | FileCheck --check-prefix=CHECK-STRICT-ALIASING %s
// RUN: %clang -S -### -cl-single-precision-constant %s 2>&1 | FileCheck --check-prefix=CHECK-SINGLE-PRECISION-CONST %s
// RUN: %clang -S -### -cl-finite-math-only %s 2>&1 | FileCheck --check-prefix=CHECK-FINITE-MATH-ONLY %s
// RUN: %clang -S -### -cl-kernel-arg-info %s 2>&1 | FileCheck --check-prefix=CHECK-KERNEL-ARG-INFO %s
// RUN: %clang -S -### -cl-unsafe-math-optimizations %s 2>&1 | FileCheck --check-prefix=CHECK-UNSAFE-MATH-OPT %s
// RUN: %clang -S -### -cl-fast-relaxed-math %s 2>&1 | FileCheck --check-prefix=CHECK-FAST-RELAXED-MATH %s
// RUN: %clang -S -### -cl-mad-enable %s 2>&1 | FileCheck --check-prefix=CHECK-MAD-ENABLE %s
// RUN: %clang -S -### -cl-no-signed-zeros %s 2>&1 | FileCheck --check-prefix=CHECK-NO-SIGNED-ZEROS %s
// RUN: %clang -S -### -cl-denorms-are-zero %s 2>&1 | FileCheck --check-prefix=CHECK-DENORMS-ARE-ZERO %s
// RUN: not %clang -cl-std=c99 -DOPENCL %s 2>&1 | FileCheck --check-prefix=CHECK-C99 %s
// RUN: not %clang -cl-std=invalid -DOPENCL %s 2>&1 | FileCheck --check-prefix=CHECK-INVALID %s

// CHECK-CL: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-std=CL"
// CHECK-CL11: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-std=CL1.1"
// CHECK-CL12: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-std=CL1.2"
// CHECK-CL20: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-std=CL2.0"
// CHECK-OPT-DISABLE: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-opt-disable"
// CHECK-STRICT-ALIASING: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-strict-aliasing"
// CHECK-SINGLE-PRECISION-CONST: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-single-precision-constant"
// CHECK-FINITE-MATH-ONLY: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-finite-math-only"
// CHECK-KERNEL-ARG-INFO: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-kernel-arg-info"
// CHECK-UNSAFE-MATH-OPT: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-unsafe-math-optimizations"
// CHECK-FAST-RELAXED-MATH: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-fast-relaxed-math"
// CHECK-MAD-ENABLE: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-mad-enable"
// CHECK-NO-SIGNED-ZEROS: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-no-signed-zeros"
// CHECK-DENORMS-ARE-ZERO: {{.*}}clang{{.*}} "-cc1" {{.*}} "-cl-denorms-are-zero"
// CHECK-C99: error: invalid value 'c99' in '-cl-std=c99'
// CHECK-INVALID: error: invalid value 'invalid' in '-cl-std=invalid'

kernel void func(void);


updated 3x3

updated 3x3

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)

#define MLO_IN_VERT_READS (MLO_IN_HEIGHT)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 1
#if MLO_HW_WAVE_ID_SETTING //&& MLO_COMPILER_AMD_OPENCL_HSAIL==1

// FIXME Conduct enabling from the host code.
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
#endif
static inline int getWaveId()
{
	int wave_id = 0;

#if MLO_HW_WAVE_ID_SETTING //&& MLO_COMPILER_AMD_OPENCL_HSAIL==1

	wave_id = __hsail_get_dynwave_id();
	wave_id &= MLO_N_WAVES_MASK;
#if 0
#elif MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_LC==1 && MLO_GRP_SZ1==1 && MLO_GRP_SZ2==1 && (MLO_GRP_SZ % (1 << MLO_LG2_WAVE_SZ))==0
	// (local_id/wavesize) has the same value in all workitems.
	// Make it scalar to enable scalarization optimizations.
	wave_id = __llvm_amdgcn_readfirstlane((uint)(get_local_id(0) >> MLO_LG2_WAVE_SZ));
	// Alternate implementation:
	//__asm__ ("v_readfirstlane_b32 %0, %1" : "=s" (wave_id) : "v" ((int)(get_local_id(0) >> MLO_LG2_WAVE_SZ)) );
#endif
#else
	wave_id = (get_local_id(0) >> MLO_LG2_WAVE_SZ);
#endif
	return(wave_id);
}

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_WAVE_SZ) << MLO_LG2_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*********************************************************************************************************
// wrw algorithm for 7x7 and smallee
// idea:
// layout
// each wave keeps WAVE_SZ output maps
// 1 input map
// alg
// for  all batches

// read output maps in VGPRS 1 map per wk_item
// for number of waves
// read full input map in SGPRs
// convolve internal pixels
// convolve border pixels
// exchange input maps
// rof
// rof

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW_7x7(
	const __global _FLOAT * top_df,
	__constant _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(0); // input map index base

	int o_idx_base = get_group_id(1); // output map index base

	int ib_base = get_group_id(2); //batch index

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_WAVES* MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * MLO_N_WAVES * MLO_HW_WAVE_SZ* MLO_N_LCL_OUT_MAPS; // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

// out address based on wave and batch
	int out_map = lcl_wv_id;

	gbl_out_off +=  (wave_id * MLO_HW_WAVE_SZ  + out_map) * MLO_IN_CHANNEL_STRIDE;

#define MLO_DAT_SZ (MLO_IN_HEIGHT * MLO_IN_WIDTH)

	__private _FLOAT top_dat[MLO_DAT_SZ];

/*
	for (int i = 0; i < MLO_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

*/
	__private _FLOAT bot_dat[MLO_DAT_SZ];

/*
	for (int i = 0; i < MLO_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}
*/

#define MLO_ACCUM_SZ (MLO_N_WAVES * MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		// read output map

		for (int j = 0; j < MLO_DAT_SZ; ++j)
		{
			top_dat[j] = top_df[gbl_out_off + j];
		}
		for (int w = 0; w < MLO_N_WAVES; ++w)
		{
			// in address based on wave_id

			int wave_id_r = w; //((wave_id + w) < MLO_N_WAVES) ?  (wave_id + w) : (wave_id + w) - MLO_N_WAVES;
			int gbl_in_off_r = gbl_in_off + wave_id_r * MLO_IN_CHANNEL_STRIDE;

			// read input map presumably to SGPRs

			for (int j = 0; j < MLO_DAT_SZ; ++j)
			{
				bot_dat[j] = bot[gbl_in_off_r + j];
			}

// convolve
// internals
			for (int m = 0; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
				{
					for (int j = MLO_FILTER_PAD1; j < MLO_IN_HEIGHT - MLO_FILTER_PAD1; ++j)
					{
						for (int i = MLO_FILTER_PAD0; i < MLO_IN_WIDTH - MLO_FILTER_PAD0; ++i)
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1) *MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
 				} // l

			} // m

#if 1

// top
			for (int m = MLO_FILTER_PAD1; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = 0;
					{
						for (int i = MLO_FILTER_PAD0; i < MLO_IN_WIDTH - MLO_FILTER_PAD0; ++i)
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// bot
			for (int m = 0; m < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = MLO_IN_HEIGHT - 1;
					{
						for (int i = MLO_FILTER_PAD0; i < MLO_IN_WIDTH - MLO_FILTER_PAD0; ++i)
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// left
			for (int m = 0; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = MLO_FILTER_PAD0; l < MLO_FILTER_SIZE0; ++l)
				{
					for (int j = MLO_FILTER_PAD1; j < MLO_IN_HEIGHT - MLO_FILTER_PAD1; ++j)
					{
						int i = 0;
						{
							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1) *MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// right
			for (int m = 0; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0 - MLO_FILTER_PAD0; ++l)
				{
					for (int j = MLO_FILTER_PAD1; j < MLO_IN_HEIGHT - MLO_FILTER_PAD1; ++j)
					{
						int i = MLO_IN_WIDTH -1;
						{
							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1) *MLO_IN_WIDTH + i + l];
						}  // i
					} // j
				} // l

			} // m

// top - left
			for (int m = MLO_FILTER_PAD1; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = MLO_FILTER_PAD0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = 0;
					{
						int i = 0;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// top - right
			for (int m = MLO_FILTER_PAD1; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0 - MLO_FILTER_PAD0; ++l)
				{
					int j = 0;
					{
						int i = MLO_IN_WIDTH - 1;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1)*MLO_IN_WIDTH + i + l];
						}  // i
					} // j
				} // l

			} // m

// bot - left
			for (int m = 0; m < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1; ++m)
			{
				for (int l = MLO_FILTER_PAD0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = MLO_IN_WIDTH - 1;
					{
						int i = 0;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// bot - right
			for (int m = 0; m < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0 - MLO_FILTER_PAD0; ++l)
				{
					int j = MLO_IN_HEIGHT - 1;
					{
						int i = MLO_IN_WIDTH - 1;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								+= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m)* MLO_IN_WIDTH + i + l];
						}  // i
					} // j
				} // l

			} // m

#endif

		} // w

	} // 	for (int b = 0;

// output
// inputs are outputs
// TODO : for more than 1 input
	int c = 0;

	int wei_df_off = (wave_id * MLO_HW_WAVE_SZ + out_map) * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	for (int w = 0; w < MLO_N_WAVES && (wave_id * MLO_HW_WAVE_SZ + out_map) < MLO_N_OUTPUTS && (c_idx + w) < MLO_N_INPUTS; ++w)
	{
		for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
		{
			weights_df[wei_df_off + w * MLO_WEI_CHANNEL_STRIDE + i] = pvt_accum[w*(MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0) + i];
		}

	}

}

#if 0
// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

#endif

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)

#define MLO_IN_VERT_READS (MLO_IN_HEIGHT)
// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

#define MLO_HW_WAVE_ID_SETTING 0

static inline int getWaveId()
{
	int wave_id = 0;

#if MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_HSAIL==1
	wave_id = __hsail_get_dynwave_id();
	wave_id &= MLO_N_WAVES_MASK;
#elif MLO_HW_WAVE_ID_SETTING && MLO_COMPILER_AMD_OPENCL_LC==1 && MLO_GRP_SZ1==1 && MLO_GRP_SZ2==1 && (MLO_GRP_SZ % (1 << MLO_LG2_WAVE_SZ))==0
	// (local_id/wavesize) has the same value in all workitems.
	// Make it scalar to enable scalarization optimizations.
	wave_id = __llvm_amdgcn_readfirstlane((uint)(get_local_id(0) >> MLO_LG2_WAVE_SZ));
	// Alternate implementation:
	//__asm__ ("v_readfirstlane_b32 %0, %1" : "=s" (wave_id) : "v" ((int)(get_local_id(0) >> MLO_LG2_WAVE_SZ)) );
#else
	wave_id = (get_local_id(0) >> MLO_LG2_WAVE_SZ);
#endif
	return(wave_id);
}

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_WAVE_SZ) << MLO_LG2_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
static inline void readInput(int lcl_id, int gbl_in_scan_off, const __global _FLOAT * bot, __local _FLOAT *lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];
// TODO : more than 1 input
		int c = 0;
		int c_scan = iDiv(p4, (MLO_N_IN_HORIZ_READS));

		int c_pix4 = iMod(p4, c_scan, (MLO_N_IN_HORIZ_READS));

//		if (c < MLO_N_INPUTS)

		{

#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT + i];
				}

				for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{
				*(MLO_READ_TYPE*)in_rd_data = *(__global MLO_READ_TYPE*)&bot[gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT];
			}

// stack of inputs, each has 1 line
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan* MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}

/*
	core processing loop
	bot - input, from local (1 span)
	top - output diff, from global (array of spans, filters vertical size)

	loop over filter vertical size

*/
static inline void Processing(int sc, int sc_lcl_off, int top_lim, int bot_lim, __private _FLOAT * pvt_accum, __local _FLOAT * lcl_bot, __private _FLOAT * top_dat)
{
	for (int l = top_lim; l >= bot_lim; --l)
	{
		for (int m = 0; m < MLO_IN_TILE0; ++m)
		{
			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				_FLOAT bot_val = lcl_bot[sc_lcl_off + n + m];
				_FLOAT top_val = top_dat[(top_lim - l) * MLO_IN_TILE0 + m];
				pvt_accum[l*MLO_FILTER_SIZE0 + n]
					// each wk-item process an input
					+= bot_val * top_val;
#if 0
				if (/*bot_val * top_val != 0 && */get_global_id(1) == 0 && get_global_id(2) == 0 /*&& get_local_id(0) == 0*/ && l == 2 && n == 1)
				{
					printf("G: %d %d %d  %f %f %f %f\n",
						MLO_OUT_N_PIXS_OFF,
						sc,
						sc_lcl_off,
						pvt_accum[l*MLO_FILTER_SIZE0 + n],
						bot_val * top_val,
						bot_val,
						top_val
					);
				}
#endif

			}

		}

	}

}
/*********************************************************************************************************
// wrw algorithm for 7x7 and smallee
// idea:
// layout
// each wave keeps WAVE_SZ output maps
// 1 input map
// alg
// for  all batches

// read output maps in VGPRS 1 map per wk_item
// for number of waves
// read full input map in SGPRs
// convolve internal pixels
// convolve border pixels
// exchange input maps
// rof
// rof

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW_7x7(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int c_idx_base = get_group_id(0); // input map index base

	int o_idx_base = get_group_id(1); // output map index base

	int ib_base = get_group_id(2); //batch index

	int ib = ib_base*MLO_N_LCL_BATCHS;

	int c_idx = c_idx_base * MLO_N_WAVES* MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * MLO_N_WAVES * MLO_HW_WAVE_SZ* MLO_N_LCL_OUT_MAPS; // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

// out address based on wave and batch
	int out_map = lcl_wv_id;

	gbl_out_off +=  (wave_id * MLO_HW_WAVE_SZ  + out_map) * MLO_IN_CHANNEL_STRIDE;

#define MLO_DAT_SZ (MLO_IN_HEIGHT * MLO_IN_WIDTH)

	__private _FLOAT top_dat[MLO_DAT_SZ];

	for (int i = 0; i < MLO_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

	__private _FLOAT bot_dat[MLO_DAT_SZ];

	for (int i = 0; i < MLO_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_WAVES * MLO_N_LCL_OUT_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		// read output map

		for (int j = 0; j < MLO_DAT_SZ; ++j)
		{
			top_dat[j] = top_df[gbl_out_off + j];
		}

		for (int w = 0; w < MLO_N_WAVES; ++w)
		{
			// in address based on wave_id

			int wave_id_r = ((wave_id + w) & MLO_N_WAVES_MASK);
			int gbl_in_off_r = gbl_in_off + wave_id_r * MLO_IN_CHANNEL_STRIDE;

			// read input map presumably to SGPRs

			for (int j = 0; j < MLO_DAT_SZ; ++j)
			{
				top_dat[j] = top_df[gbl_in_off_r + j];
			}

// convolve
// internals
			for (int m = 0; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
				{
					for (int j = MLO_FILTER_PAD1; j < MLO_IN_HEIGHT - MLO_FILTER_PAD1; ++j)
					{
						for (int i = MLO_FILTER_PAD0; i < MLO_IN_WIDTH - MLO_FILTER_PAD0; ++i)
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1) *MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
 				} // l

			} // m

// top
			for (int m = MLO_FILTER_PAD1; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = 0;
					{
						for (int i = MLO_FILTER_PAD0; i < MLO_IN_WIDTH - MLO_FILTER_PAD0; ++i)
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// bot
			for (int m = 0; m < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = MLO_IN_HEIGHT - 1;
					{
						for (int i = MLO_FILTER_PAD0; i < MLO_IN_WIDTH - MLO_FILTER_PAD0; ++i)
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// left
			for (int m = 0; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = MLO_FILTER_PAD0; l < MLO_FILTER_SIZE0; ++l)
				{
					for (int j = MLO_FILTER_PAD1; j < MLO_IN_HEIGHT - MLO_FILTER_PAD1; ++j)
					{
						int i = 0;
						{
							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1) *MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// right
			for (int m = 0; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0 - MLO_FILTER_PAD0; ++l)
				{
					for (int j = MLO_FILTER_PAD1; j < MLO_IN_HEIGHT - MLO_FILTER_PAD1; ++j)
					{
						int i = MLO_IN_WIDTH -1;
						{
							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1) *MLO_IN_WIDTH + i + l];
						}  // i
					} // j
				} // l

			} // m

// top - left
			for (int m = MLO_FILTER_PAD1; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = MLO_FILTER_PAD0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = 0;
					{
						int i = 0;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// top - right
			for (int m = MLO_FILTER_PAD1; m < MLO_FILTER_SIZE1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0 - MLO_FILTER_PAD0; ++l)
				{
					int j = 0;
					{
						int i = MLO_IN_WIDTH - 1;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m - MLO_FILTER_PAD1)*MLO_IN_WIDTH + i + l];
						}  // i
					} // j
				} // l

			} // m

// bot - left
			for (int m = 0; m < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1; ++m)
			{
				for (int l = MLO_FILTER_PAD0; l < MLO_FILTER_SIZE0; ++l)
				{
					int j = MLO_IN_WIDTH - 1;
					{
						int i = 0;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m)*MLO_IN_WIDTH + i + l - MLO_FILTER_PAD0];
						}  // i
					} // j
				} // l

			} // m

// bot - right
			for (int m = 0; m < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1; ++m)
			{
				for (int l = 0; l < MLO_FILTER_SIZE0 - MLO_FILTER_PAD0; ++l)
				{
					int j = MLO_IN_HEIGHT - 1;
					{
						int i = MLO_IN_WIDTH - 1;
						{

							pvt_accum[(wave_id_r * MLO_FILTER_SIZE1 + m) * MLO_FILTER_SIZE0 + l]
								= top_dat[j*MLO_IN_WIDTH + i] * bot_dat[(j + m)* MLO_IN_WIDTH + i + l];
						}  // i
					} // j
				} // l

			} // m

		} // w

	} // 	for (int b = 0;

// output
// inputs are outputs
// TODO : for more than 1 input
	int c = 0;

	int wei_df_off = (wave_id * MLO_HW_WAVE_SZ + out_map) * MLO_WEI_BATCH_STRIDE + c_idx * MLO_WEI_CHANNEL_STRIDE;

	for (int w = 0; w < MLO_N_WAVES && (wave_id * MLO_HW_WAVE_SZ + out_map) < MLO_N_OUTPUTS && (c_idx + w) < MLO_N_INPUTS; ++w)
	{
		for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
		{
			weights_df[wei_df_off + w * MLO_WEI_CHANNEL_STRIDE + i] = pvt_accum[i];
		}

	}

}

#if 0
// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

#endif


Correct sub_sat() implementation

Correct sub_sat() implementation

/* OpenCL built-in library: sub_sat()

   Copyright (c) 2011 Universidad Rey Juan Carlos

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "templates.h"

// Available SSE2 builtins:
//    char     __builtin_ia32_psubsb128
//    short    __builtin_ia32_psubsw128
//    uchar    __builtin_ia32_psubusb128
//    ushort   __builtin_ia32_psubusw128
// Other types don't seem to be supported.

DEFINE_EXPR_G_GG(sub_sat,
                 (sgtype)-1 < (sgtype)0 ?
                 /* signed */
                 ({
                   int bits = CHAR_BIT * sizeof(sgtype);
                   gtype min = (sgtype)1 << (sgtype)(bits-1);
                   gtype max = min - (sgtype)1;
                   (a^b) >= (gtype)0 ?
                     /* same sign: no overflow/underflow */
                     a-b :
                     a >= (gtype)0 ?
                     /* a positive, b negative: can overflow */
                     (a > max+b ? max : a-b) :
                     /* a negative, b positive: can underflow */
                     (a < min+b ? min : a-b);
                 }) :
                 /* unsigned */
                 ({
                   gtype min = (sgtype)0;
                   a < min+b ? min : a-b;
                 }))

/* OpenCL built-in library: sub_sat()

   Copyright (c) 2011 Universidad Rey Juan Carlos

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

#include "templates.h"

// Available SSE2 builtins:
//    char     __builtin_ia32_psubsb128
//    short    __builtin_ia32_psubsw128
//    uchar    __builtin_ia32_psubusb128
//    ushort   __builtin_ia32_psubusw128
// Other types don't seem to be supported.

DEFINE_EXPR_G_GG(sub_sat,
                 (sgtype)-1 < (sgtype)0 ?
                 /* signed */
                 ({
                   int bits = CHAR_BIT * sizeof(sgtype);
                   gtype min = (sgtype)1 << (sgtype)(bits-1);
                   gtype max = min + (sgtype)1;
                   (a^b) >= (gtype)0 ?
                     /* same sign: no overflow/underflow */
                     a-b :
                     a >= (gtype)0 ?
                     /* a positive, b negative: can overflow */
                     (a > max+b ? max : a-b) :
                     /* a negative, b positive: can underflow */
                     (a < min+b ? min : a-b);
                 }) :
                 /* unsigned */
                 ({
                   gtype min = (sgtype)0;
                   a < min+b ? min : a-b;
                 }))