Revert "remove local caching"

Revert "remove local caching"

This reverts commit 51d441ad789a74179dab0f56c5c85d5a670dcfce.

#ifdef HALF_AS_FLOAT
#define half float
#define half2 float2
#define half3 float3
#define half4 float4
#else
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#endif

#define UV_WIDTH RGB_WIDTH / 2
#define UV_HEIGHT RGB_HEIGHT / 2
#define U_OFFSET RGB_WIDTH * RGB_HEIGHT
#define V_OFFSET RGB_WIDTH * RGB_HEIGHT + UV_WIDTH * UV_HEIGHT

#define RGB_TO_Y(r, g, b) ((((mul24(b, 13) + mul24(g, 65) + mul24(r, 33)) + 64) >> 7) + 16)
#define RGB_TO_U(r, g, b) ((mul24(b, 56) - mul24(g, 37) - mul24(r, 19) + 0x8080) >> 8)
#define RGB_TO_V(r, g, b) ((mul24(r, 56) - mul24(g, 47) - mul24(b, 9) + 0x8080) >> 8)
#define AVERAGE(x, y, z, w) ((convert_ushort(x) + convert_ushort(y) + convert_ushort(z) + convert_ushort(w) + 1) >> 1)

// post wb CCM
const __constant half3 color_correction_0 = (half3)(1.82717181, -0.31231438, 0.07307673);
const __constant half3 color_correction_1 = (half3)(-0.5743977, 1.36858544, -0.53183455);
const __constant half3 color_correction_2 = (half3)(-0.25277411, -0.05627105, 1.45875782);

// tone mapping params
const half gamma_k = 0.75;
const half gamma_b = 0.125;
const half mp = 0.01; // ideally midpoint should be adaptive
const half rk = 9 - 100*mp;

inline half3 gamma_apply(half3 x) {
  // poly approximation for s curve
  return (x > mp) ?
    ((rk * (x-mp) * (1-(gamma_k*mp+gamma_b)) * (1+1/(rk*(1-mp))) / (1+rk*(x-mp))) + gamma_k*mp + gamma_b) :
    ((rk * (x-mp) * (gamma_k*mp+gamma_b) * (1+1/(rk*mp)) / (1-rk*(x-mp))) + gamma_k*mp + gamma_b);
}

inline half3 color_correct(half3 rgb) {
  half3 ret = (half)rgb.x * color_correction_0;
  ret += (half)rgb.y * color_correction_1;
  ret += (half)rgb.z * color_correction_2;
  return gamma_apply(ret);
}

inline half get_vignetting_s(float r) {
  if (r < 62500) {
    return (half)(1.0f + 0.0000008f*r);
  } else if (r < 490000) {
    return (half)(0.9625f + 0.0000014f*r);
  } else if (r < 1102500) {
    return (half)(1.26434f + 0.0000000000016f*r*r);
  } else {
    return (half)(0.53503625f + 0.0000000000022f*r*r);
  }
}

inline half val_from_10(const uchar * source, int gx, int gy, half black_level) {
  // parse 12bit
  int start = gy * FRAME_STRIDE + (3 * (gx / 2)) + (FRAME_STRIDE * FRAME_OFFSET);
  int offset = gx % 2;
  uint major = (uint)source[start + offset] << 4;
  uint minor = (source[start + 2] >> (4 * offset)) & 0xf;
  half pv = ((half)(major + minor)) / 4.0;

  // normalize
  pv = max((half)0.0, pv - black_level);
  pv /= (1024.0 - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    pv *= get_vignetting_s(gx*gx + gy*gy);
  }

  pv = clamp(pv, (half)0.0, (half)1.0);
  return pv;
}

inline half get_k(half a, half b, half c, half d) {
  return 2.0 - (fabs(a - b) + fabs(c - d));
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        __local half * cached,
                        float black_level
                       )
{
  const int gid_x = get_global_id(0);
  const int gid_y = get_global_id(1);

  const int lid_x = get_local_id(0);
  const int lid_y = get_local_id(1);

  const int localRowLen = mad24(get_local_size(0), 2, 2); // 2 padding
  const int localColLen = mad24(get_local_size(1), 2, 2);

  const int x_global = mul24(gid_x, 2);
  const int y_global = mul24(gid_y, 2);

  const int x_local = mad24(lid_x, 2, 1);
  const int y_local = mad24(lid_y, 2, 1);

  const int x_global_mod = (gid_x == 0 || gid_x == get_global_size(0) - 1) ? -1: 1;
  const int y_global_mod = (gid_y == 0 || gid_y == get_global_size(1) - 1) ? -1: 1;

  int localColOffset = 0;
  int globalColOffset;

  cached[mad24(y_local + 0, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + 0, black_level);
  cached[mad24(y_local + 0, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + 0, black_level);
  cached[mad24(y_local + 1, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + 1, black_level);
  cached[mad24(y_local + 1, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + 1, black_level);

  if (lid_x == 0) {  // left edge
    localColOffset = -1;
    globalColOffset = -x_global_mod;
    cached[mad24(y_local + 0, localRowLen, x_local - 1)] = val_from_10(in, x_global - x_global_mod, y_global + 0, black_level);
    cached[mad24(y_local + 1, localRowLen, x_local - 1)] = val_from_10(in, x_global - x_global_mod, y_global + 1, black_level);
  } else if (lid_x == get_local_size(0) - 1) {  // right edge
    localColOffset = 2;
    globalColOffset = x_global_mod + 1;
    cached[mad24(y_local + 0, localRowLen, x_local + 2)] = val_from_10(in, x_global + x_global_mod + 1, y_global + 0, black_level);
    cached[mad24(y_local + 1, localRowLen, x_local + 2)] = val_from_10(in, x_global + x_global_mod + 1, y_global + 1, black_level);
  }

  if (lid_y == 0) {  // top row
    cached[mad24(y_local - 1, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global - y_global_mod, black_level);
    cached[mad24(y_local - 1, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global - y_global_mod, black_level);
    if (localColOffset != 0) {  // cache corners
      cached[mad24(y_local - 1, localRowLen, x_local + localColOffset)] = val_from_10(in, x_global + globalColOffset, y_global - y_global_mod, black_level);
    }
  } else if (lid_y == get_local_size(1) - 1) {  // bottom row
    cached[mad24(y_local + 2, localRowLen, x_local + 0)] = val_from_10(in, x_global + 0, y_global + y_global_mod + 1, black_level);
    cached[mad24(y_local + 2, localRowLen, x_local + 1)] = val_from_10(in, x_global + 1, y_global + y_global_mod + 1, black_level);
    if (localColOffset != 0) {  // cache corners
      cached[mad24(y_local + 2, localRowLen, x_local + localColOffset)] = val_from_10(in, x_global + globalColOffset, y_global + y_global_mod + 1, black_level);
    }
  }

  // sync
  barrier(CLK_LOCAL_MEM_FENCE);

  half3 rgb;
  uchar3 rgb_out[4];

  const half4 va = vload4(0, cached + mad24(lid_y * 2 + 0, localRowLen, lid_x * 2));
  const half4 vb = vload4(0, cached + mad24(lid_y * 2 + 1, localRowLen, lid_x * 2));
  const half4 vc = vload4(0, cached + mad24(lid_y * 2 + 2, localRowLen, lid_x * 2));
  const half4 vd = vload4(0, cached + mad24(lid_y * 2 + 3, localRowLen, lid_x * 2));

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  const half k01 = get_k(va.s0, vb.s1, va.s2, vb.s1);
  const half k02 = get_k(va.s2, vb.s1, vc.s2, vb.s1);
  const half k03 = get_k(vc.s0, vb.s1, vc.s2, vb.s1);
  const half k04 = get_k(va.s0, vb.s1, vc.s0, vb.s1);
  rgb.x = (k02*vb.s2+k04*vb.s0)/(k02+k04); // R_G1
  rgb.y = vb.s1; // G1(R)
  rgb.z = (k01*va.s1+k03*vc.s1)/(k01+k03); // B_G1
  rgb_out[0] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k11 = get_k(va.s1, vc.s1, va.s3, vc.s3);
  const half k12 = get_k(va.s2, vb.s1, vb.s3, vc.s2);
  const half k13 = get_k(va.s1, va.s3, vc.s1, vc.s3);
  const half k14 = get_k(va.s2, vb.s3, vc.s2, vb.s1);
  rgb.x = vb.s2; // R
  rgb.y = (k11*(va.s2+vc.s2)*0.5+k13*(vb.s3+vb.s1)*0.5)/(k11+k13); // G_R
  rgb.z = (k12*(va.s3+vc.s1)*0.5+k14*(va.s1+vc.s3)*0.5)/(k12+k14); // B_R
  rgb_out[1] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k21 = get_k(vb.s0, vd.s0, vb.s2, vd.s2);
  const half k22 = get_k(vb.s1, vc.s0, vc.s2, vd.s1);
  const half k23 = get_k(vb.s0, vb.s2, vd.s0, vd.s2);
  const half k24 = get_k(vb.s1, vc.s2, vd.s1, vc.s0);
  rgb.x = (k22*(vb.s2+vd.s0)*0.5+k24*(vb.s0+vd.s2)*0.5)/(k22+k24); // R_B
  rgb.y = (k21*(vb.s1+vd.s1)*0.5+k23*(vc.s2+vc.s0)*0.5)/(k21+k23); // G_B
  rgb.z = vc.s1; // B
  rgb_out[2] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k31 = get_k(vb.s1, vc.s2, vb.s3, vc.s2);
  const half k32 = get_k(vb.s3, vc.s2, vd.s3, vc.s2);
  const half k33 = get_k(vd.s1, vc.s2, vd.s3, vc.s2);
  const half k34 = get_k(vb.s1, vc.s2, vd.s1, vc.s2);
  rgb.x = (k31*vb.s2+k33*vd.s2)/(k31+k33); // R_G2
  rgb.y = vc.s2; // G2(B)
  rgb.z = (k32*vc.s3+k34*vc.s1)/(k32+k34); // B_G2
  rgb_out[3] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  // write ys
  uchar2 yy = (uchar2)(
    RGB_TO_Y(rgb_out[0].s0, rgb_out[0].s1, rgb_out[0].s2),
    RGB_TO_Y(rgb_out[1].s0, rgb_out[1].s1, rgb_out[1].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2, RGB_WIDTH, gid_x * 2));
  yy = (uchar2)(
    RGB_TO_Y(rgb_out[2].s0, rgb_out[2].s1, rgb_out[2].s2),
    RGB_TO_Y(rgb_out[3].s0, rgb_out[3].s1, rgb_out[3].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2 + 1, RGB_WIDTH, gid_x * 2));

  // write uvs
  const short ar = AVERAGE(rgb_out[0].s0, rgb_out[1].s0, rgb_out[2].s0, rgb_out[3].s0);
  const short ag = AVERAGE(rgb_out[0].s1, rgb_out[1].s1, rgb_out[2].s1, rgb_out[3].s1);
  const short ab = AVERAGE(rgb_out[0].s2, rgb_out[1].s2, rgb_out[2].s2, rgb_out[3].s2);
  out[U_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_U(ar, ag, ab);
  out[V_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_V(ar, ag, ab);
}

#ifdef HALF_AS_FLOAT
#define half float
#define half2 float2
#define half3 float3
#define half4 float4
#else
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#endif

#define UV_WIDTH RGB_WIDTH / 2
#define UV_HEIGHT RGB_HEIGHT / 2
#define U_OFFSET RGB_WIDTH * RGB_HEIGHT
#define V_OFFSET RGB_WIDTH * RGB_HEIGHT + UV_WIDTH * UV_HEIGHT

#define RGB_TO_Y(r, g, b) ((((mul24(b, 13) + mul24(g, 65) + mul24(r, 33)) + 64) >> 7) + 16)
#define RGB_TO_U(r, g, b) ((mul24(b, 56) - mul24(g, 37) - mul24(r, 19) + 0x8080) >> 8)
#define RGB_TO_V(r, g, b) ((mul24(r, 56) - mul24(g, 47) - mul24(b, 9) + 0x8080) >> 8)
#define AVERAGE(x, y, z, w) ((convert_ushort(x) + convert_ushort(y) + convert_ushort(z) + convert_ushort(w) + 1) >> 1)

// post wb CCM
const __constant half3 color_correction_0 = (half3)(1.82717181, -0.31231438, 0.07307673);
const __constant half3 color_correction_1 = (half3)(-0.5743977, 1.36858544, -0.53183455);
const __constant half3 color_correction_2 = (half3)(-0.25277411, -0.05627105, 1.45875782);

// tone mapping params
const half gamma_k = 0.75;
const half gamma_b = 0.125;
const half mp = 0.01; // ideally midpoint should be adaptive
const half rk = 9 - 100*mp;

inline half3 gamma_apply(half3 x) {
  // poly approximation for s curve
  return (x > mp) ?
    ((rk * (x-mp) * (1-(gamma_k*mp+gamma_b)) * (1+1/(rk*(1-mp))) / (1+rk*(x-mp))) + gamma_k*mp + gamma_b) :
    ((rk * (x-mp) * (gamma_k*mp+gamma_b) * (1+1/(rk*mp)) / (1-rk*(x-mp))) + gamma_k*mp + gamma_b);
}

inline half3 color_correct(half3 rgb) {
  half3 ret = (half)rgb.x * color_correction_0;
  ret += (half)rgb.y * color_correction_1;
  ret += (half)rgb.z * color_correction_2;
  return gamma_apply(ret);
}

inline half get_vignetting_s(float r) {
  if (r < 62500) {
    return (half)(1.0f + 0.0000008f*r);
  } else if (r < 490000) {
    return (half)(0.9625f + 0.0000014f*r);
  } else if (r < 1102500) {
    return (half)(1.26434f + 0.0000000000016f*r*r);
  } else {
    return (half)(0.53503625f + 0.0000000000022f*r*r);
  }
}

inline half val_from_10(const uchar * source, int gx, int gy, half black_level) {
  // parse 12bit
  int start = gy * FRAME_STRIDE + (3 * (gx / 2)) + (FRAME_STRIDE * FRAME_OFFSET);
  int offset = gx % 2;
  uint major = (uint)source[start + offset] << 4;
  uint minor = (source[start + 2] >> (4 * offset)) & 0xf;
  half pv = ((half)(major + minor)) / 4.0;

  // normalize
  pv = max((half)0.0, pv - black_level);
  pv /= (1024.0 - black_level);

  // correct vignetting
  if (CAM_NUM == 1) { // fcamera
    gx = (gx - RGB_WIDTH/2);
    gy = (gy - RGB_HEIGHT/2);
    pv *= get_vignetting_s(gx*gx + gy*gy);
  }

  pv = clamp(pv, (half)0.0, (half)1.0);
  return pv;
}

inline half get_k(half a, half b, half c, half d) {
  return 2.0 - (fabs(a - b) + fabs(c - d));
}

__kernel void debayer10(const __global uchar * in,
                        __global uchar * out,
                        float black_level
                       )
{
  const int gid_x = get_global_id(0);
  const int gid_y = get_global_id(1);

  half3 rgb;
  uchar3 rgb_out[4];

  half4 va, vb, vc, vd;

  va.s0 = val_from_10(in, gid_x*2-1, gid_y*2-1, black_level);
  va.s1 = val_from_10(in, gid_x*2+0, gid_y*2-1, black_level);
  va.s2 = val_from_10(in, gid_x*2+1, gid_y*2-1, black_level);
  va.s3 = val_from_10(in, gid_x*2+2, gid_y*2-1, black_level);

  vb.s0 = val_from_10(in, gid_x*2-1, gid_y*2+0, black_level);
  vb.s1 = val_from_10(in, gid_x*2+0, gid_y*2+0, black_level); // G(R)
  vb.s2 = val_from_10(in, gid_x*2+1, gid_y*2+0, black_level); // R
  vb.s3 = val_from_10(in, gid_x*2+2, gid_y*2+0, black_level);

  vc.s0 = val_from_10(in, gid_x*2-1, gid_y*2+1, black_level);
  vc.s1 = val_from_10(in, gid_x*2+0, gid_y*2+1, black_level); // B
  vc.s2 = val_from_10(in, gid_x*2+1, gid_y*2+1, black_level); // G(B)
  vc.s3 = val_from_10(in, gid_x*2+2, gid_y*2+1, black_level);

  vd.s0 = val_from_10(in, gid_x*2-1, gid_y*2+2, black_level);
  vd.s1 = val_from_10(in, gid_x*2+0, gid_y*2+2, black_level);
  vd.s2 = val_from_10(in, gid_x*2+1, gid_y*2+2, black_level);
  vd.s3 = val_from_10(in, gid_x*2+2, gid_y*2+2, black_level);

  // a simplified version of https://opensignalprocessingjournal.com/contents/volumes/V6/TOSIGPJ-6-1/TOSIGPJ-6-1.pdf
  const half k01 = get_k(va.s0, vb.s1, va.s2, vb.s1);
  const half k02 = get_k(va.s2, vb.s1, vc.s2, vb.s1);
  const half k03 = get_k(vc.s0, vb.s1, vc.s2, vb.s1);
  const half k04 = get_k(va.s0, vb.s1, vc.s0, vb.s1);
  rgb.x = (k02*vb.s2+k04*vb.s0)/(k02+k04); // R_G1
  rgb.y = vb.s1; // G1(R)
  rgb.z = (k01*va.s1+k03*vc.s1)/(k01+k03); // B_G1
  rgb_out[0] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k11 = get_k(va.s1, vc.s1, va.s3, vc.s3);
  const half k12 = get_k(va.s2, vb.s1, vb.s3, vc.s2);
  const half k13 = get_k(va.s1, va.s3, vc.s1, vc.s3);
  const half k14 = get_k(va.s2, vb.s3, vc.s2, vb.s1);
  rgb.x = vb.s2; // R
  rgb.y = (k11*(va.s2+vc.s2)*0.5+k13*(vb.s3+vb.s1)*0.5)/(k11+k13); // G_R
  rgb.z = (k12*(va.s3+vc.s1)*0.5+k14*(va.s1+vc.s3)*0.5)/(k12+k14); // B_R
  rgb_out[1] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k21 = get_k(vb.s0, vd.s0, vb.s2, vd.s2);
  const half k22 = get_k(vb.s1, vc.s0, vc.s2, vd.s1);
  const half k23 = get_k(vb.s0, vb.s2, vd.s0, vd.s2);
  const half k24 = get_k(vb.s1, vc.s2, vd.s1, vc.s0);
  rgb.x = (k22*(vb.s2+vd.s0)*0.5+k24*(vb.s0+vd.s2)*0.5)/(k22+k24); // R_B
  rgb.y = (k21*(vb.s1+vd.s1)*0.5+k23*(vc.s2+vc.s0)*0.5)/(k21+k23); // G_B
  rgb.z = vc.s1; // B
  rgb_out[2] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  const half k31 = get_k(vb.s1, vc.s2, vb.s3, vc.s2);
  const half k32 = get_k(vb.s3, vc.s2, vd.s3, vc.s2);
  const half k33 = get_k(vd.s1, vc.s2, vd.s3, vc.s2);
  const half k34 = get_k(vb.s1, vc.s2, vd.s1, vc.s2);
  rgb.x = (k31*vb.s2+k33*vd.s2)/(k31+k33); // R_G2
  rgb.y = vc.s2; // G2(B)
  rgb.z = (k32*vc.s3+k34*vc.s1)/(k32+k34); // B_G2
  rgb_out[3] = convert_uchar3_sat(color_correct(clamp(rgb, 0.0, 1.0)) * 255.0);

  // write ys
  uchar2 yy = (uchar2)(
    RGB_TO_Y(rgb_out[0].s0, rgb_out[0].s1, rgb_out[0].s2),
    RGB_TO_Y(rgb_out[1].s0, rgb_out[1].s1, rgb_out[1].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2, RGB_WIDTH, gid_x * 2));
  yy = (uchar2)(
    RGB_TO_Y(rgb_out[2].s0, rgb_out[2].s1, rgb_out[2].s2),
    RGB_TO_Y(rgb_out[3].s0, rgb_out[3].s1, rgb_out[3].s2)
  );
  vstore2(yy, 0, out + mad24(gid_y * 2 + 1, RGB_WIDTH, gid_x * 2));

  // write uvs
  const short ar = AVERAGE(rgb_out[0].s0, rgb_out[1].s0, rgb_out[2].s0, rgb_out[3].s0);
  const short ag = AVERAGE(rgb_out[0].s1, rgb_out[1].s1, rgb_out[2].s1, rgb_out[3].s1);
  const short ab = AVERAGE(rgb_out[0].s2, rgb_out[1].s2, rgb_out[2].s2, rgb_out[3].s2);
  out[U_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_U(ar, ag, ab);
  out[V_OFFSET + mad24(gid_y, UV_WIDTH, gid_x)] = RGB_TO_V(ar, ag, ab);
}


Fix some signed char overflows and further optimize the vp8_filter function.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline void vp8_mbfilter(signed char mask,signed char hev,local uc*);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
#endif

char vp8_char_clamp(int in){
    if (in > 127)
        return 127;
    if (in < -128)
        return -128;

    return in;
}

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = convert_char4(base) ^ 0x80;

    char vp8_filter;
    char2 Filter;
    char4 u;

    /* add outer taps if we have high edge variance */
    vp8_filter = sub_sat(pq.s0, pq.s3);
    vp8_filter &= hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = add_sat((char2)vp8_filter, (char2){4,3});
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    /* outer tap adjustments */
    vp8_filter = Filter.s0 + 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    //u.s01 = add_sat(pq.s01, (char2){vp8_filter, Filter.s1});
    //u.s23 = sub_sat(pq.s23, (char2){Filter.s0, vp8_filter});
    u.s0 = clamp(pq.s0 + vp8_filter, -128, 127);
    u.s1 = clamp(pq.s1 + Filter.s1, -128, 127);
    u.s2 = clamp(pq.s2 - Filter.s0, -128, 127);
    u.s3 = clamp(pq.s3 - vp8_filter, -128, 127);

    return convert_uchar4(u ^ 0x80);

}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);
    int filter_level = filters[block];

    if (filters[num_blocks*filter_type + block] > 0){
        if (cur_iter == 0 || plane == 0){
            if (cur_iter > 0){
                num_planes = 1;
            }

            if (filter_level){
                int p = pitches[plane];
                int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
                int s_off = offsets[block_offset];

                size_t i = get_global_id(0);

                int  hev = 0; /* high edge variance */
                signed char mask = 0;

                global signed char *limit, *flimit, *thresh;
                global loop_filter_info *lf_info;

                if (i < threads[plane]){
                    lf_info = &lfi[filter_level];
                    flimit = lf_info->flim;
                    limit = lf_info->lim;
                    thresh = lf_info->thr;

                    s_off += i;

                    uchar8 data;
                    data.s0 = s_base[s_off-4*p];
                    data.s1 = s_base[s_off-3*p];
                    data.s2 = s_base[s_off-2*p];
                    data.s3 = s_base[s_off-p];
                    data.s4 = s_base[s_off];
                    data.s5 = s_base[s_off+p];
                    data.s6 = s_base[s_off+2*p];
                    data.s7 = s_base[s_off+3*p];

                    mask = vp8_filter_mask(limit[i], flimit[i], data);

                    hev = vp8_hevmask(thresh[i], data.s2345);

                    data.s2345 = vp8_filter(mask, hev, data.s2345);

                    s_base[s_off - 2*p] = data.s2;
                    s_base[s_off - p  ] = data.s3;
                    s_base[s_off      ] = data.s4;
                    s_base[s_off + p  ] = data.s5;

                }
            }
        }
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);

    if ((cur_iter ==1 || plane == 0) && filters[num_blocks*filter_type + block] > 0){
        if (cur_iter > 1){
            num_planes = 1;
        }

        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset];

            int  hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i = get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->flim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;

                uchar8 data = vload8(0, &s_base[s_off-4]);

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                data.s2345 = vp8_filter(mask, hev, data.s2345);

                vstore4(data.s2345, 0, &s_base[s_off-2]);

            }
        }
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*ROWS_LOCATION + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = 8*num_blocks + block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += i;

#if 1
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4*p];
                s_data[1] = s_base[s_off - 3*p];
                s_data[2] = s_base[s_off - 2*p];
                s_data[3] = s_base[s_off - 1*p];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + p];
                s_data[6] = s_base[s_off + 2*p];
                s_data[7] = s_base[s_off + 3*p];
                uchar8 data = vload8(0, s_data);
#else
                uchar8 data;
                data.s0 = s_base[s_off-4*p];
                data.s1 = s_base[s_off-3*p];
                data.s2 = s_base[s_off-2*p];
                data.s3 = s_base[s_off-p];
                data.s4 = s_base[s_off];
                data.s5 = s_base[s_off+p];
                data.s6 = s_base[s_off+2*p];
                data.s7 = s_base[s_off+3*p];
#endif

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3*p] = s_data[1];
                s_base[s_off - 2*p] = s_data[2];
                s_base[s_off - 1*p] = s_data[3];
                s_base[s_off      ] = s_data[4];
                s_base[s_off + p  ] = s_data[5];
                s_base[s_off + 2*p] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim, //unused for normal filters
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = 0;
    local unsigned char s_data[16*8*3];

    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);
}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];

                uchar8 data = vload8(0, s_data);
                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3] = s_data[1];
                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];
                s_base[s_off + 2] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){

    int block_offset = block_offsets[priority_level];
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    //Prefetch vertical edge source pixels into global cache
    int num_blocks = get_global_size(2);
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+p*thread-4], 8);
        }
    }

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

    barrier(CLK_GLOBAL_MEM_FENCE);

    //Prefetch horizontal source pixels
#if 0
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+thread-4*p], 8*p);
        }
    }
#endif

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);

}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = 0;
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

}

kernel void vp8_loop_filter_simple_horizontal_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = 0;
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = 0;
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

//Inline and non-kernel functions follow.

__inline void vp8_mbfilter(
    signed char mask,
    signed char hev,
    local uc *base
)
{
    signed char s, u;
    signed char vp8_filter;

    local uc *op2 = base;
    local uc *op1 = base + 1;
    local uc *op0 = base + 2;
    local uc *oq0 = base + 3;
    local uc *oq1 = base + 4;
    local uc *oq2 = base + 5;

    char2 filter;

    char4 ps = { *op0, *op1, *op2, 0 };
    ps ^= (char4){0x80, 0x80, 0x80, 0x80};

    char4 qs = { *oq0, *oq1, *oq2, 0 };
    qs ^= (char4){0x80, 0x80, 0x80, 0x80};

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps.s1 - qs.s1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (qs.s0 - ps.s0), -128, 127);
    vp8_filter &= mask;

    filter.s1 = vp8_filter;
    filter.s1 &= hev;
    filter.s0 = filter.s1;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter.s0 = clamp(filter.s0 + 4, -128, 127);
    filter.s1 = clamp(filter.s1 + 3, -128, 127);
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    qs.s0 = clamp(qs.s0 - filter.s0, -128, 127);
    ps.s0 = clamp(ps.s0 + filter.s1, -128, 127);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th difference across boundary */
    u = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    s = clamp(qs.s0 - u, -128, 127);
    *oq0 = s ^ 0x80;
    s = clamp(ps.s0 + u, -128, 127);
    *op0 = s ^ 0x80;

    /* roughly 2/7th difference across boundary */
    u = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    s = clamp(qs.s1 - u, -128, 127);
    *oq1 = s ^ 0x80;
    s = clamp(ps.s1 + u, -128, 127);
    *op1 = s ^ 0x80;

    /* roughly 1/7th difference across boundary */
    u = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    s = clamp(qs.s2 - u, -128, 127);
    *oq2 = s ^ 0x80;
    s = clamp(ps.s2 + u, -128, 127);
    *op2 = s ^ 0x80;
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    signed char hev = 0;
    hev  |= (abs(pq.s0 - pq.s1) > thresh) * -1;
    hev  |= (abs(pq.s3 - pq.s2) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    signed char mask = 0;

#if 1
    mask |= (abs(pq.s0 - pq.s1) > limit) * -1;
    mask |= (abs(pq.s1 - pq.s2) > limit) * -1;
    mask |= (abs(pq.s2 - pq.s3) > limit) * -1;
    mask |= (abs(pq.s5 - pq.s4) > limit) * -1;
    mask |= (abs(pq.s6 - pq.s5) > limit) * -1;
    mask |= (abs(pq.s7 - pq.s6) > limit) * -1;
    mask |= (abs(pq.s3 - pq.s4) * 2 + abs(pq.s2 - pq.s5) / 2  > flimit * 2 + limit) * -1;
    mask = ~mask;
    return mask;
#else
    //Only apply the filter if the difference is LESS than 'limit'
    mask |= (abs(pq.s0 - pq.s1) > limit);
    mask |= (abs(pq.s1 - pq.s2) > limit);
    mask |= (abs(pq.s2 - pq.s3) > limit);
    mask |= (abs(pq.s5 - pq.s4) > limit);
    mask |= (abs(pq.s6 - pq.s5) > limit);
    mask |= (abs(pq.s7 - pq.s6) > limit);
    mask |= (abs(pq.s3 - pq.s4) * 2 + abs(pq.s2 - pq.s5) / 2  > flimit * 2 + limit);

    return (mask != 0 ? 0 : -1);
#endif

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs(p0 - q0) * 2 + abs(p1 - q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter, Filter1, Filter2;
    signed char p1 = (signed char) * op1 ^ 0x80;
    signed char p0 = (signed char) * op0 ^ 0x80;
    signed char q0 = (signed char) * oq0 ^ 0x80;
    signed char q1 = (signed char) * oq1 ^ 0x80;
    signed char u;

    vp8_filter = clamp(p1 - q1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (q0 - p0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter1 >>= 3;
    u = clamp(q0 - Filter1, -128, 127);
    *oq0  = u ^ 0x80;

    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter2 >>= 3;
    u = clamp(p0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
#pragma OPENCL EXTENSION cl_amd_printf : enable

typedef unsigned char uc;
typedef signed char sc;

__inline signed char vp8_filter_mask(sc, sc, uchar8);
__inline signed char vp8_simple_filter_mask(signed char, signed char, uc, uc, uc, uc);
__inline signed char vp8_hevmask(signed char, uchar4);

__inline void vp8_mbfilter(signed char mask,signed char hev,local uc*);

void vp8_simple_filter(signed char mask,global uc *base, int op1_off,int op0_off,int oq0_off,int oq1_off);

constant int threads[3] = {16, 8, 8};

#ifndef __CL_VERSION_1_0__
#define __CL_VERSION_1_0__ 100
#endif

#if __OPENCL_VERSION__ == __CL_VERSION_1_0__
#define clamp(x,y,z) vp8_char_clamp(x)
#endif

char vp8_char_clamp(int in){
    if (in > 127)
        return 127;
    if (in < -128)
        return -128;

    return in;
}

typedef struct
{
    signed char lim[16];
    signed char flim[16];
    signed char thr[16];
    signed char mbflim[16];
} loop_filter_info;

typedef struct
{
    int mb_rows;
    int mb_cols;
} frame_info;

uchar4 vp8_filter(
    signed char mask,
    signed char hev,
    uchar4 base
)
{
    char4 pq = convert_char4(base);

    char vp8_filter;
    char2 Filter;
    char4 u;

    pq ^= (char4)0x80;

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(pq.s0 - pq.s3, -128, 127) & hev;

    /* inner taps */
    vp8_filter = clamp(vp8_filter + 3 * (pq.s2 - pq.s1), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3
     * if it equals 4 we'll set to adjust by -1 to account for the fact
     * we'd round 3 the other way
     */
    Filter = (char2)vp8_filter + (char2){4,3};
    Filter.s0 >>= 3;
    Filter.s1 >>= 3;

    vp8_filter = Filter.s0;

    /* outer tap adjustments */
    vp8_filter += 1;
    vp8_filter >>= 1;
    vp8_filter &= ~hev;

    u.s0 = clamp(pq.s0 + vp8_filter, -128, 127);
    u.s1 = clamp(pq.s1 + Filter.s1, -128, 127);
    u.s2 = clamp(pq.s2 - Filter.s0, -128, 127);
    u.s3 = clamp(pq.s3 - vp8_filter, -128, 127);

    pq = u ^ (char4)0x80;

    return convert_uchar4(pq);
}

// Filters horizontal edges of inner blocks in a Macroblock
void vp8_loop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);
    int filter_level = filters[block];

    if (filters[num_blocks*filter_type + block] > 0){
        if (cur_iter == 0 || plane == 0){
            if (cur_iter > 0){
                num_planes = 1;
            }

            if (filter_level){
                int p = pitches[plane];
                int block_offset = num_blocks*11 + cur_iter*num_blocks*num_planes + block*num_planes+plane;
                int s_off = offsets[block_offset];

                size_t i = get_global_id(0);

                int  hev = 0; /* high edge variance */
                signed char mask = 0;

                global signed char *limit, *flimit, *thresh;
                global loop_filter_info *lf_info;

                if (i < threads[plane]){
                    lf_info = &lfi[filter_level];
                    flimit = lf_info->flim;
                    limit = lf_info->lim;
                    thresh = lf_info->thr;

                    s_off += i;

                    uchar8 data;
                    data.s0 = s_base[s_off-4*p];
                    data.s1 = s_base[s_off-3*p];
                    data.s2 = s_base[s_off-2*p];
                    data.s3 = s_base[s_off-p];
                    data.s4 = s_base[s_off];
                    data.s5 = s_base[s_off+p];
                    data.s6 = s_base[s_off+2*p];
                    data.s7 = s_base[s_off+3*p];

                    mask = vp8_filter_mask(limit[i], flimit[i], data);

                    hev = vp8_hevmask(thresh[i], data.s2345);

                    data.s2345 = vp8_filter(mask, hev, data.s2345);

                    s_base[s_off - 2*p] = data.s2;
                    s_base[s_off - p  ] = data.s3;
                    s_base[s_off      ] = data.s4;
                    s_base[s_off + p  ] = data.s5;

                }
            }
        }
    }
}

void vp8_loop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    int cur_iter
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes = 3;
    size_t num_blocks = get_global_size(2);

    if ((cur_iter ==1 || plane == 0) && filters[num_blocks*filter_type + block] > 0){
        if (cur_iter > 1){
            num_planes = 1;
        }

        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset];

            int  hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i = get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->flim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;

                uchar8 data = vload8(0, &s_base[s_off-4]);

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                data.s2345 = vp8_filter(mask, hev, data.s2345);

                vstore4(data.s2345, 0, &s_base[s_off-2]);

            }
        }
    }
}

void vp8_mbloop_filter_horizontal_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = 3;
    num_blocks = get_global_size(2);

    if (filters[num_blocks*ROWS_LOCATION + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = 8*num_blocks + block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += i;

#if 1
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4*p];
                s_data[1] = s_base[s_off - 3*p];
                s_data[2] = s_base[s_off - 2*p];
                s_data[3] = s_base[s_off - 1*p];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + p];
                s_data[6] = s_base[s_off + 2*p];
                s_data[7] = s_base[s_off + 3*p];
                uchar8 data = vload8(0, s_data);
#else
                uchar8 data;
                data.s0 = s_base[s_off-4*p];
                data.s1 = s_base[s_off-3*p];
                data.s2 = s_base[s_off-2*p];
                data.s3 = s_base[s_off-p];
                data.s4 = s_base[s_off];
                data.s5 = s_base[s_off+p];
                data.s6 = s_base[s_off+2*p];
                data.s7 = s_base[s_off+3*p];
#endif

                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3*p] = s_data[1];
                s_base[s_off - 2*p] = s_data[2];
                s_base[s_off - 1*p] = s_data[3];
                s_base[s_off      ] = s_data[4];
                s_base[s_off + p  ] = s_data[5];
                s_base[s_off + 2*p] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_horizontal_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches, /* pitch */
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim, //unused for normal filters
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = 0;
    local unsigned char s_data[16*8*3];

    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);
}

void vp8_mbloop_filter_vertical_edge_worker(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters,
    int filter_type,
    local unsigned char *s_data
){
    size_t plane = get_global_id(1);
    size_t block = get_global_id(2);
    size_t num_planes;
    size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = block*num_planes+plane;
            int s_off = offsets[block_offset];

            signed char hev = 0; /* high edge variance */
            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit, *thresh;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                flimit = lf_info->mbflim;

                limit = lf_info->lim;
                thresh = lf_info->thr;

                s_off += p * i;
                s_data += plane*128+i*8;

                s_data[0] = s_base[s_off - 4];
                s_data[1] = s_base[s_off - 3];
                s_data[2] = s_base[s_off - 2];
                s_data[3] = s_base[s_off - 1];
                s_data[4] = s_base[s_off];
                s_data[5] = s_base[s_off + 1];
                s_data[6] = s_base[s_off + 2];
                s_data[7] = s_base[s_off + 3];

                uchar8 data = vload8(0, s_data);
                mask = vp8_filter_mask(limit[i], flimit[i], data);

                hev = vp8_hevmask(thresh[i], data.s2345);

                vp8_mbfilter(mask, hev, s_data + 1);

                s_base[s_off - 3] = s_data[1];
                s_base[s_off - 2] = s_data[2];
                s_base[s_off - 1] = s_data[3];
                s_base[s_off    ] = s_data[4];
                s_base[s_off + 1] = s_data[5];
                s_base[s_off + 2] = s_data[6];
            }
        }
    }
}

kernel void vp8_loop_filter_all_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){

    int block_offset = block_offsets[priority_level];
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    //Prefetch vertical edge source pixels into global cache
    int num_blocks = get_global_size(2);
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+p*thread-4], 8);
        }
    }

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

    barrier(CLK_GLOBAL_MEM_FENCE);

    //Prefetch horizontal source pixels
#if 0
    for(int plane = 0; plane < 3; plane++){
        int p = pitches[plane];
        int offset = get_global_id(2)*3+plane;
        int s_off = offsets[offset+priority_offset];
        for (int thread = 0; thread < 16; thread++){
            prefetch(&s_base[s_off+thread-4*p], 8*p);
        }
    }
#endif

    vp8_mbloop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi,
            filters, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 0, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 3, s_data);
    vp8_loop_filter_horizontal_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 4, s_data);

}

kernel void vp8_loop_filter_vertical_edges_kernel(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
){
    int block_offset = 0;
    local unsigned char s_data[16*8*3];
    int filter_offset = 4*block_offset;
    int priority_offset = 16*block_offset;

    global int *filters = &filters_in[filter_offset];

    vp8_mbloop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            COLS_LOCATION, s_data);

    //YUV planes, then 2 more passes of Y plane
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 1);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 6);
    vp8_loop_filter_vertical_edge_worker(s_base, &offsets[priority_offset], pitches, lfi, filters,
            DC_DIFFS_LOCATION, 7);

}

kernel void vp8_loop_filter_simple_horizontal_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets,
    global int *pitches,
    global loop_filter_info *lfi,
    global int *filters_in,
    int use_mbflim,
    int filter_type,
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = 0;
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[num_blocks*filter_type + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2*p], s_base[s_off-p], s_base[s_off], s_base[s_off+p]);
                vp8_simple_filter(mask, s_base, s_off - 2 * p, s_off - 1 * p, s_off, s_off + 1 * p);
            }
        }
    }
}

kernel void vp8_loop_filter_simple_vertical_edge_kernel
(
    global unsigned char *s_base,
    global int *offsets, /* Y or YUV offsets for EACH block being processed*/
    global int *pitches, /* 1 or 3 values for Y or YUV pitches*/
    global loop_filter_info *lfi, /* Single struct for the frame */
    global int *filters_in, /* Filters for each block being processed */
    int use_mbflim, /* Use lfi->flim or lfi->mbflim, need once per kernel call */
    int filter_type, /* Should dc_diffs, rows, or cols be used?*/
    int cur_iter,
    int priority_level,
    global int *block_offsets,
    global int *priority_num_blocks
)
{
    int block_offset = 0;
    int filter_offset = 4*block_offset;
    int priority_offset = 8*block_offset;
    private size_t plane = get_global_id(1);
    private size_t block = get_global_id(2);
    local size_t num_planes;
    local size_t num_blocks;
    num_planes = get_global_size(1);
    num_blocks = get_global_size(2);

    global int *filters = &filters_in[filter_offset];

    if (filters[filter_type * num_blocks + block] > 0){
        int filter_level = filters[block];
        if (filter_level){
            int p = pitches[plane];
            int block_offset = cur_iter*num_blocks*num_planes + block*num_planes+plane;
            int s_off = offsets[block_offset+priority_offset];

            signed char mask = 0;
            size_t i= get_global_id(0);

            global signed char *limit, *flimit;
            global loop_filter_info *lf_info;

            if (i < threads[plane]){
                lf_info = &lfi[filter_level];
                if (use_mbflim == 0){
                    flimit = lf_info->flim;
                } else {
                    flimit = lf_info->mbflim;
                }

                limit = lf_info->lim;

                s_off += p * i;
                mask = vp8_simple_filter_mask(limit[i], flimit[i], s_base[s_off-2], s_base[s_off-1], s_base[s_off], s_base[s_off+1]);
                vp8_simple_filter(mask, s_base, s_off - 2, s_off - 1, s_off, s_off + 1);
            }
        }
    }
}

//Inline and non-kernel functions follow.

__inline void vp8_mbfilter(
    signed char mask,
    signed char hev,
    local uc *base
)
{
    signed char s, u;
    signed char vp8_filter;

    local uc *op2 = base;
    local uc *op1 = base + 1;
    local uc *op0 = base + 2;
    local uc *oq0 = base + 3;
    local uc *oq1 = base + 4;
    local uc *oq2 = base + 5;

    char2 filter;

    char4 ps = { *op0, *op1, *op2, 0 };
    ps ^= (char4){0x80, 0x80, 0x80, 0x80};

    char4 qs = { *oq0, *oq1, *oq2, 0 };
    qs ^= (char4){0x80, 0x80, 0x80, 0x80};

    /* add outer taps if we have high edge variance */
    vp8_filter = clamp(ps.s1 - qs.s1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (qs.s0 - ps.s0), -128, 127);
    vp8_filter &= mask;

    filter.s1 = vp8_filter;
    filter.s1 &= hev;
    filter.s0 = filter.s1;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    filter.s0 = clamp(filter.s0 + 4, -128, 127);
    filter.s1 = clamp(filter.s1 + 3, -128, 127);
    filter.s0 >>= 3;
    filter.s1 >>= 3;

    qs.s0 = clamp(qs.s0 - filter.s0, -128, 127);
    ps.s0 = clamp(ps.s0 + filter.s1, -128, 127);

    /* only apply wider filter if not high edge variance */
    vp8_filter &= ~hev;
    filter.s1 = vp8_filter;

    /* roughly 3/7th difference across boundary */
    u = clamp((63 + filter.s1 * 27) >> 7, -128, 127);
    s = clamp(qs.s0 - u, -128, 127);
    *oq0 = s ^ 0x80;
    s = clamp(ps.s0 + u, -128, 127);
    *op0 = s ^ 0x80;

    /* roughly 2/7th difference across boundary */
    u = clamp((63 + filter.s1 * 18) >> 7, -128, 127);
    s = clamp(qs.s1 - u, -128, 127);
    *oq1 = s ^ 0x80;
    s = clamp(ps.s1 + u, -128, 127);
    *op1 = s ^ 0x80;

    /* roughly 1/7th difference across boundary */
    u = clamp((63 + filter.s1 * 9) >> 7, -128, 127);
    s = clamp(qs.s2 - u, -128, 127);
    *oq2 = s ^ 0x80;
    s = clamp(ps.s2 + u, -128, 127);
    *op2 = s ^ 0x80;
}

/* is there high variance internal edge ( 11111111 yes, 00000000 no) */
__inline signed char vp8_hevmask(signed char thresh, uchar4 pq)
{
    signed char hev = 0;
    hev  |= (abs(pq.s0 - pq.s1) > thresh) * -1;
    hev  |= (abs(pq.s3 - pq.s2) > thresh) * -1;
    return hev;
}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_filter_mask( signed char limit, signed char flimit,
        uchar8 pq)
{
    signed char mask = 0;

    //Only apply the filter if the difference is LESS than 'limit'
    mask |= (abs(pq.s0 - pq.s1) > limit);
    mask |= (abs(pq.s1 - pq.s2) > limit);
    mask |= (abs(pq.s2 - pq.s3) > limit);
    mask |= (abs(pq.s5 - pq.s4) > limit);
    mask |= (abs(pq.s6 - pq.s5) > limit);
    mask |= (abs(pq.s7 - pq.s6) > limit);
    mask |= (abs(pq.s3 - pq.s4) * 2 + abs(pq.s2 - pq.s5) / 2  > flimit * 2 + limit);

    return (mask != 0 ? 0 : -1);

}

/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
__inline signed char vp8_simple_filter_mask(
    signed char limit,
    signed char flimit,
    uc p1,
    uc p0,
    uc q0,
    uc q1
)
{
    signed char mask = (abs(p0 - q0) * 2 + abs(p1 - q1) / 2  <= flimit * 2 + limit) * -1;
    return mask;
}

void vp8_simple_filter(
    signed char mask,
    global uc *base,
    int op1_off,
    int op0_off,
    int oq0_off,
    int oq1_off
)
{

    global uc *op1 = base + op1_off;
    global uc *op0 = base + op0_off;
    global uc *oq0 = base + oq0_off;
    global uc *oq1 = base + oq1_off;

    signed char vp8_filter, Filter1, Filter2;
    signed char p1 = (signed char) * op1 ^ 0x80;
    signed char p0 = (signed char) * op0 ^ 0x80;
    signed char q0 = (signed char) * oq0 ^ 0x80;
    signed char q1 = (signed char) * oq1 ^ 0x80;
    signed char u;

    vp8_filter = clamp(p1 - q1, -128, 127);
    vp8_filter = clamp(vp8_filter + 3 * (q0 - p0), -128, 127);
    vp8_filter &= mask;

    /* save bottom 3 bits so that we round one side +4 and the other +3 */
    Filter1 = clamp(vp8_filter + 4, -128, 127);
    Filter1 >>= 3;
    u = clamp(q0 - Filter1, -128, 127);
    *oq0  = u ^ 0x80;

    Filter2 = clamp(vp8_filter + 3, -128, 127);
    Filter2 >>= 3;
    u = clamp(p0 + Filter2, -128, 127);
    *op0 = u ^ 0x80;
}


fix type inference to work with iterative-deepening using 'neq' builtin

member(Key, cons(Head, Tail), Val) :-
    neq(Key,Head),
    member(Key, Tail, Val).
member(Key, cons(def(Key, Val), Tail), just(Val)).
member(Key, nil, nothing).

append(nil, L, L).
append(cons(H, T1), L, cons(H, T2)) :-
    append(T1, L, T2).

unifyall(nil, T).
unifyall(cons(T, Tail), T) :-
    unifyall(Tail, T).

split(X, nil, nil, nil).
split(X, cons(def(X, T), Tail1), cons(T, Tail2), Tail3 ) :-
    split(X, Tail1, Tail2, Tail3).
split(X, cons(def(Y, T), Tail1), Tail2, cons(def(Y, T), Tail3)) :-
    neq(X, Y),
    split(X, Tail1, Tail2, Tail3).

typing(Env, lam(Var, Body), Cxt, arrow(A, B)) :-
    typing(Env, Body, C1, B),
    split(Var, C1, Def, Cxt),
    unifyall(Def, A).
typing(Env, app(Fun, Arg), Cxt, B) :-
    typing(Env, Fun, C1, arrow(A, B)),
    typing(Env, Arg, C2, A),
    append(C1, C2, Cxt).
typing(Env, var(X), Cxt, Type) :-
    member(X, Env, just(ty2(Cxt, Type))).
typing(Env, var(X), cons(def(X, Type), nil), Type) :-
    member(X, Env, nothing).

#:- typing(cons(def(i1, ty2(nil, int)), nil), var(i1), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil), var(x), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil), app(var(x), var(i1)), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil),              lam(x, app(x, i1)), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil),               app(f, app(x, i1)), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil),        lam(x, app(f, app(x, i1))), C, T).
:- typing(cons(def(i1, ty2(nil, int)), nil), lam(f, lam(x, app(var(f), app(var(x), var(i1))))), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), cons(def(b1, ty2(nil, bool)), nil)),
#    lam(f, lam(x, lam(x, app(app(var(f), app(var(x), var(i1))), app(var(x), var(b1)))))), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), cons(def(b1, ty2(nil, bool)), nil)),
#    lam(f, lam(x, app(app(var(f), app(var(x), lam(z, var(z)))), app(var(x), lam(u, lam(v, var(u))))))), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), cons(def(b1, ty2(nil, bool)), nil)),
#    lam(x, app(var(x), var(x))), C, T).

member(Key, cons(def(Key, Val), Tail), just(Val)).
member(Key, nil, nothing).
member(Key, cons(Head, Tail), Val) :-
    member(Key, Tail, Val).

append(nil, L, L).
append(cons(H, T1), L, cons(H, T2)) :-
    append(T1, L, T2).

unifyall(nil, T).
unifyall(cons(T, Tail), T) :-
    unifyall(Tail, T).

split(X, nil, nil, nil).
split(X, cons(def(X, T), Tail1), cons(T, Tail2), Tail3 ) :-
    split(X, Tail1, Tail2, Tail3).
split(X, cons(def(Y, T), Tail1), Tail2, cons(def(Y, T), Tail3)) :-
    split(X, Tail1, Tail2, Tail3).

typing(Env, lam(Var, Body), Cxt, arrow(A, B)) :-
    typing(Env, Body, C1, B),
    split(Var, C1, Def, Cxt),
    unifyall(Def, A).
typing(Env, app(Fun, Arg), Cxt, B) :-
    typing(Env, Fun, C1, arrow(A, B)),
    typing(Env, Arg, C2, A),
    append(C1, C2, Cxt).
typing(Env, var(X), Cxt, Type) :-
    member(X, Env, just(ty2(Cxt, Type))).
typing(Env, var(X), cons(def(X, Type), nil), Type) :-
    member(X, Env, nothing).

#:- typing(cons(def(i1, ty2(nil, int)), nil), var(i1), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil), var(x), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil), app(var(x), var(i1)), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil),              lam(x, app(x, i1)), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil),               app(f, app(x, i1)), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), nil),        lam(x, app(f, app(x, i1))), C, T).
:- typing(cons(def(i1, ty2(nil, int)), nil), lam(f, lam(x, app(var(f), app(var(x), var(i1))))), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), cons(def(b1, ty2(nil, bool)), nil)),
#    lam(f, lam(x, lam(x, app(app(var(f), app(var(x), var(i1))), app(var(x), var(b1)))))), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), cons(def(b1, ty2(nil, bool)), nil)),
#    lam(f, lam(x, app(app(var(f), app(var(x), lam(z, var(z)))), app(var(x), lam(u, lam(v, var(u))))))), C, T).
#:- typing(cons(def(i1, ty2(nil, int)), cons(def(b1, ty2(nil, bool)), nil)),
#    lam(x, app(var(x), var(x))), C, T).


OpenCL: fix test for lack of names in release builds

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@222853 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -ffake-address-space-map -cl-std=CL2.0 -emit-llvm -o - | FileCheck %s

// test that we generate address space casts everywhere we need conversions of
// pointers to different address spaces

void test(global int *arg_glob, generic int *arg_gen) {
  int var_priv;
  arg_gen = arg_glob; // implicit cast global -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
  arg_gen = &var_priv; // implicit cast with obtaining adr, private -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32* %var_priv to i32 addrspace(4)*
  arg_glob = (global int *)arg_gen; // explicit cast
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(4)* %{{[0-9]+}} to i32 addrspace(1)*
  global int *var_glob =
      (global int *)arg_glob; // explicit cast in the same address space
  // CHECK-NOT: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(1)*
  var_priv = arg_gen - arg_glob; // arithmetic operation
  // CHECK: %{{.*}} = ptrtoint i32 addrspace(4)* %{{.*}} to i64
  // CHECK: %{{.*}} = ptrtoint i32 addrspace(1)* %{{.*}} to i64
  var_priv = arg_gen > arg_glob; // comparison
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
}

// RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -O0 -ffake-address-space-map -cl-std=CL2.0 -emit-llvm -o - | FileCheck %s

// test that we generate address space casts everywhere we need conversions of
// pointers to different address spaces

void test(global int *arg_glob, generic int *arg_gen) {
  int var_priv;
  arg_gen = arg_glob; // implicit cast global -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
  arg_gen = &var_priv; // implicit cast with obtaining adr, private -> generic
  // CHECK: %{{[0-9]+}} = addrspacecast i32* %var_priv to i32 addrspace(4)*
  arg_glob = (global int *)arg_gen; // explicit cast
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(4)* %{{[0-9]+}} to i32 addrspace(1)*
  global int *var_glob =
      (global int *)arg_glob; // explicit cast in the same address space
  // CHECK-NOT: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(1)*
  var_priv = arg_gen - arg_glob; // arithmetic operation
  // CHECK: %sub.ptr.lhs.cast = ptrtoint i32 addrspace(4)* %6 to i64
  // CHECK: %sub.ptr.rhs.cast = ptrtoint i32 addrspace(1)* %7 to i64
  var_priv = arg_gen > arg_glob; // comparison
  // CHECK: %{{[0-9]+}} = addrspacecast i32 addrspace(1)* %{{[0-9]+}} to i32 addrspace(4)*
}


Fixed a bug with the pre-processing and the AXPY kernel

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the Xaxpy kernel. It contains one fast vectorized version in case of unit
// strides (incx=incy=1) and no offsets (offx=offy=0). Another version is more general, but doesn't
// support vector data-types. The general version has a batched implementation as well.
//
// This kernel uses the level-1 BLAS common tuning parameters.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Full version of the kernel with offsets and strided accesses
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void Xaxpy(const int n, const real_arg arg_alpha,
           const __global real* restrict xgm, const int x_offset, const int x_inc,
           __global real* ygm, const int y_offset, const int y_inc) {
  const real alpha = GetRealArg(arg_alpha);

  // Loops over the work that needs to be done (allows for an arbitrary number of threads)
  for (int id = get_global_id(0); id < n; id += get_global_size(0)) {
    real xvalue = xgm[id*x_inc + x_offset];
    MultiplyAdd(ygm[id*y_inc + y_offset], alpha, xvalue);
  }
}

// Faster version of the kernel without offsets and strided accesses but with if-statement. Also
// assumes that 'n' is dividable by 'VW' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XaxpyFaster(const int n, const real_arg arg_alpha,
                 const __global realV* restrict xgm,
                 __global realV* ygm) {
  const real alpha = GetRealArg(arg_alpha);

  const int num_usefull_threads = n / (VW * WPT);
  if (get_global_id(0) < num_usefull_threads) {
    #pragma unroll
    for (int _w = 0; _w < WPT; _w += 1) {
      const int id = _w*num_usefull_threads + get_global_id(0);
      realV xvalue = xgm[id];
      realV yvalue = ygm[id];
      ygm[id] = MultiplyAddVector(yvalue, alpha, xvalue);
    }
  }
}

// Faster version of the kernel without offsets and strided accesses. Also assumes that 'n' is
// dividable by 'VW', 'WGS' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XaxpyFastest(const int n, const real_arg arg_alpha,
                  const __global realV* restrict xgm,
                  __global realV* ygm) {
  const real alpha = GetRealArg(arg_alpha);

  #pragma unroll
  for (int _w = 0; _w < WPT; _w += 1) {
    const int id = _w*get_global_size(0) + get_global_id(0);
    realV xvalue = xgm[id];
    realV yvalue = ygm[id];
    ygm[id] = MultiplyAddVector(yvalue, alpha, xvalue);
  }
}

// =================================================================================================

// Full version of the kernel with offsets and strided accesses: batched version
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XaxpyBatched(const int n, const __constant real_arg* arg_alphas,
                  const __global real* restrict xgm, const __constant int* x_offsets, const int x_inc,
                  __global real* ygm, const __constant int* y_offsets, const int y_inc) {
  const int batch = get_group_id(1);
  const real alpha = GetRealArg(arg_alphas[batch]);

  // Loops over the work that needs to be done (allows for an arbitrary number of threads)
  for (int id = get_global_id(0); id < n; id += get_global_size(0)) {
    real xvalue = xgm[id*x_inc + x_offsets[batch]];
    MultiplyAdd(ygm[id*y_inc + y_offsets[batch]], alpha, xvalue);
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================

// =================================================================================================
// This file is part of the CLBlast project. The project is licensed under Apache Version 2.0. This
// project loosely follows the Google C++ styleguide and uses a tab-size of two spaces and a max-
// width of 100 characters per line.
//
// Author(s):
//   Cedric Nugteren <www.cedricnugteren.nl>
//
// This file contains the Xaxpy kernel. It contains one fast vectorized version in case of unit
// strides (incx=incy=1) and no offsets (offx=offy=0). Another version is more general, but doesn't
// support vector data-types. The general version has a batched implementation as well.
//
// This kernel uses the level-1 BLAS common tuning parameters.
//
// =================================================================================================

// Enables loading of this file using the C++ pre-processor's #include (C++11 standard raw string
// literal). Comment-out this line for syntax-highlighting when developing.
R"(

// =================================================================================================

// Full version of the kernel with offsets and strided accesses
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void Xaxpy(const int n, const real_arg arg_alpha,
           const __global real* restrict xgm, const int x_offset, const int x_inc,
           __global real* ygm, const int y_offset, const int y_inc) {
  const real alpha = GetRealArg(arg_alpha);

  // Loops over the work that needs to be done (allows for an arbitrary number of threads)
  for (int id = get_global_id(0); id < n; id += get_global_size(0)) {
    real xvalue = xgm[id*x_inc + x_offset];
    MultiplyAdd(ygm[id*y_inc + y_offset], alpha, xvalue);
  }
}

// Faster version of the kernel without offsets and strided accesses but with if-statement. Also
// assumes that 'n' is dividable by 'VW' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XaxpyFaster(const int n, const real_arg arg_alpha,
                 const __global realV* restrict xgm,
                 __global realV* ygm) {
  const real alpha = GetRealArg(arg_alpha);

  const int num_worker_threads = n / (VW * WPT);
  if (get_global_id(0) < num_worker_threads) {
    #pragma unroll
    for (int _w = 0; _w < WPT; _w += 1) {
      const int id = _w*num_worker_threads + get_global_id(0);
      realV xvalue = xgm[id];
      realV yvalue = ygm[id];
      ygm[id] = MultiplyAddVector(yvalue, alpha, xvalue);
    }
  }
}

// Faster version of the kernel without offsets and strided accesses. Also assumes that 'n' is
// dividable by 'VW', 'WGS' and 'WPT'.
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XaxpyFastest(const int n, const real_arg arg_alpha,
                  const __global realV* restrict xgm,
                  __global realV* ygm) {
  const real alpha = GetRealArg(arg_alpha);

  #pragma unroll
  for (int _w = 0; _w < WPT; _w += 1) {
    const int id = _w*get_global_size(0) + get_global_id(0);
    realV xvalue = xgm[id];
    realV yvalue = ygm[id];
    ygm[id] = MultiplyAddVector(yvalue, alpha, xvalue);
  }
}

// =================================================================================================

// Full version of the kernel with offsets and strided accesses: batched version
__kernel __attribute__((reqd_work_group_size(WGS, 1, 1)))
void XaxpyBatched(const int n, const __constant real_arg* arg_alphas,
                  const __global real* restrict xgm, const __constant int* x_offsets, const int x_inc,
                  __global real* ygm, const __constant int* y_offsets, const int y_inc) {
  const int batch = get_group_id(1);
  const real alpha = GetRealArg(arg_alphas[batch]);

  // Loops over the work that needs to be done (allows for an arbitrary number of threads)
  for (int id = get_global_id(0); id < n; id += get_global_size(0)) {
    real xvalue = xgm[id*x_inc + x_offsets[batch]];
    MultiplyAdd(ygm[id*y_inc + y_offsets[batch]], alpha, xvalue);
  }
}

// =================================================================================================

// End of the C++11 raw string literal
)"

// =================================================================================================