cl-bayer: fix 3a-stats calculation

cl-bayer: fix 3a-stats calculation

 * reduce sum on 3a-stats may get wrong index if l_id_x + count larger
   than STATS_3A_GRID_SIZE which is not the buf-pitch

Signed-off-by: Wind Yuan <d3b25194a13defcddf619d66a03bde7d96f8d487@intel.com>

/*
 * function: kernel_bayer_pipe
 * params:
 *   input:    image2d_t as read only
 *   output:   image2d_t as write only
 *   blc_config: black level correction configuration
 *   wb_config: whitebalance configuration
 *   gamma_table: RGGB table
 *   stats_output: 3a stats output
 */

#define GRID_X_SIZE 2
#define GRID_Y_SIZE 2

#define SHARED_PIXEL_WIDTH 16
#define SHARED_PIXEL_HEIGHT 16
#define SHARED_PIXEL_X_OFFSET 2
#define SHARED_PIXEL_Y_OFFSET 2
#define SHARED_PIXEL_X_SIZE  (SHARED_PIXEL_WIDTH + SHARED_PIXEL_X_OFFSET * 2)
#define SHARED_PIXEL_Y_SIZE  (SHARED_PIXEL_HEIGHT + SHARED_PIXEL_Y_OFFSET * 2)

#define SHARED_GRID_WIDTH (SHARED_PIXEL_WIDTH/GRID_X_SIZE)
#define SHARED_GRID_HEIGHT (SHARED_PIXEL_HEIGHT/GRID_Y_SIZE)
#define SHARED_GRID_X_OFFSET (SHARED_PIXEL_X_OFFSET/GRID_X_SIZE)
#define SHARED_GRID_Y_OFFSET (SHARED_PIXEL_Y_OFFSET/GRID_Y_SIZE)
#define SHARED_GRID_X_SIZE (SHARED_PIXEL_X_SIZE/GRID_X_SIZE)
#define SHARED_GRID_Y_SIZE (SHARED_PIXEL_Y_SIZE/GRID_Y_SIZE)

#define WORK_ITEM_X_SIZE GRID_X_SIZE
#define WORK_ITEM_Y_SIZE GRID_Y_SIZE

#define STATS_3A_GRID_SIZE (16/GRID_X_SIZE)

#define X 0
#define Y 1
#define Z 2
#define W 3

typedef struct  {
    float  level_gr;  /* Black level for GR pixels */
    float  level_r;   /* Black level for R pixels */
    float  level_b;   /* Black level for B pixels */
    float  level_gb;  /* Black level for GB pixels */
    uint   color_bits;
} CLBLCConfig;

typedef struct
{
    float r_gain;
    float gr_gain;
    float gb_gain;
    float b_gain;
} CLWBConfig;

typedef struct
{
    unsigned int avg_y;

    unsigned int avg_r;
    unsigned int avg_gr;
    unsigned int avg_gb;
    unsigned int avg_b;
    unsigned int valid_wb_count;

    unsigned int f_value1;
    unsigned int f_value2;
} XCamGridStat;

/* BA10=> GRBG  */
inline void blc (float4 *in_out, CLBLCConfig *blc_config)
{
    float multiplier = (float)(1 << (16 - blc_config->color_bits));
    in_out->x = in_out->x * multiplier - blc_config->level_gr;
    in_out->y = in_out->y * multiplier - blc_config->level_r;
    in_out->z = in_out->z * multiplier - blc_config->level_b;
    in_out->w = in_out->w * multiplier - blc_config->level_gb;
}

inline void wb (float4 *in_out, CLWBConfig *wbconfig)
{
    in_out->x *= wbconfig->gr_gain;
    in_out->y *= wbconfig->r_gain;
    in_out->z *= wbconfig->b_gain;
    in_out->w *= wbconfig->gb_gain;
}

inline void gamma_correct(float4 *in_out, __global float *table)
{
    in_out->x = table[clamp(convert_int(in_out->x * 255.0f), 0, 255)];
    in_out->y = table[clamp(convert_int(in_out->y * 255.0f), 0, 255)];
    in_out->z = table[clamp(convert_int(in_out->z * 255.0f), 0, 255)];
    in_out->w = table[clamp(convert_int(in_out->w * 255.0f), 0, 255)];
}

inline int get_shared_pos_x (int i)
{
    return i % SHARED_GRID_X_SIZE;
}

inline int get_shared_pos_y (int i)
{
    return i / SHARED_GRID_X_SIZE;
}

inline int shared_pos (int x, int y)
{
    return mad24(y, SHARED_GRID_X_SIZE, x);
}

/* BA10=> GRBG  */
inline float4 simple_calculate (
    __local float *px, __local float *py, __local float *pz, __local float *pw,
    int index, __read_only image2d_t input, sampler_t sampler, int x_start, int y_start,
    __local float4 *stats_cache,
    CLBLCConfig *blc_config,
    CLWBConfig *wb_config,
    uint enable_gamma,
    __global float *gamma_table)
{
    float4 data;
    int x0 = get_shared_pos_x (index) * WORK_ITEM_X_SIZE + x_start;
    int y0 = get_shared_pos_y (index) * WORK_ITEM_Y_SIZE + y_start;
    //Gr
    data.x = read_imagef (input, sampler, (int2)(x0, y0)).x;
    //R
    data.y = read_imagef (input, sampler, (int2)(x0 + 1, y0)).x;
    //B
    data.z = read_imagef (input, sampler, (int2)(x0, y0 + 1)).x;
    //Gb
    data.w = read_imagef (input, sampler, (int2)(x0 + 1, y0 + 1)).x;

    blc (&data, blc_config);

    /* write back for 3a stats calculation R, G, B, Y */
    stats_cache[index] = data;

    wb (&data, wb_config);
    if (enable_gamma)
        gamma_correct (&data, gamma_table);

    px[index] = data.x;
    py[index] = data.y;
    pz[index] = data.z;
    pw[index] = data.w;
}

#define MAX_DELTA_COFF 5.0f
#define MIN_DELTA_COFF 1.0f
#define DEFAULT_DELTA_COFF 4.0f

inline float delta_coff (float delta)
{
    float coff = MAX_DELTA_COFF - 20.0f * fabs(delta);
    return fmax (1.0f, coff);
}

inline float4
demosaic_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    out_data.y = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
                  in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = in_y[shared_pos(x, y)];
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
                  in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;
    return out_data;
}

inline float4
demosaic_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
                  in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    out_data.z = in_z[shared_pos(x, y)];
    return out_data;
}

inline float4
demosaic_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    out_data.y = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_denoise_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_y[shared_pos(x, y - 1)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
             in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    coff[1] = delta_coff(in_x[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_x[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y - 1)] * coff[1] +
                  in_x[shared_pos(x - 1, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x - 1, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = in_y[shared_pos(x, y)];
    coff[1] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x, y - 1)] * coff[1] +
                  in_y[shared_pos(x - 1, y)] * coff[2] +
                  in_y[shared_pos(x + 1, y)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y - 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
             in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;

    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
             in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_w[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = in_z[shared_pos(x, y)];
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x - 1, y)] * coff[2] +
                  in_z[shared_pos(x + 1, y)] * coff[3] +
                  in_z[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;
    return out_data;
};

inline float4
demosaic_denoise_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x + 1, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x + 1, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    coff[1] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y + 1)] - value);
    out_data.y = (in_w[shared_pos(x, y - 1)] * coff[1] +
                  in_w[shared_pos(x - 1, y)] * coff[2] +
                  in_w[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    value = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y + 1)] * coff[3] +
                  in_z[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

void shared_demosaic (
    __local float *x_data_in, __local float *y_data_in, __local float *z_data_in, __local float *w_data_in,
    int in_x, int in_y,
    __write_only image2d_t out, int out_x, int out_y,
    uint has_denoise)
{
    float4 out_data;

    if (has_denoise) {
        out_data = demosaic_denoise_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_denoise_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_denoise_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_denoise_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    } else {
        out_data = demosaic_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    }

}

inline void stats_3a_calculate (
    __local float4 * input,
    __global XCamGridStat * stats_output,
    CLWBConfig *wb_config)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE / 2;

    for (; count > 0; count /= 2) {
        if ((l_id_x % STATS_3A_GRID_SIZE) + (l_id_y % STATS_3A_GRID_SIZE)* STATS_3A_GRID_SIZE < count) {
            int index1 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET);
            int index2 = shared_pos (SHARED_GRID_X_OFFSET + ((l_id_x + count) % STATS_3A_GRID_SIZE),
                                     SHARED_GRID_Y_OFFSET + l_id_y + (l_id_x + count) / STATS_3A_GRID_SIZE);
            input[index1] = (input[index1] + input[index2]) / 2.0f;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    if (l_id_x % STATS_3A_GRID_SIZE == 0 && l_id_y % STATS_3A_GRID_SIZE == 0) {
        float4 tmp_data;
        int out_index = mad24(g_id_y / STATS_3A_GRID_SIZE,  g_size_x / STATS_3A_GRID_SIZE, g_id_x / STATS_3A_GRID_SIZE);
        tmp_data = input[shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET)];
        stats_output[out_index].avg_gr = convert_uchar_sat(tmp_data.x * 255.0f);
        stats_output[out_index].avg_r = convert_uchar_sat(tmp_data.y * 255.0f);
        stats_output[out_index].avg_b = convert_uchar_sat(tmp_data.z * 255.0f);
        stats_output[out_index].avg_gb = convert_uchar_sat(tmp_data.w * 255.0f);
        stats_output[out_index].valid_wb_count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE;
        stats_output[out_index].avg_y =
            convert_uchar_sat(((tmp_data.x * wb_config->gr_gain + tmp_data.w * wb_config->gb_gain) * 0.2935f +
                               tmp_data.y * wb_config->r_gain * 0.299f + tmp_data.z * wb_config->b_gain * 0.114f) * 255.0f);
        stats_output[out_index].f_value1 = 0;
        stats_output[out_index].f_value2 = 0;
    }
}

__kernel void kernel_bayer_pipe (__read_only image2d_t input,
                                 __write_only image2d_t output,
                                 CLBLCConfig blc_config,
                                 CLWBConfig wb_config,
                                 uint has_denoise,
                                 uint enable_gamma,
                                 __global float * gamma_table,
                                 __global XCamGridStat * stats_output)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int l_size_x = get_local_size (0);
    int l_size_y = get_local_size (1);

    sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

    __local float p1_x[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_y[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_z[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_w[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 p2[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 *stats_cache = p2;

    int out_x_start, out_y_start;
    int x_start = (g_id_x - l_id_x) * WORK_ITEM_X_SIZE - SHARED_PIXEL_X_OFFSET;
    int y_start = (g_id_y - l_id_y) * WORK_ITEM_Y_SIZE - SHARED_PIXEL_Y_OFFSET;
    int i = l_id_x + l_id_y * l_size_x;

    for (; i < SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE; i += l_size_x * l_size_y) {
        simple_calculate (p1_x, p1_y, p1_z, p1_w, i,
                          input, sampler, x_start, y_start,
                          stats_cache,
                          &blc_config,
                          &wb_config,
                          enable_gamma,
                          gamma_table);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    stats_3a_calculate (stats_cache, stats_output, &wb_config);

    shared_demosaic (
        p1_x, p1_y, p1_z, p1_w, l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET,
        output, g_id_x * WORK_ITEM_X_SIZE, g_id_y * WORK_ITEM_Y_SIZE, has_denoise);
}

/*
 * function: kernel_bayer_pipe
 * params:
 *   input:    image2d_t as read only
 *   output:   image2d_t as write only
 *   blc_config: black level correction configuration
 *   wb_config: whitebalance configuration
 *   gamma_table: RGGB table
 *   stats_output: 3a stats output
 */

#define GRID_X_SIZE 2
#define GRID_Y_SIZE 2

#define SHARED_PIXEL_WIDTH 16
#define SHARED_PIXEL_HEIGHT 16
#define SHARED_PIXEL_X_OFFSET 2
#define SHARED_PIXEL_Y_OFFSET 2
#define SHARED_PIXEL_X_SIZE  (SHARED_PIXEL_WIDTH + SHARED_PIXEL_X_OFFSET * 2)
#define SHARED_PIXEL_Y_SIZE  (SHARED_PIXEL_HEIGHT + SHARED_PIXEL_Y_OFFSET * 2)

#define SHARED_GRID_WIDTH (SHARED_PIXEL_WIDTH/GRID_X_SIZE)
#define SHARED_GRID_HEIGHT (SHARED_PIXEL_HEIGHT/GRID_Y_SIZE)
#define SHARED_GRID_X_OFFSET (SHARED_PIXEL_X_OFFSET/GRID_X_SIZE)
#define SHARED_GRID_Y_OFFSET (SHARED_PIXEL_Y_OFFSET/GRID_Y_SIZE)
#define SHARED_GRID_X_SIZE (SHARED_PIXEL_X_SIZE/GRID_X_SIZE)
#define SHARED_GRID_Y_SIZE (SHARED_PIXEL_Y_SIZE/GRID_Y_SIZE)

#define WORK_ITEM_X_SIZE GRID_X_SIZE
#define WORK_ITEM_Y_SIZE GRID_Y_SIZE

#define STATS_3A_GRID_SIZE (16/GRID_X_SIZE)

#define X 0
#define Y 1
#define Z 2
#define W 3

typedef struct  {
    float  level_gr;  /* Black level for GR pixels */
    float  level_r;   /* Black level for R pixels */
    float  level_b;   /* Black level for B pixels */
    float  level_gb;  /* Black level for GB pixels */
    uint   color_bits;
} CLBLCConfig;

typedef struct
{
    float r_gain;
    float gr_gain;
    float gb_gain;
    float b_gain;
} CLWBConfig;

typedef struct
{
    unsigned int avg_y;

    unsigned int avg_r;
    unsigned int avg_gr;
    unsigned int avg_gb;
    unsigned int avg_b;
    unsigned int valid_wb_count;

    unsigned int f_value1;
    unsigned int f_value2;
} XCamGridStat;

/* BA10=> GRBG  */
inline void blc (float4 *in_out, CLBLCConfig *blc_config)
{
    float multiplier = (float)(1 << (16 - blc_config->color_bits));
    in_out->x = in_out->x * multiplier - blc_config->level_gr;
    in_out->y = in_out->y * multiplier - blc_config->level_r;
    in_out->z = in_out->z * multiplier - blc_config->level_b;
    in_out->w = in_out->w * multiplier - blc_config->level_gb;
}

inline void wb (float4 *in_out, CLWBConfig *wbconfig)
{
    in_out->x *= wbconfig->gr_gain;
    in_out->y *= wbconfig->r_gain;
    in_out->z *= wbconfig->b_gain;
    in_out->w *= wbconfig->gb_gain;
}

inline void gamma_correct(float4 *in_out, __global float *table)
{
    in_out->x = table[clamp(convert_int(in_out->x * 255.0f), 0, 255)];
    in_out->y = table[clamp(convert_int(in_out->y * 255.0f), 0, 255)];
    in_out->z = table[clamp(convert_int(in_out->z * 255.0f), 0, 255)];
    in_out->w = table[clamp(convert_int(in_out->w * 255.0f), 0, 255)];
}

inline int get_shared_pos_x (int i)
{
    return i % SHARED_GRID_X_SIZE;
}

inline int get_shared_pos_y (int i)
{
    return i / SHARED_GRID_X_SIZE;
}

inline int shared_pos (int x, int y)
{
    return mad24(y, SHARED_GRID_X_SIZE, x);
}

/* BA10=> GRBG  */
inline float4 simple_calculate (
    __local float *px, __local float *py, __local float *pz, __local float *pw,
    int index, __read_only image2d_t input, sampler_t sampler, int x_start, int y_start,
    __local float4 *stats_cache,
    CLBLCConfig *blc_config,
    CLWBConfig *wb_config,
    uint enable_gamma,
    __global float *gamma_table)
{
    float4 data;
    int x0 = get_shared_pos_x (index) * WORK_ITEM_X_SIZE + x_start;
    int y0 = get_shared_pos_y (index) * WORK_ITEM_Y_SIZE + y_start;
    //Gr
    data.x = read_imagef (input, sampler, (int2)(x0, y0)).x;
    //R
    data.y = read_imagef (input, sampler, (int2)(x0 + 1, y0)).x;
    //B
    data.z = read_imagef (input, sampler, (int2)(x0, y0 + 1)).x;
    //Gb
    data.w = read_imagef (input, sampler, (int2)(x0 + 1, y0 + 1)).x;

    blc (&data, blc_config);

    /* write back for 3a stats calculation R, G, B, Y */
    stats_cache[index] = data;

    wb (&data, wb_config);
    if (enable_gamma)
        gamma_correct (&data, gamma_table);

    px[index] = data.x;
    py[index] = data.y;
    pz[index] = data.z;
    pw[index] = data.w;
}

#define MAX_DELTA_COFF 5.0f
#define MIN_DELTA_COFF 1.0f
#define DEFAULT_DELTA_COFF 4.0f

inline float delta_coff (float delta)
{
    float coff = MAX_DELTA_COFF - 20.0f * fabs(delta);
    return fmax (1.0f, coff);
}

inline float4
demosaic_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    out_data.y = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
                  in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = in_y[shared_pos(x, y)];
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
                  in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;
    return out_data;
}

inline float4
demosaic_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
                  in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    out_data.z = in_z[shared_pos(x, y)];
    return out_data;
}

inline float4
demosaic_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    out_data.y = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_denoise_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_y[shared_pos(x, y - 1)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
             in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    coff[1] = delta_coff(in_x[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_x[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y - 1)] * coff[1] +
                  in_x[shared_pos(x - 1, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x - 1, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = in_y[shared_pos(x, y)];
    coff[1] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x, y - 1)] * coff[1] +
                  in_y[shared_pos(x - 1, y)] * coff[2] +
                  in_y[shared_pos(x + 1, y)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y - 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
             in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;

    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
             in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_w[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = in_z[shared_pos(x, y)];
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x - 1, y)] * coff[2] +
                  in_z[shared_pos(x + 1, y)] * coff[3] +
                  in_z[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;
    return out_data;
};

inline float4
demosaic_denoise_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x + 1, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x + 1, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    coff[1] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y + 1)] - value);
    out_data.y = (in_w[shared_pos(x, y - 1)] * coff[1] +
                  in_w[shared_pos(x - 1, y)] * coff[2] +
                  in_w[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    value = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y + 1)] * coff[3] +
                  in_z[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

void shared_demosaic (
    __local float *x_data_in, __local float *y_data_in, __local float *z_data_in, __local float *w_data_in,
    int in_x, int in_y,
    __write_only image2d_t out, int out_x, int out_y,
    uint has_denoise)
{
    float4 out_data;

    if (has_denoise) {
        out_data = demosaic_denoise_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_denoise_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_denoise_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_denoise_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    } else {
        out_data = demosaic_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    }

}

inline void stats_3a_calculate (
    __local float4 * input,
    __global XCamGridStat * stats_output,
    CLWBConfig *wb_config)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE / 2;

    for (; count > 0; count /= 2) {
        if ((l_id_x % STATS_3A_GRID_SIZE) + (l_id_y % STATS_3A_GRID_SIZE)* STATS_3A_GRID_SIZE < count) {
            int index1 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET);
            int index2 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET + count % STATS_3A_GRID_SIZE,
                                     l_id_y + SHARED_GRID_Y_OFFSET + count / STATS_3A_GRID_SIZE);
            //input[index1].x = (input[index1].x + input[index2].x) / 2.0f;
            //input[index1].y = (input[index1].y + input[index2].y) / 2.0f;
            //input[index1].z = (input[index1].z + input[index2].z) / 2.0f;
            //input[index1].w = (input[index1].w + input[index2].w) / 2.0f;
            input[index1] = (input[index1] + input[index2]) / 2.0f;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    if (l_id_x % STATS_3A_GRID_SIZE == 0 && l_id_y % STATS_3A_GRID_SIZE == 0) {
        float4 tmp_data;
        int out_index = mad24(g_id_y / STATS_3A_GRID_SIZE,  g_size_x / STATS_3A_GRID_SIZE, g_id_x / STATS_3A_GRID_SIZE);
        tmp_data = input[shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET)];
        stats_output[out_index].avg_gr = convert_uchar_sat(tmp_data.x * 255.0f);
        stats_output[out_index].avg_r = convert_uchar_sat(tmp_data.y * 255.0f);
        stats_output[out_index].avg_b = convert_uchar_sat(tmp_data.z * 255.0f);
        stats_output[out_index].avg_gb = convert_uchar_sat(tmp_data.w * 255.0f);
        stats_output[out_index].valid_wb_count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE;
        stats_output[out_index].avg_y =
            convert_uchar_sat(((tmp_data.x * wb_config->gr_gain + tmp_data.w * wb_config->gb_gain) * 0.2935f +
                               tmp_data.y * wb_config->r_gain * 0.299f + tmp_data.z * wb_config->b_gain * 0.114f) * 255.0f);
        stats_output[out_index].f_value1 = 0;
        stats_output[out_index].f_value2 = 0;
    }
}

__kernel void kernel_bayer_pipe (__read_only image2d_t input,
                                 __write_only image2d_t output,
                                 CLBLCConfig blc_config,
                                 CLWBConfig wb_config,
                                 uint has_denoise,
                                 uint enable_gamma,
                                 __global float * gamma_table,
                                 __global XCamGridStat * stats_output)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int l_size_x = get_local_size (0);
    int l_size_y = get_local_size (1);

    sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

    __local float p1_x[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_y[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_z[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_w[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 p2[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 *stats_cache = p2;

    int out_x_start, out_y_start;
    int x_start = (g_id_x - l_id_x) * WORK_ITEM_X_SIZE - SHARED_PIXEL_X_OFFSET;
    int y_start = (g_id_y - l_id_y) * WORK_ITEM_Y_SIZE - SHARED_PIXEL_Y_OFFSET;
    int i = l_id_x + l_id_y * l_size_x;

    for (; i < SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE; i += l_size_x * l_size_y) {
        simple_calculate (p1_x, p1_y, p1_z, p1_w, i,
                          input, sampler, x_start, y_start,
                          stats_cache,
                          &blc_config,
                          &wb_config,
                          enable_gamma,
                          gamma_table);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    stats_3a_calculate (stats_cache, stats_output, &wb_config);

    shared_demosaic (
        p1_x, p1_y, p1_z, p1_w, l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET,
        output, g_id_x * WORK_ITEM_X_SIZE, g_id_y * WORK_ITEM_Y_SIZE, has_denoise);
}


try moving const assign

try moving const assign

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [imageid][outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
void kernel propagate_4_by_n_outplane_smallercache( const int batchSize,
    const int pixelsPerThread,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube, local float *_pixelSums ) {
    const int globalId = get_global_id(0);

    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int n = workgroupId / gNumFilters;
    const int outPlane = workgroupId % gNumFilters;

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int numFilterPixelsPerThread = ( gFilterSizeSquared + workgroupSize - 1 ) / workgroupSize;

    local float *_myPixelSums = _pixelSums + pixelsPerThread * localId;

    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        _myPixelSums[pixel] = 0.0f;
    }

    const int outputRow = localId / gOutputBoardSize;
    const int outputCol = localId % gOutputBoardSize;

    const int minu = gPadZeros ? max( -gHalfFilterSize, -outputRow ) : -gHalfFilterSize;
    const int maxu = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputRow  - evenPadding) : gHalfFilterSize - evenPadding;
    const int minv = gPadZeros ? max( -gHalfFilterSize, -outputCol ) : - gHalfFilterSize;
    const int maxv = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputCol - evenPadding) : gHalfFilterSize - evenPadding;

    for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
        int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int i = 0; i < numUpstreamsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gInputBoardSizeSquared ) {
                _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
            }
        }
        const int filterGlobalOffset = ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared;
        for( int i = 0; i < numFilterPixelsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gFilterSizeSquared ) {
                _filterCube[thisOffset] = filters[filterGlobalOffset + thisOffset];
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
            const int virtualLocalId = localId + pixel * workgroupSize;

            float thissum = 0;
            if( virtualLocalId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u + ( gPadZeros ? 0 : gHalfFilterSize );
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v + ( gPadZeros ? 0 : gHalfFilterSize );
                        thissum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
            _myPixelSums[pixel] += thissum;
//            if( globalId == 0 ) results[0] = _upstreamBoard[pixel];
        }
    }
    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        const int virtualLocalId = localId + pixel * workgroupSize;
        if( virtualLocalId < gOutputBoardSizeSquared ) {
            float sum = _myPixelSums[pixel];
            #ifdef BIASED
                sum += biases[outPlane];
            #endif
            // results are organized like [imageid][filterid][row][col]
            int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + virtualLocalId;
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
            // results[resultIndex ] = 123;
            //if( globalId == 0 ) results[0] += 0.000001f + _perPixelSums[0];
        }
    }
}
#endif
#endif

// Copyright Hugh Perkins 2014, 2015 hughperkins at gmail
//
// This Source Code Form is subject to the terms of the Mozilla Public License,
// v. 2.0. If a copy of the MPL was not distributed with this file, You can
// obtain one at http://mozilla.org/MPL/2.0/.

// expected defines:
// one of: [ TANH | RELU | LINEAR ]
// BIASED (or not)

#ifdef TANH
    #define ACTIVATION_FUNCTION(output) (tanh(output))
#elif defined SCALEDTANH
    #define ACTIVATION_FUNCTION(output) ( 1.7159f * tanh( 0.66667f * output))
#elif SIGMOID
    #define ACTIVATION_FUNCTION(output) (1.0f / (1 + exp(-output)))
#elif defined RELU
    #define ACTIVATION_FUNCTION(output) (output> 0 ? output : 0)
#elif defined LINEAR
    #define ACTIVATION_FUNCTION(output) (output)
#endif

#ifdef gOutputBoardSize // for previous tests that dont define it
#ifdef ACTIVATION_FUNCTION // protect against not defined
// workgroup id organized like: [imageid][outplane]
// local id organized like: [outrow][outcol]
// each thread iterates over: [upstreamplane][filterrow][filtercol]
// number workgroups = 32
// one filter plane takes up 5 * 5 * 4 = 100 bytes
// one filter cube (corresponding to one outplane) = 5*5 * 32 * 4 = 3.2KB (ok)
// all filter cubes = 3.2KB * 32 = 102KB (too big)
// results are organized like [imageid][filterid][row][col]
void kernel propagate_4_by_n_outplane_smallercache( const int batchSize,
    const int pixelsPerThread,
      global const float *images, global const float *filters,
        #ifdef BIASED
            global const float*biases,
        #endif
    global float *results,
    local float *_upstreamBoard, local float *_filterCube, local float *_pixelSums ) {
    const int globalId = get_global_id(0);

    const int evenPadding = gFilterSize % 2 == 0 ? 1 : 0;

    const int localId = get_local_id(0);
    const int workgroupId = get_group_id(0);
    const int workgroupSize = get_local_size(0);
    const int n = workgroupId / gNumFilters;
    const int outPlane = workgroupId % gNumFilters;

    const int numUpstreamsPerThread = ( gInputBoardSizeSquared + workgroupSize - 1 ) / workgroupSize;
    const int numFilterPixelsPerThread = ( gFilterSizeSquared + workgroupSize - 1 ) / workgroupSize;

    local float *_myPixelSums = _pixelSums + pixelsPerThread * localId;

    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        _myPixelSums[pixel] = 0.0f;
    }

    for( int upstreamPlane = 0; upstreamPlane < gInputPlanes; upstreamPlane++ ) {
        int thisUpstreamBoardOffset = ( n * gInputPlanes + upstreamPlane ) * gInputBoardSizeSquared;
        barrier(CLK_LOCAL_MEM_FENCE);
        for( int i = 0; i < numUpstreamsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gInputBoardSizeSquared ) {
                _upstreamBoard[ thisOffset ] = images[ thisUpstreamBoardOffset + thisOffset ];
            }
        }
        const int filterGlobalOffset = ( outPlane * gInputPlanes + upstreamPlane ) * gFilterSizeSquared;
        for( int i = 0; i < numFilterPixelsPerThread; i++ ) {
            int thisOffset = workgroupSize * i + localId;
            if( thisOffset < gFilterSizeSquared ) {
                _filterCube[thisOffset] = filters[filterGlobalOffset + thisOffset];
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
            const int virtualLocalId = localId + pixel * workgroupSize;

            const int outputRow = virtualLocalId / gOutputBoardSize;
            const int outputCol = virtualLocalId % gOutputBoardSize;

            const int minu = gPadZeros ? max( -gHalfFilterSize, -outputRow ) : -gHalfFilterSize;
            const int maxu = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputRow  - evenPadding) : gHalfFilterSize - evenPadding;
            const int minv = gPadZeros ? max( -gHalfFilterSize, -outputCol ) : - gHalfFilterSize;
            const int maxv = gPadZeros ? min( gHalfFilterSize - evenPadding, gOutputBoardSize - 1 - outputCol - evenPadding) : gHalfFilterSize - evenPadding;

            float thissum = 0;
            if( virtualLocalId < gOutputBoardSizeSquared ) {
                for( int u = minu; u <= maxu; u++ ) {
                    int inputRow = outputRow + u + ( gPadZeros ? 0 : gHalfFilterSize );
                    int inputboardrowoffset = inputRow * gInputBoardSize;
                    int filterrowoffset = (u+gHalfFilterSize) * gFilterSize + gHalfFilterSize;
                    for( int v = minv; v <= maxv; v++ ) {
                        int inputCol = outputCol + v + ( gPadZeros ? 0 : gHalfFilterSize );
                        thissum += _upstreamBoard[ inputboardrowoffset + inputCol] * _filterCube[ filterrowoffset + v ];
                    }
                }
            }
            _myPixelSums[pixel] += thissum;
//            if( globalId == 0 ) results[0] = _upstreamBoard[pixel];
        }
    }
    for( int pixel = 0; pixel < pixelsPerThread; pixel++ ) {
        const int virtualLocalId = localId + pixel * workgroupSize;
        if( virtualLocalId < gOutputBoardSizeSquared ) {
            float sum = _myPixelSums[pixel];
            #ifdef BIASED
                sum += biases[outPlane];
            #endif
            // results are organized like [imageid][filterid][row][col]
            int resultIndex = ( n * gNumFilters + outPlane ) * gOutputBoardSizeSquared + virtualLocalId;
            results[resultIndex ] = ACTIVATION_FUNCTION(sum);
            // results[resultIndex ] = 123;
            //if( globalId == 0 ) results[0] += 0.000001f + _perPixelSums[0];
        }
    }
}
#endif
#endif


correctness fix

correctness fix


removed a comment

removed a comment

#ifdef TYPE_FLOAT
#define TYPE float4
#define BUFFER_TYPE float
#define READ_IMAGE read_imagef
#define WRITE_IMAGE write_imagef
#define MAX_VALUE FLT_MAX
#define MIN_VALUE FLT_MIN
#elif TYPE_UINT8
#define TYPE uint4
#define BUFFER_TYPE uchar
#define READ_IMAGE read_imageui
#define WRITE_IMAGE write_imageui
#define MAX_VALUE UCHAR_MAX
#define MIN_VALUE 0
#elif TYPE_INT8
#define TYPE int4
#define BUFFER_TYPE char
#define READ_IMAGE read_imagei
#define WRITE_IMAGE write_imagei
#define MAX_VALUE CHAR_MAX
#define MIN_VALUE CHAR_MIN
#elif TYPE_UINT16
#define TYPE uint4
#define BUFFER_TYPE ushort
#define READ_IMAGE read_imageui
#define WRITE_IMAGE write_imageui
#define MAX_VALUE USHRT_MAX
#define MIN_VALUE 0
#else
#define TYPE int4
#define BUFFER_TYPE short
#define READ_IMAGE read_imagei
#define WRITE_IMAGE write_imagei
#define MAX_VALUE SHRT_MAX
#define MIN_VALUE SHRT_MIN
#endif

__constant int2 offset2D[4] = {
                   {0,0},
                   {0,1},
                   {1,0},
                   {1,1}
};

__constant int4 offset3D[8] = {
                   {0,0,0,0},
                   {1,0,0,0},
                   {0,1,0,0},
                   {0,0,1,0},
                   {1,0,1,0},
                   {1,1,1,0},
                   {0,1,1,0},
                   {1,1,0,0}
};

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST;

__kernel void createFirstMinMaxImage2DLevel(
        __read_only image2d_t image,
        __write_only image2d_t firstLevel
        ) {
    const int2 pos = {get_global_id(0), get_global_id(1)};
    const int2 readPos = pos*2;
    const int2 size = {get_image_width(image), get_image_height(image)};

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    for(int i = 0; i < 4; i++) {
        TYPE temp;
        temp = READ_IMAGE(image, sampler, select((int2)(0,0), readPos+offset2D[i], readPos+offset2D[i] < size));
        result.x = min(result.x, temp.x);
        result.y = max(result.y, temp.x);
    }

    WRITE_IMAGE(firstLevel, pos, result);
}

__kernel void createMinMaxImage2DLevel(
        __read_only image2d_t readLevel,
        __write_only image2d_t writeLevel
        ) {
    const int2 pos = {get_global_id(0), get_global_id(1)};
    const int2 readPos = pos*2;

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    TYPE temp;
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[0]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[1]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[2]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[3]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);

    WRITE_IMAGE(writeLevel, pos, result);
}

#ifdef cl_khr_3d_image_writes
__kernel void createFirstMinMaxImage3DLevel(
        __read_only image3d_t image,
        __write_only image3d_t firstLevel
        ) {
    const int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    const int4 readPos = pos*2;
    const int4 size = {get_image_width(image), get_image_height(image), get_image_depth(image), 1};

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    for(int i = 0; i < 8; i++) {
        TYPE temp;
        temp = READ_IMAGE(image, sampler,  select((int4)(0,0,0,0), readPos+offset3D[i], readPos+offset3D[i] < size));
        result.x = min(result.x, temp.x);
        result.y = max(result.y, temp.x);
    }

    WRITE_IMAGE(firstLevel, pos, result);
}

__kernel void createMinMaxImage3DLevel(
        __read_only image3d_t readLevel,
        __write_only image3d_t writeLevel
        ) {
    const int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    const int4 readPos = pos*2;

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    for(int i = 0; i < 8; i++) {
        TYPE temp;
        temp = READ_IMAGE(readLevel, sampler, readPos + offset3D[i]);
        result.x = min(result.x, temp.x);
        result.y = max(result.y, temp.y);
    }

    WRITE_IMAGE(writeLevel, pos, result);
}
#endif

__kernel void reduce(
        __global BUFFER_TYPE* buffer,
        __local BUFFER_TYPE* minScratch,
        __local BUFFER_TYPE* maxScratch,
        __private int length,
        __private int X,
        __global BUFFER_TYPE* result) {

    int global_index = get_global_id(0)*X;
    BUFFER_TYPE minAccumulator = MAX_VALUE;
    BUFFER_TYPE maxAccumulator = MIN_VALUE;
    // Loop sequentially over chunks of input vector
    for(int i = 0; i < X && global_index < length; i++) {
        float element = buffer[global_index];
        minAccumulator = (minAccumulator < element) ? minAccumulator : element;
        maxAccumulator = (maxAccumulator > element) ? maxAccumulator : element;
        global_index += 1;
    }

    // Perform parallel reduction
    int local_index = get_local_id(0);
    minScratch[local_index] = minAccumulator;
    maxScratch[local_index] = maxAccumulator;
    barrier(CLK_LOCAL_MEM_FENCE);
    for(int offset = get_local_size(0) / 2; offset > 0; offset = offset / 2) {
        if(local_index < offset) {
          BUFFER_TYPE other = minScratch[local_index + offset];
          BUFFER_TYPE mine = minScratch[local_index];
          minScratch[local_index] = (mine < other) ? mine : other;
          other = maxScratch[local_index + offset];
          mine = maxScratch[local_index];
          maxScratch[local_index] = (mine > other) ? mine : other;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    if(local_index == 0) {
        result[get_group_id(0)*2] = minScratch[0];
        result[get_group_id(0)*2+1] = maxScratch[0];
    }
}

#ifdef TYPE_FLOAT
#define TYPE float4
#define BUFFER_TYPE float
#define READ_IMAGE read_imagef
#define WRITE_IMAGE write_imagef
#define MAX_VALUE FLT_MAX
#define MIN_VALUE FLT_MIN
#elif TYPE_UINT8
#define TYPE uint4
#define BUFFER_TYPE uchar
#define READ_IMAGE read_imageui
#define WRITE_IMAGE write_imageui
#define MAX_VALUE UCHAR_MAX
#define MIN_VALUE 0
#elif TYPE_INT8
#define TYPE int4
#define BUFFER_TYPE char
#define READ_IMAGE read_imagei
#define WRITE_IMAGE write_imagei
#define MAX_VALUE CHAR_MAX
#define MIN_VALUE CHAR_MIN
#elif TYPE_UINT16
#define TYPE uint4
#define BUFFER_TYPE ushort
#define READ_IMAGE read_imageui
#define WRITE_IMAGE write_imageui
#define MAX_VALUE USHRT_MAX
#define MIN_VALUE 0
#else
#define TYPE int4
#define BUFFER_TYPE short
#define READ_IMAGE read_imagei
#define WRITE_IMAGE write_imagei
#define MAX_VALUE SHRT_MAX
#define MIN_VALUE SHRT_MIN
#endif

__constant int2 offset2D[4] = {
                   {0,0},
                   {0,1},
                   {1,0},
                   {1,1}
};

__constant int4 offset3D[8] = {
                   {0,0,0,0},
                   {1,0,0,0},
                   {0,1,0,0},
                   {0,0,1,0},
                   {1,0,1,0},
                   {1,1,1,0},
                   {0,1,1,0},
                   {1,1,0,0}
};

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST;

__kernel void createFirstMinMaxImage2DLevel(
        __read_only image2d_t image,
        __write_only image2d_t firstLevel
        ) {
    const int2 pos = {get_global_id(0), get_global_id(1)};
    const int2 readPos = pos*2;
    const int2 size = {get_image_width(image), get_image_height(image)};

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    // Check if in bounds
    for(int i = 0; i < 4; i++) {
        TYPE temp;
        temp = READ_IMAGE(image, sampler, select((int2)(0,0), readPos+offset2D[i], readPos+offset2D[i] < size));
        result.x = min(result.x, temp.x);
        result.y = max(result.y, temp.x);
    }

    WRITE_IMAGE(firstLevel, pos, result);
}

__kernel void createMinMaxImage2DLevel(
        __read_only image2d_t readLevel,
        __write_only image2d_t writeLevel
        ) {
    const int2 pos = {get_global_id(0), get_global_id(1)};
    const int2 readPos = pos*2;

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    TYPE temp;
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[0]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[1]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[2]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);
    temp = READ_IMAGE(readLevel, sampler, readPos + offset2D[3]);
    result.x = min(result.x, temp.x);
    result.y = max(result.y, temp.y);

    WRITE_IMAGE(writeLevel, pos, result);
}

#ifdef cl_khr_3d_image_writes
__kernel void createFirstMinMaxImage3DLevel(
        __read_only image3d_t image,
        __write_only image3d_t firstLevel
        ) {
    const int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    const int4 readPos = pos*2;
    const int4 size = {get_image_width(image), get_image_height(image), get_image_depth(image), 1};

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    for(int i = 0; i < 8; i++) {
        TYPE temp;
        temp = READ_IMAGE(image, sampler,  select((int4)(0,0,0,0), readPos+offset3D[i], readPos+offset3D[i] < size));
        result.x = min(result.x, temp.x);
        result.y = max(result.y, temp.x);
    }

    WRITE_IMAGE(firstLevel, pos, result);
}

__kernel void createMinMaxImage3DLevel(
        __read_only image3d_t readLevel,
        __write_only image3d_t writeLevel
        ) {
    const int4 pos = {get_global_id(0), get_global_id(1), get_global_id(2), 0};
    const int4 readPos = pos*2;

    TYPE result = {MAX_VALUE, MIN_VALUE,0,0};
    for(int i = 0; i < 8; i++) {
        TYPE temp;
        temp = READ_IMAGE(readLevel, sampler, readPos + offset3D[i]);
        result.x = min(result.x, temp.x);
        result.y = max(result.y, temp.y);
    }

    WRITE_IMAGE(writeLevel, pos, result);
}
#endif

__kernel void reduce(
        __global BUFFER_TYPE* buffer,
        __local BUFFER_TYPE* minScratch,
        __local BUFFER_TYPE* maxScratch,
        __private int length,
        __private int X,
        __global BUFFER_TYPE* result) {

    int global_index = get_global_id(0)*X;
    BUFFER_TYPE minAccumulator = MAX_VALUE;
    BUFFER_TYPE maxAccumulator = MIN_VALUE;
    // Loop sequentially over chunks of input vector
    for(int i = 0; i < X && global_index < length; i++) {
        float element = buffer[global_index];
        minAccumulator = (minAccumulator < element) ? minAccumulator : element;
        maxAccumulator = (maxAccumulator > element) ? maxAccumulator : element;
        global_index += 1;
    }

    // Perform parallel reduction
    int local_index = get_local_id(0);
    minScratch[local_index] = minAccumulator;
    maxScratch[local_index] = maxAccumulator;
    barrier(CLK_LOCAL_MEM_FENCE);
    for(int offset = get_local_size(0) / 2; offset > 0; offset = offset / 2) {
        if(local_index < offset) {
          BUFFER_TYPE other = minScratch[local_index + offset];
          BUFFER_TYPE mine = minScratch[local_index];
          minScratch[local_index] = (mine < other) ? mine : other;
          other = maxScratch[local_index + offset];
          mine = maxScratch[local_index];
          maxScratch[local_index] = (mine > other) ? mine : other;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
    if(local_index == 0) {
        result[get_group_id(0)*2] = minScratch[0];
        result[get_group_id(0)*2+1] = maxScratch[0];
    }
}


Delete prngcl_xor128.cl

Delete prngcl_xor128.cl

/******************************************************************************
 * @file     prngcl_xor128.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.0
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of XOR128 pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   George Marsaglia,
 *   "Xorshift RNGs",
 *   Journal of Statistical Software, 8, 14 (2003), 1--6.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_XOR128_CL
#define PRNGCL_XOR128_CL

#include "prngcl_common.cl"

#define XOR128_m_FP   (4294967296.0f)
#define XOR128_min    1
#define XOR128_max    4294967295
#define XOR128_k      (2.3283064365386962890625E-10) // 1/2^32

//________________________________________________________________________________________________________ XOR128 PRNG
__attribute__((always_inline)) uint4
xor128_step(uint4 seed)
{
    uint4 result;
    uint t = (seed.x^(seed.x<<11));

	result.x = seed.y;
	result.y = seed.z;
	result.z = seed.w;
	result.w = (seed.w^(seed.w>>19))^(t^(t>>8));

    return result;
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double
xor128_step_double(uint4* seed)
{
    hgpu_double rnd_double;
    uint rnd1, rnd2;
    uint4 sed = (*seed);
    bool flag = true;
    while (flag) {
        sed = xor128_step(sed);
        rnd1 = sed.w;
        if ((rnd1>XOR128_min) && (rnd1<XOR128_max)) {
            sed = xor128_step(sed);
            rnd2 = sed.w;
            rnd_double = hgpu_uint_to_double(rnd1,rnd2,XOR128_min,XOR128_max,XOR128_k);
            flag = false;
        }
    }
    (*seed) = sed;
    return rnd_double;
}
#endif

__kernel void
xor128(__global uint4* seed_table,
                     __global hgpu_float4* randoms,
                     const uint N)
{
    uint giddst = GID;
#ifdef PRECISION_DOUBLE  // if double precision is defined
    hgpu_double4 result;
#else
    float4 result;
    float4 normal = (float4) XOR128_m_FP;
#endif
    uint4 seed = seed_table[GID];
    for (uint i = 0; i < N; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        result.x = xor128_step_double(&seed);
        result.y = xor128_step_double(&seed);
        result.z = xor128_step_double(&seed);
        result.w = xor128_step_double(&seed);
        randoms[giddst] = result;
#else
        seed = xor128_step(seed);
        result.x = (float) seed.w;
        seed = xor128_step(seed);
        result.y = (float) seed.w;
        seed = xor128_step(seed);
        result.z = (float) seed.w;
        seed = xor128_step(seed);
        result.w = (float) seed.w;
        randoms[giddst] = result / normal;
#endif
        giddst += GID_SIZE;
    }
    seed_table[GID] = seed;
}

#endif


Backend: add double support for select.

Backend: add double support for select.

Signed-off-by: rander <5299db4a9bf784920a3e558099f6736bcd8939e6@intel.com>
Tested-by: Yang Rong <78759e37dc7114caed18f74e7515c5cb519dbf6d@intel.com>

/*
 * Copyright © 2012 - 2014 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 */
#include "ocl_relational.h"
#include "ocl_float.h"

OVERLOADABLE int isequal(float x, float y) {
  return x == y;
}

OVERLOADABLE int isnotequal(float x, float y) {
  return x != y;
}

OVERLOADABLE int isgreater(float x, float y) {
  return x > y;
}

OVERLOADABLE int isgreaterequal(float x, float y) {
  return x >= y;
}

OVERLOADABLE int isless(float x, float y) {
  return x < y;
}

OVERLOADABLE int islessequal(float x, float y) {
  return x <= y;
}

OVERLOADABLE int islessgreater(float x, float y) {
  return (x < y) || (x > y);
}

OVERLOADABLE int isfinite(float x) {
  union { uint u; float f; } u;
  u.f = x;
  return (u.u & 0x7FFFFFFF) < 0x7F800000;
}

OVERLOADABLE int isinf(float x) {
  union { uint u; float f; } u;
  u.f = x;
  return (u.u & 0x7FFFFFFF) == 0x7F800000;
}

OVERLOADABLE int isnan(float x) {
  return x != x;
}

OVERLOADABLE int isnormal(float x) {
  union { uint u; float f; } u;
  u.f = x;
  u.u &= 0x7FFFFFFF;
  return (u.u < 0x7F800000) && (u.u >= 0x800000);
}

OVERLOADABLE int isordered(float x, float y) {
  return isequal(x, x) && isequal(y, y);
}
OVERLOADABLE int isunordered(float x, float y) {
  return isnan(x) || isnan(y);
}
OVERLOADABLE int signbit(float x) {
  union { uint u; float f; } u;
  u.f = x;
  return u.u >> 31;
}

// Half float version.
OVERLOADABLE int isequal(half x, half y) {
  return x == y;
}

OVERLOADABLE int isnotequal(half x, half y) {
  return x != y;
}

OVERLOADABLE int isgreater(half x, half y) {
  return x > y;
}

OVERLOADABLE int isgreaterequal(half x, half y) {
  return x >= y;
}

OVERLOADABLE int isless(half x, half y) {
  return x < y;
}

OVERLOADABLE int islessequal(half x, half y) {
  return x <= y;
}

OVERLOADABLE int islessgreater(half x, half y) {
  return (x < y) || (x > y);
}

OVERLOADABLE int isfinite(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  return (u.u & 0x7FFF) < 0x7C00;
}

OVERLOADABLE int isinf(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  return (u.u & 0x7FFF) == 0x7C00;
}

OVERLOADABLE int isnan(half x) {
  return x != x;
}

OVERLOADABLE int isnormal(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  u.u &= 0x7FFF;
  return (u.u < 0x7C00) && (u.u >= 0x400);
}

OVERLOADABLE int isordered(half x, half y) {
  return isequal(x, x) && isequal(y, y);
}
OVERLOADABLE int isunordered(half x, half y) {
  return isnan(x) || isnan(y);
}
OVERLOADABLE int signbit(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  return u.u >> 15;
}

OVERLOADABLE int isequal(double x, double y) {
  return x == y;
}

OVERLOADABLE int isnotequal(double x, double y) {
  return x != y;
}

OVERLOADABLE int isgreater(double x, double y) {
  return x > y;
}

OVERLOADABLE int isgreaterequal(double x, double y) {
  return x >= y;
}

OVERLOADABLE int isless(double x, double y) {
  return x < y;
}

OVERLOADABLE int islessequal(double x, double y) {
  return x <= y;
}

OVERLOADABLE int islessgreater(double x, double y) {
   return (x < y) || (x > y);
}

OVERLOADABLE int isfinite(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  return (u.u & DF_ABS_MASK) < DF_POSITIVE_INF;
}

OVERLOADABLE int isinf(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  return (u.u & DF_ABS_MASK) == DF_POSITIVE_INF;
}

OVERLOADABLE int isnan(double x) {
  long lx = as_long(x);
  return ((lx & DF_EXP_MASK) == DF_POSITIVE_INF) && (lx & DF_MAN_MASK);
}

OVERLOADABLE int isnormal(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  u.u &= DF_ABS_MASK;
  return (u.u < DF_POSITIVE_INF) && (u.u >= DF_MIN_NORMAL);
}

OVERLOADABLE int isordered(double x, double y) {
  return isequal(x, x) && isequal(y, y);
}
OVERLOADABLE int isunordered(double x, double y) {
  return isnan(x) || isnan(y);
}
OVERLOADABLE int signbit(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  return u.u >> 63;
}

// any
#define DEC1(type) OVERLOADABLE int any(type a) { return a<0; }
#define DEC2(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0; }
#define DEC3(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0; }
#define DEC4(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0 || a.s3<0; }
#define DEC8(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0 || a.s3<0 || a.s4<0 || a.s5<0 || a.s6<0 || a.s7<0; }
#define DEC16(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0 || a.s3<0 || a.s4<0 || a.s5<0 || a.s6<0 || a.s7<0 || a.s8<0 || a.s9<0 || a.sA<0 || a.sB<0 || a.sC<0 || a.sD<0 || a.sE<0 || a.sF<0; }
DEC1(char);
DEC1(short);
DEC1(int);
DEC1(long);
#define DEC(n) DEC##n(char##n); DEC##n(short##n); DEC##n(int##n); DEC##n(long##n);
DEC(2);
DEC(3);
DEC(4);
DEC(8);
DEC(16);
#undef DEC
#undef DEC1
#undef DEC2
#undef DEC3
#undef DEC4
#undef DEC8
#undef DEC16

// all
#define DEC1(type) OVERLOADABLE int all(type a) { return a<0; }
#define DEC2(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0; }
#define DEC3(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0; }
#define DEC4(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0 && a.s3<0; }
#define DEC8(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0 && a.s3<0 && a.s4<0 && a.s5<0 && a.s6<0 && a.s7<0; }
#define DEC16(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0 && a.s3<0 && a.s4<0 && a.s5<0 && a.s6<0 && a.s7<0 && a.s8<0 && a.s9<0 && a.sA<0 && a.sB<0 && a.sC<0 && a.sD<0 && a.sE<0 && a.sF<0; }
DEC1(char);
DEC1(short);
DEC1(int);
DEC1(long);
#define DEC(n) DEC##n(char##n); DEC##n(short##n); DEC##n(int##n); DEC##n(long##n);
DEC(2);
DEC(3);
DEC(4);
DEC(8);
DEC(16);
#undef DEC
#undef DEC1
#undef DEC2
#undef DEC3
#undef DEC4
#undef DEC8
#undef DEC16

#define DEF(type) OVERLOADABLE type bitselect(type a, type b, type c) { return (a & ~c) | (b & c); }
DEF(char); DEF(uchar); DEF(short); DEF(ushort); DEF(int); DEF(uint)
DEF(long); DEF(ulong)
#undef DEF
OVERLOADABLE float bitselect(float a, float b, float c) {
  return as_float(bitselect(as_int(a), as_int(b), as_int(c)));
}
OVERLOADABLE double bitselect(double a, double b, double c) {
  return as_double(bitselect(as_long(a), as_long(b), as_long(c)));
}

// select
#define DEF(TYPE1, TYPE2) \
OVERLOADABLE TYPE1 select(TYPE1 src0, TYPE1 src1, TYPE2 cond) { \
  return cond ? src1 : src0; \
}
DEF(char, char)
DEF(char, uchar)
DEF(uchar, char)
DEF(uchar, uchar)
DEF(short, short)
DEF(short, ushort)
DEF(ushort, short)
DEF(ushort, ushort)
DEF(int, int)
DEF(int, uint)
DEF(uint, int)
DEF(uint, uint)
DEF(long, long)
DEF(long, ulong)
DEF(ulong, long)
DEF(ulong, ulong)
DEF(float, int)
DEF(float, uint)
DEF(double, long)
DEF(double, ulong)
#undef DEF

/*
 * Copyright © 2012 - 2014 Intel Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library. If not, see <http://www.gnu.org/licenses/>.
 *
 */
#include "ocl_relational.h"
#include "ocl_float.h"

OVERLOADABLE int isequal(float x, float y) {
  return x == y;
}

OVERLOADABLE int isnotequal(float x, float y) {
  return x != y;
}

OVERLOADABLE int isgreater(float x, float y) {
  return x > y;
}

OVERLOADABLE int isgreaterequal(float x, float y) {
  return x >= y;
}

OVERLOADABLE int isless(float x, float y) {
  return x < y;
}

OVERLOADABLE int islessequal(float x, float y) {
  return x <= y;
}

OVERLOADABLE int islessgreater(float x, float y) {
  return (x < y) || (x > y);
}

OVERLOADABLE int isfinite(float x) {
  union { uint u; float f; } u;
  u.f = x;
  return (u.u & 0x7FFFFFFF) < 0x7F800000;
}

OVERLOADABLE int isinf(float x) {
  union { uint u; float f; } u;
  u.f = x;
  return (u.u & 0x7FFFFFFF) == 0x7F800000;
}

OVERLOADABLE int isnan(float x) {
  return x != x;
}

OVERLOADABLE int isnormal(float x) {
  union { uint u; float f; } u;
  u.f = x;
  u.u &= 0x7FFFFFFF;
  return (u.u < 0x7F800000) && (u.u >= 0x800000);
}

OVERLOADABLE int isordered(float x, float y) {
  return isequal(x, x) && isequal(y, y);
}
OVERLOADABLE int isunordered(float x, float y) {
  return isnan(x) || isnan(y);
}
OVERLOADABLE int signbit(float x) {
  union { uint u; float f; } u;
  u.f = x;
  return u.u >> 31;
}

// Half float version.
OVERLOADABLE int isequal(half x, half y) {
  return x == y;
}

OVERLOADABLE int isnotequal(half x, half y) {
  return x != y;
}

OVERLOADABLE int isgreater(half x, half y) {
  return x > y;
}

OVERLOADABLE int isgreaterequal(half x, half y) {
  return x >= y;
}

OVERLOADABLE int isless(half x, half y) {
  return x < y;
}

OVERLOADABLE int islessequal(half x, half y) {
  return x <= y;
}

OVERLOADABLE int islessgreater(half x, half y) {
  return (x < y) || (x > y);
}

OVERLOADABLE int isfinite(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  return (u.u & 0x7FFF) < 0x7C00;
}

OVERLOADABLE int isinf(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  return (u.u & 0x7FFF) == 0x7C00;
}

OVERLOADABLE int isnan(half x) {
  return x != x;
}

OVERLOADABLE int isnormal(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  u.u &= 0x7FFF;
  return (u.u < 0x7C00) && (u.u >= 0x400);
}

OVERLOADABLE int isordered(half x, half y) {
  return isequal(x, x) && isequal(y, y);
}
OVERLOADABLE int isunordered(half x, half y) {
  return isnan(x) || isnan(y);
}
OVERLOADABLE int signbit(half x) {
  union { ushort u; half h; } u;
  u.h = x;
  return u.u >> 15;
}

OVERLOADABLE int isequal(double x, double y) {
  return x == y;
}

OVERLOADABLE int isnotequal(double x, double y) {
  return x != y;
}

OVERLOADABLE int isgreater(double x, double y) {
  return x > y;
}

OVERLOADABLE int isgreaterequal(double x, double y) {
  return x >= y;
}

OVERLOADABLE int isless(double x, double y) {
  return x < y;
}

OVERLOADABLE int islessequal(double x, double y) {
  return x <= y;
}

OVERLOADABLE int islessgreater(double x, double y) {
   return (x < y) || (x > y);
}

OVERLOADABLE int isfinite(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  return (u.u & DF_ABS_MASK) < DF_POSITIVE_INF;
}

OVERLOADABLE int isinf(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  return (u.u & DF_ABS_MASK) == DF_POSITIVE_INF;
}

OVERLOADABLE int isnan(double x) {
  long lx = as_long(x);
  return ((lx & DF_EXP_MASK) == DF_POSITIVE_INF) && (lx & DF_MAN_MASK);
}

OVERLOADABLE int isnormal(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  u.u &= DF_ABS_MASK;
  return (u.u < DF_POSITIVE_INF) && (u.u >= DF_MIN_NORMAL);
}

OVERLOADABLE int isordered(double x, double y) {
  return isequal(x, x) && isequal(y, y);
}
OVERLOADABLE int isunordered(double x, double y) {
  return isnan(x) || isnan(y);
}
OVERLOADABLE int signbit(double x) {
  union { ulong u; double f; } u;
  u.f = x;
  return u.u >> 63;
}

// any
#define DEC1(type) OVERLOADABLE int any(type a) { return a<0; }
#define DEC2(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0; }
#define DEC3(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0; }
#define DEC4(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0 || a.s3<0; }
#define DEC8(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0 || a.s3<0 || a.s4<0 || a.s5<0 || a.s6<0 || a.s7<0; }
#define DEC16(type) OVERLOADABLE int any(type a) { return a.s0<0 || a.s1<0 || a.s2<0 || a.s3<0 || a.s4<0 || a.s5<0 || a.s6<0 || a.s7<0 || a.s8<0 || a.s9<0 || a.sA<0 || a.sB<0 || a.sC<0 || a.sD<0 || a.sE<0 || a.sF<0; }
DEC1(char);
DEC1(short);
DEC1(int);
DEC1(long);
#define DEC(n) DEC##n(char##n); DEC##n(short##n); DEC##n(int##n); DEC##n(long##n);
DEC(2);
DEC(3);
DEC(4);
DEC(8);
DEC(16);
#undef DEC
#undef DEC1
#undef DEC2
#undef DEC3
#undef DEC4
#undef DEC8
#undef DEC16

// all
#define DEC1(type) OVERLOADABLE int all(type a) { return a<0; }
#define DEC2(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0; }
#define DEC3(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0; }
#define DEC4(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0 && a.s3<0; }
#define DEC8(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0 && a.s3<0 && a.s4<0 && a.s5<0 && a.s6<0 && a.s7<0; }
#define DEC16(type) OVERLOADABLE int all(type a) { return a.s0<0 && a.s1<0 && a.s2<0 && a.s3<0 && a.s4<0 && a.s5<0 && a.s6<0 && a.s7<0 && a.s8<0 && a.s9<0 && a.sA<0 && a.sB<0 && a.sC<0 && a.sD<0 && a.sE<0 && a.sF<0; }
DEC1(char);
DEC1(short);
DEC1(int);
DEC1(long);
#define DEC(n) DEC##n(char##n); DEC##n(short##n); DEC##n(int##n); DEC##n(long##n);
DEC(2);
DEC(3);
DEC(4);
DEC(8);
DEC(16);
#undef DEC
#undef DEC1
#undef DEC2
#undef DEC3
#undef DEC4
#undef DEC8
#undef DEC16

#define DEF(type) OVERLOADABLE type bitselect(type a, type b, type c) { return (a & ~c) | (b & c); }
DEF(char); DEF(uchar); DEF(short); DEF(ushort); DEF(int); DEF(uint)
DEF(long); DEF(ulong)
#undef DEF
OVERLOADABLE float bitselect(float a, float b, float c) {
  return as_float(bitselect(as_int(a), as_int(b), as_int(c)));
}
OVERLOADABLE double bitselect(double a, double b, double c) {
  return as_double(bitselect(as_long(a), as_long(b), as_long(c)));
}

// select
#define DEF(TYPE1, TYPE2) \
OVERLOADABLE TYPE1 select(TYPE1 src0, TYPE1 src1, TYPE2 cond) { \
  return cond ? src1 : src0; \
}
DEF(char, char)
DEF(char, uchar)
DEF(uchar, char)
DEF(uchar, uchar)
DEF(short, short)
DEF(short, ushort)
DEF(ushort, short)
DEF(ushort, ushort)
DEF(int, int)
DEF(int, uint)
DEF(uint, int)
DEF(uint, uint)
DEF(long, long)
DEF(long, ulong)
DEF(ulong, long)
DEF(ulong, ulong)
DEF(float, int)
DEF(float, uint)
#undef DEF