cl-bayer: tune nr delta coff

cl-bayer: tune nr delta coff

Signed-off-by: Wind Yuan <d3b25194a13defcddf619d66a03bde7d96f8d487@intel.com>

/*
 * function: kernel_bayer_pipe
 * params:
 *   input:    image2d_t as read only
 *   output:   image2d_t as write only
 *   blc_config: black level correction configuration
 *   wb_config: whitebalance configuration
 *   gamma_table: RGGB table
 *   stats_output: 3a stats output
 */

#define GRID_X_SIZE 2
#define GRID_Y_SIZE 2

#define SHARED_PIXEL_WIDTH 16
#define SHARED_PIXEL_HEIGHT 16
#define SHARED_PIXEL_X_OFFSET 2
#define SHARED_PIXEL_Y_OFFSET 2
#define SHARED_PIXEL_X_SIZE  (SHARED_PIXEL_WIDTH + SHARED_PIXEL_X_OFFSET * 2)
#define SHARED_PIXEL_Y_SIZE  (SHARED_PIXEL_HEIGHT + SHARED_PIXEL_Y_OFFSET * 2)

#define SHARED_GRID_WIDTH (SHARED_PIXEL_WIDTH/GRID_X_SIZE)
#define SHARED_GRID_HEIGHT (SHARED_PIXEL_HEIGHT/GRID_Y_SIZE)
#define SHARED_GRID_X_OFFSET (SHARED_PIXEL_X_OFFSET/GRID_X_SIZE)
#define SHARED_GRID_Y_OFFSET (SHARED_PIXEL_Y_OFFSET/GRID_Y_SIZE)
#define SHARED_GRID_X_SIZE (SHARED_PIXEL_X_SIZE/GRID_X_SIZE)
#define SHARED_GRID_Y_SIZE (SHARED_PIXEL_Y_SIZE/GRID_Y_SIZE)

#define WORK_ITEM_X_SIZE GRID_X_SIZE
#define WORK_ITEM_Y_SIZE GRID_Y_SIZE

#define STATS_3A_GRID_SIZE (16/GRID_X_SIZE)

#define X 0
#define Y 1
#define Z 2
#define W 3

typedef struct  {
    float  level_gr;  /* Black level for GR pixels */
    float  level_r;   /* Black level for R pixels */
    float  level_b;   /* Black level for B pixels */
    float  level_gb;  /* Black level for GB pixels */
    uint   color_bits;
} CLBLCConfig;

typedef struct
{
    float r_gain;
    float gr_gain;
    float gb_gain;
    float b_gain;
} CLWBConfig;

typedef struct
{
    unsigned int avg_y;

    unsigned int avg_r;
    unsigned int avg_gr;
    unsigned int avg_gb;
    unsigned int avg_b;
    unsigned int valid_wb_count;

    unsigned int f_value1;
    unsigned int f_value2;
} XCamGridStat;

/* BA10=> GRBG  */
inline void blc (float4 *in_out, CLBLCConfig *blc_config)
{
    float multiplier = (float)(1 << (16 - blc_config->color_bits));
    in_out->x = in_out->x * multiplier - blc_config->level_gr;
    in_out->y = in_out->y * multiplier - blc_config->level_r;
    in_out->z = in_out->z * multiplier - blc_config->level_b;
    in_out->w = in_out->w * multiplier - blc_config->level_gb;
}

inline void wb (float4 *in_out, CLWBConfig *wbconfig)
{
    in_out->x *= wbconfig->gr_gain;
    in_out->y *= wbconfig->r_gain;
    in_out->z *= wbconfig->b_gain;
    in_out->w *= wbconfig->gb_gain;
}

inline void gamma_correct(float4 *in_out, __global float *table)
{
    in_out->x = table[clamp(convert_int(in_out->x * 255.0f), 0, 255)];
    in_out->y = table[clamp(convert_int(in_out->y * 255.0f), 0, 255)];
    in_out->z = table[clamp(convert_int(in_out->z * 255.0f), 0, 255)];
    in_out->w = table[clamp(convert_int(in_out->w * 255.0f), 0, 255)];
}

inline int get_shared_pos_x (int i)
{
    return i % SHARED_GRID_X_SIZE;
}

inline int get_shared_pos_y (int i)
{
    return i / SHARED_GRID_X_SIZE;
}

inline int shared_pos (int x, int y)
{
    return mad24(y, SHARED_GRID_X_SIZE, x);
}

/* BA10=> GRBG  */
inline float4 simple_calculate (
    __local float *px, __local float *py, __local float *pz, __local float *pw,
    int index, __read_only image2d_t input, sampler_t sampler, int x_start, int y_start,
    __local float4 *stats_cache,
    CLBLCConfig *blc_config,
    CLWBConfig *wb_config,
    __global float *gamma_table)
{
    float4 data;
    int x0 = get_shared_pos_x (index) * WORK_ITEM_X_SIZE + x_start;
    int y0 = get_shared_pos_y (index) * WORK_ITEM_Y_SIZE + y_start;
    //Gr
    data.x = read_imagef (input, sampler, (int2)(x0, y0)).x;
    //R
    data.y = read_imagef (input, sampler, (int2)(x0 + 1, y0)).x;
    //B
    data.z = read_imagef (input, sampler, (int2)(x0, y0 + 1)).x;
    //Gb
    data.w = read_imagef (input, sampler, (int2)(x0 + 1, y0 + 1)).x;

    blc (&data, blc_config);

    /* write back for 3a stats calculation R, G, B, Y */
    stats_cache[index] = data;

    wb (&data, wb_config);
    gamma_correct (&data, gamma_table);

    px[index] = data.x;
    py[index] = data.y;
    pz[index] = data.z;
    pw[index] = data.w;
}

#define MAX_DELTA_COFF 5.0f
#define MIN_DELTA_COFF 1.0f
#define DEFAULT_DELTA_COFF 4.0f

inline float delta_coff (float delta)
{
    float coff = MAX_DELTA_COFF - 20.0f * fabs(delta);
    return fmax (1.0f, coff);
}

inline float4
demosaic_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    out_data.y = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
                  in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = in_y[shared_pos(x, y)];
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
                  in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;
    return out_data;
}

inline float4
demosaic_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
                  in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    out_data.z = in_z[shared_pos(x, y)];
    return out_data;
}

inline float4
demosaic_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    out_data.y = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_denoise_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_y[shared_pos(x, y - 1)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
             in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    coff[1] = delta_coff(in_x[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_x[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y - 1)] * coff[1] +
                  in_x[shared_pos(x - 1, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x - 1, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = in_y[shared_pos(x, y)];
    coff[1] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x, y - 1)] * coff[1] +
                  in_y[shared_pos(x - 1, y)] * coff[2] +
                  in_y[shared_pos(x + 1, y)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y - 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
             in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;

    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
             in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_w[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = in_z[shared_pos(x, y)];
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x - 1, y)] * coff[2] +
                  in_z[shared_pos(x + 1, y)] * coff[3] +
                  in_z[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;
    return out_data;
};

inline float4
demosaic_denoise_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = DEFAULT_DELTA_COFF;

    value = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x + 1, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x + 1, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    coff[1] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y + 1)] - value);
    out_data.y = (in_w[shared_pos(x, y - 1)] * coff[1] +
                  in_w[shared_pos(x - 1, y)] * coff[2] +
                  in_w[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    value = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y + 1)] * coff[3] +
                  in_z[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

void shared_demosaic (
    __local float *x_data_in, __local float *y_data_in, __local float *z_data_in, __local float *w_data_in,
    int in_x, int in_y,
    __write_only image2d_t out, int out_x, int out_y,
    uint has_denoise)
{
    float4 out_data;

    if (has_denoise) {
        out_data = demosaic_denoise_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_denoise_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_denoise_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_denoise_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    } else {
        out_data = demosaic_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    }

}

inline void stats_3a_calculate (
    __local float4 * input,
    __global XCamGridStat * stats_output,
    CLWBConfig *wb_config)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE / 2;

    for (; count > 0; count /= 2) {
        if ((l_id_x % STATS_3A_GRID_SIZE) + (l_id_y % STATS_3A_GRID_SIZE)* STATS_3A_GRID_SIZE < count) {
            int index1 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET);
            int index2 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET + count % STATS_3A_GRID_SIZE,
                                     l_id_y + SHARED_GRID_Y_OFFSET + count / STATS_3A_GRID_SIZE);
            //input[index1].x = (input[index1].x + input[index2].x) / 2.0f;
            //input[index1].y = (input[index1].y + input[index2].y) / 2.0f;
            //input[index1].z = (input[index1].z + input[index2].z) / 2.0f;
            //input[index1].w = (input[index1].w + input[index2].w) / 2.0f;
            input[index1] = (input[index1] + input[index2]) / 2.0f;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    if (l_id_x % STATS_3A_GRID_SIZE == 0 && l_id_y % STATS_3A_GRID_SIZE == 0) {
        float4 tmp_data;
        int out_index = mad24(g_id_y / STATS_3A_GRID_SIZE,  g_size_x / STATS_3A_GRID_SIZE, g_id_x / STATS_3A_GRID_SIZE);
        tmp_data = input[shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET)];
        stats_output[out_index].avg_gr = convert_uchar_sat(tmp_data.x * 255.0f);
        stats_output[out_index].avg_r = convert_uchar_sat(tmp_data.y * 255.0f);
        stats_output[out_index].avg_b = convert_uchar_sat(tmp_data.z * 255.0f);
        stats_output[out_index].avg_gb = convert_uchar_sat(tmp_data.w * 255.0f);
        stats_output[out_index].valid_wb_count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE;
        stats_output[out_index].avg_y =
            convert_uchar_sat(((tmp_data.x * wb_config->gr_gain + tmp_data.w * wb_config->gb_gain) * 0.2935f +
                               tmp_data.y * wb_config->r_gain * 0.299f + tmp_data.z * wb_config->b_gain * 0.114f) * 255.0f);
        stats_output[out_index].f_value1 = 0;
        stats_output[out_index].f_value2 = 0;
    }
}

__kernel void kernel_bayer_pipe (__read_only image2d_t input,
                                 __write_only image2d_t output,
                                 CLBLCConfig blc_config,
                                 CLWBConfig wb_config,
                                 uint has_denoise,
                                 __global float * gamma_table,
                                 __global XCamGridStat * stats_output)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int l_size_x = get_local_size (0);
    int l_size_y = get_local_size (1);

    sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

    __local float p1_x[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_y[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_z[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_w[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 p2[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 *stats_cache = p2;

    int out_x_start, out_y_start;
    int x_start = (g_id_x - l_id_x) * WORK_ITEM_X_SIZE - SHARED_PIXEL_X_OFFSET;
    int y_start = (g_id_y - l_id_y) * WORK_ITEM_Y_SIZE - SHARED_PIXEL_Y_OFFSET;
    int i = l_id_x + l_id_y * l_size_x;

    for (; i < SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE; i += l_size_x * l_size_y) {
        simple_calculate (p1_x, p1_y, p1_z, p1_w, i,
                          input, sampler, x_start, y_start,
                          stats_cache,
                          &blc_config,
                          &wb_config,
                          gamma_table);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    stats_3a_calculate (stats_cache, stats_output, &wb_config);

    shared_demosaic (
        p1_x, p1_y, p1_z, p1_w, l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET,
        output, g_id_x * WORK_ITEM_X_SIZE, g_id_y * WORK_ITEM_Y_SIZE, has_denoise);
}

/*
 * function: kernel_bayer_pipe
 * params:
 *   input:    image2d_t as read only
 *   output:   image2d_t as write only
 *   blc_config: black level correction configuration
 *   wb_config: whitebalance configuration
 *   gamma_table: RGGB table
 *   stats_output: 3a stats output
 */

#define GRID_X_SIZE 2
#define GRID_Y_SIZE 2

#define SHARED_PIXEL_WIDTH 16
#define SHARED_PIXEL_HEIGHT 16
#define SHARED_PIXEL_X_OFFSET 2
#define SHARED_PIXEL_Y_OFFSET 2
#define SHARED_PIXEL_X_SIZE  (SHARED_PIXEL_WIDTH + SHARED_PIXEL_X_OFFSET * 2)
#define SHARED_PIXEL_Y_SIZE  (SHARED_PIXEL_HEIGHT + SHARED_PIXEL_Y_OFFSET * 2)

#define SHARED_GRID_WIDTH (SHARED_PIXEL_WIDTH/GRID_X_SIZE)
#define SHARED_GRID_HEIGHT (SHARED_PIXEL_HEIGHT/GRID_Y_SIZE)
#define SHARED_GRID_X_OFFSET (SHARED_PIXEL_X_OFFSET/GRID_X_SIZE)
#define SHARED_GRID_Y_OFFSET (SHARED_PIXEL_Y_OFFSET/GRID_Y_SIZE)
#define SHARED_GRID_X_SIZE (SHARED_PIXEL_X_SIZE/GRID_X_SIZE)
#define SHARED_GRID_Y_SIZE (SHARED_PIXEL_Y_SIZE/GRID_Y_SIZE)

#define WORK_ITEM_X_SIZE GRID_X_SIZE
#define WORK_ITEM_Y_SIZE GRID_Y_SIZE

#define STATS_3A_GRID_SIZE (16/GRID_X_SIZE)

#define X 0
#define Y 1
#define Z 2
#define W 3

typedef struct  {
    float  level_gr;  /* Black level for GR pixels */
    float  level_r;   /* Black level for R pixels */
    float  level_b;   /* Black level for B pixels */
    float  level_gb;  /* Black level for GB pixels */
    uint   color_bits;
} CLBLCConfig;

typedef struct
{
    float r_gain;
    float gr_gain;
    float gb_gain;
    float b_gain;
} CLWBConfig;

typedef struct
{
    unsigned int avg_y;

    unsigned int avg_r;
    unsigned int avg_gr;
    unsigned int avg_gb;
    unsigned int avg_b;
    unsigned int valid_wb_count;

    unsigned int f_value1;
    unsigned int f_value2;
} XCamGridStat;

/* BA10=> GRBG  */
inline void blc (float4 *in_out, CLBLCConfig *blc_config)
{
    float multiplier = (float)(1 << (16 - blc_config->color_bits));
    in_out->x = in_out->x * multiplier - blc_config->level_gr;
    in_out->y = in_out->y * multiplier - blc_config->level_r;
    in_out->z = in_out->z * multiplier - blc_config->level_b;
    in_out->w = in_out->w * multiplier - blc_config->level_gb;
}

inline void wb (float4 *in_out, CLWBConfig *wbconfig)
{
    in_out->x *= wbconfig->gr_gain;
    in_out->y *= wbconfig->r_gain;
    in_out->z *= wbconfig->b_gain;
    in_out->w *= wbconfig->gb_gain;
}

inline void gamma_correct(float4 *in_out, __global float *table)
{
    in_out->x = table[clamp(convert_int(in_out->x * 255.0f), 0, 255)];
    in_out->y = table[clamp(convert_int(in_out->y * 255.0f), 0, 255)];
    in_out->z = table[clamp(convert_int(in_out->z * 255.0f), 0, 255)];
    in_out->w = table[clamp(convert_int(in_out->w * 255.0f), 0, 255)];
}

inline int get_shared_pos_x (int i)
{
    return i % SHARED_GRID_X_SIZE;
}

inline int get_shared_pos_y (int i)
{
    return i / SHARED_GRID_X_SIZE;
}

inline int shared_pos (int x, int y)
{
    return mad24(y, SHARED_GRID_X_SIZE, x);
}

/* BA10=> GRBG  */
inline float4 simple_calculate (
    __local float *px, __local float *py, __local float *pz, __local float *pw,
    int index, __read_only image2d_t input, sampler_t sampler, int x_start, int y_start,
    __local float4 *stats_cache,
    CLBLCConfig *blc_config,
    CLWBConfig *wb_config,
    __global float *gamma_table)
{
    float4 data;
    int x0 = get_shared_pos_x (index) * WORK_ITEM_X_SIZE + x_start;
    int y0 = get_shared_pos_y (index) * WORK_ITEM_Y_SIZE + y_start;
    //Gr
    data.x = read_imagef (input, sampler, (int2)(x0, y0)).x;
    //R
    data.y = read_imagef (input, sampler, (int2)(x0 + 1, y0)).x;
    //B
    data.z = read_imagef (input, sampler, (int2)(x0, y0 + 1)).x;
    //Gb
    data.w = read_imagef (input, sampler, (int2)(x0 + 1, y0 + 1)).x;

    blc (&data, blc_config);

    /* write back for 3a stats calculation R, G, B, Y */
    stats_cache[index] = data;

    wb (&data, wb_config);
    gamma_correct (&data, gamma_table);

    px[index] = data.x;
    py[index] = data.y;
    pz[index] = data.z;
    pw[index] = data.w;
}

inline float delta_coff (float delta)
{
    float coff = 3.0f - 10.0f * fabs(delta);
    return fmax (0.3f, coff);
}

inline float4
demosaic_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    out_data.y = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
                  in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = in_y[shared_pos(x, y)];
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    out_data.z = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
                  in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;
    return out_data;
}

inline float4
demosaic_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
                  in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    out_data.y = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
                  in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    out_data.z = in_z[shared_pos(x, y)];
    return out_data;
}

inline float4
demosaic_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    out_data.x = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    out_data.y = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
                  in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    out_data.z = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    return out_data;
}

inline float4
demosaic_denoise_x0y0_gr (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = delta_coff(0.0f);

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_y[shared_pos(x, y - 1)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] * 4.0f + in_w[shared_pos(x - 1, y - 1)] +
             in_w[shared_pos(x, y - 1)] + in_w[shared_pos(x - 1, y)] + in_w[shared_pos(x, y)]) * 0.125f;
    coff[1] = delta_coff(in_x[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_x[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y - 1)] * coff[1] +
                  in_x[shared_pos(x - 1, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x - 1, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x - 1, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x - 1, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x1y0_r (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = delta_coff(0.0f);

    value = in_y[shared_pos(x, y)];
    coff[1] = delta_coff(in_y[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x, y - 1)] * coff[1] +
                  in_y[shared_pos(x - 1, y)] * coff[2] +
                  in_y[shared_pos(x + 1, y)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_w[shared_pos(x, y - 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_x[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_x[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y - 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_z[shared_pos(x, y - 1)] + in_z[shared_pos(x + 1, y - 1)] +
             in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.25f;

    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y)] * coff[3] +
                  in_z[shared_pos(x + 1, y)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

inline float4
demosaic_denoise_x0y1_b (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = delta_coff(0.0f);

    value = (in_y[shared_pos(x - 1, y)] + in_y[shared_pos(x, y)] +
             in_y[shared_pos(x - 1, y + 1)] + in_y[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_x[shared_pos(x, y)] + in_w[shared_pos(x, y)] +
             in_w[shared_pos(x - 1, y)] + in_x[shared_pos(x, y + 1)]) * 0.25f;
    coff[1] = delta_coff(in_x[shared_pos(x, y)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[4] = delta_coff(in_x[shared_pos(x, y + 1)] - value);
    out_data.y = (in_x[shared_pos(x, y)] * coff[1] +
                  in_w[shared_pos(x, y)] * coff[2] +
                  in_w[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_x[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = in_z[shared_pos(x, y)];
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x - 1, y)] * coff[2] +
                  in_z[shared_pos(x + 1, y)] * coff[3] +
                  in_z[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;
    return out_data;
};

inline float4
demosaic_denoise_x1y1_gb (__local float *in_x, __local float *in_y, __local float *in_z, __local float *in_w, int x, int y)
{
    float4 out_data;
    float value;
    float coff[5];

    coff[0] = delta_coff(0.0f);

    value = (in_y[shared_pos(x, y)] + in_y[shared_pos(x, y + 1)]) * 0.5f;
    coff[1] = delta_coff(in_y[shared_pos(x - 1, y)] - value);
    coff[2] = delta_coff(in_y[shared_pos(x + 1, y)] - value);
    coff[3] = delta_coff(in_y[shared_pos(x - 1, y + 1)] - value);
    coff[4] = delta_coff(in_y[shared_pos(x + 1, y + 1)] - value);
    out_data.x = (in_y[shared_pos(x - 1, y)] * coff[1] +
                  in_y[shared_pos(x + 1, y)] * coff[2] +
                  in_y[shared_pos(x - 1, y + 1)] * coff[3] +
                  in_y[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);

    value = (in_w[shared_pos(x, y)] * 4.0f + in_x[shared_pos(x, y)] +
             in_x[shared_pos(x + 1, y)] + in_x[shared_pos(x, y + 1)] + in_x[shared_pos(x + 1, y + 1)]) * 0.125f;
    coff[1] = delta_coff(in_w[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_w[shared_pos(x - 1, y)] - value);
    coff[3] = delta_coff(in_w[shared_pos(x + 1, y)] - value);
    coff[4] = delta_coff(in_w[shared_pos(x, y + 1)] - value);
    out_data.y = (in_w[shared_pos(x, y - 1)] * coff[1] +
                  in_w[shared_pos(x - 1, y)] * coff[2] +
                  in_w[shared_pos(x + 1, y)] * coff[3] +
                  in_w[shared_pos(x, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    value = (in_z[shared_pos(x, y)] + in_z[shared_pos(x + 1, y)]) * 0.5f;
    coff[1] = delta_coff(in_z[shared_pos(x, y - 1)] - value);
    coff[2] = delta_coff(in_z[shared_pos(x + 1, y - 1)] - value);
    coff[3] = delta_coff(in_z[shared_pos(x, y + 1)] - value);
    coff[4] = delta_coff(in_z[shared_pos(x + 1, y + 1)] - value);
    out_data.z = (in_z[shared_pos(x, y - 1)] * coff[1] +
                  in_z[shared_pos(x + 1, y - 1)] * coff[2] +
                  in_z[shared_pos(x, y + 1)] * coff[3] +
                  in_z[shared_pos(x + 1, y + 1)] * coff[4] +
                  value * coff[0]) /
                 (coff[0] + coff[1] + coff[2] + coff[3] + coff[4]);
    out_data.w = 0.0f;

    return out_data;
}

void shared_demosaic (
    __local float *x_data_in, __local float *y_data_in, __local float *z_data_in, __local float *w_data_in,
    int in_x, int in_y,
    __write_only image2d_t out, int out_x, int out_y,
    uint has_denoise)
{
    float4 out_data;

    if (has_denoise) {
        out_data = demosaic_denoise_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_denoise_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_denoise_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_denoise_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    } else {
        out_data = demosaic_x0y0_gr (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y), out_data);

        out_data = demosaic_x1y0_r (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y), out_data);

        out_data = demosaic_x0y1_b (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x, out_y + 1), out_data);

        out_data = demosaic_x1y1_gb (x_data_in, y_data_in, z_data_in, w_data_in, in_x, in_y);
        write_imagef(out, (int2)(out_x + 1, out_y + 1), out_data);
    }

}

inline void stats_3a_calculate (
    __local float4 * input,
    __global XCamGridStat * stats_output,
    CLWBConfig *wb_config)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE / 2;

    for (; count > 0; count /= 2) {
        if ((l_id_x % STATS_3A_GRID_SIZE) + (l_id_y % STATS_3A_GRID_SIZE)* STATS_3A_GRID_SIZE < count) {
            int index1 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET);
            int index2 = shared_pos (l_id_x + SHARED_GRID_X_OFFSET + count % STATS_3A_GRID_SIZE,
                                     l_id_y + SHARED_GRID_Y_OFFSET + count / STATS_3A_GRID_SIZE);
            //input[index1].x = (input[index1].x + input[index2].x) / 2.0f;
            //input[index1].y = (input[index1].y + input[index2].y) / 2.0f;
            //input[index1].z = (input[index1].z + input[index2].z) / 2.0f;
            //input[index1].w = (input[index1].w + input[index2].w) / 2.0f;
            input[index1] = (input[index1] + input[index2]) / 2.0f;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    if (l_id_x % STATS_3A_GRID_SIZE == 0 && l_id_y % STATS_3A_GRID_SIZE == 0) {
        float4 tmp_data;
        int out_index = mad24(g_id_y / STATS_3A_GRID_SIZE,  g_size_x / STATS_3A_GRID_SIZE, g_id_x / STATS_3A_GRID_SIZE);
        tmp_data = input[shared_pos (l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET)];
        stats_output[out_index].avg_gr = convert_uchar_sat(tmp_data.x * 255.0f);
        stats_output[out_index].avg_r = convert_uchar_sat(tmp_data.y * 255.0f);
        stats_output[out_index].avg_b = convert_uchar_sat(tmp_data.z * 255.0f);
        stats_output[out_index].avg_gb = convert_uchar_sat(tmp_data.w * 255.0f);
        stats_output[out_index].valid_wb_count = STATS_3A_GRID_SIZE * STATS_3A_GRID_SIZE;
        stats_output[out_index].avg_y =
            convert_uchar_sat(((tmp_data.x * wb_config->gr_gain + tmp_data.w * wb_config->gb_gain) * 0.2935f +
                               tmp_data.y * wb_config->r_gain * 0.299f + tmp_data.z * wb_config->b_gain * 0.114f) * 255.0f);
        stats_output[out_index].f_value1 = 0;
        stats_output[out_index].f_value2 = 0;
    }
}

__kernel void kernel_bayer_pipe (__read_only image2d_t input,
                                 __write_only image2d_t output,
                                 CLBLCConfig blc_config,
                                 CLWBConfig wb_config,
                                 uint has_denoise,
                                 __global float * gamma_table,
                                 __global XCamGridStat * stats_output)
{
    int g_id_x = get_global_id (0);
    int g_id_y = get_global_id (1);
    int g_size_x = get_global_size (0);
    int g_size_y = get_global_size (1);

    int l_id_x = get_local_id(0);
    int l_id_y = get_local_id(1);
    int l_size_x = get_local_size (0);
    int l_size_y = get_local_size (1);

    sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

    __local float p1_x[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_y[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_z[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE], p1_w[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 p2[SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE];
    __local float4 *stats_cache = p2;

    int out_x_start, out_y_start;
    int x_start = (g_id_x - l_id_x) * WORK_ITEM_X_SIZE - SHARED_PIXEL_X_OFFSET;
    int y_start = (g_id_y - l_id_y) * WORK_ITEM_Y_SIZE - SHARED_PIXEL_Y_OFFSET;
    int i = l_id_x + l_id_y * l_size_x;

    for (; i < SHARED_GRID_X_SIZE * SHARED_GRID_Y_SIZE; i += l_size_x * l_size_y) {
        simple_calculate (p1_x, p1_y, p1_z, p1_w, i,
                          input, sampler, x_start, y_start,
                          stats_cache,
                          &blc_config,
                          &wb_config,
                          gamma_table);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    stats_3a_calculate (stats_cache, stats_output, &wb_config);

    shared_demosaic (
        p1_x, p1_y, p1_z, p1_w, l_id_x + SHARED_GRID_X_OFFSET, l_id_y + SHARED_GRID_Y_OFFSET,
        output, g_id_x * WORK_ITEM_X_SIZE, g_id_y * WORK_ITEM_Y_SIZE, has_denoise);
}


modifies tests with templates

modifies tests with templates


returned cl

returned cl

#define LSIZE 8

__kernel void KNearestFor(			  unsigned dim, unsigned hst_ptrtest_patterns_dim1, __global float * dist_matrix,
			  __global float * train_patterns, unsigned hst_ptrtrain_patterns_dim1, __global float * test_patterns,
			  unsigned hst_ptrdist_matrix_dim1)
{
  __local float test_patterns_local[8*8];
  __local float train_patterns_local[8*8];
  unsigned li = get_local_id(1);
  unsigned lj = get_local_id(0);

  float d = 0.0;
  for (unsigned k = 0; k < dim; k+=8)
    {

      test_patterns_local[(li * 8) + get_local_id(0)] = test_patterns[(get_global_id(1) * hst_ptrtest_patterns_dim1) + (k + get_local_id(0))];
      train_patterns_local[(li * 8) + get_local_id(0)] = train_patterns[((k + get_local_id(1)) * hst_ptrtrain_patterns_dim1) + get_global_id(0)];
      barrier(CLK_LOCAL_MEM_FENCE);

      /* for (unsigned kk = 0; kk < 8; kk++) */
      /*   { */
          float tmp = test_patterns_local[(li * 8) + 0] - train_patterns_local[(0 * 8) + lj];
          d += tmp * tmp;
          tmp = test_patterns_local[(li * 8) + 1] - train_patterns_local[(1 * 8) + lj];
          d += tmp * tmp;
          tmp = test_patterns_local[(li * 8) + 2] - train_patterns_local[(2 * 8) + lj];
          d += tmp * tmp;
          tmp = test_patterns_local[(li * 8) + 3] - train_patterns_local[(3 * 8) + lj];
          d += tmp * tmp;
          tmp = test_patterns_local[(li * 8) + 4] - train_patterns_local[(4 * 8) + lj];
          d += tmp * tmp;
          tmp = test_patterns_local[(li * 8) + 5] - train_patterns_local[(5 * 8) + lj];
          d += tmp * tmp;
          tmp = test_patterns_local[(li * 8) + 6] - train_patterns_local[(6 * 8) + lj];
          d += tmp * tmp;
          tmp = test_patterns_local[(li * 8) + 7] - train_patterns_local[(7 * 8) + lj];
          d += tmp * tmp;
        /* } */
      barrier(CLK_LOCAL_MEM_FENCE);
    }

  dist_matrix[(get_global_id(1) * hst_ptrdist_matrix_dim1) + get_global_id(0)] = d;
}

#define LSIZE 8

__kernel void KNearestFor(			  unsigned dim, unsigned hst_ptrtest_patterns_dim1, __global float * dist_matrix,
			  __global float * train_patterns, unsigned hst_ptrtrain_patterns_dim1, __global float * test_patterns,
			  unsigned hst_ptrdist_matrix_dim1)
{
  __local float test_patterns_local[8*8];
  __local float train_patterns_local[8*8];
  unsigned li = get_local_id(1);
  unsigned lj = get_local_id(0);

  float d = 0.0;
  for (unsigned k = 0; k < dim; k+=8)
    {

      test_patterns_local[(li * 8) + get_local_id(0)] = test_patterns[(get_global_id(1) * hst_ptrtest_patterns_dim1) + (k + get_local_id(0))];
      train_patterns_local[(li * 8) + get_local_id(0)] = train_patterns[((k + get_local_id(1)) * hst_ptrtrain_patterns_dim1) + get_global_id(0)];
      barrier(CLK_LOCAL_MEM_FENCE);

      /* for (unsigned kk = 0; kk < 8; kk++) */
      /*   { */
          float tmp = test_patterns_local[(li * 8) + 0] - train_patterns_local[(0 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
          tmp = test_patterns_local[(li * 8) + 1] - train_patterns_local[(1 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
          tmp = test_patterns_local[(li * 8) + 2] - train_patterns_local[(2 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
          tmp = test_patterns_local[(li * 8) + 3] - train_patterns_local[(3 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
          tmp = test_patterns_local[(li * 8) + 4] - train_patterns_local[(4 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
          tmp = test_patterns_local[(li * 8) + 5] - train_patterns_local[(5 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
          tmp = test_patterns_local[(li * 8) + 6] - train_patterns_local[(6 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
          tmp = test_patterns_local[(li * 8) + 7] - train_patterns_local[(7 * 8) + lj];
          d += tmp * tmp;
          d += tmp * tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp* tmp;
        /* } */
      barrier(CLK_LOCAL_MEM_FENCE);
    }

  dist_matrix[(get_global_id(1) * hst_ptrdist_matrix_dim1) + get_global_id(0)] = d;
}


OpenCL implementation of MRG32k3a PRNG

OpenCL implementation of MRG32k3a PRNG

/******************************************************************************
 * @file     prngcl_mrg32k3a.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of MRG32k3a pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   Pierre L'Ecuyer,
 *   "Good Parameter Sets for Combined Multiple Recursive Random Number Generators",
 *   Operations Research, 47, 1 (1999), 159--164.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, 2014 Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_MRG32K3A_CL
#define PRNGCL_MRG32K3A_CL

#include "prngcl_common.cl"

#define MRG32k3a_min    (1.0/4294967088.0)
#define MRG32k3a_max    (4294967087.0/4294967088.0)
#define MRG32k3a_k      (2.3283065492957279e-010) // 1/(2^32-208)
#define MRG32k3a_left   (MRG32k3a_min+MRG32k3a_k*MRG32k3a_max)
#define MRG32k3a_right  (MRG32k3a_max+MRG32k3a_k*MRG32k3a_min)

#define MRG32k3a_m     (2.3283065492957276882397381481401e-10) // 1/(2^32-208) for MRG32k3a generator
#define MRG32k3a_m1    4294967087
#define MRG32k3a_m2    4294944443
#define MRG32k3a_a11   1403580L
#define MRG32k3a_a12   810728L
#define MRG32k3a_a21   527612L
#define MRG32k3a_a22   1370589L

//________________________________________________________________________________________________________ MRG32k3a PRNG
__attribute__((always_inline)) void
mrg32k3a_step(uint4* seed1,uint4* seed2,float* rnd)
{
    long x1n, x2n;
    long k;

    x1n = MRG32k3a_a11 * (*seed1).y - MRG32k3a_a12 * (*seed1).x;
    k = x1n / MRG32k3a_m1;
    x1n -= k * MRG32k3a_m1;
    if (x1n < 0) x1n += MRG32k3a_m1;
    (*seed1).x = (*seed1).y;
    (*seed1).y = (*seed1).z;
    (*seed1).z = x1n;

    x2n = MRG32k3a_a21 * (*seed2).z - MRG32k3a_a22 * (*seed2).x;
    k = x2n / MRG32k3a_m2;
    x2n -= k * MRG32k3a_m2;
    if (x2n < 0) x2n += MRG32k3a_m2;
    (*seed2).x = (*seed2).y;
    (*seed2).y = (*seed2).z;
    (*seed2).z = x2n;

    if (x1n>x2n) (*rnd) = (float) ((x1n - x2n) * MRG32k3a_m);
    else         (*rnd) = (float) ((x1n - x2n + MRG32k3a_m1) * MRG32k3a_m);
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double
mrg32k3a_step_double(uint4* seed1,uint4* seed2)
{
    hgpu_double rnd_double;
    float rnd1, rnd2;
    mrg32k3a_step(seed1,seed2,&rnd1);

#ifndef PRNG_SKIP_CHECK
    bool flag = true;
    while (flag) {
#endif
        mrg32k3a_step(seed1,seed2,&rnd2);
        rnd_double = hgpu_float_to_double(rnd1,rnd2,MRG32k3a_min,MRG32k3a_max,MRG32k3a_k);

#ifndef PRNG_SKIP_CHECK
        if ((rnd_double>=MRG32k3a_left) && (rnd_double<MRG32k3a_right))
            flag = false;
        else
            rnd1 = rnd2;
    }
#endif
    return rnd_double;
}
#endif

__kernel void
mrg32k3a(__global uint4* seed_table,
         __global hgpu_float4* randoms,
            const uint N)
{
    uint giddst = GID;
#ifdef PRECISION_DOUBLE  // if double precision is defined
    float rnd;
    hgpu_double4 result;
#else
    float rnd;
    float4 result;
#endif

    uint4 seed1 = seed_table[GID];
    uint4 seed2 = seed_table[GID + GID_SIZE];

    for (uint i = 0; i < N; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        // MRG32k3a PRNG returns values [1;4294967087]/4294967088, or (0.0;1.0)
        result.x = mrg32k3a_step_double(&seed1,&seed2);
        result.y = mrg32k3a_step_double(&seed1,&seed2);
        result.z = mrg32k3a_step_double(&seed1,&seed2);
        result.w = mrg32k3a_step_double(&seed1,&seed2);
        randoms[giddst] = result;
#else
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.x = rnd;
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.y = rnd;
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.z = rnd;
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.w = rnd;
        randoms[giddst] = result;
#endif
        giddst += GID_SIZE;
    }
    seed_table[GID] = seed1;
    seed_table[GID + GID_SIZE] = seed2;
}

#endif

/******************************************************************************
 * @file     prngcl_mrg32k3a.cl
 * @author   Vadim Demchik <vadimdi@yahoo.com>
 * @version  1.1
 *
 * @brief    [PRNGCL library]
 *           contains OpenCL implementation of MRG32k3a pseudo-random number generator
 *
 *
 * @section  CREDITS
 *
 *   Pierre L'Ecuyer,
 *   "Good Parameter Sets for Combined Multiple Recursive Random Number Generators",
 *   Operations Research, 47, 1 (1999), 159--164.
 *
 *
 * @section  LICENSE
 *
 * Copyright (c) 2013, Vadim Demchik
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *    Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *    Redistributions in binary form must reproduce the above copyright notice,
 *      this list of conditions and the following disclaimer in the documentation
 *      and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/

#ifndef PRNGCL_MRG32K3A_CL
#define PRNGCL_MRG32K3A_CL

#include "prngcl_common.cl"

#define MRG32k3a_min    (1.0/4294967088.0)
#define MRG32k3a_max    (4294967087.0/4294967088.0)
#define MRG32k3a_k      (2.3283065492957279e-010) // 1/(2^32-208)
#define MRG32k3a_left   (MRG32k3a_min+MRG32k3a_k*MRG32k3a_max)
#define MRG32k3a_right  (MRG32k3a_max+MRG32k3a_k*MRG32k3a_min)

#define MRG32k3a_m     (2.3283065492957276882397381481401e-10) // 1/(2^32-208) for MRG32k3a generator
#define MRG32k3a_m1    4294967087
#define MRG32k3a_m2    4294944443
#define MRG32k3a_a11   1403580L
#define MRG32k3a_a12   810728L
#define MRG32k3a_a21   527612L
#define MRG32k3a_a22   1370589L

//________________________________________________________________________________________________________ MRG32k3a PRNG
__attribute__((always_inline)) void
mrg32k3a_step(uint4* seed1,uint4* seed2,float* rnd)
{
    long x1n, x2n;
    long k;

    x1n = MRG32k3a_a11 * (*seed1).y - MRG32k3a_a12 * (*seed1).x;
    k = x1n / MRG32k3a_m1;
    x1n -= k * MRG32k3a_m1;
    if (x1n < 0) x1n += MRG32k3a_m1;
    (*seed1).x = (*seed1).y;
    (*seed1).y = (*seed1).z;
    (*seed1).z = x1n;

    x2n = MRG32k3a_a21 * (*seed2).z - MRG32k3a_a22 * (*seed2).x;
    k = x2n / MRG32k3a_m2;
    x2n -= k * MRG32k3a_m2;
    if (x2n < 0) x2n += MRG32k3a_m2;
    (*seed2).x = (*seed2).y;
    (*seed2).y = (*seed2).z;
    (*seed2).z = x2n;

    if (x1n>x2n) (*rnd) = (float) ((x1n - x2n) * MRG32k3a_m);
    else         (*rnd) = (float) ((x1n - x2n + MRG32k3a_m1) * MRG32k3a_m);
}

#ifdef PRECISION_DOUBLE  // if double precision is defined
__attribute__((always_inline)) hgpu_double
mrg32k3a_step_double(uint4* seed1,uint4* seed2)
{
    hgpu_double rnd_double;
    float rnd1, rnd2;
    bool flag = true;
    mrg32k3a_step(seed1,seed2,&rnd1);
    while (flag) {
        mrg32k3a_step(seed1,seed2,&rnd2);
        rnd_double = hgpu_float_to_double(rnd1,rnd2,MRG32k3a_min,MRG32k3a_max,MRG32k3a_k);
        if ((rnd_double>=MRG32k3a_left) && (rnd_double<MRG32k3a_right))
            flag = false;
        else
            rnd1 = rnd2;
    }
    return rnd_double;
}
#endif

__kernel void
mrg32k3a(__global uint4* seed_table,
         __global hgpu_float4* randoms,
            const uint N)
{
    uint giddst = GID;
#ifdef PRECISION_DOUBLE  // if double precision is defined
    float rnd;
    hgpu_double4 result;
#else
    float rnd;
    float4 result;
#endif

    uint4 seed1 = seed_table[GID];
    uint4 seed2 = seed_table[GID + GID_SIZE];

    for (uint i = 0; i < N; i++) {
#ifdef PRECISION_DOUBLE // if double precision is defined
        // MRG32k3a PRNG returns values [1;4294967087]/4294967088, or (0.0;1.0)
        result.x = mrg32k3a_step_double(&seed1,&seed2);
        result.y = mrg32k3a_step_double(&seed1,&seed2);
        result.z = mrg32k3a_step_double(&seed1,&seed2);
        result.w = mrg32k3a_step_double(&seed1,&seed2);
        randoms[giddst] = result;
#else
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.x = rnd;
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.y = rnd;
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.z = rnd;
        mrg32k3a_step(&seed1,&seed2,&rnd);
            result.w = rnd;
        randoms[giddst] = result;
#endif
        giddst += GID_SIZE;
    }
    seed_table[GID] = seed1;
    seed_table[GID + GID_SIZE] = seed2;
}

#endif


Correct range for pixels to be convolved with PSF

// compute image
kernel void render(constant char* data, float4 pcs,
                   constant float2* qq, constant float2* ww,
                   global float* value, global float* error)
{
    // get pixel index
    int k = get_global_id(0);

    // compute pixel flux if pixel is in image
    if(k < IMAGE_SIZE)
    {
        // pixel position
        float2 x = pcs.xy + pcs.zw*(float2)(k%IMAGE_WIDTH, k/IMAGE_WIDTH);

        // value and error of quadrature
        float2 f = 0;

        // apply quadrature rule to computed surface brightness
        for(size_t n = 0; n < QUAD_POINTS; ++n)
            f += ww[n]*compute(data, x + qq[n]);

        // done
        value[k] = f.s0;
        error[k] = f.s1;
    }
}

// calculate log-likelihood of computed model
kernel void loglike(global const float* image, global const float* weight,
                    global const float* model, global float* loglike)
{
    // get pixel index
    int k = get_global_id(0);

    // compute chi^2 value if pixel is in image
    if(k < IMAGE_SIZE)
    {
        float d = model[k] - image[k];
        loglike[k] = weight[k]*d*d;
    }
}

// convolve input with PSF
kernel void convolve(global float* input, constant float* psf,
                     local float* input2, local float* psf2,
                     global float* output)
{
    int i, j;

    // pixel indices for output
    int gi = get_global_id(0);
    int gj = get_global_id(1);

    // local indices, size and origin
    int li = get_local_id(0);
    int lj = get_local_id(1);
    int lw = get_local_size(0);
    int lh = get_local_size(1);
    int ls = lw*lh;

    // cache size and origin
    int cw = PSF_WIDTH/2 + lw + PSF_WIDTH/2;
    int ch = PSF_HEIGHT/2 + lh + PSF_HEIGHT/2;
    int cs = cw*ch;
    int cx = mad24((int)get_group_id(0), lw, -PSF_WIDTH/2);
    int cy = mad24((int)get_group_id(1), lh, -PSF_HEIGHT/2);

    // fill cache
    for(i = mad24(lj, lw, li); i < cs; i += ls)
        input2[i] = input[clamp(cy + i/cw, 0, IMAGE_HEIGHT-1)*IMAGE_WIDTH + clamp(cx + i%cw, 0, IMAGE_WIDTH-1)];
    for(i = mad24(lj, lw, li); i < PSF_WIDTH*PSF_HEIGHT; i += ls)
        psf2[i] = psf[i];

    // wait for all items to finish cache filling
    barrier(CLK_LOCAL_MEM_FENCE);

    // check if pixel is in image
    if(gi < IMAGE_WIDTH && gj < IMAGE_HEIGHT)
    {
        // convolved value for pixel
        float x = 0;

        // convolve
        for(j = 0; j < PSF_HEIGHT; ++j)
            for(i = 0; i < PSF_WIDTH; ++i)
                x += psf2[mad24(j, PSF_WIDTH, i)]*input2[mad24(lj + PSF_HEIGHT - j - 1, cw, li + PSF_WIDTH - i - 1)];

        // store convolved value
        output[mad24(gj, IMAGE_WIDTH, gi)] = x;
    }
}

// compute image
kernel void render(constant char* data, float4 pcs,
                   constant float2* qq, constant float2* ww,
                   global float* value, global float* error)
{
    // get pixel index
    int k = get_global_id(0);

    // compute pixel flux if pixel is in image
    if(k < IMAGE_SIZE)
    {
        // pixel position
        float2 x = pcs.xy + pcs.zw*(float2)(k%IMAGE_WIDTH, k/IMAGE_WIDTH);

        // value and error of quadrature
        float2 f = 0;

        // apply quadrature rule to computed surface brightness
        for(size_t n = 0; n < QUAD_POINTS; ++n)
            f += ww[n]*compute(data, x + qq[n]);

        // done
        value[k] = f.s0;
        error[k] = f.s1;
    }
}

// calculate log-likelihood of computed model
kernel void loglike(global const float* image, global const float* weight,
                    global const float* model, global float* loglike)
{
    // get pixel index
    int k = get_global_id(0);

    // compute chi^2 value if pixel is in image
    if(k < IMAGE_SIZE)
    {
        float d = model[k] - image[k];
        loglike[k] = weight[k]*d*d;
    }
}

// convolve input with PSF
kernel void convolve(global float* input, constant float* psf,
                     local float* input2, local float* psf2,
                     global float* output)
{
    int i, j;

    // pixel indices for output
    int gi = get_global_id(0);
    int gj = get_global_id(1);

    // local indices, size and origin
    int li = get_local_id(0);
    int lj = get_local_id(1);
    int lw = get_local_size(0);
    int lh = get_local_size(1);
    int ls = lw*lh;

    // cache size and origin
    int cw = PSF_WIDTH/2 + lw + PSF_WIDTH/2;
    int ch = PSF_HEIGHT/2 + lh + PSF_HEIGHT/2;
    int cs = cw*ch;
    int cx = mad24((int)get_group_id(0), lw, -PSF_WIDTH/2);
    int cy = mad24((int)get_group_id(1), lh, -PSF_HEIGHT/2);

    // fill cache
    for(i = mad24(lj, lw, li); i < cs; i += ls)
        input2[i] = input[clamp(cy + i/cw, 0, IMAGE_HEIGHT-1)*IMAGE_WIDTH + clamp(cx + i%cw, 0, IMAGE_WIDTH-1)];
    for(i = mad24(lj, lw, li); i < PSF_WIDTH*PSF_HEIGHT; i += ls)
        psf2[i] = psf[i];

    // wait for all items to finish cache filling
    barrier(CLK_LOCAL_MEM_FENCE);

    // check if pixel is in image
    if(gi < IMAGE_WIDTH && gj < IMAGE_HEIGHT)
    {
        // convolved value for pixel
        float x = 0;

        // convolve
        for(j = 0; j < PSF_HEIGHT; ++j)
            for(i = 0; i < PSF_WIDTH; ++i)
                x += psf2[mad24(j, PSF_WIDTH, i)]*input2[mad24(lj + PSF_HEIGHT - j, cw, li + PSF_WIDTH - i)];

        // store convolved value
        output[mad24(gj, IMAGE_WIDTH, gi)] = x;
    }
}


removed error cond check, enforced by CLAMP_TO_EDGE for 2d

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

__kernel void GVF2DInit(__read_only image2d_t volume, __write_only image2d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int2 pos = {get_global_id(0), get_global_id(1)};

    float f10 = read_imagef(volume, sampler, pos + (int2)(1,0)).x;
    float f_10 = read_imagef(volume, sampler, pos - (int2)(1,0)).x;
    float f01 = read_imagef(volume, sampler, pos + (int2)(0,1)).x;
    float f0_1 = read_imagef(volume, sampler, pos - (int2)(0,1)).x;

    float4 gradient = {
        0.5f*(f10-f_10),
        0.5f*(f01-f0_1),
        0,
        0
    };

    write_imagef(vector_field, pos, gradient);
}

__kernel __attribute__((reqd_work_group_size(16,16,1))) void GVF2DIteration(__read_only image2d_t init_vector_field, __read_only image2d_t read_vector_field, __write_only image2d_t write_vector_field, __private float mu) {

    int2 writePos = {get_global_id(0), get_global_id(1)};
    // Enforce mirror boundary conditions
    //int2 size = {get_global_size(0), get_global_size(1)};
    int2 pos = writePos;
    //pos = select(pos, (int2)(2,2), pos == (int2)(0,0));
    //pos = select(pos, size-3, pos >= size-1);

    float2 v = read_imagef(read_vector_field, sampler, pos).xy;

        // Load data from shared memory and do calculations
        float2 init_vector = read_imagef(init_vector_field, sampler, pos).xy;

        float2 fx1, fx_1, fy1, fy_1;
        fx1 = read_imagef(read_vector_field, sampler, pos + (int2)(1,0)).xy;
        fy1 = read_imagef(read_vector_field, sampler, pos + (int2)(0,1)).xy;
        fx_1 = read_imagef(read_vector_field, sampler, pos - (int2)(1,0)).xy;
        fy_1 = read_imagef(read_vector_field, sampler, pos - (int2)(0,1)).xy;

        // Update the vector field: Calculate Laplacian using a 3D central difference scheme
        float2 laplacian = -4*v + fx1 + fx_1 + fy1 + fy_1;

        v += mu * laplacian - (v - init_vector)*(init_vector.x*init_vector.x+init_vector.y*init_vector.y);

        write_imagef(write_vector_field, writePos, (float4)(v.x,v.y,0,0));
}

__kernel void GVF2DResult(__write_only image2d_t result, __read_only image2d_t vectorField, __write_only image2d_t vectorFieldFinal) {
    int2 pos = {get_global_id(0), get_global_id(1)};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.z = 0;
    vector.w = 0;
    write_imagef(vectorFieldFinal, pos, vector);
    write_imagef(result, pos, length(vector));
}

__constant sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;

__kernel void GVF2DInit(__read_only image2d_t volume, __write_only image2d_t vector_field ) {
    // Calculate gradient using a 1D central difference for each dimension, with spacing 1
    int2 pos = {get_global_id(0), get_global_id(1)};

    float f10 = read_imagef(volume, sampler, pos + (int2)(1,0)).x;
    float f_10 = read_imagef(volume, sampler, pos - (int2)(1,0)).x;
    float f01 = read_imagef(volume, sampler, pos + (int2)(0,1)).x;
    float f0_1 = read_imagef(volume, sampler, pos - (int2)(0,1)).x;

    float4 gradient = {
        0.5f*(f10-f_10),
        0.5f*(f01-f0_1),
        0,
        0
    };

    write_imagef(vector_field, pos, gradient);
}

__kernel __attribute__((reqd_work_group_size(16,16,1))) void GVF2DIteration(__read_only image2d_t init_vector_field, __read_only image2d_t read_vector_field, __write_only image2d_t write_vector_field, __private float mu) {

    int2 writePos = {get_global_id(0), get_global_id(1)};
    // Enforce mirror boundary conditions
    int2 size = {get_global_size(0), get_global_size(1)};
    int2 pos = writePos;
    pos = select(pos, (int2)(2,2), pos == (int2)(0,0));
    pos = select(pos, size-3, pos >= size-1);

    float2 v = read_imagef(read_vector_field, sampler, pos).xy;

        // Load data from shared memory and do calculations
        float2 init_vector = read_imagef(init_vector_field, sampler, pos).xy;

        float2 fx1, fx_1, fy1, fy_1;
        fx1 = read_imagef(read_vector_field, sampler, pos + (int2)(1,0)).xy;
        fy1 = read_imagef(read_vector_field, sampler, pos + (int2)(0,1)).xy;
        fx_1 = read_imagef(read_vector_field, sampler, pos - (int2)(1,0)).xy;
        fy_1 = read_imagef(read_vector_field, sampler, pos - (int2)(0,1)).xy;

        // Update the vector field: Calculate Laplacian using a 3D central difference scheme
        float2 laplacian = -4*v + fx1 + fx_1 + fy1 + fy_1;

        v += mu * laplacian - (v - init_vector)*(init_vector.x*init_vector.x+init_vector.y*init_vector.y);

        write_imagef(write_vector_field, writePos, (float4)(v.x,v.y,0,0));
}

__kernel void GVF2DResult(__write_only image2d_t result, __read_only image2d_t vectorField, __write_only image2d_t vectorFieldFinal) {
    int2 pos = {get_global_id(0), get_global_id(1)};
    float4 vector = read_imagef(vectorField, sampler, pos);
    vector.z = 0;
    vector.w = 0;
    write_imagef(vectorFieldFinal, pos, vector);
    write_imagef(result, pos, length(vector));
}


faster using private pvector

faster using private pvector

/*
    ##########################################################
    #                                                        #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #   !!                                              !!   #
    #   !!  KEEP THIS CODE CONSISTENT WITH KERNELPY.PY  !!   #
    #   !!                                              !!   #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #                                                        #
    ##########################################################
*/

#ifndef _PAR_BLOCK_ // protected block so we can include this code twice.
#define _PAR_BLOCK_

typedef struct {
#if MAX_PD > 0
    int32_t pd_par[MAX_PD];     // id of the nth polydispersity variable
    int32_t pd_length[MAX_PD];  // length of the nth polydispersity weight vector
    int32_t pd_offset[MAX_PD];  // offset of pd weights in the value & weight vector
    int32_t pd_stride[MAX_PD];  // stride to move to the next index at this level
#endif // MAX_PD > 0
    int32_t pd_prod;            // total number of voxels in hypercube
    int32_t pd_sum;             // total length of the weights vector
    int32_t num_active;         // number of non-trivial pd loops
    int32_t theta_par;          // id of spherical correction variable
} ProblemDetails;

typedef struct {
    PARAMETER_TABLE;
} ParameterBlock;
#endif

kernel
void KERNEL_NAME(
    int32_t nq,                 // number of q values
    const int32_t pd_start,     // where we are in the polydispersity loop
    const int32_t pd_stop,      // where we are stopping in the polydispersity loop
    global const ProblemDetails *details,
    global const double *values,
    global const double *q, // nq q values, with padding to boundary
    global double *result,  // nq+1 return values, again with padding
    const double cutoff     // cutoff in the polydispersity weight product
    )
{
  // Storage for the current parameter values.  These will be updated as we
  // walk the polydispersity cube.  local_values will be aliased to pvec.
  ParameterBlock local_values;
  double *pvec = (double *)&local_values;

  // who we are and what element we are working with
  const int q_index = get_global_id(0);

  // Fill in the initial variables
  for (int i=0; i < NPARS; i++) {
    pvec[i] = values[2+i];
  }

  // Monodisperse computation
  if (details->num_active == 0) {
    double norm, scale, background;
    // TODO: only needs to be done by one process...
    #ifdef INVALID
    if (INVALID(local_values)) { return; }
    #endif

    norm = CALL_VOLUME(local_values);
    scale = values[0];
    background = values[1];

    if (q_index < nq) {
      double scattering = CALL_IQ(q, q_index, local_values);
      result[q_index] = (norm>0. ? scale*scattering/norm + background : background);
    }
    return;
  }

#if MAX_PD > 0

  double this_result;

  //printf("Entering polydispersity from %d to %d\n", pd_start, pd_stop);

  global const double *pd_value = values+2+NPARS;
  global const double *pd_weight = pd_value+details->pd_sum;

  // need product of weights at every Iq calc, so keep product of
  // weights from the outer loops so that weight = partial_weight * fast_weight
  double pd_norm;
  double partial_weight; // product of weight w4*w3*w2 but not w1
  double spherical_correction; // cosine correction for latitude variation
  double weight; // product of partial_weight*w1*spherical_correction
  int p0_par;
  int p0_length;
  int p0_offset;
  int p0_is_theta;
  int p0_index;

  // Number of elements in the longest polydispersity loop
  p0_par = details->pd_par[0];
  p0_length = details->pd_length[0];
  p0_offset = details->pd_offset[0];
  p0_is_theta = (p0_par == details->theta_par);

  // Trigger the reset behaviour that happens at the end the fast loop
  // by setting the initial index >= weight vector length.
  p0_index = p0_length;

  // Default the spherical correction to 1.0 in case it is not otherwise set
  spherical_correction = 1.0;
  weight=1.0;

  // Since we are no longer looping over the entire polydispersity hypercube
  // for each q, we need to track the result and normalization values between
  // calls.  This means initializing them to 0 at the start and accumulating
  // them between calls.
  pd_norm = pd_start == 0 ? 0.0 : result[nq];

  if (q_index < nq) {
    this_result = pd_start == 0 ? 0.0 : result[q_index];
  }

  // Loop over the weights then loop over q, accumulating values
  for (int loop_index=pd_start; loop_index < pd_stop; loop_index++) {
    // check if fast loop needs to be reset
    if (p0_index == p0_length) {
      //printf("should be here with %d active\n", num_active);

      // Compute position in polydispersity hypercube and partial weight
      partial_weight = 1.0;
      for (int k=1; k < details->num_active; k++) {
        int pk = details->pd_par[k];
        int index = details->pd_offset[k] + (loop_index/details->pd_stride[k])%details->pd_length[k];
        pvec[pk] = pd_value[index];
        partial_weight *= pd_weight[index];
        //printf("index[%d] = %d\n",k,index);
        if (pk == details->theta_par) {
          spherical_correction = fmax(fabs(cos(M_PI_180*pvec[pk])), 1.e-6);
        }
      }
      p0_index = loop_index%p0_length;
      //printf("\n");
    }

    // Update parameter p0
    weight = partial_weight*pd_weight[p0_offset + p0_index];
    pvec[p0_par] = pd_value[p0_offset + p0_index];
    if (p0_is_theta) {
      spherical_correction = fmax(fabs(cos(M_PI_180*pvec[p0_par])), 1.e-6);
    }
    p0_index++;
    //printf("\n");

    // Increment fast index

    #ifdef INVALID
    if (INVALID(local_values)) continue;
    #endif

    // Accumulate I(q)
    // Note: weight==0 must always be excluded
    if (weight > cutoff) {
      // spherical correction has some nasty effects when theta is +90 or -90
      // where it becomes zero.  If the entirety of the correction
      weight *= spherical_correction;
      pd_norm += weight * CALL_VOLUME(local_values);

      const double scattering = CALL_IQ(q, q_index, local_values);
      this_result += weight*scattering;
    }
  }

  if (q_index < nq) {
    if (pd_stop >= details->pd_prod) {
      // End of the PD loop we can normalize
      double scale, background;
      scale = values[0];
      background = values[1];
      result[q_index] = (pd_norm>0. ? scale*this_result/pd_norm + background : background);
    } else {
      // Partial result, so remember it but don't normalize it.
      result[q_index] = this_result;
    }

    // Remember the updated norm.
    if (q_index == 0) result[nq] = pd_norm;
  }

#endif // MAX_PD > 0
}

/*
    ##########################################################
    #                                                        #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #   !!                                              !!   #
    #   !!  KEEP THIS CODE CONSISTENT WITH KERNELPY.PY  !!   #
    #   !!                                              !!   #
    #   !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!   #
    #                                                        #
    ##########################################################
*/

#ifndef _PAR_BLOCK_ // protected block so we can include this code twice.
#define _PAR_BLOCK_

typedef struct {
#if MAX_PD > 0
    int32_t pd_par[MAX_PD];     // id of the nth polydispersity variable
    int32_t pd_length[MAX_PD];  // length of the nth polydispersity weight vector
    int32_t pd_offset[MAX_PD];  // offset of pd weights in the value & weight vector
    int32_t pd_stride[MAX_PD];  // stride to move to the next index at this level
#endif // MAX_PD > 0
    int32_t pd_prod;            // total number of voxels in hypercube
    int32_t pd_sum;             // total length of the weights vector
    int32_t num_active;         // number of non-trivial pd loops
    int32_t theta_par;          // id of spherical correction variable
} ProblemDetails;

typedef struct {
    PARAMETER_TABLE;
} ParameterBlock;
#endif

kernel
void KERNEL_NAME(
    int32_t nq,                 // number of q values
    const int32_t pd_start,     // where we are in the polydispersity loop
    const int32_t pd_stop,      // where we are stopping in the polydispersity loop
    global const ProblemDetails *details,
    global const double *values,
    global const double *q, // nq q values, with padding to boundary
    global double *result,  // nq+1 return values, again with padding
    const double cutoff     // cutoff in the polydispersity weight product
    )
{
  // Storage for the current parameter values.  These will be updated as we
  // walk the polydispersity cube.  local_values will be aliased to pvec.
  local ParameterBlock local_values;

  // who we are and what element we are working with
  const int q_index = get_global_id(0);
  const int thread = get_local_id(0);

  // Fill in the initial variables
  event_t e = async_work_group_copy((local double *)&local_values, values+2, NPARS, 0);
  wait_group_events(1, &e);

  // Monodisperse computation
  if (details->num_active == 0) {
    double norm, scale, background;
    // TODO: only needs to be done by one process...
    #ifdef INVALID
    if (INVALID(local_values)) { return; }
    #endif

    norm = CALL_VOLUME(local_values);
    scale = values[0];
    background = values[1];

    if (q_index < nq) {
      double scattering = CALL_IQ(q, q_index, local_values);
      result[q_index] = (norm>0. ? scale*scattering/norm + background : background);
    }
    return;
  }

#if MAX_PD > 0

  double this_result;

  //printf("Entering polydispersity from %d to %d\n", pd_start, pd_stop);
  // norm will be shared across all threads.

  // "values" is global and can't be assigned to a local, so even though only
  // the alias is only needed for thread 0 it is allocated in all threads.
  global const double *pd_value = values+2+NPARS;
  global const double *pd_weight = pd_value+details->pd_sum;

  // need product of weights at every Iq calc, so keep product of
  // weights from the outer loops so that weight = partial_weight * fast_weight
  local double pd_norm;
  local double partial_weight; // product of weight w4*w3*w2 but not w1
  local double spherical_correction; // cosine correction for latitude variation
  local double weight; // product of partial_weight*w1*spherical_correction
  local double *pvec;
  local int p0_par;
  local int p0_length;
  local int p0_offset;
  local int p0_is_theta;
  local int p0_index;

  // Number of elements in the longest polydispersity loop
  barrier(CLK_LOCAL_MEM_FENCE);
  if (thread == 0) {
    pvec = (local double *)(&local_values);

    // Number of elements in the longest polydispersity loop
    p0_par = details->pd_par[0];
    p0_length = details->pd_length[0];
    p0_offset = details->pd_offset[0];
    p0_is_theta = (p0_par == details->theta_par);

    // Trigger the reset behaviour that happens at the end the fast loop
    // by setting the initial index >= weight vector length.
    p0_index = p0_length;

    // Default the spherical correction to 1.0 in case it is not otherwise set
    spherical_correction = 1.0;

    // Since we are no longer looping over the entire polydispersity hypercube
    // for each q, we need to track the result and normalization values between
    // calls.  This means initializing them to 0 at the start and accumulating
    // them between calls.
    pd_norm = pd_start == 0 ? 0.0 : result[nq];
  }
  barrier(CLK_LOCAL_MEM_FENCE);

  if (q_index < nq) {
    this_result = pd_start == 0 ? 0.0 : result[q_index];
  }

  // Loop over the weights then loop over q, accumulating values
  for (int loop_index=pd_start; loop_index < pd_stop; loop_index++) {
    barrier(CLK_LOCAL_MEM_FENCE);
    if (thread == 0) {
      // check if fast loop needs to be reset
      if (p0_index == p0_length) {
        //printf("should be here with %d active\n", num_active);

        // Compute position in polydispersity hypercube and partial weight
        partial_weight = 1.0;
        for (int k=1; k < details->num_active; k++) {
          int pk = details->pd_par[k];
          int index = details->pd_offset[k] + (loop_index/details->pd_stride[k])%details->pd_length[k];
          pvec[pk] = pd_value[index];
          partial_weight *= pd_weight[index];
          //printf("index[%d] = %d\n",k,index);
          if (pk == details->theta_par) {
            spherical_correction = fmax(fabs(cos(M_PI_180*pvec[pk])), 1.e-6);
          }
        }
        p0_index = loop_index%p0_length;
        //printf("\n");
      }

      // Update parameter p0
      weight = partial_weight*pd_weight[p0_offset + p0_index];
      pvec[p0_par] = pd_value[p0_offset + p0_index];
      if (p0_is_theta) {
        spherical_correction = fmax(fabs(cos(M_PI_180*pvec[p0_par])), 1.e-6);
      }
      p0_index++;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    //printf("\n");

    // Increment fast index

    #ifdef INVALID
    if (INVALID(local_values)) continue;
    #endif

    // Accumulate I(q)
    // Note: weight==0 must always be excluded
    if (weight > cutoff) {
      // spherical correction has some nasty effects when theta is +90 or -90
      // where it becomes zero.  If the entirety of the correction
      weight *= spherical_correction;
      pd_norm += weight * CALL_VOLUME(local_values);

      const double scattering = CALL_IQ(q, q_index, local_values);
      this_result += weight*scattering;
    }
  }

  if (q_index < nq) {
    if (pd_stop >= details->pd_prod) {
      // End of the PD loop we can normalize
      double scale, background;
      scale = values[0];
      background = values[1];
      result[q_index] = (pd_norm>0. ? scale*this_result/pd_norm + background : background);
    } else {
      // Partial result, so remember it but don't normalize it.
      result[q_index] = this_result;
    }

    // Remember the updated norm.
    if (q_index == 0) result[nq] = pd_norm;
  }

#endif // MAX_PD > 0
}


Remove commented out random generator in nn

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

typedef float Scalar;
typedef float4 Scalar4;

Scalar sigmoid(Scalar x) {
    return (Scalar)1.0/((Scalar)1.0 + exp(-x));
}

kernel void sigmoidPredict(global Scalar *x) {
    size_t i = get_global_id(0);
    x[i] = sigmoid(x[i]);
}

kernel void sigmoidFeedforward(global Scalar *x, global Scalar *derivative) {
    size_t i = get_global_id(0);
    Scalar y = sigmoid(x[i]);
    x[i] = y;
    derivative[i] = y*((Scalar)1.0 - y);
}

kernel void reluPredict(global Scalar *x) {
    size_t i = get_global_id(0);
    x[i] = max(x[i], (Scalar)0.0);
}

kernel void reluFeedforward(global Scalar *x, global Scalar *derivative) {
    size_t i = get_global_id(0);
    if (x[i] > 0.0) {
        derivative[i] = 1.0;
    } else {
        x[i] = 0.0;
        derivative[i] = 0.0;
    }
}

kernel void meanSquaredError(global Scalar *prediction, global Scalar *y, global Scalar *output) {
    size_t i = get_global_id(0);
    Scalar diff = y[i] - prediction[i];
    output[i] += diff * diff;
}

kernel void crossEntropyError(global Scalar *prediction, global Scalar *y, global Scalar *output) {
    size_t i = get_global_id(0);
    Scalar err = -(y[i]*log(prediction[i]) + ((Scalar)1.0 - y[i])*log((Scalar)1.0 - prediction[i]));
    output[i] += isnan(err)? (Scalar)0.0 : err;
}

// The "responsibility" of the last layer with MSE criterion.
kernel void computeMSELayerError(global Scalar *prediction, global Scalar *y, global Scalar *derivative, global Scalar *errorTerm) {
    size_t i = get_global_id(0);
    errorTerm[i] = (prediction[i] - y[i]) * derivative[i];
}

// The "responsibility" of the last layer with cross entropy error.
kernel void computeCrossEntropyLayerError(global Scalar *prediction, global Scalar *y, global Scalar *errorTerm) {
    size_t i = get_global_id(0);
    errorTerm[i] = prediction[i] - y[i];
}

// gradients = error * input'
// bias gradients are just added
kernel void computeWeightGradient(global Scalar *errorTerm, global Scalar *input, global Scalar *weightGradients) {
    size_t row = get_global_id(0);
    size_t column = get_global_id(1);
    size_t columns = get_global_size(1);
    weightGradients[row*columns + column] += errorTerm[row] * input[column];
}

kernel void computeWeightGradient4(global Scalar *errorTerm, global Scalar4 *input, global Scalar4 *weightGradients) {
    size_t row = get_global_id(0);
    size_t column = get_global_id(1);
    size_t columns = get_global_size(1);
    weightGradients[row*columns + column] += errorTerm[row] * input[column];
}

kernel void computeWeightGradientParallel(global Scalar *errorTerm, global Scalar *input, global Scalar *weightGradients, const uint count) {
    size_t row = get_global_id(0);
    size_t rows = get_global_size(0);
    size_t column = get_global_id(1);
    size_t columns = get_global_size(1);
    Scalar sum = 0.0;
    for (size_t i = 0; i < count; ++i) {
        sum += errorTerm[i*rows + row] * input[i*columns + column];
    }
    weightGradients[row*columns + column] += sum;
}

kernel void computeWeightGradient4Parallel(global Scalar *errorTerm, global Scalar4 *input, global Scalar4 *weightGradients, const uint count) {
    size_t row = get_global_id(0);
    size_t rows = get_global_size(0);
    size_t column = get_global_id(1);
    size_t columns = get_global_size(1);
    Scalar4 sum = 0.0;
    for (size_t i = 0; i < count; ++i) {
        sum += errorTerm[i*rows + row] * input[i*columns + column];
    }
    weightGradients[row*columns + column] += sum;
}

kernel void computeBiasGradient(global Scalar *errorTerm, const uint count, global Scalar *biasGradients) {
    size_t i = get_global_id(0);
    Scalar sum = 0.0;
    const global Scalar *errorTermEl = errorTerm + i;
    for (size_t j = 0; j < count; ++j) {
        sum += errorTermEl[j*get_global_size(0)];
    }
    biasGradients[i] += sum;
}

kernel void evaluateClassification(global Scalar *outputs, const uint size, global ushort *labels, global uchar *dest) {
    size_t part = get_global_id(0);

    const global Scalar *output = outputs + size*(get_global_id(0) - get_global_offset(0));
    Scalar maxValue = output[0];
    size_t maxIndex = 0;
    for (size_t i = 1; i < size; ++i) {
        if (output[i] > maxValue) {
            maxValue = output[i];
            maxIndex = i;
        }
    }
    dest[part] = maxIndex == labels[part];
}

/*
TODO:

float parkMillerRandom(int *seed) {
    double const a    = 16807;      //ie 7**5
    double const m    = 2147483647; //ie 2**31-1
    double const reciprocal_m = 1.0/m;

    double temp = *seed * a;
    seed = (int)(temp - m * floor(temp * reciprocal_m));
    return float(temp * reciprocal_m);
}

kernel void dropout(global Scalar *x, global int* seeds, const float discardProbability) {
    size_t i = get_global_id(0);
    int seed = seeds[i];
    x[i] = parkMillerRandom(&seed);
    seeds[i] = seed;
}*/


Proper sphere handling

Proper sphere handling

//
//  render.cl
//  Grower
//
//  Created by Lukasz Kwoska on 10/12/14.
//  Copyright (c) 2014 Spinal Development. All rights reserved.
//

typedef struct {
  float3 origin;
  float padding1;
  //Normalized direction vector
  float3 dir;
  float padding2;
} ray;

float intersect_sphere(ray r, float4 s) {

  float3 origin_center = r.origin - s.xyz;

  float dirs_dot = dot(r.dir, origin_center);
  float origin_center_len = length(origin_center);

  float delta = dirs_dot * dirs_dot + s.w * s.w - origin_center_len * origin_center_len;

  if (delta < 0) {
    return -1;
  }

  if (delta == 0) {
    return -dirs_dot;
  }

  delta = sqrt(delta);

  float d1 = -dirs_dot - delta;
  float d2 = -dirs_dot + delta;

  if ((d2 < d1 || d1 < 0) && d2 > 0) {
    d1 = d2;
  }

  return d1;
}

//sphere is defined by it's position float3 and radius
__kernel void render(int width, int height, __global uchar4 *pixels, __global float4 *spheres) {
  int x = get_global_id(0);
  int y = get_global_id(1);

  ray r;
  r.origin = (float3)(x, y, 0);
  r.dir = (float3)(0, 0, 1);

  float4 s = (float4)(0, 0, 0, 100);

  uchar4 color = intersect_sphere(r, s) > 0 ? (uchar4)(255, 255, 255, 255) : (uchar4)(0,0,0,255);

  pixels[y * width + x] = color;

}

//
//  render.cl
//  Grower
//
//  Created by Lukasz Kwoska on 10/12/14.
//  Copyright (c) 2014 Spinal Development. All rights reserved.
//

typedef struct {
  float3 origin;
  float padding1;
  //Normalized direction vector
  float3 dir;
  float padding2;
} ray;

float intersect_sphere(ray r, float4 s) {

  float3 origin_center = r.origin - s.xyz;

  float dirs_dot = dot(r.dir, origin_center);
  float origin_center_len = length(origin_center);

  float delta = dirs_dot * dirs_dot + s.w * s.w - origin_center_len * origin_center_len;

  if (delta < 0) {
    return -1;
  }

  if (delta == 0) {
    return -dirs_dot;
  }

  delta = sqrt(delta);

  float d1 = -dirs_dot - delta;
  float d2 = -dirs_dot + delta;

  if (d2 < d1 && d2 > 0) {
    d1 = d2;
  }

  return d1;
}

//sphere is defined by it's position float3 and radius
__kernel void render(int width, int height, __global uchar4 *pixels, __global float4 *spheres) {
  int x = get_global_id(0);
  int y = get_global_id(1);

  ray r;
  r.origin = (float3)(x, y, 0);
  r.dir = (float3)(0, 0, 1);

  float4 s = (float4)(0, 0, 200, 100);

  uchar4 color = intersect_sphere(r, s) > 0 ? (uchar4)(255, 255, 255, 255) : (uchar4)(0,0,0,255);

  pixels[y * width + x] = color;

}


adding rad8 version of 4096

adding rad8 version of 4096


Correct sign for Sérsic magnitude.

Correct sign for Sérsic magnitude.

OBJECT(sersic) = SOURCE;

PARAMS(sersic) = {
    { "x" },
    { "y" },
    { "r" },
    { "mag" },
    { "n" },
    { "f" },
    { "pa", true },
};

struct sersic
{
    float x;
    float y;
    float q;
    float cos;
    float sin;
    float log0;
    float log1;
    float m;
};

static float sersic(constant struct sersic* src, float2 y)
{
    float u = (y.x - src->x)*src->cos - (y.y - src->y)*src->sin;
    float v = (y.y - src->y)*src->cos + (y.x - src->x)*src->sin;

    float r2 = u*u + v*v/src->q/src->q;

    return exp(src->log0 - exp(src->log1 + src->m*log(r2)));
}

static void set_sersic(global struct sersic* src, float x, float y, float r, float mag, float n, float q, float pa)
{
    float b = 1.9992f*n - 0.3271f; // approximation valid for 0.5 < n < 8

    src->x = x;
    src->y = y;
    src->q = q;

    src->cos = cos(pa);
    src->sin = sin(pa);
    src->log0 = -0.4f*mag*LOG_10 + 2*n*log(b) - LOG_PI - log(q) - 2*log(r) - log(tgamma(2*n+1));
    src->log1 = log(b) - log(r)/n;
    src->m = 0.5f/n;
}

OBJECT(sersic) = SOURCE;

PARAMS(sersic) = {
    { "x" },
    { "y" },
    { "r" },
    { "mag" },
    { "n" },
    { "f" },
    { "pa", true },
};

struct sersic
{
    float x;
    float y;
    float q;
    float cos;
    float sin;
    float log0;
    float log1;
    float m;
};

static float sersic(constant struct sersic* src, float2 y)
{
    float u = (y.x - src->x)*src->cos - (y.y - src->y)*src->sin;
    float v = (y.y - src->y)*src->cos + (y.x - src->x)*src->sin;

    float r2 = u*u + v*v/src->q/src->q;

    return exp(src->log0 - exp(src->log1 + src->m*log(r2)));
}

static void set_sersic(global struct sersic* src, float x, float y, float r, float mag, float n, float q, float pa)
{
    float b = 1.9992f*n - 0.3271f; // approximation valid for 0.5 < n < 8

    src->x = x;
    src->y = y;
    src->q = q;

    src->cos = cos(pa);
    src->sin = sin(pa);
    src->log0 = 0.4f*mag*LOG_10 + 2*n*log(b) - LOG_PI - log(q) - 2*log(r) - log(tgamma(2*n+1));
    src->log1 = log(b) - log(r)/n;
    src->m = 0.5f/n;
}