Add condition checking of residuals because it may be NULL.

Signed-off-by: Yan Wang <8ca54580088a7ffe1dd243d1c16c6d1f04463d55@linux.intel.com>
Reviewed-by: Chuanbo Weng <4145c40a11b367664079fb2725d5e44cc6d84d1d@intel.com>

typedef struct _motion_estimation_desc_intel {
  uint mb_block_type;
  uint subpixel_mode;
  uint sad_adjust_mode;
  uint search_path_type;
} accelerator_intel_t;

__kernel __attribute__((reqd_work_group_size(16,1,1)))
void block_motion_estimate_intel(accelerator_intel_t accel,
                                 __read_only  image2d_t src_image,
                                 __read_only  image2d_t ref_image,
                                 __global short2 * prediction_motion_vector_buffer,
                                 __global short2 * motion_vector_buffer,
                                 __global ushort * residuals){

  uint src_grf0_dw7;
  uint src_grf0_dw6;
  uint src_grf0_dw5;
  uint src_grf0_dw4;
  uint src_grf0_dw3;
  uint src_grf0_dw2;
  uint src_grf0_dw1;
  uint src_grf0_dw0;
  uint src_grf1_dw7;
  uint src_grf1_dw6;
  uint src_grf1_dw5;
  uint src_grf1_dw4;
  uint src_grf1_dw3;
  uint src_grf1_dw2;
  uint src_grf1_dw1;
  uint src_grf1_dw0;
  uint src_grf2_dw7;
  uint src_grf2_dw6;
  uint src_grf2_dw5;
  uint src_grf2_dw4;
  uint src_grf2_dw3;
  uint src_grf2_dw2;
  uint src_grf2_dw1;
  uint src_grf2_dw0;
  uint src_grf3_dw7;
  uint src_grf3_dw6;
  uint src_grf3_dw5;
  uint src_grf3_dw4;
  uint src_grf3_dw3;
  uint src_grf3_dw2;
  uint src_grf3_dw1;
  uint src_grf3_dw0;
  uint src_grf4_dw7;
  uint src_grf4_dw6;
  uint src_grf4_dw5;
  uint src_grf4_dw4;
  uint src_grf4_dw3;
  uint src_grf4_dw2;
  uint src_grf4_dw1;
  uint src_grf4_dw0;

  uint8 vme_result = (0, 0, 0, 0, 0, 0, 0, 0);

  int lgid_x = get_group_id(0);
  int lgid_y = get_group_id(1);

  int num_groups_x = get_num_groups(0);
  int index = lgid_y * num_groups_x + lgid_x;

  uint2 srcCoord = 0;
  short2 predict_mv = 0;
  if(prediction_motion_vector_buffer != NULL){
    predict_mv = prediction_motion_vector_buffer[index];
    predict_mv.x = predict_mv.x / 4;
    predict_mv.y = predict_mv.y / 4;
  }

  srcCoord.x = lgid_x * 16;
  srcCoord.y = lgid_y * 16;

  //CL_ME_SEARCH_PATH_RADIUS_2_2_INTEL
  if(accel.search_path_type == 0x0){
    //src_grf0_dw5 = (Ref_Height << 24) | (Ref_Width << 16) | (Ignored << 8) | (Dispatch_Id);
    src_grf0_dw5 =   (20 << 24)         | (20 << 16)        | (0 << 8)       | (0);
    //src_grf0_dw1 = (Ref1Y << 16)  | (Ref1X);
    src_grf0_dw1 =   ((-2 + predict_mv.y) << 16 ) | ((-2 + predict_mv.x) & 0x0000ffff);
    //src_grf0_dw0 = (Ref0Y << 16)  | (Ref0X);
    src_grf0_dw0 =   ((-2 + predict_mv.y) << 16 ) | ((-2 + predict_mv.x) & 0x0000ffff);
    //src_grf1_dw2 = (Start1Y << 28)                  | (Start1X << 24)                | (Start0Y << 20)
    src_grf1_dw2 =   (0 << 28)                        | (0 << 24)                      | (0 << 20)
                   //| (Start0X << 16)               | (Max_Num_SU << 8)              | (LenSP);
                     | (0 << 16)                     | (2 << 8)                       | (2);
  }
  //CL_ME_SEARCH_PATH_RADIUS_4_4_INTEL
  else if(accel.search_path_type == 0x1){
    src_grf0_dw5 =   (24 << 24)         | (24 << 16)        | (0 << 8)       | (0);
    src_grf0_dw1 =   ((-4 + predict_mv.y) << 16 ) | ((-4 + predict_mv.x) & 0x0000ffff);
    src_grf0_dw0 =   ((-4 + predict_mv.y) << 16 ) | ((-4 + predict_mv.x) & 0x0000ffff);
    src_grf1_dw2 =   (0 << 28)                        | (0 << 24)                      | (0 << 20)
                     | (0 << 16)                     | (48 << 8)                       | (48);
  }
  //CL_ME_SEARCH_PATH_RADIUS_16_12_INTEL
  else if(accel.search_path_type == 0x5){
    src_grf0_dw5 =   (40 << 24)         | (48 << 16)        | (0 << 8)       | (0);
    src_grf0_dw1 =   ((-12 + predict_mv.y) << 16 ) | ((-16 + predict_mv.x) & 0x0000ffff);
    src_grf0_dw0 =   ((-12 + predict_mv.y) << 16 ) | ((-16 +  + predict_mv.x) & 0x0000ffff);
    src_grf1_dw2 =   (0 << 28)                        | (0 << 24)                      | (0 << 20)
                     | (0 << 16)                     | (48 << 8)                       | (48);
  }

  /*Deal with mb_block_type & sad_adjust_mode & subpixel_mode*/
  uchar sub_mb_part_mask = 0;
  //CL_ME_MB_TYPE_16x16_INTEL
  if(accel.mb_block_type == 0x0)
    sub_mb_part_mask = 0x7e;
  //CL_ME_MB_TYPE_8x8_INTEL
  else if(accel.mb_block_type == 0x1)
    sub_mb_part_mask = 0x77;
  //CL_ME_MB_TYPE_4x4_INTEL
  else if(accel.mb_block_type == 0x2)
    sub_mb_part_mask = 0x3f;

  uchar inter_sad = 0;
  //CL_ME_SAD_ADJUST_MODE_NONE_INTEL
  if(accel.sad_adjust_mode == 0x0)
    inter_sad = 0;
  //CL_ME_SAD_ADJUST_MODE_HAAR_INTEL
  else if(accel.sad_adjust_mode == 0x1)
    inter_sad = 2;

  uchar sub_pel_mode = 0;
  //CL_ME_SUBPIXEL_MODE_INTEGER_INTEL
  if(accel.subpixel_mode == 0x0)
    sub_pel_mode = 0;
  //CL_ME_SUBPIXEL_MODE_HPEL_INTEL
  else if(accel.subpixel_mode == 0x1)
    sub_pel_mode = 1;
  //CL_ME_SUBPIXEL_MODE_QPEL_INTEL
  else if(accel.subpixel_mode == 0x2)
    sub_pel_mode = 3;

  //src_grf0_dw3 = (Reserved << 31)                | (Sub_Mb_Part_Mask << 24)       | (Intra_SAD << 22)
  src_grf0_dw3 =   (0 << 31)                       | (sub_mb_part_mask << 24)       | (0 << 22)
                 //| (Inter_SAD << 20)             | (BB_Skip_Enabled << 19)        | (Reserverd << 18)
                   | (inter_sad << 20)             | (0 << 19)                      | (0 << 18)
                 //| (Dis_Aligned_Src_Fetch << 17) | (Dis_Aligned_Ref_Fetch << 16)  | (Dis_Field_Cache_Alloc << 15)
                   | (0 << 17)                     | (0 << 16)                      | (0 << 15)
                 //| (Skip_Type << 14)             | (Sub_Pel_Mode << 12)           | (Dual_Search_Path_Opt << 11)
                   | (0 << 14)                     | (sub_pel_mode << 12)           | (0 << 11)
                 //| (Search_Ctrl << 8)            | (Ref_Access << 7)              | (SrcAccess << 6)
                   | (0 << 8)                      | (0 << 7)                       | (0 << 6)
                 //| (Mb_Type_Remap << 4)          | (Reserved_Workaround << 3)     | (Reserved_Workaround << 2)
                   | (0 << 4)                      | (0 << 3)                       | (0 << 2)
                 //| (Src_Size);
                   | (0);

  //src_grf0_dw7 = Debug;
  src_grf0_dw7 = 0;
  //src_grf0_dw6 = Debug;
  src_grf0_dw6 = 0;
  //src_grf0_dw5 = (Ref_Height << 24) | (Ref_Width << 16) | (Ignored << 8) | (Dispatch_Id?);
  //src_grf0_dw4 = Ignored;
  src_grf0_dw4 = 0;

  //src_grf0_dw2 = (SrcY << 16) | (SrcX);
  src_grf0_dw2 = (srcCoord.y << 16)  | (srcCoord.x);
  //src_grf0_dw1 = (Ref1Y << 16)  | (Ref1X);
  //src_grf0_dw0 = (Ref0Y << 16)  | (Ref0X);
  /*src_grf1_dw7 = (Skip_Center_Mask << 24)         | (Reserved << 22)               | (Ref1_Field_Polarity << 21)
                 | (Ref0_Field_Polarity << 20)   | (Src_Field_Polarity << 19)     | (Bilinear_Enable << 18)
                 | (MV_Cost_Scale_Factor << 16)  | (Mb_Intra_Struct << 8)         | (Intra_Corner_Swap << 7)
                 | (Non_Skip_Mode_Added << 6)    | (Non_Skip_ZMv_Added << 5)      | (IntraPartMask);*/
  src_grf1_dw7 = 0;
  //src_grf1_dw6 = Reserved;
  src_grf1_dw6 = 0;
  /*src_grf1_dw5 = (Cost_Center1Y << 16)  | (Cost_Center1X);
  src_grf1_dw4 = (Cost_Center0Y << 16)  | (Cost_Center0X);
  src_grf1_dw3 = (Ime_Too_Good << 24 )  | (Ime_Too_Bad << 16)  | (Part_Tolerance_Thrhd << 8) | (FBPrunThrhd);*/
  src_grf1_dw5 = 0;
  src_grf1_dw4 = 0;
  src_grf1_dw3 = 0;
  //src_grf1_dw2 = (Start1Y << 28)                  | (Start1X << 24)                | (Start0Y << 20)
                 //| (Start0X << 16)               | (Max_Num_SU << 8)              | (LenSP);
  /*src_grf1_dw1 = (RepartEn << 31)                 | (FBPrunEn << 30)               | (AdaptiveValidationControl << 29)
                 | (Uni_Mix_Disable << 28)       | (Bi_Sub_Mb_Part_Mask << 24)    | (Reserverd << 22)
                 | (Bi_Weight << 16)             | (Reserved << 6)                | (MaxNumMVs);*/
  //src_grf1_dw1 = (0 << 24) | (2);
  src_grf1_dw1 = (0 << 24) | (16);
  /*src_grf1_dw0 = (Early_Ime_Stop << 24)           | (Early_Fme_Success << 16)      | (Skip_Success << 8)
                 | (T8x8_Flag_For_Inter_En << 7) | (Quit_Inter_En << 6)           | (Early_Ime_Success_En << 5)
                 | (Early_Success_En << 4)       | (Part_Candidate_En << 3)       | (Bi_Mix_Dis << 2)
                 | (Adaptive_En  << 1)           | (SkipModeEn);*/
  src_grf1_dw0 = 0;
  /*src_grf2_dw7 = Ref1_SkipCenter_3_Delta_XY;
  src_grf2_dw6 = Ref0_SkipCenter_3_Delta_XY;
  src_grf2_dw5 = Ref1_SkipCenter_2_Delta_XY;
  src_grf2_dw4 = Ref0_SkipCenter_3_Delta_XY;
  src_grf2_dw3 = Ref1_SkipCenter_1_Delta_XY;
  src_grf2_dw2 = Ref0_SkipCenter_1_Delta_XY;
  src_grf2_dw1 = Ref1_SkipCenter_0_Delta_XY;
  src_grf2_dw0 = (Ref0_Skip_Center_0_Delta_Y << 16)  | (Ref0_Skip_Center_0_Delta_X);
  src_grf3_dw7 = Neighbor pixel Luma value [23, -1] to [20, -1];
  src_grf3_dw6 = Neighbor pixel Luma value [19, -1] to [16, -1];
  src_grf3_dw5 = Neighbor pixel Luma value [15, -1] to [12, -1];
  src_grf3_dw4 = Neighbor pixel Luma value [11, -1] to [8, -1];
  src_grf3_dw3 = Neighbor pixel Luma value [7, -1] to [4, -1];
  src_grf3_dw2 = (Neighbor pixel Luma value [3, -1] << 24)    | (Neighbor pixel Luma value [2, -1] << 16)
                 | (Neighbor pixel Luma value [1, -1] << 8)  | (Neighbor pixel Luma value [0, -1]);
  //src_grf3_dw1 = (?)  | (Reserved)  | ((Intra_16x16_Mode_Mask);
  src_grf3_dw0 = (Reserved<<25)  | (Intra_16x16_Mode_Mask << 16)  | (Reserved)  | (Intra_16x16_Mode_Mask);
  src_grf4_dw7 = Reserved;
  src_grf4_dw6 = Reserved;
  src_grf4_dw5 = Reserved;
  src_grf4_dw4 = (Intra_MxM_Pred_Mode_B15 << 28)    | (Intra_MxM_Pred_Mode_B14 << 24)  | (Intra_MxM_Pred_Mode_B11 << 20)
                 | (Intra_MxM_Pred_Mode_B10 << 16) | (Intra_MxM_Pred_Mode_A15 << 12)  | (Intra_MxM_Pred_Mode_A13 << 8)
                 | (Intra_MxM_Pred_Mode_A7 << 4)   | (Intra_MxM_Pred_Mode_A5);
  //src_grf4_dw3 = (?)  | (Neighbor pixel Luma value [-1, 14] to [-1, 12]);
  src_grf4_dw2 = Neighbor pixel Luma value [-1, 11] to [-1, 8];
  src_grf4_dw1 = Neighbor pixel Luma value [-1, 7] to [-1, 4];
  src_grf4_dw0 = (Neighbor pixel Luma value [-1, 3] << 24)    | (Neighbor pixel Luma value [-1, 2] << 16)
                 | (Neighbor pixel Luma value [-1, 1] << 8)  | (Neighbor pixel Luma value [-1, 0]);*/
  src_grf2_dw7 = 0;
  src_grf2_dw6 = 0;
  src_grf2_dw5 = 0;
  src_grf2_dw4 = 0;
  src_grf2_dw3 = 0;
  src_grf2_dw2 = 0;
  src_grf2_dw1 = 0;
  src_grf2_dw0 = 0;
  src_grf3_dw7 = 0;
  src_grf3_dw6 = 0;
  src_grf3_dw5 = 0;
  src_grf3_dw4 = 0;
  src_grf3_dw3 = 0;
  src_grf3_dw2 = 0;
  src_grf3_dw1 = 0;
  src_grf3_dw0 = 0;
  src_grf4_dw7 = 0;
  src_grf4_dw6 = 0;
  src_grf4_dw5 = 0;
  src_grf4_dw4 = 0;
  src_grf4_dw3 = 0;
  src_grf4_dw2 = 0;
  src_grf4_dw1 = 0;
  src_grf4_dw0 = 0;

  int lid_x = get_local_id(0);

  vme_result = __gen_ocl_vme(src_image, ref_image,
                src_grf0_dw7, src_grf0_dw6, src_grf0_dw5, src_grf0_dw4,
                src_grf0_dw3, src_grf0_dw2, src_grf0_dw1, src_grf0_dw0,
                src_grf1_dw7, src_grf1_dw6, src_grf1_dw5, src_grf1_dw4,
                src_grf1_dw3, src_grf1_dw2, src_grf1_dw1, src_grf1_dw0,
                src_grf2_dw7, src_grf2_dw6, src_grf2_dw5, src_grf2_dw4,
                src_grf2_dw3, src_grf2_dw2, src_grf2_dw1, src_grf2_dw0,
                src_grf3_dw7, src_grf3_dw6, src_grf3_dw5, src_grf3_dw4,
                src_grf3_dw3, src_grf3_dw2, src_grf3_dw1, src_grf3_dw0,
                src_grf4_dw7, src_grf4_dw6, src_grf4_dw5, src_grf4_dw4,
                src_grf4_dw3, src_grf4_dw2, src_grf4_dw1, src_grf4_dw0,
                //msg_type, vme_search_path_lut, lut_sub,
                1, 0, 0);

  barrier(CLK_LOCAL_MEM_FENCE);

  short2 mv[16];
  ushort res[16];

  uint write_back_dwx;
  uint simd_width = get_max_sub_group_size();

  /* In simd 8 mode, one kernel variable 'uint' map to 8 dword.
   * In simd 16 mode, one kernel variable 'uint' map to 16 dword.
   * That's why we should treat simd8 and simd16 differently when
   * use __gen_ocl_region.
   * */
  if(simd_width == 8){
    write_back_dwx = __gen_ocl_region(0, vme_result.s1);
    mv[0] = as_short2( write_back_dwx );

    if(accel.mb_block_type > 0x0){
      for(int i = 2, j = 1; j < 4; i += 2, j++){
        write_back_dwx = __gen_ocl_region(i, vme_result.s1);
        mv[j] = as_short2( write_back_dwx );
      }
      if(accel.mb_block_type > 0x1){
        for(int i = 0, j = 4; j < 8; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s2);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 8; j < 12; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s3);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 12; j < 16; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s4);
          mv[j] = as_short2( write_back_dwx );
        }
      }
    }
    ushort2 temp_res;
    for(int i = 0; i < 8; i++){
      write_back_dwx = __gen_ocl_region(i, vme_result.s5);
      temp_res = as_ushort2(write_back_dwx);
      res[i*2] = temp_res.s0;
      res[i*2+1] = temp_res.s1;
    }
  }
  else if(simd_width == 16){
    write_back_dwx = __gen_ocl_region(0 + 8, vme_result.s0);
    mv[0] = as_short2( write_back_dwx );

    if(accel.mb_block_type > 0x0){
      for(int i = 2, j = 1; j < 4; i += 2, j++){
        write_back_dwx = __gen_ocl_region(i + 8, vme_result.s0);
        mv[j] = as_short2( write_back_dwx );
      }
      if(accel.mb_block_type > 0x1){
        for(int i = 0, j = 4; j < 8; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s1);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 8; j < 12; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i + 8, vme_result.s1);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 12; j < 16; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s2);
          mv[j] = as_short2( write_back_dwx );
        }
      }
    }
    ushort2 temp_res;
    for(int i = 0; i < 8; i++){
      write_back_dwx = __gen_ocl_region(i + 8, vme_result.s2);
      temp_res = as_ushort2(write_back_dwx);
      res[i*2] = temp_res.s0;
      res[i*2+1] = temp_res.s1;
    }
  }

  int mv_index;

  //CL_ME_MB_TYPE_16x16_INTEL
  if(accel.mb_block_type == 0x0){
    mv_index = index * 1;
    if( lid_x == 0 ){
      motion_vector_buffer[mv_index] = mv[lid_x];
      if(residuals)
        residuals[mv_index] = 2 * res[lid_x];
    }
  }
  //CL_ME_MB_TYPE_8x8_INTEL
  else if(accel.mb_block_type == 0x1){
    if(lid_x < 4){
      mv_index = lgid_y * num_groups_x * 4 + lgid_x * 2;
      mv_index = mv_index + num_groups_x * 2 * (lid_x / 2) + (lid_x % 2);
      motion_vector_buffer[mv_index] = mv[lid_x];
      if(residuals)
        residuals[mv_index] = 2 * res[lid_x];
    }
  }
  //CL_ME_MB_TYPE_4x4_INTEL
  else if(accel.mb_block_type == 0x2){
    if(lid_x < 16){
      mv_index = lgid_y * num_groups_x * 16 + lgid_x * 4;
      mv_index = mv_index + num_groups_x * 4 * (lid_x / 4) + (lid_x % 4);
      motion_vector_buffer[mv_index] = mv[lid_x];
      if(residuals)
        residuals[mv_index] = 2 * res[lid_x];
    }
  }

}

typedef struct _motion_estimation_desc_intel {
  uint mb_block_type;
  uint subpixel_mode;
  uint sad_adjust_mode;
  uint search_path_type;
} accelerator_intel_t;

__kernel __attribute__((reqd_work_group_size(16,1,1)))
void block_motion_estimate_intel(accelerator_intel_t accel,
                                 __read_only  image2d_t src_image,
                                 __read_only  image2d_t ref_image,
                                 __global short2 * prediction_motion_vector_buffer,
                                 __global short2 * motion_vector_buffer,
                                 __global ushort * residuals){

  uint src_grf0_dw7;
  uint src_grf0_dw6;
  uint src_grf0_dw5;
  uint src_grf0_dw4;
  uint src_grf0_dw3;
  uint src_grf0_dw2;
  uint src_grf0_dw1;
  uint src_grf0_dw0;
  uint src_grf1_dw7;
  uint src_grf1_dw6;
  uint src_grf1_dw5;
  uint src_grf1_dw4;
  uint src_grf1_dw3;
  uint src_grf1_dw2;
  uint src_grf1_dw1;
  uint src_grf1_dw0;
  uint src_grf2_dw7;
  uint src_grf2_dw6;
  uint src_grf2_dw5;
  uint src_grf2_dw4;
  uint src_grf2_dw3;
  uint src_grf2_dw2;
  uint src_grf2_dw1;
  uint src_grf2_dw0;
  uint src_grf3_dw7;
  uint src_grf3_dw6;
  uint src_grf3_dw5;
  uint src_grf3_dw4;
  uint src_grf3_dw3;
  uint src_grf3_dw2;
  uint src_grf3_dw1;
  uint src_grf3_dw0;
  uint src_grf4_dw7;
  uint src_grf4_dw6;
  uint src_grf4_dw5;
  uint src_grf4_dw4;
  uint src_grf4_dw3;
  uint src_grf4_dw2;
  uint src_grf4_dw1;
  uint src_grf4_dw0;

  uint8 vme_result = (0, 0, 0, 0, 0, 0, 0, 0);

  int lgid_x = get_group_id(0);
  int lgid_y = get_group_id(1);

  int num_groups_x = get_num_groups(0);
  int index = lgid_y * num_groups_x + lgid_x;

  uint2 srcCoord = 0;
  short2 predict_mv = 0;
  if(prediction_motion_vector_buffer != NULL){
    predict_mv = prediction_motion_vector_buffer[index];
    predict_mv.x = predict_mv.x / 4;
    predict_mv.y = predict_mv.y / 4;
  }

  srcCoord.x = lgid_x * 16;
  srcCoord.y = lgid_y * 16;

  //CL_ME_SEARCH_PATH_RADIUS_2_2_INTEL
  if(accel.search_path_type == 0x0){
    //src_grf0_dw5 = (Ref_Height << 24) | (Ref_Width << 16) | (Ignored << 8) | (Dispatch_Id);
    src_grf0_dw5 =   (20 << 24)         | (20 << 16)        | (0 << 8)       | (0);
    //src_grf0_dw1 = (Ref1Y << 16)  | (Ref1X);
    src_grf0_dw1 =   ((-2 + predict_mv.y) << 16 ) | ((-2 + predict_mv.x) & 0x0000ffff);
    //src_grf0_dw0 = (Ref0Y << 16)  | (Ref0X);
    src_grf0_dw0 =   ((-2 + predict_mv.y) << 16 ) | ((-2 + predict_mv.x) & 0x0000ffff);
    //src_grf1_dw2 = (Start1Y << 28)                  | (Start1X << 24)                | (Start0Y << 20)
    src_grf1_dw2 =   (0 << 28)                        | (0 << 24)                      | (0 << 20)
                   //| (Start0X << 16)               | (Max_Num_SU << 8)              | (LenSP);
                     | (0 << 16)                     | (2 << 8)                       | (2);
  }
  //CL_ME_SEARCH_PATH_RADIUS_4_4_INTEL
  else if(accel.search_path_type == 0x1){
    src_grf0_dw5 =   (24 << 24)         | (24 << 16)        | (0 << 8)       | (0);
    src_grf0_dw1 =   ((-4 + predict_mv.y) << 16 ) | ((-4 + predict_mv.x) & 0x0000ffff);
    src_grf0_dw0 =   ((-4 + predict_mv.y) << 16 ) | ((-4 + predict_mv.x) & 0x0000ffff);
    src_grf1_dw2 =   (0 << 28)                        | (0 << 24)                      | (0 << 20)
                     | (0 << 16)                     | (48 << 8)                       | (48);
  }
  //CL_ME_SEARCH_PATH_RADIUS_16_12_INTEL
  else if(accel.search_path_type == 0x5){
    src_grf0_dw5 =   (40 << 24)         | (48 << 16)        | (0 << 8)       | (0);
    src_grf0_dw1 =   ((-12 + predict_mv.y) << 16 ) | ((-16 + predict_mv.x) & 0x0000ffff);
    src_grf0_dw0 =   ((-12 + predict_mv.y) << 16 ) | ((-16 +  + predict_mv.x) & 0x0000ffff);
    src_grf1_dw2 =   (0 << 28)                        | (0 << 24)                      | (0 << 20)
                     | (0 << 16)                     | (48 << 8)                       | (48);
  }

  /*Deal with mb_block_type & sad_adjust_mode & subpixel_mode*/
  uchar sub_mb_part_mask = 0;
  //CL_ME_MB_TYPE_16x16_INTEL
  if(accel.mb_block_type == 0x0)
    sub_mb_part_mask = 0x7e;
  //CL_ME_MB_TYPE_8x8_INTEL
  else if(accel.mb_block_type == 0x1)
    sub_mb_part_mask = 0x77;
  //CL_ME_MB_TYPE_4x4_INTEL
  else if(accel.mb_block_type == 0x2)
    sub_mb_part_mask = 0x3f;

  uchar inter_sad = 0;
  //CL_ME_SAD_ADJUST_MODE_NONE_INTEL
  if(accel.sad_adjust_mode == 0x0)
    inter_sad = 0;
  //CL_ME_SAD_ADJUST_MODE_HAAR_INTEL
  else if(accel.sad_adjust_mode == 0x1)
    inter_sad = 2;

  uchar sub_pel_mode = 0;
  //CL_ME_SUBPIXEL_MODE_INTEGER_INTEL
  if(accel.subpixel_mode == 0x0)
    sub_pel_mode = 0;
  //CL_ME_SUBPIXEL_MODE_HPEL_INTEL
  else if(accel.subpixel_mode == 0x1)
    sub_pel_mode = 1;
  //CL_ME_SUBPIXEL_MODE_QPEL_INTEL
  else if(accel.subpixel_mode == 0x2)
    sub_pel_mode = 3;

  //src_grf0_dw3 = (Reserved << 31)                | (Sub_Mb_Part_Mask << 24)       | (Intra_SAD << 22)
  src_grf0_dw3 =   (0 << 31)                       | (sub_mb_part_mask << 24)       | (0 << 22)
                 //| (Inter_SAD << 20)             | (BB_Skip_Enabled << 19)        | (Reserverd << 18)
                   | (inter_sad << 20)             | (0 << 19)                      | (0 << 18)
                 //| (Dis_Aligned_Src_Fetch << 17) | (Dis_Aligned_Ref_Fetch << 16)  | (Dis_Field_Cache_Alloc << 15)
                   | (0 << 17)                     | (0 << 16)                      | (0 << 15)
                 //| (Skip_Type << 14)             | (Sub_Pel_Mode << 12)           | (Dual_Search_Path_Opt << 11)
                   | (0 << 14)                     | (sub_pel_mode << 12)           | (0 << 11)
                 //| (Search_Ctrl << 8)            | (Ref_Access << 7)              | (SrcAccess << 6)
                   | (0 << 8)                      | (0 << 7)                       | (0 << 6)
                 //| (Mb_Type_Remap << 4)          | (Reserved_Workaround << 3)     | (Reserved_Workaround << 2)
                   | (0 << 4)                      | (0 << 3)                       | (0 << 2)
                 //| (Src_Size);
                   | (0);

  //src_grf0_dw7 = Debug;
  src_grf0_dw7 = 0;
  //src_grf0_dw6 = Debug;
  src_grf0_dw6 = 0;
  //src_grf0_dw5 = (Ref_Height << 24) | (Ref_Width << 16) | (Ignored << 8) | (Dispatch_Id?);
  //src_grf0_dw4 = Ignored;
  src_grf0_dw4 = 0;

  //src_grf0_dw2 = (SrcY << 16) | (SrcX);
  src_grf0_dw2 = (srcCoord.y << 16)  | (srcCoord.x);
  //src_grf0_dw1 = (Ref1Y << 16)  | (Ref1X);
  //src_grf0_dw0 = (Ref0Y << 16)  | (Ref0X);
  /*src_grf1_dw7 = (Skip_Center_Mask << 24)         | (Reserved << 22)               | (Ref1_Field_Polarity << 21)
                 | (Ref0_Field_Polarity << 20)   | (Src_Field_Polarity << 19)     | (Bilinear_Enable << 18)
                 | (MV_Cost_Scale_Factor << 16)  | (Mb_Intra_Struct << 8)         | (Intra_Corner_Swap << 7)
                 | (Non_Skip_Mode_Added << 6)    | (Non_Skip_ZMv_Added << 5)      | (IntraPartMask);*/
  src_grf1_dw7 = 0;
  //src_grf1_dw6 = Reserved;
  src_grf1_dw6 = 0;
  /*src_grf1_dw5 = (Cost_Center1Y << 16)  | (Cost_Center1X);
  src_grf1_dw4 = (Cost_Center0Y << 16)  | (Cost_Center0X);
  src_grf1_dw3 = (Ime_Too_Good << 24 )  | (Ime_Too_Bad << 16)  | (Part_Tolerance_Thrhd << 8) | (FBPrunThrhd);*/
  src_grf1_dw5 = 0;
  src_grf1_dw4 = 0;
  src_grf1_dw3 = 0;
  //src_grf1_dw2 = (Start1Y << 28)                  | (Start1X << 24)                | (Start0Y << 20)
                 //| (Start0X << 16)               | (Max_Num_SU << 8)              | (LenSP);
  /*src_grf1_dw1 = (RepartEn << 31)                 | (FBPrunEn << 30)               | (AdaptiveValidationControl << 29)
                 | (Uni_Mix_Disable << 28)       | (Bi_Sub_Mb_Part_Mask << 24)    | (Reserverd << 22)
                 | (Bi_Weight << 16)             | (Reserved << 6)                | (MaxNumMVs);*/
  //src_grf1_dw1 = (0 << 24) | (2);
  src_grf1_dw1 = (0 << 24) | (16);
  /*src_grf1_dw0 = (Early_Ime_Stop << 24)           | (Early_Fme_Success << 16)      | (Skip_Success << 8)
                 | (T8x8_Flag_For_Inter_En << 7) | (Quit_Inter_En << 6)           | (Early_Ime_Success_En << 5)
                 | (Early_Success_En << 4)       | (Part_Candidate_En << 3)       | (Bi_Mix_Dis << 2)
                 | (Adaptive_En  << 1)           | (SkipModeEn);*/
  src_grf1_dw0 = 0;
  /*src_grf2_dw7 = Ref1_SkipCenter_3_Delta_XY;
  src_grf2_dw6 = Ref0_SkipCenter_3_Delta_XY;
  src_grf2_dw5 = Ref1_SkipCenter_2_Delta_XY;
  src_grf2_dw4 = Ref0_SkipCenter_3_Delta_XY;
  src_grf2_dw3 = Ref1_SkipCenter_1_Delta_XY;
  src_grf2_dw2 = Ref0_SkipCenter_1_Delta_XY;
  src_grf2_dw1 = Ref1_SkipCenter_0_Delta_XY;
  src_grf2_dw0 = (Ref0_Skip_Center_0_Delta_Y << 16)  | (Ref0_Skip_Center_0_Delta_X);
  src_grf3_dw7 = Neighbor pixel Luma value [23, -1] to [20, -1];
  src_grf3_dw6 = Neighbor pixel Luma value [19, -1] to [16, -1];
  src_grf3_dw5 = Neighbor pixel Luma value [15, -1] to [12, -1];
  src_grf3_dw4 = Neighbor pixel Luma value [11, -1] to [8, -1];
  src_grf3_dw3 = Neighbor pixel Luma value [7, -1] to [4, -1];
  src_grf3_dw2 = (Neighbor pixel Luma value [3, -1] << 24)    | (Neighbor pixel Luma value [2, -1] << 16)
                 | (Neighbor pixel Luma value [1, -1] << 8)  | (Neighbor pixel Luma value [0, -1]);
  //src_grf3_dw1 = (?)  | (Reserved)  | ((Intra_16x16_Mode_Mask);
  src_grf3_dw0 = (Reserved<<25)  | (Intra_16x16_Mode_Mask << 16)  | (Reserved)  | (Intra_16x16_Mode_Mask);
  src_grf4_dw7 = Reserved;
  src_grf4_dw6 = Reserved;
  src_grf4_dw5 = Reserved;
  src_grf4_dw4 = (Intra_MxM_Pred_Mode_B15 << 28)    | (Intra_MxM_Pred_Mode_B14 << 24)  | (Intra_MxM_Pred_Mode_B11 << 20)
                 | (Intra_MxM_Pred_Mode_B10 << 16) | (Intra_MxM_Pred_Mode_A15 << 12)  | (Intra_MxM_Pred_Mode_A13 << 8)
                 | (Intra_MxM_Pred_Mode_A7 << 4)   | (Intra_MxM_Pred_Mode_A5);
  //src_grf4_dw3 = (?)  | (Neighbor pixel Luma value [-1, 14] to [-1, 12]);
  src_grf4_dw2 = Neighbor pixel Luma value [-1, 11] to [-1, 8];
  src_grf4_dw1 = Neighbor pixel Luma value [-1, 7] to [-1, 4];
  src_grf4_dw0 = (Neighbor pixel Luma value [-1, 3] << 24)    | (Neighbor pixel Luma value [-1, 2] << 16)
                 | (Neighbor pixel Luma value [-1, 1] << 8)  | (Neighbor pixel Luma value [-1, 0]);*/
  src_grf2_dw7 = 0;
  src_grf2_dw6 = 0;
  src_grf2_dw5 = 0;
  src_grf2_dw4 = 0;
  src_grf2_dw3 = 0;
  src_grf2_dw2 = 0;
  src_grf2_dw1 = 0;
  src_grf2_dw0 = 0;
  src_grf3_dw7 = 0;
  src_grf3_dw6 = 0;
  src_grf3_dw5 = 0;
  src_grf3_dw4 = 0;
  src_grf3_dw3 = 0;
  src_grf3_dw2 = 0;
  src_grf3_dw1 = 0;
  src_grf3_dw0 = 0;
  src_grf4_dw7 = 0;
  src_grf4_dw6 = 0;
  src_grf4_dw5 = 0;
  src_grf4_dw4 = 0;
  src_grf4_dw3 = 0;
  src_grf4_dw2 = 0;
  src_grf4_dw1 = 0;
  src_grf4_dw0 = 0;

  int lid_x = get_local_id(0);

  vme_result = __gen_ocl_vme(src_image, ref_image,
                src_grf0_dw7, src_grf0_dw6, src_grf0_dw5, src_grf0_dw4,
                src_grf0_dw3, src_grf0_dw2, src_grf0_dw1, src_grf0_dw0,
                src_grf1_dw7, src_grf1_dw6, src_grf1_dw5, src_grf1_dw4,
                src_grf1_dw3, src_grf1_dw2, src_grf1_dw1, src_grf1_dw0,
                src_grf2_dw7, src_grf2_dw6, src_grf2_dw5, src_grf2_dw4,
                src_grf2_dw3, src_grf2_dw2, src_grf2_dw1, src_grf2_dw0,
                src_grf3_dw7, src_grf3_dw6, src_grf3_dw5, src_grf3_dw4,
                src_grf3_dw3, src_grf3_dw2, src_grf3_dw1, src_grf3_dw0,
                src_grf4_dw7, src_grf4_dw6, src_grf4_dw5, src_grf4_dw4,
                src_grf4_dw3, src_grf4_dw2, src_grf4_dw1, src_grf4_dw0,
                //msg_type, vme_search_path_lut, lut_sub,
                1, 0, 0);

  barrier(CLK_LOCAL_MEM_FENCE);

  short2 mv[16];
  ushort res[16];

  uint write_back_dwx;
  uint simd_width = get_max_sub_group_size();

  /* In simd 8 mode, one kernel variable 'uint' map to 8 dword.
   * In simd 16 mode, one kernel variable 'uint' map to 16 dword.
   * That's why we should treat simd8 and simd16 differently when
   * use __gen_ocl_region.
   * */
  if(simd_width == 8){
    write_back_dwx = __gen_ocl_region(0, vme_result.s1);
    mv[0] = as_short2( write_back_dwx );

    if(accel.mb_block_type > 0x0){
      for(int i = 2, j = 1; j < 4; i += 2, j++){
        write_back_dwx = __gen_ocl_region(i, vme_result.s1);
        mv[j] = as_short2( write_back_dwx );
      }
      if(accel.mb_block_type > 0x1){
        for(int i = 0, j = 4; j < 8; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s2);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 8; j < 12; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s3);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 12; j < 16; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s4);
          mv[j] = as_short2( write_back_dwx );
        }
      }
    }
    ushort2 temp_res;
    for(int i = 0; i < 8; i++){
      write_back_dwx = __gen_ocl_region(i, vme_result.s5);
      temp_res = as_ushort2(write_back_dwx);
      res[i*2] = temp_res.s0;
      res[i*2+1] = temp_res.s1;
    }
  }
  else if(simd_width == 16){
    write_back_dwx = __gen_ocl_region(0 + 8, vme_result.s0);
    mv[0] = as_short2( write_back_dwx );

    if(accel.mb_block_type > 0x0){
      for(int i = 2, j = 1; j < 4; i += 2, j++){
        write_back_dwx = __gen_ocl_region(i + 8, vme_result.s0);
        mv[j] = as_short2( write_back_dwx );
      }
      if(accel.mb_block_type > 0x1){
        for(int i = 0, j = 4; j < 8; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s1);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 8; j < 12; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i + 8, vme_result.s1);
          mv[j] = as_short2( write_back_dwx );
        }
        for(int i = 0, j = 12; j < 16; i += 2, j++){
          write_back_dwx = __gen_ocl_region(i, vme_result.s2);
          mv[j] = as_short2( write_back_dwx );
        }
      }
    }
    ushort2 temp_res;
    for(int i = 0; i < 8; i++){
      write_back_dwx = __gen_ocl_region(i + 8, vme_result.s2);
      temp_res = as_ushort2(write_back_dwx);
      res[i*2] = temp_res.s0;
      res[i*2+1] = temp_res.s1;
    }
  }

  int mv_index;

  //CL_ME_MB_TYPE_16x16_INTEL
  if(accel.mb_block_type == 0x0){
    mv_index = index * 1;
    if( lid_x == 0 ){
      motion_vector_buffer[mv_index] = mv[lid_x];
      residuals[mv_index] = 2 * res[lid_x];
    }
  }
  //CL_ME_MB_TYPE_8x8_INTEL
  else if(accel.mb_block_type == 0x1){
    if(lid_x < 4){
      mv_index = lgid_y * num_groups_x * 4 + lgid_x * 2;
      mv_index = mv_index + num_groups_x * 2 * (lid_x / 2) + (lid_x % 2);
      motion_vector_buffer[mv_index] = mv[lid_x];
      residuals[mv_index] = 2 * res[lid_x];
    }
  }
  //CL_ME_MB_TYPE_4x4_INTEL
  else if(accel.mb_block_type == 0x2){
    if(lid_x < 16){
      mv_index = lgid_y * num_groups_x * 16 + lgid_x * 4;
      mv_index = mv_index + num_groups_x * 4 * (lid_x / 4) + (lid_x % 4);
      motion_vector_buffer[mv_index] = mv[lid_x];
      residuals[mv_index] = 2 * res[lid_x];
    }
  }

}


MIOpenConvBwdWrW_LxG_P53 perf improvements

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
#if MLO_FILTER_PAD1 > 0

#if  MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS - 2)
#define MLO_IN_VERT_READS (MLO_IN_EXTENT1 + MLO_FILTER_PAD1)
#else
#define MLO_N_GENERIC_LOOPS 0
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

#else
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS)
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

__attribute__((always_inline))
uint iDiv(uint v, uint d)
{
	uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
	return(r);
}

__attribute__((always_inline))
uint iMod(uint v, uint u, uint d)
{
	uint r = v - mul24((uint)u, (uint)d);
	return(r);
}

__attribute__((always_inline))
void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, uint lcl_id, uint scan_lcl, uint sum_stride, uint unit_len, UNUSED bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (uint i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (uint j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (uint i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
__attribute__((always_inline))
void readInput(uint lcl_id, uint gbl_in_scan_off, uint n_v_reads, const __global _FLOAT * __restrict bot, __local _FLOAT * __restrict lcl_bot)
{
	for (uint p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];
// TODO : more than 1 input
		uint c = 0;
		uint t_p4 = p4;
#if MLO_N_LCL_IN_MAPS > 1
		c = iDiv(p4, (MLO_N_IN_HORIZ_READS * n_v_reads));
		t_p4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS * n_v_reads));
#endif

#if MLO_N_IN_HORIZ_READS & (MLO_N_IN_HORIZ_READS - 1)
		uint c_scan = iDiv(t_p4, (MLO_N_IN_HORIZ_READS));
		uint c_pix4 = iMod(t_p4, c_scan, (MLO_N_IN_HORIZ_READS));
#else
		uint c_scan = t_p4 / (MLO_N_IN_HORIZ_READS);
		uint c_pix4 = t_p4 & (MLO_N_IN_HORIZ_READS - 1);
#endif

//		if (c < MLO_N_INPUTS)

		{
			uint bot_off = gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
			const __global _FLOAT * bot_p = &bot[bot_off];
#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (uint i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}

				for (uint i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{

				for (uint i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = bot_p[i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}
			}

// MLO_N_LCL_IN_MAPS inputs
			for (uint i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan* MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}

/*
	core processing loop
	bot - input, from local (1 span)
	top - output diff, from global (array of spans, filters vertical size)

	loop over filter vertical size

*/
__attribute__((always_inline))
void Processing(UNUSED uint sc, uint sc_lcl_off, uint top_lim, int bot_lim, __private _FLOAT * __restrict pvt_accum, __local _FLOAT * __restrict lcl_bot, __private _FLOAT * __restrict top_dat)
{
	for (int l = top_lim; l >= bot_lim; --l)
	{
		for (uint m = 0; m < MLO_IN_TILE0; ++m)
		{
			for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					int bot_off = sc_lcl_off + c*MLO_IN_LCL_SZ + n + m;
					_FLOAT bot_val = lcl_bot[bot_off];
					for(uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
					{
						uint pvt_top_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
						uint pvt_accum_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;

						_FLOAT top_val = top_dat[pvt_top_off];

						pvt_accum[pvt_accum_off]
						// each wk-it process an input
							+= bot_val*top_val;
#if 0
						if (/*bot_val * top_val != 0 && */l == 0 && n == 1 && get_local_id(0) == 0 && get_local_id(1) == 0 && get_local_id(2) == 0 && k == 0)
						{
							printf("K: %d %d %d %f %f %f %f\n",
								sc,
								bot_off,
								pvt_top_off,
								pvt_accum[pvt_accum_off],
								bot_val * top_val,
								bot_val,
								top_val
							);
						}
#endif

					}
				}
			}

		}

	}

}

__attribute__((always_inline))
void moveOutputUp(__private _FLOAT * __restrict top_dat)
{
	// move up output to reduce overfetch
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
		{
			for (uint i = 0; i < MLO_IN_TILE0; ++i)
			{
				uint pvt_off_n = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0 + i;
				uint pvt_off_o = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) *MLO_IN_TILE0 + i;
				top_dat[pvt_off_n] = top_dat[pvt_off_o];
			}
		}
	}
}

__attribute__((always_inline))

__attribute__((always_inline))
void spanReadingOutput(int spn, int k, int j, int top_df_off, _FLOAT mask,
	__private _FLOAT * __restrict top_dat, const __global _FLOAT * __restrict top_df)
{
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
	const __global _FLOAT * top_df_p = &top_df[top_df_off];
#if MLO_OUT_N_PIXS_OFF > 0
	if (spn == MLO_N_SPANS_PER_SCAN - 1)
	{
		uint i = 0;
		for (; i < MLO_OUT_N_PIXS_OFF; ++i)
		{
			top_dat[pvt_off + i] = top_df_p[i] * mask;

#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif
		}
		for (; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = 0;
		}
	}
	else
#else
	(void)spn;
#endif
	{
		for (uint i = 0; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = top_df_p[i] * mask;
#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * __restrict bias_df,
#endif
	UNUSED _FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ) + 1];
	__local _FLOAT * lcl_bot = lcl;

	uint lcl_id = get_local_id(0);

	uint c_idx_base = get_group_id(0); // input map index base

	uint o_idx_base = get_group_id(1);// output map index base

	uint ib_base = get_group_id(2);

	uint ib = ib_base*(MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);

	uint c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	uint o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	uint gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	uint gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
	// 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
	// TODO: more than 1 input
#if MLO_N_SPANS_PER_SCAN & (MLO_N_SPANS_PER_SCAN - 1)
	uint o = iDiv(lcl_id, MLO_N_SPANS_PER_SCAN);
	uint spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);
#else
	uint o = lcl_id / MLO_N_SPANS_PER_SCAN;
	uint spn = lcl_id & (MLO_N_SPANS_PER_SCAN - 1);
#endif
	//	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);

	uint lcl_bot_off = spn * MLO_IN_TILE0;
	uint out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
	gbl_out_off += out_wk_item_off;
// no output out of range
	gbl_out_off = (o_idx + o < MLO_N_OUTPUTS && o < MLO_OUT_STACKS) ? gbl_out_off : 0;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_N_LCL_IN_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (uint i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (uint i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	// over all batches
	for (uint b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// top border input block
		uint gbl_in_scan_off = gbl_in_off;
		uint gbl_out_scan_off = gbl_out_off;

		// read input map
		readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);
		// move input pointer
		gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

		for (uint i = 0; i < MLO_TOP_DAT_SZ; ++i)
		{
			top_dat[i] = 0;
		}

		// prefetch output

		int gbl_out_scan_off1 = gbl_out_scan_off;
		for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
		{
			for (uint j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				// loop around all output maps
				uint top_df_off = gbl_out_scan_off1 + j*MLO_OUT_STRIDE;
				_FLOAT mask = 1;
#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

					spanReadingOutput(spn, k, j, top_df_off, mask,
						top_dat, top_df);

			}
		}

		gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

		uint sc = 0;
		uint sc_lcl_off = lcl_bot_off;

		// prolog
		// handling padding

		// top padding
		for (; sc < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
		}

		for (; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
			- MLO_FILTER_PAD1
#endif
			; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
						top_dat, top_df);

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		}

// non-border input blocks
		for (uint i_loop = 0;i_loop < MLO_N_GENERIC_LOOPS; ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);

			readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < (i_loop + 2) * MLO_IN_EXTENT1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// bottom border block

//#if (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1) != 0
		for (int i_loop = 0; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1); ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
//#endif
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read 1 scan line less
// padding processing takes care of the bottom border.

#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))

			readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

			// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					uint top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

		// epilog
		// handling padding

		for (; sc < MLO_IN_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, (MLO_FILTER_PAD1 + 1 - (MLO_IN_HEIGHT - sc)), pvt_accum, lcl_bot, top_dat);
			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

	// final summation over all output maps and each filter row
	// this coudl be done with log but it negligeble anyway
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
		{

			for (uint l = 0; l < MLO_FILTER_SIZE1; ++l)
			{

				barrier(CLK_LOCAL_MEM_FENCE);
				for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
				{
					uint pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
					lcl[lcl_id * MLO_FILTER_SIZE0 + n] =
						pvt_accum[pvt_off];

				}

				barrier(CLK_LOCAL_MEM_FENCE);

				if (spn == 0)
				{
					for (uint s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
					{

						for (uint n = 0; n < MLO_FILTER_SIZE0; ++n)
						{
							uint pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
							pvt_accum[pvt_off]
								+= lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
						}

					}
				}

			}
		}
	}

// output
// inputs are outputs
// TODO : for more than 1 input

	uint wei_df_off = (((ib/MLO_N_BATCH_LOOPS) * MLO_N_OUTPUTS + o_idx + o) * (uint)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24(c_idx, (uint)MLO_WEI_CHANNEL_STRIDE);
	for (uint k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		if (spn == 0 && o_idx + o + k*MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
		{
			for (uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
			{
				for (uint i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
				{
					weights_df[wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE + i] = pvt_accum[(k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + i];
				}
			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * __restrict weight_df_tmp,
	__global _FLOAT * __restrict weights_df
)
{
	uint gbl_id = get_global_id(0);
	uint wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

#if MLO_WEI_CHANNEL_STRIDE & (MLO_WEI_CHANNEL_STRIDE - 1)
	uint wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	uint wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);
#else
	uint wei_blk_idx = wei_idx0 / MLO_WEI_CHANNEL_STRIDE;
	uint wei_idx = wei_idx0 & (MLO_WEI_CHANNEL_STRIDE - 1);
#endif

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (uint i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (uint i = 0; i < batch_loop; ++i)
	{
		for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (uint j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}

/*
 * Copyright (c) 2017 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

#define MLO_N_OUT_HORIZ_READS ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1) / MLO_IN_TILE0)
#define MLO_N_SPANS_PER_SCAN (MLO_N_OUT_HORIZ_READS)
#define MLO_N_OUT_HORIZ_PIX_READS (MLO_N_OUT_HORIZ_READS * MLO_IN_TILE0)
#define MLO_OUT_N_PIXS_OFF (MLO_OUT_WIDTH - ((MLO_OUT_WIDTH / MLO_IN_TILE0)*MLO_IN_TILE0))
#define MLO_N_OUT_VERTICAL_READS (MLO_FILTER_SIZE1)
// won't run non-border blocks if  MLO_IN_N_VERT_LOOPS < 2
#if MLO_FILTER_PAD1 > 0

#if  MLO_IN_N_VERT_LOOPS >= 2
#define MLO_N_GENERIC_LOOPS ((int)(MLO_IN_N_VERT_LOOPS - 2))
#define MLO_IN_VERT_READS (MLO_IN_EXTENT1 + MLO_FILTER_PAD1)
#else
#define MLO_N_GENERIC_LOOPS 0
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

#else
#define MLO_N_GENERIC_LOOPS (MLO_IN_N_VERT_LOOPS)
#define MLO_IN_VERT_READS MLO_IN_EXTENT1
#endif

// there is an assumption that the scanline fits into LDS
#define MLO_N_IN_HORIZ_PIX_READS (MLO_IN_WIDTH)
#define MLO_N_IN_HORIZ_READS ((MLO_N_IN_HORIZ_PIX_READS + MLO_READ_UNIT - 1) / MLO_READ_UNIT)
#define MLO_IN_N_PIXS_OFF  (MLO_N_IN_HORIZ_PIX_READS - (MLO_N_IN_HORIZ_PIX_READS/MLO_READ_UNIT)*MLO_READ_UNIT)
#define MLO_IN_LCL_WIDTH (MLO_N_IN_HORIZ_READS * MLO_READ_UNIT + 2* MLO_FILTER_PAD0)
#define MLO_IN_LCL_HEIGHT MLO_IN_VERT_READS
#define MLO_IN_LCL_SZ (MLO_IN_LCL_WIDTH * MLO_IN_LCL_HEIGHT)
#define MLO_TOTAL_IN_LCL_SZ (MLO_N_LCL_BATCHS*MLO_N_LCL_IN_MAPS*MLO_IN_LCL_SZ)

#define MLO_WEI_LCL_SZ (MLO_GRP_SZ * MLO_FILTER_SIZE0)
#if MLO_TOTAL_IN_LCL_SZ > MLO_WEI_LCL_SZ
#define MLO_LCL_SZ (MLO_TOTAL_IN_LCL_SZ)
#else
#define MLO_LCL_SZ (MLO_WEI_LCL_SZ)
#endif

__attribute__((always_inline))
int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

__attribute__((always_inline))
int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

__attribute__((always_inline))
void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, UNUSED bool debug)
{
// read first half
	if (scan_lcl < (sum_stride >> 1))
	{
		for (int i = 0; i < unit_len; ++i)
		{
			weights_accum[i] = lcl_blob[(lcl_id + scan_lcl) * unit_len + i];

		}

	}
// add second half
// appload accumulated value so far
	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];
			}

		}
	}
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
__attribute__((always_inline))
void readInput(int lcl_id, int gbl_in_scan_off, int n_v_reads, const __global _FLOAT * __restrict bot, __local _FLOAT * __restrict lcl_bot)
{
	for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * n_v_reads;
		p4 += MLO_GRP_SZ)
	{
		__private _FLOAT in_rd_data[MLO_READ_UNIT];
// TODO : more than 1 input
		int c = 0;
		int t_p4 = p4;
#if MLO_N_LCL_IN_MAPS > 1
		c = iDiv(p4, (MLO_N_IN_HORIZ_READS * n_v_reads));
		t_p4 = iMod(p4, c, (MLO_N_IN_HORIZ_READS * n_v_reads));
#endif

		int c_scan = iDiv(t_p4, (MLO_N_IN_HORIZ_READS));

		int c_pix4 = iMod(t_p4, c_scan, (MLO_N_IN_HORIZ_READS));

//		if (c < MLO_N_INPUTS)

		{
			int bot_off = gbl_in_scan_off + c*MLO_IN_CHANNEL_STRIDE + c_scan* MLO_IN_STRIDE + c_pix4*MLO_READ_UNIT;
#if MLO_IN_N_PIXS_OFF > 0

			if (c_pix4 == MLO_N_IN_HORIZ_READS - 1)
			{
				for (int i = 0; i < MLO_IN_N_PIXS_OFF; ++i)
				{

					in_rd_data[i] = bot[bot_off + i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}

				for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = 0;
				}

			}
			else
#endif
			{

				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					in_rd_data[i] = bot[bot_off + i];
#if DBG_OUT_OF_RNGE
					if (bot_off + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
					{
						printf("k:err:in-of-range\n");
					}
#endif
				}
			}

// MLO_N_LCL_IN_MAPS inputs
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				int lcl_in_off = c*MLO_IN_LCL_SZ + c_scan* MLO_IN_LCL_WIDTH + MLO_FILTER_PAD0 + c_pix4*MLO_READ_UNIT + i;
				lcl_bot[lcl_in_off] = in_rd_data[i];
			}
		}

	} // for (int p4 = lcl_id; p4 < MLO_N_LCL_IN_MAPS * MLO_N_IN_HORIZ_READS * MLO_IN_VERT_READS;

	barrier(CLK_LOCAL_MEM_FENCE);

}

/*
	core processing loop
	bot - input, from local (1 span)
	top - output diff, from global (array of spans, filters vertical size)

	loop over filter vertical size

*/
__attribute__((always_inline))
void Processing(UNUSED int sc, int sc_lcl_off, int top_lim, int bot_lim, __private _FLOAT * __restrict pvt_accum, __local _FLOAT * __restrict lcl_bot, __private _FLOAT * __restrict top_dat)
{
	for (int l = top_lim; l >= bot_lim; --l)
	{
		for (int m = 0; m < MLO_IN_TILE0; ++m)
		{
			for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
			{
				for(int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					int bot_off = sc_lcl_off + c*MLO_IN_LCL_SZ + n + m;
					_FLOAT bot_val = lcl_bot[bot_off];
					for(int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
					{
						int pvt_top_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (top_lim - l) * MLO_IN_TILE0 + m;
						int pvt_accum_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;

						_FLOAT top_val = top_dat[pvt_top_off];

						pvt_accum[pvt_accum_off]
						// each wk-it process an input
							+= bot_val*top_val;
#if 0
						if (/*bot_val * top_val != 0 && */l == 0 && n == 1 && get_local_id(0) == 0 && get_local_id(1) == 0 && get_local_id(2) == 0 && k == 0)
						{
							printf("K: %d %d %d %f %f %f %f\n",
								sc,
								bot_off,
								pvt_top_off,
								pvt_accum[pvt_accum_off],
								bot_val * top_val,
								bot_val,
								top_val
							);
						}
#endif

					}
				}
			}

		}

	}

}

__attribute__((always_inline))
void moveOutputUp(__private _FLOAT * __restrict top_dat)
{
	// move up output to reduce overfetch
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
		{
			for (int i = 0; i < MLO_IN_TILE0; ++i)
			{
				int pvt_off_n = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0 + i;
				int pvt_off_o = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + (j + 1) *MLO_IN_TILE0 + i;
				top_dat[pvt_off_n] = top_dat[pvt_off_o];
			}
		}
	}
}

__attribute__((always_inline))

__attribute__((always_inline))
void spanReadingOutput(int spn, int k, int j, int top_df_off, _FLOAT mask,
	__private _FLOAT * __restrict top_dat, const __global _FLOAT * __restrict top_df)
{
	int pvt_off = k*MLO_IN_TILE0 * MLO_FILTER_SIZE1 + j *MLO_IN_TILE0;
#if MLO_OUT_N_PIXS_OFF > 0
	if (spn == MLO_N_SPANS_PER_SCAN - 1)
	{
		int i = 0;
		for (; i < MLO_OUT_N_PIXS_OFF; ++i)
		{
			top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;

#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif
		}
		for (; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = 0;
		}
	}
	else
#else
	(void)spn;
#endif
	{
		for (int i = 0; i < MLO_IN_TILE0; ++i)
		{
			top_dat[pvt_off + i] = top_df[top_df_off + i] * mask;
#if DBG_OUT_OF_RNGE
			if (top_df_off + i >= MLO_OUT_BATCH_STRIDE * MLO_BATCH_SZ)
			{
				printf("k:err:out-of-range\n");
			}
#endif

		}
	}
}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// split output scan-line on number of spans by the  MLO_IN_TILE0 (2 for example)
// 1 scan-line has ((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) spans
// group will process MLO_GRP_SZ/((MLO_OUT_WIDTH + MLO_IN_TILE0 - 1/MLO_IN_TILE0) output maps

// alg
// load a block of input map (or full map) into LDS
// loop
// read MLO_FILTER_SIZE1 number of spans from output map into VGPRs (for example 5 *2 = 10)
// read 1 input line for  maps into LDS
// accumulate

// accumulate all spans at the end
// start new loop for the next batch (if defined)
// write out

// kerenl handles 5x5, 3x3 with padding
// small images in 1 short- MLO_N_GENERIC_LOOPS == 0
// big images  in 2 blocks - MLO_IN_N_VERT_LOOPS == 2 or multiple blocks - MLO_IN_N_VERT_LOOPS > 2
// there are prolog and apilog that deal with top/bottom padding.
// left/right padding handles as a LDS border pixels zeroed at the beginning.

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MIOpenCvBwdWrW(
	const __global _FLOAT * __restrict top_df,
	const __global _FLOAT * __restrict bot,
	__global _FLOAT * __restrict weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * __restrict bias_df,
#endif
	UNUSED _FLOAT padding_val
)
{

	// input/output tiles + reduce buffer

	__local _FLOAT lcl[(MLO_LCL_SZ) + 1];
	__local _FLOAT * lcl_bot = lcl;

	int lcl_id = get_local_id(0);

	int c_idx_base = get_group_id(0); // input map index base

	int o_idx_base = get_group_id(1);// output map index base

	int ib_base = get_group_id(2);

	int ib = ib_base*(MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);

	int c_idx = c_idx_base * MLO_N_LCL_IN_MAPS; // input map index

	int o_idx = o_idx_base * (MLO_N_LCL_OUT_MAPS * MLO_OUT_STACKS); // output map index

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = o_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;
	// 1 span per wk_item, total scanline with MLO_N_SPANS_PER_SCAN spans
	// TODO: more than 1 input
	int o = iDiv(lcl_id, MLO_N_SPANS_PER_SCAN);
	//	bool scan_lead = (o*MLO_N_SPANS_PER_SCAN == lcl_id);
	int spn = iMod(lcl_id, o, MLO_N_SPANS_PER_SCAN);

	int lcl_bot_off = spn * MLO_IN_TILE0;
	int out_wk_item_off = o * MLO_OUT_CHANNEL_STRIDE + lcl_bot_off;
	gbl_out_off += out_wk_item_off;
// no output out of range
	gbl_out_off = (o_idx + o < MLO_N_OUTPUTS && o < MLO_OUT_STACKS) ? gbl_out_off : 0;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_IN_TILE0 * MLO_FILTER_SIZE1)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS * MLO_N_LCL_IN_MAPS * MLO_FILTER_SIZE1*MLO_FILTER_SIZE0)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	// zero out LDS
	for (int i = lcl_id; i < (MLO_LCL_SZ); i += MLO_GRP_SZ)
	{
		lcl[i] = 0;
	}

	// over all batches
	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// top border input block
		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input map
		readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);
		// move input pointer
		gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1;

		for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
		{
			top_dat[i] = 0;
		}

		// prefetch output

		int gbl_out_scan_off1 = gbl_out_scan_off;
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k, gbl_out_scan_off1 += MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE)
		{
			for (int j = 0; j < MLO_FILTER_SIZE1 - 1; ++j)
			{
				// loop around all output maps
				int top_df_off = gbl_out_scan_off1 + j*MLO_OUT_STRIDE;
				_FLOAT mask = 1;
#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = (j < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = (j < MLO_OUT_HEIGHT) ? 1 : 0;
#endif

					spanReadingOutput(spn, k, j, top_df_off, mask,
						top_dat, top_df);

			}
		}

		gbl_out_scan_off += (MLO_FILTER_SIZE1 - 1) * MLO_OUT_STRIDE;

		int sc = 0;
		int sc_lcl_off = lcl_bot_off;

		// prolog
		// handling padding

		// top padding
		for (; sc < MLO_FILTER_SIZE1 - MLO_FILTER_PAD1 - 1; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{
			Processing(sc, sc_lcl_off, sc + MLO_FILTER_PAD1, 0, pvt_accum, lcl_bot, top_dat);
		}

		for (; sc < MLO_IN_EXTENT1
#if MLO_IN_N_VERT_LOOPS == 1
			- MLO_FILTER_PAD1
#endif
			; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
			{
				int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
				_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT || MLO_FILTER_SIZE1 - 1 > MLO_OUT_HEIGHT
				top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
				mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
					spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
						top_dat, top_df);

			}

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		}

// non-border input blocks
		for (int i_loop = 0;i_loop < MLO_N_GENERIC_LOOPS; ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);

			readInput(lcl_id, gbl_in_scan_off, MLO_IN_VERT_READS, bot, lcl_bot);

// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < (i_loop + 2) * MLO_IN_EXTENT1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

// bottom border block

		for (int i_loop = 0; i_loop < (MLO_IN_N_VERT_LOOPS - MLO_N_GENERIC_LOOPS - 1); ++i_loop, gbl_in_scan_off += MLO_IN_STRIDE * MLO_IN_EXTENT1)
		{
			barrier(CLK_LOCAL_MEM_FENCE);
// read 1 scan line less
// padding processing takes care of the bottom border.

#define MLO_LAST_VERT_READS (MLO_IN_HEIGHT - MLO_IN_EXTENT1 * (MLO_IN_N_VERT_LOOPS - 1))

            readInput(lcl_id, gbl_in_scan_off, MLO_LAST_VERT_READS, bot, lcl_bot);

			// point to the start of the local buffer

			sc_lcl_off = lcl_bot_off;

			for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1
				; ++sc, gbl_out_scan_off += MLO_OUT_STRIDE, sc_lcl_off += MLO_IN_LCL_WIDTH)
			{

				for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
				{
					int top_df_off = gbl_out_scan_off + k*MLO_OUT_STACKS * MLO_OUT_CHANNEL_STRIDE;
					_FLOAT mask = 1;

#if MLO_IN_HEIGHT !=  MLO_OUT_HEIGHT
					top_df_off = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? top_df_off : 0;
					mask = ((sc + MLO_FILTER_PAD1) < MLO_OUT_HEIGHT) ? 1 : 0;
#endif
						spanReadingOutput(spn, k, (MLO_FILTER_SIZE1 - 1), top_df_off, mask,
							top_dat, top_df);

				}

				// processing
				Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, 0, pvt_accum, lcl_bot, top_dat);

				// move up output to reduce overfetch
				moveOutputUp(top_dat);

			}
		}

		// epilog
		// handling padding

		for (; sc < MLO_IN_HEIGHT; ++sc, sc_lcl_off += MLO_IN_LCL_WIDTH)
		{

			// processing
			Processing(sc, sc_lcl_off, MLO_FILTER_SIZE1 - 1, (MLO_FILTER_PAD1 + 1 - (MLO_IN_HEIGHT - sc)), pvt_accum, lcl_bot, top_dat);
			// move up output to reduce overfetch
			moveOutputUp(top_dat);

		} // for (; sc < MLO_OUT_HEIGHT - MLO_FILTER_PAD1 + 2; ++sc, gbl_out_scan_off += MLO_OUT_CHANNEL_STRIDE, gbl_in_scan_off += MLO_IN_CHANNEL_STRIDE)

	} // 	for (int b = 0;

	// final summation over all output maps and each filter row
	// this coudl be done with log but it negligeble anyway
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for(int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
		{

			for (int l = 0; l < MLO_FILTER_SIZE1; ++l)
			{

				barrier(CLK_LOCAL_MEM_FENCE);
				for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
				{
					int pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
					lcl[lcl_id * MLO_FILTER_SIZE0 + n] =
						pvt_accum[pvt_off];

				}

				barrier(CLK_LOCAL_MEM_FENCE);

				if (spn == 0)
				{
					for (int s = 0; s < MLO_N_SPANS_PER_SCAN - 1; ++s)
					{

						for (int n = 0; n < MLO_FILTER_SIZE0; ++n)
						{
							int pvt_off = (k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + l*MLO_FILTER_SIZE0 + n;
							pvt_accum[pvt_off]
								+= lcl[(lcl_id + s + 1) * MLO_FILTER_SIZE0 + n];
						}

					}
				}

			}
		}
	}

// output
// inputs are outputs
// TODO : for more than 1 input

	int wei_df_off = (((ib/MLO_N_BATCH_LOOPS) * MLO_N_OUTPUTS + o_idx + o) * (int)MLO_WEI_BATCH_STRIDE)
		// this input channel
		+ mul24(c_idx, (int)MLO_WEI_CHANNEL_STRIDE);
	for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
	{
		for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
		{
			for (int i = 0; i < (MLO_FILTER_SIZE1 * MLO_FILTER_SIZE0); ++i)
			{
				if (spn == 0 && o_idx + o + k*MLO_OUT_STACKS < MLO_N_OUTPUTS && o < MLO_OUT_STACKS)
				{
					weights_df[wei_df_off + k*MLO_OUT_STACKS*MLO_WEI_BATCH_STRIDE + c*MLO_WEI_CHANNEL_STRIDE + i] = pvt_accum[(k*MLO_N_LCL_IN_MAPS + c) *MLO_FILTER_SIZE1*MLO_FILTER_SIZE0 + i];
				}
			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MIOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		for (int j = 0; j < MLO_UT_READ_UNIT; ++j)
		{
			pvt_accum_wei[j]
				+= weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE) + wei_idx + j];
		}
	}

	for (int j = 0; j < MLO_UT_READ_UNIT; ++j)
	{
		weights_df[wei_idx0 + j] = pvt_accum_wei[j];
	}

}


1x1 is correct

1x1 is correct

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
static inline void readData(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * g_data, __private _FLOAT * p_data, bool last_pixel)
{
	for (int j = 0; j < n; ++j)
	{
		int gbl_data_off0 = (j*map_stride + map_base < map_limit) ? gbl_data_off + j*map_stride*gbl_data_stride : 0;

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = 0;
			}

		}
		else
#endif
		{
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}
		}

	} // for (int j = 0; j < n; ++j)

}

static inline void readDataFlex(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * g_data, __local _FLOAT * l_data)
{

	for( int l = get_local_id(0); l < n*map_stride* MLO_MAP_WK_SZ; l += MLO_GRP_SZ)
	{

	    int r = 0;
		int k0 = l;
#if MLO_N_LCL_OUT_MAPS > 1
		r = iDiv(l, map_stride* MLO_MAP_WK_SZ);  // maps row
		k0 = iMod(l, r, map_stride* MLO_MAP_WK_SZ);
#endif
		int k = iDiv(k0, MLO_MAP_WK_SZ);  // map column
		int p4 = iMod(k0, k, MLO_MAP_WK_SZ); // pixel block

		bool last_pixel = (p4 == MLO_MAP_WK_SZ -1);

		int gbl_data_off0 = (r*map_stride + k + map_base < map_limit) ? gbl_data_off + (r*map_stride + k)*gbl_data_stride + p4*MLO_READ_UNIT : 0;
		__private _FLOAT p_data[MLO_READ_UNIT];

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = 0;
			}

		}
		else
#endif
		{
//			*(MLO_READ_TYPE*)&p_data[j*MLO_READ_UNIT] = *(__global MLO_READ_TYPE*)&g_data[gbl_data_off + j*gbl_data_stride*MLO_OUT_STACKS];
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = g_data[gbl_data_off0 + i];
			}
		}

		for(int i = 0; i < MLO_READ_UNIT; ++i)
		{
			l_data[(k*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i] = p_data[i];
		}

	}

}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS per group, MLO_N_LCL_IN_MAPS per wk_item input maps
// read MLO_OUT_STACKS per group, MLO_N_LCL_OUT_MAPS per wk_item output maps

// alg
// loop in MLO_N_LCL_OUT_MAPS
// load MLO_OUT_STACKS of output into LDS
loop in MLO_OUT_STACKS
// convolve with MLO_N_LCL_IN_MAPS per wk-item

// reduce

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
	// reduction memory.
	// ceil pow2 of the number of wk-items keeping the map
#if (MLO_POW2_MAP_WK_SZ * MLO_OUT_STACKS) > (MLO_OUT_STACKS * MLO_MAP_WK_SZ * MLO_READ_UNIT)
#define MLO_LCL_MEM_SZ (MLO_POW2_MAP_WK_SZ * MLO_OUT_STACKS)
#else
#define MLO_LCL_MEM_SZ (MLO_OUT_STACKS * MLO_MAP_WK_SZ * MLO_READ_UNIT)
#endif

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;
	__local _FLOAT * proc_mem = lcl_mem;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int k_idx = get_group_id(0) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_IN_STACKS * MLO_N_LCL_IN_MAPS); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	// map id inside group
	int m_idx = iDiv(lcl_id, MLO_MAP_WK_SZ);
	// read pixel inside the map
	int p4 = iMod(lcl_id, m_idx, MLO_MAP_WK_SZ);

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);
//	bool out_of_range = false; // (m_idx >= MLO_OUT_STACKS);
//	bool out_of_range_in = (m_idx + c_idx >= MLO_N_INPUTS);
//	bool out_of_range_out = (m_idx + k_idx >= MLO_N_OUTPUTS);

	gbl_in_off += m_idx * MLO_IN_CHANNEL_STRIDE + p4*MLO_READ_UNIT;
//	gbl_out_off += m_idx * MLO_OUT_CHANNEL_STRIDE + p4*MLO_READ_UNIT;

	// read guards
//	gbl_in_off = (out_of_range_in || out_of_range) ? 0 : gbl_in_off;
//	gbl_out_off = (out_of_range_out || out_of_range) ? 0 : gbl_out_off;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS * MLO_OUT_STACKS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}
	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input maps
		readData(MLO_N_LCL_IN_MAPS, gbl_in_scan_off, MLO_IN_CHANNEL_STRIDE, MLO_IN_STACKS, (m_idx + c_idx), MLO_N_INPUTS, bot, bot_dat, last_pixel);

		// read output maps
		readDataFlex(MLO_N_LCL_OUT_MAPS, gbl_out_scan_off, MLO_OUT_CHANNEL_STRIDE, MLO_OUT_STACKS, (k_idx), MLO_N_OUTPUTS, top_df, proc_mem);

		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
#if 0
	// move 1 set of output maps into LDS
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				proc_mem[lcl_id*MLO_READ_UNIT + i] = top_dat[k*MLO_READ_UNIT + i];
			}

			barrier(CLK_LOCAL_MEM_FENCE);
#endif
			/*
			core processing loop
			bot - input
			top - output diff

			do convolution with all available input maps

			*/

			for (int n = 0; n < MLO_OUT_STACKS; ++n)
			{
				__private _FLOAT pvt_top[MLO_READ_UNIT];
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_top[i] = proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i]; //top_df[gbl_out_scan_off + (k*MLO_OUT_STACKS + n0)*MLO_IN_CHANNEL_STRIDE + i]; //proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
				}

				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c]
							+= bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i];
#if 0

						if (k_idx + k*MLO_OUT_STACKS + n == 0 && c_idx + c * MLO_IN_STACKS + m_idx == 16 && m_idx < MLO_IN_STACKS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS)
						{
							printf("K:s: %d %d %d %d %d %d %d %d %f %f %f %f\n",
								MLO_OUT_STACKS,
								n,
								m_idx,
								gbl_out_scan_off,
								get_group_id(1),
								get_local_id(0),
								k,
								c,
								pvt_accum[(n* MLO_N_LCL_OUT_MAPS + k)*MLO_N_LCL_IN_MAPS + c],
								bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i],
								bot_dat[c*MLO_READ_UNIT + i],
								pvt_top[i]
							);
						}
#endif

					}

				}
//				Processing(MLO_N_LCL_OUT_MAPS, o_idx, m_idx, MLO_N_LCL_IN_MAPS, c_idx, m_idx, pvt_accum, bot_dat, top_dat, (p4 == 0 && (lcl_id == 0 || lcl_id == 1)));
			}
		}

	}

	barrier(CLK_LOCAL_MEM_FENCE);

#if 0
	int red_base_off = m_idx * MLO_MAP_WK_SZ*MLO_ACCUM_SZ;
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		// write data
		red_mem[red_base_off + p4*MLO_ACCUM_SZ + l] = pvt_accum[l];

	}
	barrier(CLK_LOCAL_MEM_FENCE);
	__private _FLOAT final_sum[1] = { 0 };
	for (int s = 0; s < MLO_MAP_WK_SZ; ++s)
	{
		final_sum[0] += red_mem[red_base_off + p4*MLO_MAP_WK_SZ + s];
	}
#else

#if 0 //MLO_IN_WIDTH > 24
	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		red_mem[i] = 0;
	}

	int red_base_off = (m_idx >= MLO_OUT_STACKS) ? MLO_LCL_MEM_SZ : m_idx * MLO_POW2_MAP_WK_SZ;
	// final summation over each filter row
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// write data
		red_mem[red_base_off + p4] = pvt_accum[l];

		// barrier inside
		ReduceKernel(&red_mem[red_base_off], &pvt_accum[l], p4, p4, MLO_POW2_MAP_WK_SZ, 1, false);

	}

	barrier(CLK_LOCAL_MEM_FENCE);
#else

	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		lcl_mem[lcl_id] = pvt_accum[l];
		barrier(CLK_LOCAL_MEM_FENCE);
		if (p4 == 0)
		{
			for (int i = 1; i < MLO_MAP_WK_SZ; ++i)
			{
				pvt_accum[l] += lcl_mem[lcl_id + i];
			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

#endif

#endif

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + m_idx) * MLO_WEI_CHANNEL_STRIDE);

	for (int n = 0; n < MLO_OUT_STACKS && p4 == 0 && m_idx < MLO_IN_STACKS; ++n)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS; ++c)
			{
				weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c];

			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define MLO_HW_WAVE_ID_SETTING 0

#if MLO_HW_WAVE_ID_SETTING
extern __attribute__((const)) uint __hsail_get_dynwave_id(void);
static inline int getWaveId()
{
	int wave_id = 0;

	wave_id = __hsail_get_dynwave_id();
	wave_id = wave_id & MLO_N_PHYS_WAVES_MASK;
	return(wave_id);
}
#else
inline int getWaveId()
{
	int wave_id = 0;

	wave_id = (get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ);

	return(wave_id);
}
#endif

inline int gePhysLocalId()
{
	int lcl_wave_id = get_local_id(0) - ((get_local_id(0) >> MLO_LG2_PHYS_WAVE_SZ) << MLO_LG2_PHYS_WAVE_SZ);
	return(lcl_wave_id);
}

inline int iDiv(int v, int d)
{
	int r = (int)((float)v / d + 0.00001f);
	return(r);
}

inline int iMod(int v, int u, int d)
{
	int r = v - mul24((int)u, (int)d);
	return(r);
}

inline void ReduceKernel(__local _FLOAT * lcl_blob, __private _FLOAT *weights_accum, int lcl_id, int scan_lcl, int sum_stride, int unit_len, bool debug)
{

	for (int j = (sum_stride >> 1); j > 0; j >>= 1)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		if (scan_lcl < j)
		{
			for (int i = 0; i < unit_len; ++i)
			{
				weights_accum[i] += lcl_blob[(lcl_id + j) * unit_len + i];

				lcl_blob[lcl_id * unit_len + i] = weights_accum[i];

			}

		}
	}
}

static inline void  Kahan_summation(__private _FLOAT *sum, __private _FLOAT * c, _FLOAT v)
{
	_FLOAT y = v - *c;    //So far, so good: c is zero.
	_FLOAT t = *sum + y;         //Alas, sum is big, y small, so low-order digits of y are lost.
	*c = (t - *sum) - y;   //(t - sum) recovers the high-order part of y; subtracting y recovers -(low part of y)
	*sum = t;             //Algebraically, c should always be zero. Beware eagerly optimising compilers!
}

/*
	group cooperative read
	read by MLO_READ_UNIT
	handle out of range both horizontally and vertically (by fixed number of veryical reads)

	no guard against number of inputs
*/
static inline void readData(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * g_data, __private _FLOAT * p_data, bool last_pixel)
{
	for (int j = 0; j < n; ++j)
	{
		int gbl_data_off0 = (j*map_stride + map_base < map_limit) ? gbl_data_off + j*map_stride*gbl_data_stride : 0;

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = 0;
			}

		}
		else
#endif
		{
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[j*MLO_READ_UNIT + i] = g_data[gbl_data_off0 + i];
			}
		}

	} // for (int j = 0; j < n; ++j)

}

static inline void readDataFlex(int n, int gbl_data_off, int gbl_data_stride, int map_stride, int map_base,  int map_limit, const __global _FLOAT * g_data, __local _FLOAT * l_data)
{

	for( int l = get_local_id(0); l < n*map_stride* MLO_MAP_WK_SZ; l += MLO_GRP_SZ)
	{

	    int r = 0;
		int k0 = l;
#if MLO_N_LCL_OUT_MAPS > 1
		r = iDiv(l, map_stride* MLO_MAP_WK_SZ);  // maps row
		k0 = iMod(l, r, map_stride* MLO_MAP_WK_SZ);
#endif
		int k = iDiv(k0, MLO_MAP_WK_SZ);  // map column
		int p4 = iMod(k0, k, MLO_MAP_WK_SZ); // pixel block

		bool last_pixel = (p4 == MLO_MAP_WK_SZ -1);

		int gbl_data_off0 = (r*map_stride + k + map_base < map_limit) ? gbl_data_off + (r*map_stride + k)*gbl_data_stride + p4*MLO_READ_UNIT : 0;
		__private _FLOAT p_data[MLO_READ_UNIT];

#if MLO_N_PIXS_OFF > 0

		if (last_pixel)
		{
			for (int i = 0; i < MLO_N_PIXS_OFF; ++i)
			{

				p_data[i] = g_data[gbl_data_off0 + i];
			}

			for (int i = MLO_IN_N_PIXS_OFF; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = 0;
			}

		}
		else
#endif
		{
//			*(MLO_READ_TYPE*)&p_data[j*MLO_READ_UNIT] = *(__global MLO_READ_TYPE*)&g_data[gbl_data_off + j*gbl_data_stride*MLO_OUT_STACKS];
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				p_data[i] = g_data[gbl_data_off0 + i];
			}
		}

		for(int i = 0; i < MLO_READ_UNIT; ++i)
		{
			l_data[(k*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i] = p_data[i];
		}

	}

}

/*********************************************************************************************************
// wrw algorithm for large filters
// idea:
// read MLO_OUT_STACKS per group, MLO_N_LCL_IN_MAPS per wk_item input maps
// read MLO_OUT_STACKS per group, MLO_N_LCL_OUT_MAPS per wk_item output maps

// alg
// loop in MLO_N_LCL_OUT_MAPS
// load MLO_OUT_STACKS of output into LDS
loop in MLO_OUT_STACKS
// convolve with MLO_N_LCL_IN_MAPS per wk-item

// reduce

// write out

**********************************************************************************************************/

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2)))
__kernel void MLOpenCvBwdWrW(
	const __global _FLOAT * top_df,
	const __global _FLOAT * bot,
	__global _FLOAT * weights_df,
#if MLO_CONV_BIAS
	__global _FLOAT * bias_df,
#endif
	_FLOAT padding_val
)
{
	// reduction memory.
	// ceil pow2 of the number of wk-items keeping the map
#if (MLO_POW2_MAP_WK_SZ * MLO_OUT_STACKS) > (MLO_OUT_STACKS * MLO_MAP_WK_SZ * MLO_READ_UNIT)
#define MLO_LCL_MEM_SZ (MLO_POW2_MAP_WK_SZ * MLO_OUT_STACKS)
#else
#define MLO_LCL_MEM_SZ (MLO_OUT_STACKS * MLO_MAP_WK_SZ * MLO_READ_UNIT)
#endif

	__local _FLOAT lcl_mem[MLO_LCL_MEM_SZ];
	__local _FLOAT * red_mem = lcl_mem;
	__local _FLOAT * proc_mem = lcl_mem;

	// guarnteeing an uniformity over a wave
	int wave_id = getWaveId();
	int lcl_id = get_local_id(0);
	int lcl_wv_id = gePhysLocalId();

	int k_idx = get_group_id(0) * (MLO_OUT_STACKS * MLO_N_LCL_OUT_MAPS); // output map index base

	int c_idx = get_group_id(1) * (MLO_IN_STACKS * MLO_N_LCL_IN_MAPS); // input map index based

	int ib = get_group_id(2); // batch id

	int gbl_in_off = c_idx * MLO_IN_CHANNEL_STRIDE + ib * MLO_IN_BATCH_STRIDE;
	int gbl_out_off = k_idx * MLO_OUT_CHANNEL_STRIDE + ib * MLO_OUT_BATCH_STRIDE;

	// map id inside group
	int m_idx = iDiv(lcl_id, MLO_MAP_WK_SZ);
	// read pixel inside the map
	int p4 = iMod(lcl_id, m_idx, MLO_MAP_WK_SZ);

	bool last_pixel = (p4 == MLO_MAP_WK_SZ - 1);
//	bool out_of_range = false; // (m_idx >= MLO_OUT_STACKS);
//	bool out_of_range_in = (m_idx + c_idx >= MLO_N_INPUTS);
//	bool out_of_range_out = (m_idx + k_idx >= MLO_N_OUTPUTS);

	gbl_in_off += m_idx * MLO_IN_CHANNEL_STRIDE + p4*MLO_READ_UNIT;
//	gbl_out_off += m_idx * MLO_OUT_CHANNEL_STRIDE + p4*MLO_READ_UNIT;

	// read guards
//	gbl_in_off = (out_of_range_in || out_of_range) ? 0 : gbl_in_off;
//	gbl_out_off = (out_of_range_out || out_of_range) ? 0 : gbl_out_off;

#define MLO_TOP_DAT_SZ (MLO_N_LCL_OUT_MAPS * MLO_READ_UNIT)

	__private _FLOAT top_dat[MLO_TOP_DAT_SZ];

	for (int i = 0; i < MLO_TOP_DAT_SZ; ++i)
	{
		top_dat[i] = 0;
	}

#define MLO_BOT_DAT_SZ (MLO_N_LCL_IN_MAPS * MLO_READ_UNIT)

	__private _FLOAT bot_dat[MLO_BOT_DAT_SZ];

	for (int i = 0; i < MLO_BOT_DAT_SZ; ++i)
	{
		bot_dat[i] = 0;
	}

#define MLO_ACCUM_SZ (MLO_N_LCL_OUT_MAPS* MLO_N_LCL_IN_MAPS * MLO_OUT_STACKS)

	__private _FLOAT pvt_accum[MLO_ACCUM_SZ];

	for (int i = 0; i < MLO_ACCUM_SZ; ++i)
	{
		pvt_accum[i] = 0;
	}

	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		lcl_mem[i] = 0;
	}
	// over all batches

	for (int b = 0;
		b < MLO_N_BATCH_LOOPS;
		++b,
		gbl_in_off += MLO_N_LCL_BATCHS*MLO_IN_BATCH_STRIDE,
		gbl_out_off += MLO_N_LCL_BATCHS*MLO_OUT_BATCH_STRIDE
		)
	{

		int gbl_in_scan_off = gbl_in_off;
		int gbl_out_scan_off = gbl_out_off;

		// read input maps
		readData(MLO_N_LCL_IN_MAPS, gbl_in_scan_off, MLO_IN_CHANNEL_STRIDE, MLO_IN_STACKS, (m_idx + c_idx), MLO_N_INPUTS, bot, bot_dat, last_pixel);

		// read output maps
		readDataFlex(MLO_N_LCL_OUT_MAPS, gbl_out_scan_off, MLO_OUT_CHANNEL_STRIDE, MLO_OUT_STACKS, (k_idx), MLO_N_OUTPUTS, top_df, proc_mem);

		for (int k = 0; k < MLO_N_LCL_OUT_MAPS; ++k)
		{

			barrier(CLK_LOCAL_MEM_FENCE);
#if 0
	// move 1 set of output maps into LDS
			for (int i = 0; i < MLO_READ_UNIT; ++i)
			{
				proc_mem[lcl_id*MLO_READ_UNIT + i] = top_dat[k*MLO_READ_UNIT + i];
			}

			barrier(CLK_LOCAL_MEM_FENCE);
#endif
			/*
			core processing loop
			bot - input
			top - output diff

			do convolution with all available input maps

			*/

			for (int n = 0; n < MLO_OUT_STACKS; ++n)
			{
				__private _FLOAT pvt_top[MLO_READ_UNIT];
				for (int i = 0; i < MLO_READ_UNIT; ++i)
				{
					pvt_top[i] = proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i]; //top_df[gbl_out_scan_off + (k*MLO_OUT_STACKS + n0)*MLO_IN_CHANNEL_STRIDE + i]; //proc_mem[(n*MLO_MAP_WK_SZ + p4)*MLO_READ_UNIT + i];
				}

				for (int c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
				{
					for (int i = 0; i < MLO_READ_UNIT; ++i)
					{
						pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c]
							+= bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i];
#if 0

						if (k_idx + k*MLO_OUT_STACKS + n == 0 && c_idx + c * MLO_IN_STACKS + m_idx == 16 && m_idx < MLO_IN_STACKS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS)
						{
							printf("K:s: %d %d %d %d %d %d %d %d %f %f %f %f\n",
								MLO_OUT_STACKS,
								n,
								m_idx,
								gbl_out_scan_off,
								get_group_id(1),
								get_local_id(0),
								k,
								c,
								pvt_accum[(n* MLO_N_LCL_OUT_MAPS + k)*MLO_N_LCL_IN_MAPS + c],
								bot_dat[c*MLO_READ_UNIT + i] * pvt_top[i],
								bot_dat[c*MLO_READ_UNIT + i],
								pvt_top[i]
							);
						}
#endif

					}

				}
//				Processing(MLO_N_LCL_OUT_MAPS, o_idx, m_idx, MLO_N_LCL_IN_MAPS, c_idx, m_idx, pvt_accum, bot_dat, top_dat, (p4 == 0 && (lcl_id == 0 || lcl_id == 1)));
			}
		}

	}

	barrier(CLK_LOCAL_MEM_FENCE);
#if 1
#if 0 //MLO_IN_WIDTH > 24
	for (int i = lcl_id; i < MLO_LCL_MEM_SZ; i += MLO_GRP_SZ)
	{
		red_mem[i] = 0;
	}

	int red_base_off = (m_idx >= MLO_OUT_STACKS) ? MLO_LCL_MEM_SZ : m_idx * MLO_POW2_MAP_WK_SZ;
	// final summation over each filter row
	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		barrier(CLK_LOCAL_MEM_FENCE);
		// write data
		red_mem[red_base_off + p4] = pvt_accum[l];

		// barrier inside
		ReduceKernel(&red_mem[red_base_off], &pvt_accum[l], p4, p4, MLO_POW2_MAP_WK_SZ, 1, false);

	}

	barrier(CLK_LOCAL_MEM_FENCE);
#else

	for (int l = 0; l < MLO_ACCUM_SZ; ++l)
	{
		lcl_mem[lcl_id] = pvt_accum[l];
		barrier(CLK_LOCAL_MEM_FENCE);
		if (p4 == 0)
		{
			for (int i = 1; i < MLO_MAP_WK_SZ; ++i)
			{
				pvt_accum[l] += lcl_mem[lcl_id + i];
			}

		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

#endif

#endif

	// write out
	// inputs are outputs
	int wei_df_off = ((ib * MLO_N_OUTPUTS + k_idx) * (int)MLO_WEI_BATCH_STRIDE) + ((c_idx + m_idx) * MLO_WEI_CHANNEL_STRIDE);

	for (int n = 0; n < MLO_OUT_STACKS && p4 == 0 && m_idx < MLO_IN_STACKS; ++n)
	{
		for (int k = 0; k < MLO_N_LCL_OUT_MAPS && k_idx + k*MLO_OUT_STACKS + n < MLO_N_OUTPUTS; ++k)
		{
			for (int c = 0; c < MLO_N_LCL_IN_MAPS && c_idx + m_idx + c*MLO_IN_STACKS < MLO_N_INPUTS; ++c)
			{
				weights_df[wei_df_off + (k*MLO_OUT_STACKS + n)*MLO_WEI_BATCH_STRIDE + c*MLO_IN_STACKS*MLO_WEI_CHANNEL_STRIDE] = pvt_accum[(k*MLO_OUT_STACKS + n)*MLO_N_LCL_IN_MAPS + c];

			}

		}
	}

}

// final reduction kernel
// add filters over batches
__attribute__((reqd_work_group_size(MLO_UT_GRP_SZ0, 1, 1)))
__kernel void MLOpenCvBwdWrW_rdc(
	const __global _FLOAT * weight_df_tmp,
	__global _FLOAT * weights_df
)
{
	int gbl_id = get_global_id(0);
	int wei_idx0 = gbl_id * MLO_UT_READ_UNIT;

	int wei_blk_idx = iDiv(wei_idx0, MLO_WEI_CHANNEL_STRIDE);
	int wei_idx = iMod(wei_idx0, wei_blk_idx, MLO_WEI_CHANNEL_STRIDE);

	_FLOAT pvt_accum_wei[MLO_UT_READ_UNIT];
	for (int i = 0; i < MLO_UT_READ_UNIT; ++i)
	{
		pvt_accum_wei[i] = 0;
	}

	int batch_loop = (MLO_BATCH_SZ + (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS) - 1) / (MLO_N_BATCH_LOOPS*MLO_N_LCL_BATCHS);
	for (int i = 0; i < batch_loop; ++i)
	{
		*(MLO_UT_READ_TYPE*)pvt_accum_wei
			+= *(__global MLO_UT_READ_TYPE*)&weight_df_tmp[(wei_blk_idx * MLO_WEI_CHANNEL_STRIDE + i* MLO_N_OUTPUTS*MLO_WEI_BATCH_STRIDE)  + wei_idx];
	}

	*(__global MLO_UT_READ_TYPE*)&weights_df[wei_idx0] = *(MLO_UT_READ_TYPE*)pvt_accum_wei;

}