[OpenCL] Add block argument CodeGen test

r326937 ("[OpenCL] Remove block invoke function from emitted block
literal struct", 2018-03-07) broke block argument handling.  In
particular the commit was causing a crash during code generation, see
the discussion in https://reviews.llvm.org/D43783 .

The offending commit has just been reverted; add a test to avoid
breaking this again in the future.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@343583 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -triple spir-unknown-unknown | FileCheck -check-prefixes=COMMON,SPIR %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -triple amdgcn-amd-amdhsa | FileCheck -check-prefixes=COMMON,AMDGCN %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -debug-info-kind=limited -triple spir-unknown-unknown | FileCheck -check-prefixes=CHECK-DEBUG %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -debug-info-kind=limited -triple amdgcn-amd-amdhsa | FileCheck -check-prefixes=CHECK-DEBUG %s

// SPIR: %struct.__opencl_block_literal_generic = type { i32, i32, i8 addrspace(4)* }
// AMDGCN: %struct.__opencl_block_literal_generic = type { i32, i32, i8* }
// SPIR: @__block_literal_global = internal addrspace(1) constant { i32, i32, i8 addrspace(4)* } { i32 12, i32 4, i8 addrspace(4)* addrspacecast (i8* bitcast (void (i8 addrspace(4)*, i8 addrspace(3)*)* @block_A_block_invoke to i8*) to i8 addrspace(4)*) }
// AMDGCN: @__block_literal_global = internal addrspace(1) constant { i32, i32, i8* } { i32 16, i32 8, i8* bitcast (void (i8*, i8 addrspace(3)*)* @block_A_block_invoke to i8*) }
// COMMON-NOT: .str

// SPIR-LABEL: define internal {{.*}}void @block_A_block_invoke(i8 addrspace(4)* %.block_descriptor, i8 addrspace(3)* %a)
// AMDGCN-LABEL: define internal {{.*}}void @block_A_block_invoke(i8* %.block_descriptor, i8 addrspace(3)* %a)
void (^block_A)(local void *) = ^(local void *a) {
  return;
};

// COMMON-LABEL: define {{.*}}void @foo()
void foo(){
  int i;
  // COMMON-NOT: %block.isa
  // COMMON-NOT: %block.flags
  // COMMON-NOT: %block.reserved
  // COMMON-NOT: %block.descriptor
  // SPIR: %[[block_size:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %block, i32 0, i32 0
  // AMDGCN: %[[block_size:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %block, i32 0, i32 0
  // SPIR: store i32 16, i32* %[[block_size]]
  // AMDGCN: store i32 20, i32 addrspace(5)* %[[block_size]]
  // SPIR: %[[block_align:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %block, i32 0, i32 1
  // AMDGCN: %[[block_align:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %block, i32 0, i32 1
  // SPIR: store i32 4, i32* %[[block_align]]
  // AMDGCN: store i32 8, i32 addrspace(5)* %[[block_align]]
  // SPIR: %[[block_invoke:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %[[block:.*]], i32 0, i32 2
  // SPIR: store i8 addrspace(4)* addrspacecast (i8* bitcast (i32 (i8 addrspace(4)*)* @__foo_block_invoke to i8*) to i8 addrspace(4)*), i8 addrspace(4)** %[[block_invoke]]
  // SPIR: %[[block_captured:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %[[block]], i32 0, i32 3
  // SPIR: %[[i_value:.*]] = load i32, i32* %i
  // SPIR: store i32 %[[i_value]], i32* %[[block_captured]],
  // SPIR: %[[blk_ptr:.*]] = bitcast <{ i32, i32, i8 addrspace(4)*, i32 }>* %[[block]] to i32 ()*
  // SPIR: %[[blk_gen_ptr:.*]] = addrspacecast i32 ()* %[[blk_ptr]] to i32 () addrspace(4)*
  // SPIR: store i32 () addrspace(4)* %[[blk_gen_ptr]], i32 () addrspace(4)** %[[block_B:.*]],
  // SPIR: %[[blk_gen_ptr:.*]] = load i32 () addrspace(4)*, i32 () addrspace(4)** %[[block_B]]
  // SPIR: %[[block_literal:.*]] = bitcast i32 () addrspace(4)* %[[blk_gen_ptr]] to %struct.__opencl_block_literal_generic addrspace(4)*
  // SPIR: %[[invoke_addr:.*]] = getelementptr inbounds %struct.__opencl_block_literal_generic, %struct.__opencl_block_literal_generic addrspace(4)* %[[block_literal]], i32 0, i32 2
  // SPIR: %[[blk_gen_ptr:.*]] = bitcast %struct.__opencl_block_literal_generic addrspace(4)* %[[block_literal]] to i8 addrspace(4)*
  // SPIR: %[[invoke_func_ptr:.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %[[invoke_addr]]
  // SPIR: %[[invoke_func:.*]] = addrspacecast i8 addrspace(4)* %[[invoke_func_ptr]] to i32 (i8 addrspace(4)*)*
  // SPIR: call {{.*}}i32 %[[invoke_func]](i8 addrspace(4)* %[[blk_gen_ptr]])
  // AMDGCN: %[[block_invoke:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %[[block:.*]], i32 0, i32 2
  // AMDGCN: store i8* bitcast (i32 (i8*)* @__foo_block_invoke to i8*), i8* addrspace(5)* %[[block_invoke]]
  // AMDGCN: %[[block_captured:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %[[block]], i32 0, i32 3
  // AMDGCN: %[[i_value:.*]] = load i32, i32 addrspace(5)* %i
  // AMDGCN: store i32 %[[i_value]], i32 addrspace(5)* %[[block_captured]],
  // AMDGCN: %[[blk_ptr:.*]] = bitcast <{ i32, i32, i8*, i32 }> addrspace(5)* %[[block]] to i32 () addrspace(5)*
  // AMDGCN: %[[blk_gen_ptr:.*]] = addrspacecast i32 () addrspace(5)* %[[blk_ptr]] to i32 ()*
  // AMDGCN: store i32 ()* %[[blk_gen_ptr]], i32 ()* addrspace(5)* %[[block_B:.*]],
  // AMDGCN: %[[blk_gen_ptr:.*]] = load i32 ()*, i32 ()* addrspace(5)* %[[block_B]]
  // AMDGCN: %[[block_literal:.*]] = bitcast i32 ()* %[[blk_gen_ptr]] to %struct.__opencl_block_literal_generic*
  // AMDGCN: %[[invoke_addr:.*]] = getelementptr inbounds %struct.__opencl_block_literal_generic, %struct.__opencl_block_literal_generic* %[[block_literal]], i32 0, i32 2
  // AMDGCN: %[[blk_gen_ptr:.*]] = bitcast %struct.__opencl_block_literal_generic* %[[block_literal]] to i8*
  // AMDGCN: %[[invoke_func_ptr:.*]] = load i8*, i8** %[[invoke_addr]]
  // AMDGCN: %[[invoke_func:.*]] = bitcast i8* %[[invoke_func_ptr]] to i32 (i8*)*
  // AMDGCN: call {{.*}}i32 %[[invoke_func]](i8* %[[blk_gen_ptr]])

  int (^ block_B)(void) = ^{
    return i;
  };
  block_B();
}

// SPIR-LABEL: define internal {{.*}}i32 @__foo_block_invoke(i8 addrspace(4)* %.block_descriptor)
// SPIR:  %[[block:.*]] = bitcast i8 addrspace(4)* %.block_descriptor to <{ i32, i32, i8 addrspace(4)*, i32 }> addrspace(4)*
// SPIR:  %[[block_capture_addr:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }> addrspace(4)* %[[block]], i32 0, i32 3
// SPIR:  %[[block_capture:.*]] = load i32, i32 addrspace(4)* %[[block_capture_addr]]
// AMDGCN-LABEL: define internal {{.*}}i32 @__foo_block_invoke(i8* %.block_descriptor)
// AMDGCN:  %[[block:.*]] = bitcast i8* %.block_descriptor to <{ i32, i32, i8*, i32 }>*
// AMDGCN:  %[[block_capture_addr:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }>* %[[block]], i32 0, i32 3
// AMDGCN:  %[[block_capture:.*]] = load i32, i32* %[[block_capture_addr]]

// COMMON-NOT: define{{.*}}@__foo_block_invoke_kernel

// Test that we support block arguments.
// COMMON-LABEL: define {{.*}} @blockArgFunc
int blockArgFunc(int (^ bl)(void)) {
  return bl();
}

// COMMON-LABEL: define {{.*}} @get21
// COMMON: define {{.*}} @__get21_block_invoke
// COMMON: ret i32 21
int get21() {
  return blockArgFunc(^{return 21;});
}

// COMMON-LABEL: define {{.*}} @get42
// COMMON: define {{.*}} @__get42_block_invoke
// COMMON: ret i32 42
int get42() {
  return blockArgFunc(^{return 42;});
}

// CHECK-DEBUG: !DIDerivedType(tag: DW_TAG_member, name: "__size"
// CHECK-DEBUG: !DIDerivedType(tag: DW_TAG_member, name: "__align"

// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__isa"
// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__flags"
// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__reserved"
// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__FuncPtr"

// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -triple spir-unknown-unknown | FileCheck -check-prefixes=COMMON,SPIR %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -triple amdgcn-amd-amdhsa | FileCheck -check-prefixes=COMMON,AMDGCN %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -debug-info-kind=limited -triple spir-unknown-unknown | FileCheck -check-prefixes=CHECK-DEBUG %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -O0 -debug-info-kind=limited -triple amdgcn-amd-amdhsa | FileCheck -check-prefixes=CHECK-DEBUG %s

// SPIR: %struct.__opencl_block_literal_generic = type { i32, i32, i8 addrspace(4)* }
// AMDGCN: %struct.__opencl_block_literal_generic = type { i32, i32, i8* }
// SPIR: @__block_literal_global = internal addrspace(1) constant { i32, i32, i8 addrspace(4)* } { i32 12, i32 4, i8 addrspace(4)* addrspacecast (i8* bitcast (void (i8 addrspace(4)*, i8 addrspace(3)*)* @block_A_block_invoke to i8*) to i8 addrspace(4)*) }
// AMDGCN: @__block_literal_global = internal addrspace(1) constant { i32, i32, i8* } { i32 16, i32 8, i8* bitcast (void (i8*, i8 addrspace(3)*)* @block_A_block_invoke to i8*) }
// COMMON-NOT: .str

// SPIR-LABEL: define internal {{.*}}void @block_A_block_invoke(i8 addrspace(4)* %.block_descriptor, i8 addrspace(3)* %a)
// AMDGCN-LABEL: define internal {{.*}}void @block_A_block_invoke(i8* %.block_descriptor, i8 addrspace(3)* %a)
void (^block_A)(local void *) = ^(local void *a) {
  return;
};

// COMMON-LABEL: define {{.*}}void @foo()
void foo(){
  int i;
  // COMMON-NOT: %block.isa
  // COMMON-NOT: %block.flags
  // COMMON-NOT: %block.reserved
  // COMMON-NOT: %block.descriptor
  // SPIR: %[[block_size:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %block, i32 0, i32 0
  // AMDGCN: %[[block_size:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %block, i32 0, i32 0
  // SPIR: store i32 16, i32* %[[block_size]]
  // AMDGCN: store i32 20, i32 addrspace(5)* %[[block_size]]
  // SPIR: %[[block_align:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %block, i32 0, i32 1
  // AMDGCN: %[[block_align:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %block, i32 0, i32 1
  // SPIR: store i32 4, i32* %[[block_align]]
  // AMDGCN: store i32 8, i32 addrspace(5)* %[[block_align]]
  // SPIR: %[[block_invoke:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %[[block:.*]], i32 0, i32 2
  // SPIR: store i8 addrspace(4)* addrspacecast (i8* bitcast (i32 (i8 addrspace(4)*)* @__foo_block_invoke to i8*) to i8 addrspace(4)*), i8 addrspace(4)** %[[block_invoke]]
  // SPIR: %[[block_captured:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }>* %[[block]], i32 0, i32 3
  // SPIR: %[[i_value:.*]] = load i32, i32* %i
  // SPIR: store i32 %[[i_value]], i32* %[[block_captured]],
  // SPIR: %[[blk_ptr:.*]] = bitcast <{ i32, i32, i8 addrspace(4)*, i32 }>* %[[block]] to i32 ()*
  // SPIR: %[[blk_gen_ptr:.*]] = addrspacecast i32 ()* %[[blk_ptr]] to i32 () addrspace(4)*
  // SPIR: store i32 () addrspace(4)* %[[blk_gen_ptr]], i32 () addrspace(4)** %[[block_B:.*]],
  // SPIR: %[[blk_gen_ptr:.*]] = load i32 () addrspace(4)*, i32 () addrspace(4)** %[[block_B]]
  // SPIR: %[[block_literal:.*]] = bitcast i32 () addrspace(4)* %[[blk_gen_ptr]] to %struct.__opencl_block_literal_generic addrspace(4)*
  // SPIR: %[[invoke_addr:.*]] = getelementptr inbounds %struct.__opencl_block_literal_generic, %struct.__opencl_block_literal_generic addrspace(4)* %[[block_literal]], i32 0, i32 2
  // SPIR: %[[blk_gen_ptr:.*]] = bitcast %struct.__opencl_block_literal_generic addrspace(4)* %[[block_literal]] to i8 addrspace(4)*
  // SPIR: %[[invoke_func_ptr:.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %[[invoke_addr]]
  // SPIR: %[[invoke_func:.*]] = addrspacecast i8 addrspace(4)* %[[invoke_func_ptr]] to i32 (i8 addrspace(4)*)*
  // SPIR: call {{.*}}i32 %[[invoke_func]](i8 addrspace(4)* %[[blk_gen_ptr]])
  // AMDGCN: %[[block_invoke:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %[[block:.*]], i32 0, i32 2
  // AMDGCN: store i8* bitcast (i32 (i8*)* @__foo_block_invoke to i8*), i8* addrspace(5)* %[[block_invoke]]
  // AMDGCN: %[[block_captured:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }> addrspace(5)* %[[block]], i32 0, i32 3
  // AMDGCN: %[[i_value:.*]] = load i32, i32 addrspace(5)* %i
  // AMDGCN: store i32 %[[i_value]], i32 addrspace(5)* %[[block_captured]],
  // AMDGCN: %[[blk_ptr:.*]] = bitcast <{ i32, i32, i8*, i32 }> addrspace(5)* %[[block]] to i32 () addrspace(5)*
  // AMDGCN: %[[blk_gen_ptr:.*]] = addrspacecast i32 () addrspace(5)* %[[blk_ptr]] to i32 ()*
  // AMDGCN: store i32 ()* %[[blk_gen_ptr]], i32 ()* addrspace(5)* %[[block_B:.*]],
  // AMDGCN: %[[blk_gen_ptr:.*]] = load i32 ()*, i32 ()* addrspace(5)* %[[block_B]]
  // AMDGCN: %[[block_literal:.*]] = bitcast i32 ()* %[[blk_gen_ptr]] to %struct.__opencl_block_literal_generic*
  // AMDGCN: %[[invoke_addr:.*]] = getelementptr inbounds %struct.__opencl_block_literal_generic, %struct.__opencl_block_literal_generic* %[[block_literal]], i32 0, i32 2
  // AMDGCN: %[[blk_gen_ptr:.*]] = bitcast %struct.__opencl_block_literal_generic* %[[block_literal]] to i8*
  // AMDGCN: %[[invoke_func_ptr:.*]] = load i8*, i8** %[[invoke_addr]]
  // AMDGCN: %[[invoke_func:.*]] = bitcast i8* %[[invoke_func_ptr]] to i32 (i8*)*
  // AMDGCN: call {{.*}}i32 %[[invoke_func]](i8* %[[blk_gen_ptr]])

  int (^ block_B)(void) = ^{
    return i;
  };
  block_B();
}

// SPIR-LABEL: define internal {{.*}}i32 @__foo_block_invoke(i8 addrspace(4)* %.block_descriptor)
// SPIR:  %[[block:.*]] = bitcast i8 addrspace(4)* %.block_descriptor to <{ i32, i32, i8 addrspace(4)*, i32 }> addrspace(4)*
// SPIR:  %[[block_capture_addr:.*]] = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i32 }>, <{ i32, i32, i8 addrspace(4)*, i32 }> addrspace(4)* %[[block]], i32 0, i32 3
// SPIR:  %[[block_capture:.*]] = load i32, i32 addrspace(4)* %[[block_capture_addr]]
// AMDGCN-LABEL: define internal {{.*}}i32 @__foo_block_invoke(i8* %.block_descriptor)
// AMDGCN:  %[[block:.*]] = bitcast i8* %.block_descriptor to <{ i32, i32, i8*, i32 }>*
// AMDGCN:  %[[block_capture_addr:.*]] = getelementptr inbounds <{ i32, i32, i8*, i32 }>, <{ i32, i32, i8*, i32 }>* %[[block]], i32 0, i32 3
// AMDGCN:  %[[block_capture:.*]] = load i32, i32* %[[block_capture_addr]]

// COMMON-NOT: define{{.*}}@__foo_block_invoke_kernel

// CHECK-DEBUG: !DIDerivedType(tag: DW_TAG_member, name: "__size"
// CHECK-DEBUG: !DIDerivedType(tag: DW_TAG_member, name: "__align"

// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__isa"
// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__flags"
// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__reserved"
// CHECK-DEBUG-NOT: !DIDerivedType(tag: DW_TAG_member, name: "__FuncPtr"


make typedefs macro based, include common

#include "clcommons/common.h"
#include "clcommons/image.h"

#ifndef PIXELT
#define PIXELT uint
#endif
#ifndef LABELT
#define LABELT uint
#endif
#ifndef CONNECTIVITYPIXELT
#define CONNECTIVITYPIXELT uint
#endif
#ifndef LDSPIXELT
#define LDSPIXELT uint
#endif
#ifndef LDSLABELT
#define LDSLABELT uint
#endif
#ifndef LDSCONNECTIVITYPIXELT
#define LDSCONNECTIVITYPIXELT uint
#endif

typedef PIXELT PixelT;
typedef LDSPIXELT LDSPixelT;
typedef LABELT LabelT;
typedef LDSLABELT LDSLabelT;
typedef CONNECTIVITYPIXELT ConnectivityPixelT;
typedef LDSCONNECTIVITYPIXELT LDSConnectivityPixelT;

#ifndef BG_VALUE
#define BG_VALUE 0
#endif

#ifndef CONNECTIVITY  //4 or 8
#define CONNECTIVITY 8
#endif

#ifndef WORKGROUP_TILE_SIZE_X
#define WORKGROUP_TILE_SIZE_X 32
#endif
#ifndef WORKGROUP_TILE_SIZE_Y
#define WORKGROUP_TILE_SIZE_Y 8
#endif

#ifndef WORKITEM_REPEAT_X
#define WORKITEM_REPEAT_X 1
#endif
#ifndef WORKITEM_REPEAT_Y
#define WORKITEM_REPEAT_Y 4
#endif

#define TILE_COLS (WORKGROUP_TILE_SIZE_X * WORKITEM_REPEAT_X)
#define TILE_ROWS (WORKGROUP_TILE_SIZE_Y * WORKITEM_REPEAT_Y)

enum ConnectivityEnum {
    UP = (1<<0),
    DOWN = (1<<2),
    LEFT = (1<<3),
    RIGHT = (1<<4),
    LEFT_UP = (1<<5),
    LEFT_DOWN = (1<<6),
    RIGHT_UP = (1<<7),
    RIGHT_DOWN = (1<<8)
};

#define isConnected(p1, p2) ((p1) == (p2))

#define pixel_at(type, basename, r, c) image_pixel_at(type, PASTE2(basename, _p), im_rows, im_cols, PASTE2(basename, _pitch), (r), (c))

//global dimensions: divUp(image.cols, block.x), divUp(image.rows, block.y);
//__attribute__((reqd_work_group_size(WORKGROUP_TILE_SIZE_X, WORKGROUP_TILE_SIZE_Y, 1)))
//__kernel void
//make_connectivity_image(uint im_rows, uint im_cols, __global PixelT *image_p, uint image_pitch, __global ConnectivityPixelT *connectivityim_p, uint connectivityim_pitch){
//    const uint c = get_global_id(1);
//    const uint r = get_global_id(0);
//    const bool valid_pixel_task = (c < im_cols) & (r < im_rows);
//
//    if(valid_pixel_task){
//        PixelT pixel = pixel_at(PixelT, image, r, c);
//        ConnectivityPixelT connectivity = 0;
//
//#if CONNECTIVITY == 8
//        connectivity |= c > 0 && r > 0                     && isConnected(pixel, pixel_at(PixelT, image, r-1, c - 1)) ? LEFT_UP : 0
//        connectivity |= c > 0                              && isConnected(pixel, pixel_at(PixelT, image, r, c - 1)) ? LEFT : 0
//        connectivity |= c > 0 && r < im_rows - 1           && isConnected(pixel, pixel_at(PixelT, image, r+1, c - 1)) ? LEFT_DOWN : 0
//        connectivity |=          r < im_rows - 1           && isConnected(pixel, pixel_at(PixelT, image, r+1, c)) ? DOWN : 0
//        connectivity |= c < im_cols - 1 && r < im_rows - 1 && isConnected(pixel, pixel_at(PixelT, image, r+1, c + 1)) ? RIGHT_DOWN : 0
//        connectivity |= c < im_cols - 1                    && isConnected(pixel, pixel_at(PixelT, image, r, c + 1)) ? RIGHT : 0
//        connectivity |= c < im_cols - 1 && r > 0           && isConnected(pixel, pixel_at(PixelT, image, r-1, c + 1)) ? RIGHT_UP : 0
//        connectivity |=          r > 0                     && isConnected(pixel, pixel_at(PixelT, image, r-1, c)) ? UP : 0
//#else
//        connectivity |= c > 0                              && isConnected(pixel, pixel_at(PixelT, image, r, c - 1)) ? LEFT : 0
//        connectivity |=          r < im_rows - 1           && isConnected(pixel, pixel_at(PixelT, image, r+1, c)) ? DOWN : 0
//        connectivity |= c < im_cols - 1                    && isConnected(pixel, pixel_at(PixelT, image, r, c + 1)) ? RIGHT : 0
//        connectivity |=          r > 0                     && isConnected(pixel, pixel_at(PixelT, image, r-1, c)) ? UP : 0
//#endif
//
//        pixel_at(ConnectivityPixelT, connectivity, r, c) = connectivity;
//    }
//}

//global dimensions: divUp(im_cols, workgroup_tile_size_x), divUp(im_rows, workgroup_tile_size_y);
__attribute__((reqd_work_group_size(WORKGROUP_TILE_SIZE_X, WORKGROUP_TILE_SIZE_Y, 1)))
__kernel void
make_connectivity_image(uint im_rows, uint im_cols, __global PixelT *image_p, uint image_pitch, __global ConnectivityPixelT *connectivityim_p, uint connectivityim_pitch){
    const uint tile_col_blocksize = get_local_size(0);
    const uint tile_row_blocksize = get_local_size(1);
    const uint tile_col_block = get_group_id(0) + get_global_offset(0) / tile_col_blocksize;
    const uint tile_row_block = get_group_id(1) + get_global_offset(1) / tile_row_blocksize;
    const uint tile_col = get_local_id(0);
    const uint tile_row = get_local_id(1);
    const bool valid_pixel_task = (get_global_id(0) < im_cols) & (get_global_id(1) < im_rows);

    uint tile_rows = tile_row_blocksize;
    uint tile_cols = tile_col_blocksize;

    const uint tile_row_start = tile_row_blocksize * tile_rows;
    const uint tile_col_start = tile_col_blocksize * tile_cols;
    const uint tile_row_end = min((tile_row_blocksize + 1) * tile_rows, (uint) im_rows);
    const uint tile_col_end = min((tile_col_blocksize + 1) * tile_cols, (uint) im_cols);
    //adjust to true tile dimensions
    tile_rows = tile_row_end - tile_row_start;
    tile_cols = tile_col_end - tile_col_start;
    const uint n_tile_pixels = tile_rows * tile_cols;
    __local LDSPixelT im_tile[WORKGROUP_TILE_SIZE_Y][WORKGROUP_TILE_SIZE_X];
    __local ConnectivityPixelT connectivity_tile[WORKGROUP_TILE_SIZE_Y][WORKGROUP_TILE_SIZE_X];

    const uint tid = tile_row * tile_cols + tile_col;
    for(uint im_tile_fill_task_id = tid; im_tile_fill_task_id < n_tile_pixels; im_tile_fill_task_id += n_tile_pixels){
        const uint im_tile_row = im_tile_fill_task_id / tile_cols;
        const uint im_tile_col = im_tile_fill_task_id % tile_cols;
        const uint g_c = im_tile_col + tile_col_block * tile_col_blocksize;
        const uint g_r = im_tile_row + tile_row_block * tile_row_blocksize;

        im_tile[im_tile_row][im_tile_col] = pixel_at(PixelT, image, g_r, g_c);
    }
    lds_barrier();

    {
        const uint c = get_local_id(1);
        const uint r = get_local_id(0);
        const uint g_c = get_global_id(1);
        const uint g_r = get_global_id(0);
        PixelT pixel = valid_pixel_task ? pixel_at(PixelT, image, r, c) : 0;
        ConnectivityPixelT connectivity = 0;

#if CONNECTIVITY == 8
        connectivity |= c > 0 && r > 0                         && isConnected(pixel, im_tile[r-1][c - 1]) ? LEFT_UP : 0
        connectivity |= c > 0                                  && isConnected(pixel, im_tile[r][c - 1]) ? LEFT : 0
        connectivity |= c > 0 && r < tile_rows - 1             && isConnected(pixel, im_tile[r+1][c - 1]) ? LEFT_DOWN : 0
        connectivity |=          r < tile_rows - 1             && isConnected(pixel, im_tile[r+1][c]) ? DOWN : 0
        connectivity |= c < tile_cols - 1 && r < tile_rows - 1 && isConnected(pixel, im_tile[r+1][c + 1]) ? RIGHT_DOWN : 0
        connectivity |= c < tile_cols - 1                      && isConnected(pixel, im_tile[r][c + 1]) ? RIGHT : 0
        connectivity |= c < tile_cols - 1 && r > 0             && isConnected(pixel, im_tile[r-1][c + 1]) ? RIGHT_UP : 0
        connectivity |=          r > 0                         && isConnected(pixel, im_tile[r-1][c]) ? UP : 0
#else
        connectivity |= c > 0                                  && isConnected(pixel, im_tile[r][c - 1]) ? LEFT : 0
        connectivity |=          r < tile_rows - 1             && isConnected(pixel, im_tile[r+1][c]) ? DOWN : 0
        connectivity |= c < tile_cols - 1                      && isConnected(pixel, im_tile[r][c + 1]) ? RIGHT : 0
        connectivity |=          r > 0                         && isConnected(pixel, im_tile[r-1][c]) ? UP : 0
#endif
        connectivity_tile[r][c] = connectivity;

    }
    lds_barrier();

    for(uint im_tile_fill_task_id = tid; im_tile_fill_task_id < n_tile_pixels; im_tile_fill_task_id += n_tile_pixels){
        const uint im_tile_row = im_tile_fill_task_id / tile_cols;
        const uint im_tile_col = im_tile_fill_task_id % tile_cols;
        const uint g_c = im_tile_col + tile_col_block * tile_col_blocksize;
        const uint g_r = im_tile_row + tile_row_block * tile_row_blocksize;

        pixel_at(ConnectivityPixelT, connectivity, g_r, g_c) = connectivity_tile[im_tile_row][im_tile_col];
    }
}

__attribute__((reqd_work_group_size(WORKGROUP_TILE_SIZE_X, WORKGROUP_TILE_SIZE_Y, 1)))
__kernel void
label_tiles(uint im_rows, uint im_cols, __global ConnectivityPixelT *labelim_p, uint labelim_pitch, __global ConnectivityPixelT *connectivity_p, uint connectivityPitch){
    const uint get_local_id(0) = get_local_id(0);
    const uint get_local_id(1) = get_local_id(1);

    const uint x = get_local_id(0) + get_group_id(0) * TILE_COLS;
    const uint y = get_local_id(1) + get_group_id(1) * TILE_ROWS;

    //if (x >= im_rows || y >= im_rows) return;

    //currently x is 1
    //int bounds = ((y + WORKITEM_REPEAT_Y) < im_rows);

    __local LDSLabelT label_tile_im[TILE_ROWS][TILE_COLS];
    __local LDSConnectivityPixelT  edge_tile_im[TILE_ROWS][TILE_COLS];

    LDSLabelT new_labels[WORKITEM_REPEAT_Y][WORKITEM_REPEAT_X];
    LDSLabelT old_labels[WORKITEM_REPEAT_Y][WORKITEM_REPEAT_X];

    #pragma unroll
    for (int i = 0; i < WORKITEM_REPEAT_Y; ++i){
        #pragma unroll
        for (int j = 0; j < WORKITEM_REPEAT_X; ++j){
            const uint yloc = get_local_id(1) + WORKGROUP_TILE_SIZE_Y * i;
            const uint xloc = get_local_id(0) + WORKGROUP_TILE_SIZE_X * j;
            const bool valid_pixel_task = (xloc < im_cols) & (yloc < im_rows);
            ConnectivityPixelT c = valid_pixel_task ? pixel_at(ConnectivityPixelT, connectivity, y + WORKGROUP_TILE_SIZE_Y * i, x + WORKGROUP_TILE_SIZE_X * j) : 0;

            c = xloc == 0 ? c & ~(LEFT|LEFT_DOWN|LEFT_UP) : c;
            c = yloc == 0 ? c & ~(UP|LEFT_UP|RIGHT_UP) : c;

            c = xloc == TILE_COLS - 1 ? c & ~(RIGHT|RIGHT_DOWN|RIGHT_UP);
            c = yloc == TILE_ROWS - 1 ? c & ~(DOWN|LEFT_DOWN|RIGHT_DOWN) : c;

            new_labels[i][j] = yloc * TILE_COLS + xloc;
            edge_tile_im[yloc][xloc] = c;
        }
    }

    for (int k = 0; ;++k){
        //make copies
        #pragma unroll
        for (int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for (int j = 0; j < WORKITEM_REPEAT_X; ++j){
                const uint yloc = get_local_id(1) + WORKGROUP_TILE_SIZE_Y * i;
                const uint xloc = get_local_id(0) + WORKGROUP_TILE_SIZE_X * j;

                old_labels[i][j]          = new_labels[i][j];
                label_tile_im[yloc][xloc] = new_labels[i][j];
            }
        }
        lds_barrier();

        //take minimum label of local neighboorhood - single writer, multi reader version
        #pragma unroll
        for (int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for (int j = 0; j < WORKITEM_REPEAT_X; ++j){
                const uint yloc = get_local_id(1) + WORKGROUP_TILE_SIZE_Y * i;
                const uint xloc = get_local_id(0) + WORKGROUP_TILE_SIZE_X * j;

                const ConnectivityPixelT connectivity = edge_tile_im[yloc][xloc];
                const LDSLabelT label = new_labels[i][j];

#if CONNECTIVITY == 8
                if (connectivity & UP)
                   label = min(label, label_tile_im[yloc - 1][xloc]);
                if (connectivity & LEFT)
                   label = min(label, label_tile_im[yloc][xloc - 1]);
                if (connectivity & LEFT_UP)
                   label = min(label, label_tile_im[yloc - 1][xloc - 1]);
                if (connectivity &  LEFT_DOWN)
                   label = min(label, label_tile_im[yloc + 1][xloc - 1]);
                if (connectivity &  DOWN)
                   label = min(label, label_tile_im[yloc + 1][xloc]);
                if (connectivity & RIGHT_DOWN)
                   label = min(label, label_tile_im[yloc - 1][xloc + 1]);
                if (connectivity & RIGHT)
                   label = min(label, label_tile_im[yloc][xloc + 1]);
                if (connectivity & RIGHT_UP)
                   label = min(label, label_tile_im[yloc + 1][xloc + 1]);

#else
                if (connectivity & UP)
                   label = min(label, label_tile_im[yloc - 1][xloc]);
                if (connectivity &  DOWN)
                   label = min(label, label_tile_im[yloc + 1][xloc]);
                if (connectivity & LEFT)
                   label = min(label, label_tile_im[yloc][xloc - 1]);
                if (connectivity & RIGHT)
                   label = min(label, label_tile_im[yloc][xloc + 1]);
#endif

                new_labels[i][j] = label;
            }
        }
        lds_barrier();

        __local int changed;
        if((get_local_id(1) == 0) & (get_local_id(0) == 0)){
            changed = 0;
        }
        lds_barrier();

        const __local LDSLabelT *labels = &label_tile_im[0][0];

        int pchanged = 0;
        #pragma unroll
        for(int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for(int j = 0; j < WORKITEM_REPEAT_X; ++j){
                if(new_labels[i][j] < old_labels[i][j]){
                    pchanged++;
                    atomic_min(labels + old_labels[i][j], new_labels[i][j]);
                }
            }
        }
        atomic_add(&changed, pchanged);
        lds_barrier();

        //if there are no updates, we are finished
        if(!changed){
            break;
        }

        //Compact paths
        #pragma unroll
        for(int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for(int j = 0; j < WORKITEM_REPEAT_X; ++j){
                LDSLabelT label = new_labels[i][j];

                //find root label
                while(labels[label] < label){
                    label = labels[label];
                }

                new_labels[i][j] = label;
            }
        }
        lds_barrier();
    }

    //save with adjusted global (untiled) labels
    #pragma unroll
    for(int i = 0; i < WORKITEM_REPEAT_Y; ++i){
        #pragma unroll
        for(int j = 0; j < WORKITEM_REPEAT_X; ++j){
            const LabelT tile_label = new_labels[i][j];
            const uint yloc = (tile_label / TILE_COLS) + get_group_id(0) * TILE_COLS;
            const uint xloc = (tile_label % TILE_COLS) + get_group_id(1) * TILE_ROWS;

            const LabelT glabel = yloc * im_cols + xloc;
            const bool valid_pixel_task = (xloc < im_cols) & (yloc < im_rows);
            if(valid_pixel_task){
                pixel_at(LabelT, labelim, yloc, xloc) = glabel;
            }
        }
    }
}

inline
LabelT find_root_global(const __global *labelim, uint labelim_pitch, LabelT label, const uint im_rows, const uint im_cols){
    for(;;){
        const uint y = label / im_cols;
        const uint x = label % im_cols;
        const LabelT parent = pixel_at(LabelT, labelim, y, x);

        if(label == parent){
            break;
        }

        label = parent;
    }
    return label;
}

__kernel
void compact_paths_global(uint im_rows, uint im_cols, __global LabelT *labelim, uint labelim_pitch){
    const uint x = get_global_id(0);
    const uint y = get_global_id(1);

    if((x < im_cols) & (y < im_rows)){
        pixel_at(LabelT, labelim, y, x) = find_root_global(labelim, pixel_at(LabelT, labelim, y, x));
    }
}

inline
void merge_edge_labels(const uint im_rows, const uint im_cols, __global LabelT *labelim_p, const uint labelim_pitch, const LabelT l1, const LabelT l2, uint *changed){

    const LabelT r1 = find_root_global(comps, l1, im_rows, im_cols);
    const LabelT r2 = find_root_global(comps, l2, im_rows, im_cols);

    if (r1 == r2) return;

    const LabelT mi = min(r1, r2);
    const LabelT ma = max(r1, r2);

    const uint y = ma / im_cols;
    const uint x = ma % im_cols;

    atomic_min(&pixel_at(LabelT, labelim, y, x), mi);
    *changed = true;
}

__kernel
void cross_merge(
    const uint n_vert_tiles, const uint n_horz_tiles,
    uint tile_rows, uint tile_cols,
    const uint im_rows, const uint im_cols,
    const __global ConnectivityPixelT *connectivityim_p, uint connectivityim_pitch,
    const __global LabelT *labelim_p, uint labelim,
    const uint yIncomplete, uint xIncomplete
){
    const uint ngroups_x = get_num_groups(0);
    const uint ngroups_y = get_num_groups(1);

    const uint tid = get_local_linear_id();
    const uint stride = get_local_size(1) * get_local_size(0);

    const uint ybegin = get_group_id(1) * (n_vert_tiles * tile_rows);
    const uint yend   = ybegin + n_vert_tiles * tile_rows;

    if (get_group_id(1) == ngroups_y - 1){
        yend -= yIncomplete * tile_rows;
        yend -= tile_rows;
        tile_rows = (im_rows % tile_rows);

        yend += tile_rows;
    }

    const uint xbegin = get_group_id(0) * n_horz_tiles * tile_cols;
    const uint xend   = xbegin + n_horz_tiles * tile_cols;

    if(get_group_id(0) == ngroups_x - 1){
        if (xIncomplete) yend = ybegin;
        xend -= xIncomplete * tile_cols;
        xend -= tile_cols;
        tile_cols = (im_cols % tile_cols);

        xend += tile_cols;
    }

    if (get_group_id(1) == (ngroups_y - 1) && yIncomplete){
        xend = xbegin;
    }

    const uint tasksV = (n_horz_tiles - 1) * (yend - ybegin);
    const uint tasksH = (n_vert_tiles - 1) * (xend - xbegin);

    const uint total = tasksH + tasksV;

    __local uint changed;
    uint pchanged;
    do{
        pchanged = false;
        for(uint taskIdx = tid; taskIdx < total; taskIdx += stride){
            if(taskIdx < tasksH){
                const uint indexH = taskIdx;

                const uint row = indexH / (xend - xbegin);
                const uint col = indexH - % (xend - xbegin);

                const uint y = ybegin + (row + 1) * tile_rows;
                const uint x = xbegin + col;

                const ConnectivityPixelT e = pixel_at(ConnectivityPixelT, connectivityim, y, x);
                if(e & UP){
                    LabelT lc = pixel_at(LabelT, labelim, y, x);
                    LabelT lu = pixel_at(LabelT, labelim, y - 1, x);
                    merge_edge_labels(im_rows, im_cols, labelim, labelim_pitch, lc, lu, &pchanged);
                }
            }else{
                const uint indexV = taskIdx - tasksH;

                const uint col = indexV / (yend - ybegin);
                const uint row = indexV % (yend - ybegin);

                const uint x = xbegin + (col + 1) * tile_cols;
                const uint y = ybegin + row;

                const ConnectivityPixelT e = pixel_at(ConnectivityPixelT, connectivityim, y, x);
                if(e & LEFT){
                    LabelT lc = pixel_at(LabelT, labelim, y, x);
                    LabelT lu = pixel_at(LabelT, labelim, y, x - 1);
                    merge_edge_labels(im_rows, im_cols, labelim, labelim_pitch, lc, ll, &pchanged);
                }
            }
        }
        atomic_add(&changed, pchanged);
        lds_barrier();
        pchanged = changed;
    }while(pchanged);
}

__kernel
void mark_root_classes(
    uint im_rows, uint im_cols,
    __global PixelT *image_p, uint image_pitch
    __global const LabelT* labelim_p, const labelim_pitch,
    __global uint* is_root_class_image_p, const uint is_root_class_image_pitch,
){
    const uint c = get_global_id(0);
    const uint r = get_global_id(1);
    const bool valid_pixel_task = (c < im_cols) & (r < im_rows);

    const uint linear_index = c + r * im_cols;
    if(valid_pixel_task){
        const PixelT pixel = pixel_at(PixelT, image, r, c);
        const LabelT label = pixel_at(LabelT, labelim, r, c);
        pixel_at(uint, is_root_class_image, r, c) = (pixel != BG_VALUE) & (label == linear_index);
    }
}

__kernel
void relabel_with_scanline_order(
    uint im_rows, uint im_cols,
    __global LabelT* labelim_out_p, const uint labelim_out_pitch,
    __global const PixelT* image_p, const image_pitch,
    __global const LabelT* labelim_p, const labelim_pitch,
    __global uint* is_root_class_image_psum_p, const uint is_root_class_image_psum_pitch
){
    const uint c = get_global_id(0);
    const uint r = get_global_id(1);
    const bool valid_pixel_task = (c < im_cols) & (r < im_rows);
    const uint linear_index = c + r * im_cols;

    if(valid_pixelTask){
        const LabelT pixel = pixel_at(PixelT, image, r, c);
        LabelT final_label = 0;
        if(pixel != BG_VALUE){
            const LabelT label = pixel_at(LabelT, labelim, r, c);
            const uint label_r = label / im_cols;
            const uint label_c = label % im_cols;
            const uint scan_id = pixel_at(uint, is_root_class_image_psum, label_r, label_c);
            final_label = scan_id + 1;
        }
        image_pixel_at(LabelT, labelim, r, c) = final_label;
    }
}

#include "clcommons/image.h"

typedef int LabelT;
typedef int LDSLabelT;
typedef uint PixelT;
typedef uint LDSPixelT;
typedef uint ConnectivityPixelT;
typedef uint LDSConnectivityPixelT;

#ifndef WORKGROUP_TILE_SIZE_X
#define WORKGROUP_TILE_SIZE_X 32
#endif
#ifndef WORKGROUP_TILE_SIZE_Y
#define WORKGROUP_TILE_SIZE_Y 8
#endif

#ifndef WORKITEM_REPEAT_X
#define WORKITEM_REPEAT_X 1
#endif
#ifndef WORKITEM_REPEAT_Y
#define WORKITEM_REPEAT_Y 4
#endif

#define TILE_COLS (WORKGROUP_TILE_SIZE_X * WORKITEM_REPEAT_X)
#define TILE_ROWS (WORKGROUP_TILE_SIZE_Y * WORKITEM_REPEAT_Y)

enum ConnectivityEnum {
    UP = (1<<0),
    DOWN = (1<<2),
    LEFT = (1<<3),
    RIGHT = (1<<4),
    LEFT_UP = (1<<5),
    LEFT_DOWN = (1<<6),
    RIGHT_UP = (1<<7),
    RIGHT_DOWN = (1<<8)
};

#define BG_VALUE 0

#define isConnected(p1, p2) ((p1) == (p2))

#ifndef CONNECTIVITY
#define CONNECTIVITY 8
#endif

#define pixel_at(type, basename, r, c) image_pixel_at(type, PASTE2(basename, _p), im_rows, im_cols, PASTE2(basename, _pitch), (r), (c))

//global dimensions: divUp(image.cols, block.x), divUp(image.rows, block.y);
//__attribute__((reqd_work_group_size(WORKGROUP_TILE_SIZE_X, WORKGROUP_TILE_SIZE_Y, 1)))
//__kernel void
//make_connectivity_image(uint im_rows, uint im_cols, __global PixelT *image_p, uint image_pitch, __global ConnectivityPixelT *connectivityim_p, uint connectivityim_pitch){
//    const uint c = get_global_id(1);
//    const uint r = get_global_id(0);
//    const bool valid_pixel_task = (c < im_cols) & (r < im_rows);
//
//    if(valid_pixel_task){
//        PixelT pixel = pixel_at(PixelT, image, r, c);
//        ConnectivityPixelT connectivity = 0;
//
//#if CONNECTIVITY == 8
//        connectivity |= c > 0 && r > 0                     && isConnected(pixel, pixel_at(PixelT, image, r-1, c - 1)) ? LEFT_UP : 0
//        connectivity |= c > 0                              && isConnected(pixel, pixel_at(PixelT, image, r, c - 1)) ? LEFT : 0
//        connectivity |= c > 0 && r < im_rows - 1           && isConnected(pixel, pixel_at(PixelT, image, r+1, c - 1)) ? LEFT_DOWN : 0
//        connectivity |=          r < im_rows - 1           && isConnected(pixel, pixel_at(PixelT, image, r+1, c)) ? DOWN : 0
//        connectivity |= c < im_cols - 1 && r < im_rows - 1 && isConnected(pixel, pixel_at(PixelT, image, r+1, c + 1)) ? RIGHT_DOWN : 0
//        connectivity |= c < im_cols - 1                    && isConnected(pixel, pixel_at(PixelT, image, r, c + 1)) ? RIGHT : 0
//        connectivity |= c < im_cols - 1 && r > 0           && isConnected(pixel, pixel_at(PixelT, image, r-1, c + 1)) ? RIGHT_UP : 0
//        connectivity |=          r > 0                     && isConnected(pixel, pixel_at(PixelT, image, r-1, c)) ? UP : 0
//#else
//        connectivity |= c > 0                              && isConnected(pixel, pixel_at(PixelT, image, r, c - 1)) ? LEFT : 0
//        connectivity |=          r < im_rows - 1           && isConnected(pixel, pixel_at(PixelT, image, r+1, c)) ? DOWN : 0
//        connectivity |= c < im_cols - 1                    && isConnected(pixel, pixel_at(PixelT, image, r, c + 1)) ? RIGHT : 0
//        connectivity |=          r > 0                     && isConnected(pixel, pixel_at(PixelT, image, r-1, c)) ? UP : 0
//#endif
//
//        pixel_at(ConnectivityPixelT, connectivity, r, c) = connectivity;
//    }
//}

//global dimensions: divUp(im_cols, workgroup_tile_size_x), divUp(im_rows, workgroup_tile_size_y);
__attribute__((reqd_work_group_size(WORKGROUP_TILE_SIZE_X, WORKGROUP_TILE_SIZE_Y, 1)))
__kernel void
make_connectivity_image(uint im_rows, uint im_cols, __global PixelT *image_p, uint image_pitch, __global ConnectivityPixelT *connectivityim_p, uint connectivityim_pitch){
    const uint tile_col_blocksize = get_local_size(0);
    const uint tile_row_blocksize = get_local_size(1);
    const uint tile_col_block = get_group_id(0) + get_global_offset(0) / tile_col_blocksize;
    const uint tile_row_block = get_group_id(1) + get_global_offset(1) / tile_row_blocksize;
    const uint tile_col = get_local_id(0);
    const uint tile_row = get_local_id(1);
    const bool valid_pixel_task = (get_global_id(0) < im_cols) & (get_global_id(1) < im_rows);

    uint tile_rows = tile_row_blocksize;
    uint tile_cols = tile_col_blocksize;

    const uint tile_row_start = tile_row_blocksize * tile_rows;
    const uint tile_col_start = tile_col_blocksize * tile_cols;
    const uint tile_row_end = min((tile_row_blocksize + 1) * tile_rows, (uint) im_rows);
    const uint tile_col_end = min((tile_col_blocksize + 1) * tile_cols, (uint) im_cols);
    //adjust to true tile dimensions
    tile_rows = tile_row_end - tile_row_start;
    tile_cols = tile_col_end - tile_col_start;
    const uint n_tile_pixels = tile_rows * tile_cols;
    __local LDSPixelT im_tile[WORKGROUP_TILE_SIZE_Y][WORKGROUP_TILE_SIZE_X];
    __local ConnectivityPixelT connectivity_tile[WORKGROUP_TILE_SIZE_Y][WORKGROUP_TILE_SIZE_X];

    const uint tid = tile_row * tile_cols + tile_col;
    for(uint im_tile_fill_task_id = tid; im_tile_fill_task_id < n_tile_pixels; im_tile_fill_task_id += n_tile_pixels){
        const uint im_tile_row = im_tile_fill_task_id / tile_cols;
        const uint im_tile_col = im_tile_fill_task_id % tile_cols;
        const uint g_c = im_tile_col + tile_col_block * tile_col_blocksize;
        const uint g_r = im_tile_row + tile_row_block * tile_row_blocksize;

        im_tile[im_tile_row][im_tile_col] = pixel_at(PixelT, image, g_r, g_c);
    }
    lds_barrier();

    {
        const uint c = get_local_id(1);
        const uint r = get_local_id(0);
        const uint g_c = get_global_id(1);
        const uint g_r = get_global_id(0);
        PixelT pixel = valid_pixel_task ? pixel_at(PixelT, image, r, c) : 0;
        ConnectivityPixelT connectivity = 0;

#if CONNECTIVITY == 8
        connectivity |= c > 0 && r > 0                         && isConnected(pixel, im_tile[r-1][c - 1]) ? LEFT_UP : 0
        connectivity |= c > 0                                  && isConnected(pixel, im_tile[r][c - 1]) ? LEFT : 0
        connectivity |= c > 0 && r < tile_rows - 1             && isConnected(pixel, im_tile[r+1][c - 1]) ? LEFT_DOWN : 0
        connectivity |=          r < tile_rows - 1             && isConnected(pixel, im_tile[r+1][c]) ? DOWN : 0
        connectivity |= c < tile_cols - 1 && r < tile_rows - 1 && isConnected(pixel, im_tile[r+1][c + 1]) ? RIGHT_DOWN : 0
        connectivity |= c < tile_cols - 1                      && isConnected(pixel, im_tile[r][c + 1]) ? RIGHT : 0
        connectivity |= c < tile_cols - 1 && r > 0             && isConnected(pixel, im_tile[r-1][c + 1]) ? RIGHT_UP : 0
        connectivity |=          r > 0                         && isConnected(pixel, im_tile[r-1][c]) ? UP : 0
#else
        connectivity |= c > 0                                  && isConnected(pixel, im_tile[r][c - 1]) ? LEFT : 0
        connectivity |=          r < tile_rows - 1             && isConnected(pixel, im_tile[r+1][c]) ? DOWN : 0
        connectivity |= c < tile_cols - 1                      && isConnected(pixel, im_tile[r][c + 1]) ? RIGHT : 0
        connectivity |=          r > 0                         && isConnected(pixel, im_tile[r-1][c]) ? UP : 0
#endif
        connectivity_tile[r][c] = connectivity;

    }
    lds_barrier();

    for(uint im_tile_fill_task_id = tid; im_tile_fill_task_id < n_tile_pixels; im_tile_fill_task_id += n_tile_pixels){
        const uint im_tile_row = im_tile_fill_task_id / tile_cols;
        const uint im_tile_col = im_tile_fill_task_id % tile_cols;
        const uint g_c = im_tile_col + tile_col_block * tile_col_blocksize;
        const uint g_r = im_tile_row + tile_row_block * tile_row_blocksize;

        pixel_at(ConnectivityPixelT, connectivity, g_r, g_c) = connectivity_tile[im_tile_row][im_tile_col];
    }
}

__attribute__((reqd_work_group_size(WORKGROUP_TILE_SIZE_X, WORKGROUP_TILE_SIZE_Y, 1)))
__kernel void
label_tiles(uint im_rows, uint im_cols, __global ConnectivityPixelT *labelim_p, uint labelim_pitch, __global ConnectivityPixelT *connectivity_p, uint connectivityPitch){
    const uint get_local_id(0) = get_local_id(0);
    const uint get_local_id(1) = get_local_id(1);

    const uint x = get_local_id(0) + get_group_id(0) * TILE_COLS;
    const uint y = get_local_id(1) + get_group_id(1) * TILE_ROWS;

    //if (x >= im_rows || y >= im_rows) return;

    //currently x is 1
    //int bounds = ((y + WORKITEM_REPEAT_Y) < im_rows);

    __local LDSLabelT label_tile_im[TILE_ROWS][TILE_COLS];
    __local LDSConnectivityPixelT  edge_tile_im[TILE_ROWS][TILE_COLS];

    LDSLabelT new_labels[WORKITEM_REPEAT_Y][WORKITEM_REPEAT_X];
    LDSLabelT old_labels[WORKITEM_REPEAT_Y][WORKITEM_REPEAT_X];

    #pragma unroll
    for (int i = 0; i < WORKITEM_REPEAT_Y; ++i){
        #pragma unroll
        for (int j = 0; j < WORKITEM_REPEAT_X; ++j){
            const uint yloc = get_local_id(1) + WORKGROUP_TILE_SIZE_Y * i;
            const uint xloc = get_local_id(0) + WORKGROUP_TILE_SIZE_X * j;
            const bool valid_pixel_task = (xloc < im_cols) & (yloc < im_rows);
            ConnectivityPixelT c = valid_pixel_task ? pixel_at(ConnectivityPixelT, connectivity, y + WORKGROUP_TILE_SIZE_Y * i, x + WORKGROUP_TILE_SIZE_X * j) : 0;

            c = xloc == 0 ? c & ~(LEFT|LEFT_DOWN|LEFT_UP) : c;
            c = yloc == 0 ? c & ~(UP|LEFT_UP|RIGHT_UP) : c;

            c = xloc == TILE_COLS - 1 ? c & ~(RIGHT|RIGHT_DOWN|RIGHT_UP);
            c = yloc == TILE_ROWS - 1 ? c & ~(DOWN|LEFT_DOWN|RIGHT_DOWN) : c;

            new_labels[i][j] = yloc * TILE_COLS + xloc;
            edge_tile_im[yloc][xloc] = c;
        }
    }

    for (int k = 0; ;++k){
        //make copies
        #pragma unroll
        for (int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for (int j = 0; j < WORKITEM_REPEAT_X; ++j){
                const uint yloc = get_local_id(1) + WORKGROUP_TILE_SIZE_Y * i;
                const uint xloc = get_local_id(0) + WORKGROUP_TILE_SIZE_X * j;

                old_labels[i][j]          = new_labels[i][j];
                label_tile_im[yloc][xloc] = new_labels[i][j];
            }
        }
        lds_barrier();

        //take minimum label of local neighboorhood - single writer, multi reader version
        #pragma unroll
        for (int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for (int j = 0; j < WORKITEM_REPEAT_X; ++j){
                const uint yloc = get_local_id(1) + WORKGROUP_TILE_SIZE_Y * i;
                const uint xloc = get_local_id(0) + WORKGROUP_TILE_SIZE_X * j;

                const ConnectivityPixelT connectivity = edge_tile_im[yloc][xloc];
                const LDSLabelT label = new_labels[i][j];

#if CONNECTIVITY == 8
                if (connectivity & UP)
                   label = min(label, label_tile_im[yloc - 1][xloc]);
                if (connectivity & LEFT)
                   label = min(label, label_tile_im[yloc][xloc - 1]);
                if (connectivity & LEFT_UP)
                   label = min(label, label_tile_im[yloc - 1][xloc - 1]);
                if (connectivity &  LEFT_DOWN)
                   label = min(label, label_tile_im[yloc + 1][xloc - 1]);
                if (connectivity &  DOWN)
                   label = min(label, label_tile_im[yloc + 1][xloc]);
                if (connectivity & RIGHT_DOWN)
                   label = min(label, label_tile_im[yloc - 1][xloc + 1]);
                if (connectivity & RIGHT)
                   label = min(label, label_tile_im[yloc][xloc + 1]);
                if (connectivity & RIGHT_UP)
                   label = min(label, label_tile_im[yloc + 1][xloc + 1]);

#else
                if (connectivity & UP)
                   label = min(label, label_tile_im[yloc - 1][xloc]);
                if (connectivity &  DOWN)
                   label = min(label, label_tile_im[yloc + 1][xloc]);
                if (connectivity & LEFT)
                   label = min(label, label_tile_im[yloc][xloc - 1]);
                if (connectivity & RIGHT)
                   label = min(label, label_tile_im[yloc][xloc + 1]);
#endif

                new_labels[i][j] = label;
            }
        }
        lds_barrier();

        __local int changed;
        if((get_local_id(1) == 0) & (get_local_id(0) == 0)){
            changed = 0;
        }
        lds_barrier();

        const __local LDSLabelT *labels = &label_tile_im[0][0];

        int pchanged = 0;
        #pragma unroll
        for(int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for(int j = 0; j < WORKITEM_REPEAT_X; ++j){
                if(new_labels[i][j] < old_labels[i][j]){
                    pchanged++;
                    atomic_min(labels + old_labels[i][j], new_labels[i][j]);
                }
            }
        }
        atomic_add(&changed, pchanged);
        lds_barrier();

        //if there are no updates, we are finished
        if(!changed){
            break;
        }

        //Compact paths
        #pragma unroll
        for(int i = 0; i < WORKITEM_REPEAT_Y; ++i){
            #pragma unroll
            for(int j = 0; j < WORKITEM_REPEAT_X; ++j){
                LDSLabelT label = new_labels[i][j];

                //find root label
                while(labels[label] < label){
                    label = labels[label];
                }

                new_labels[i][j] = label;
            }
        }
        lds_barrier();
    }

    //save with adjusted global (untiled) labels
    #pragma unroll
    for(int i = 0; i < WORKITEM_REPEAT_Y; ++i){
        #pragma unroll
        for(int j = 0; j < WORKITEM_REPEAT_X; ++j){
            const LabelT tile_label = new_labels[i][j];
            const uint yloc = (tile_label / TILE_COLS) + get_group_id(0) * TILE_COLS;
            const uint xloc = (tile_label % TILE_COLS) + get_group_id(1) * TILE_ROWS;

            const LabelT glabel = yloc * im_cols + xloc;
            const bool valid_pixel_task = (xloc < im_cols) & (yloc < im_rows);
            if(valid_pixel_task){
                pixel_at(LabelT, labelim, yloc, xloc) = glabel;
            }
        }
    }
}

inline
LabelT find_root_global(const __global *labelim, uint labelim_pitch, LabelT label, const uint im_rows, const uint im_cols){
    for(;;){
        const uint y = label / im_cols;
        const uint x = label % im_cols;
        const LabelT parent = pixel_at(LabelT, labelim, y, x);

        if(label == parent){
            break;
        }

        label = parent;
    }
    return label;
}

__kernel
void compact_paths_global(uint im_rows, uint im_cols, __global LabelT *labelim, uint labelim_pitch){
    const uint x = get_global_id(0);
    const uint y = get_global_id(1);

    if((x < im_cols) & (y < im_rows)){
        pixel_at(LabelT, labelim, y, x) = find_root_global(labelim, pixel_at(LabelT, labelim, y, x));
    }
}

inline
void merge_edge_labels(const uint im_rows, const uint im_cols, __global LabelT *labelim_p, const uint labelim_pitch, const LabelT l1, const LabelT l2, uint *changed){

    const LabelT r1 = find_root_global(comps, l1, im_rows, im_cols);
    const LabelT r2 = find_root_global(comps, l2, im_rows, im_cols);

    if (r1 == r2) return;

    const LabelT mi = min(r1, r2);
    const LabelT ma = max(r1, r2);

    const uint y = ma / im_cols;
    const uint x = ma % im_cols;

    atomic_min(&pixel_at(LabelT, labelim, y, x), mi);
    *changed = true;
}

__kernel
void cross_merge(
    const uint n_vert_tiles, const uint n_horz_tiles,
    uint tile_rows, uint tile_cols,
    const uint im_rows, const uint im_cols,
    const __global ConnectivityPixelT *connectivityim_p, uint connectivityim_pitch,
    const __global LabelT *labelim_p, uint labelim,
    const uint yIncomplete, uint xIncomplete
){
    const uint ngroups_x = get_num_groups(0);
    const uint ngroups_y = get_num_groups(1);

    const uint tid = get_local_linear_id();
    const uint stride = get_local_size(1) * get_local_size(0);

    const uint ybegin = get_group_id(1) * (n_vert_tiles * tile_rows);
    const uint yend   = ybegin + n_vert_tiles * tile_rows;

    if (get_group_id(1) == ngroups_y - 1){
        yend -= yIncomplete * tile_rows;
        yend -= tile_rows;
        tile_rows = (im_rows % tile_rows);

        yend += tile_rows;
    }

    const uint xbegin = get_group_id(0) * n_horz_tiles * tile_cols;
    const uint xend   = xbegin + n_horz_tiles * tile_cols;

    if(get_group_id(0) == ngroups_x - 1){
        if (xIncomplete) yend = ybegin;
        xend -= xIncomplete * tile_cols;
        xend -= tile_cols;
        tile_cols = (im_cols % tile_cols);

        xend += tile_cols;
    }

    if (get_group_id(1) == (ngroups_y - 1) && yIncomplete){
        xend = xbegin;
    }

    const uint tasksV = (n_horz_tiles - 1) * (yend - ybegin);
    const uint tasksH = (n_vert_tiles - 1) * (xend - xbegin);

    const uint total = tasksH + tasksV;

    __local uint changed;
    uint pchanged;
    do{
        pchanged = false;
        for(uint taskIdx = tid; taskIdx < total; taskIdx += stride){
            if(taskIdx < tasksH){
                const uint indexH = taskIdx;

                const uint row = indexH / (xend - xbegin);
                const uint col = indexH - % (xend - xbegin);

                const uint y = ybegin + (row + 1) * tile_rows;
                const uint x = xbegin + col;

                const ConnectivityPixelT e = pixel_at(ConnectivityPixelT, connectivityim, y, x);
                if(e & UP){
                    LabelT lc = pixel_at(LabelT, labelim, y, x);
                    LabelT lu = pixel_at(LabelT, labelim, y - 1, x);
                    merge_edge_labels(im_rows, im_cols, labelim, labelim_pitch, lc, lu, &pchanged);
                }
            }else{
                const uint indexV = taskIdx - tasksH;

                const uint col = indexV / (yend - ybegin);
                const uint row = indexV % (yend - ybegin);

                const uint x = xbegin + (col + 1) * tile_cols;
                const uint y = ybegin + row;

                const ConnectivityPixelT e = pixel_at(ConnectivityPixelT, connectivityim, y, x);
                if(e & LEFT){
                    LabelT lc = pixel_at(LabelT, labelim, y, x);
                    LabelT lu = pixel_at(LabelT, labelim, y, x - 1);
                    merge_edge_labels(im_rows, im_cols, labelim, labelim_pitch, lc, ll, &pchanged);
                }
            }
        }
        atomic_add(&changed, pchanged);
        lds_barrier();
        pchanged = changed;
    }while(pchanged);
}

__kernel
void mark_root_classes(
    uint im_rows, uint im_cols,
    __global PixelT *image_p, uint image_pitch
    __global const LabelT* labelim_p, const labelim_pitch,
    __global uint* is_root_class_image_p, const uint is_root_class_image_pitch,
){
    const uint c = get_global_id(0);
    const uint r = get_global_id(1);
    const bool valid_pixel_task = (c < im_cols) & (r < im_rows);

    const uint linear_index = c + r * im_cols;
    if(valid_pixel_task){
        const PixelT pixel = pixel_at(PixelT, image, r, c);
        const LabelT label = pixel_at(LabelT, labelim, r, c);
        pixel_at(uint, is_root_class_image, r, c) = (pixel != BG_VALUE) & (label == linear_index);
    }
}

__kernel
void relabel_with_scanline_order(
    uint im_rows, uint im_cols,
    __global LabelT* labelim_out_p, const uint labelim_out_pitch,
    __global const PixelT* image_p, const image_pitch,
    __global const LabelT* labelim_p, const labelim_pitch,
    __global uint* is_root_class_image_psum_p, const uint is_root_class_image_psum_pitch
){
    const uint c = get_global_id(0);
    const uint r = get_global_id(1);
    const bool valid_pixel_task = (c < im_cols) & (r < im_rows);
    const uint linear_index = c + r * im_cols;

    if(valid_pixelTask){
        const LabelT pixel = pixel_at(PixelT, image, r, c);
        LabelT final_label = 0;
        if(pixel != BG_VALUE){
            const LabelT label = pixel_at(LabelT, labelim, r, c);
            const uint label_r = label / im_cols;
            const uint label_c = label % im_cols;
            const uint scan_id = pixel_at(uint, is_root_class_image_psum, label_r, label_c);
            final_label = scan_id + 1;
        }
        image_pixel_at(LabelT, labelim, r, c) = final_label;
    }
}