#### Types and constructors ####

# Plain arrays can be treated as images. Other types will have
# metadata associated, make yours a child of one of the following:
abstract AbstractImage{T,N} <: AbstractArray{T,N}         # image with metadata
"""
`AbstractImageDirect` is the supertype of all images where pixel
values are stored directly in an `AbstractArray`.  See also
`AbstractImageIndexed`.
"""
abstract AbstractImageDirect{T,N} <: AbstractImage{T,N}
"""
`AbstractImageIndexed` is the supertype of all "colormap" images,
where pixel values are accessed from a lookup table.  See also
`AbstractImageDirect`.
"""
abstract AbstractImageIndexed{T,N} <: AbstractImage{T,N}

"""
```
Image(data, [properties])
Image(data, prop1=val1, prop2=val2, ...)
```
creates a new "direct" image, one in which the values in `data`
correspond to the pixel values. In contrast with `convert`, `grayim`
and `colorim`, this does not permute the data array or attempt to
guess any of the `properties`. If `data` encodes color information
along one of the dimensions of the array (as opposed to using a
`Color` array, from the `Colors.jl` package), be sure to specify the
`"colordim"` and `"colorspace"` in `properties`.
"""
type Image{T,N,A<:AbstractArray} <: AbstractImageDirect{T,N}
    data::A
    properties::Dict{ASCIIString,Any}
end
Image(data::AbstractArray, props::Dict) = Image{eltype(data),ndims(data),typeof(data)}(data,props)
Image{T,N}(data::AbstractArray{T,N}, props::Dict) = Image{T,N,typeof(data)}(data,props)
Image(data::AbstractArray; kwargs...) = Image(data, kwargs2dict(kwargs))

"""
```
ImageCmap(data, cmap, [properties])
```
creates an indexed (colormap) image.
"""
type ImageCmap{T<:Colorant,N,A<:AbstractArray} <: AbstractImageIndexed{T,N}
    data::A
    cmap::Vector{T}
    properties::Dict{ASCIIString,Any}
end
ImageCmap{_,N}(data::AbstractArray{_,N}, cmap::AbstractVector, props::Dict) = ImageCmap{eltype(cmap),N,typeof(data)}(data, cmap, props)
ImageCmap(data::AbstractArray, cmap::AbstractVector; kwargs...) = ImageCmap(data, cmap, kwargs2dict(kwargs))

# Convenience constructors
"""
```
img = grayim(A)
```
creates a 2d or 3d _spatial_ grayscale Image from an AbstractArray
`A`, assumed to be in "horizontal-major" order (and without permuting
any dimensions). If you are working with 3d grayscale images, usage of
this function is strongly recommended. This can fix errors like any of
the following:

```
ERROR: Wrong number of spatial dimensions for plain Array, use an AbstractImage type
ERROR: Cannot infer colorspace of Array, use an AbstractImage type
ERROR: Cannot infer pixelspacing of Array, use an AbstractImage type
```

The main reason for such errors---and the reason that `grayim` is
recommended---is the Matlab-derived convention that a `m x n x 3` array is to be
interpreted as RGB.  One might then say that an `m x n x k` array, for `k`
different from 3, could be interpreted as grayscale. However, this would lead to
difficult-to-track-down surprises on the day where `k` happened to be 3 for your
grayscale image.

See also: `colorim`, `Image`, `convert(Image, A)`.
"""
grayim(A::AbstractImage) = A
grayim(A::AbstractArray{UInt8,2})  = grayim(reinterpret(UFixed8, A))
grayim(A::AbstractArray{UInt16,2}) = grayim(reinterpret(UFixed16, A))
grayim(A::AbstractArray{UInt8,3})  = grayim(reinterpret(UFixed8, A))
grayim(A::AbstractArray{UInt16,3}) = grayim(reinterpret(UFixed16, A))
grayim{T}(A::AbstractArray{T,2}) = Image(A; colorspace="Gray", spatialorder=["x","y"])
grayim{T}(A::AbstractArray{T,3}) = Image(A; colorspace="Gray", spatialorder=["x","y","z"])

"""
```
img = colorim(A, [colorspace])
```
Creates a 2d color image from an AbstractArray `A`, auto-detecting which of the
first or last dimension encodes the color and choosing between "horizontal-" and
"vertical-major" accordingly. `colorspace` defaults to `"RGB"` but could also be
e.g. `"Lab"` or `"HSV"`.  If the array represents a 4-channel image, the
`colorspace` option is mandatory since there is no way to automatically
distinguish between `"ARGB"` and `"RGBA"`.  If both the first and last
dimensions happen to be of size 3 or 4, it is impossible to guess which one
represents color and thus an error is generated.  Thus, if your code needs to be
robust to arbitrary-sized images, you should use the `Image` constructor
directly.

See also: `grayim`, `Image`, `convert(Image{RGB}, A)`.
"""
colorim(A::AbstractImage) = A
function colorim{T}(A::AbstractArray{T,3})
    if size(A, 1) == 4 || size(A, 3) == 4
        error("The array looks like a 4-channel color image. Please specify the colorspace explicitly (e.g. \"ARGB\" or \"RGBA\".)")
    end

    colorim(A, "RGB")
end

function colorim{T}(A::AbstractArray{T,3}, colorspace)
    if 3 <= size(A, 1) <= 4 && 3 <= size(A, 3) <= 4
        error("Both first and last dimensions are of size 3 or 4; impossible to guess which is for color. Use the Image constructor directly.")
    elseif 3 <= size(A, 1) <= 4  # Image as returned by imread for regular 2D RGB images
        if T<:Fractional
            CT = getcolortype(colorspace, eltype(A))
            Image(reinterpret(CT, A); spatialorder=["x","y"])
        else
            Image(A; colorspace=colorspace, colordim=1, spatialorder=["x","y"])
        end
    elseif 3 <= size(A, 3) <= 4  # "Matlab"-style image, as returned by convert(Array, im).
        Image(A; colorspace=colorspace, colordim=3, spatialorder=["y","x"])
    else
        error("Neither the first nor the last dimension is of size 3. This doesn't look like an RGB image.")
    end
end

colorim(A::AbstractArray{UInt8,3})  = colorim(reinterpret(UFixed8, A))
colorim(A::AbstractArray{UInt16,3}) = colorim(reinterpret(UFixed16, A))
colorim(A::AbstractArray{UInt8,3},  colorspace) = colorim(reinterpret(UFixed8, A), colorspace)
colorim(A::AbstractArray{UInt16,3}, colorspace) = colorim(reinterpret(UFixed16, A), colorspace)

#### Core operations ####

eltype{T}(img::AbstractImage{T}) = T

size(img::AbstractImage) = size(img.data)
size(img::AbstractImage, i::Integer) = size(img.data, i)
size(img::AbstractImage, dimname::AbstractString) = size(img.data, dimindex(img, dimname))

resize!(a::AbstractImage, nl::Integer) = resize!(a.data, nl)

ndims(img::AbstractImage) = ndims(img.data)

linearindexing(img::Image) = linearindexing(img.data)

strides(img::AbstractImage) = strides(img.data)

copy(img::Image) = Image(copy(img.data), dictcopy(img.properties))
copy(img::ImageCmap) = ImageCmap(copy(img.data), copy(img.cmap), dictcopy(img.properties))

function dictcopy(dct)
    newkeys = [copy(key) for key in keys(dct)]
    newvals = [copy(val) for val in values(dct)]
    Dict{ASCIIString,Any}(zip(newkeys,newvals))
end

"""
```
imgnew = copyproperties(img, data)
```
Creates a new image from the data array `data`, copying the properties from
Image `img`.
"""
copyproperties(img::AbstractArray, data::AbstractArray) = data

copyproperties(img::AbstractImageDirect, data::AbstractArray) = Image(data, deepcopy(img.properties))

copyproperties(img::AbstractImageIndexed, data::AbstractArray) = ImageCmap(data, copy(img.cmap), deepcopy(img.properties))

copyproperties(img::AbstractImageDirect, _data::AbstractImageDirect) = copyproperties(img, data(_data))

"""
```
imgnew = shareproperties(img, data)
```
Creates a new image from the data array `data`, *sharing* the properties of
Image `img`. Any modifications made to the properties of one will affect the
other.
"""
shareproperties(img::AbstractArray, data::AbstractArray) = data

shareproperties(img::AbstractImageDirect, data::AbstractArray) = Image(data, img.properties)

shareproperties(img::AbstractImageIndexed, data::AbstractArray) = ImageCmap(data, img.cmap, img.properties)

# similar
similar(img::AbstractImageDirect) = Image(similar(img.data), copy(img.properties))
similar(img::AbstractImageDirect, ::NTuple{0}) = Image(similar(img.data), copy(img.properties))

similar(img::AbstractImageDirect, dims::Dims) = Image(similar(img.data, dims), copy(img.properties))

similar{T}(img::AbstractImageDirect, ::Type{T}) = Image(similar(img.data, T), copy(img.properties))

similar{T}(img::AbstractImageDirect, ::Type{T}, dims::Dims) = Image(similar(img.data, T, dims), copy(img.properties))

similar(img::AbstractImageIndexed) = ImageCmap(similar(img.data), copy(img.cmap), copy(img.properties))
similar(img::AbstractImageIndexed, ::NTuple{0}) = ImageCmap(similar(img.data), copy(img.cmap), copy(img.properties))

similar(img::AbstractImageIndexed, dims::Dims) = ImageCmap(similar(img.data, dims), copy(img.cmap), copy(img.properties))

similar{T}(img::AbstractImageIndexed, ::Type{T}) = ImageCmap(similar(img.data, T), copy(img.cmap), copy(img.properties))

similar{T}(img::AbstractImageIndexed, ::Type{T}, dims::Dims) = ImageCmap(similar(img.data, T, dims), copy(img.cmap), copy(img.properties))

# copy properties
function copy!(imgdest::AbstractImage, imgsrc::AbstractImage, prop1::ASCIIString, props::ASCIIString...)
    imgdest[prop1] = imgsrc[prop1]
    for p in props
        imgdest[p] = imgsrc[p]
    end
    imgdest
end

## reinterpret: Color->T
# Arrays
reinterpret{CV1<:Colorant,CV2<:Colorant}(::Type{CV1}, A::Array{CV2,1}) = _reinterpret_cvarray(CV1, A)
reinterpret{CV1<:Colorant,CV2<:Colorant}(::Type{CV1}, A::Array{CV2})   = _reinterpret_cvarray(CV1, A)
reinterpret{T,CV<:Colorant}(::Type{T}, A::Array{CV,1}) = _reinterpret_cvarray(T, A)
reinterpret{T,CV<:Colorant}(::Type{T}, A::Array{CV})   = _reinterpret_cvarray(T, A)
reinterpret{T,CV<:Colorant}(::Type{T}, A::StridedArray{CV})   = slice(_reinterpret_cvarray(T, A.parent), A.indexes...)
function _reinterpret_cvarray{T,CV<:Colorant}(::Type{T}, A::Array{CV})
    if sizeof(T) == sizeof(CV)
        return reinterpret(T, A, size(A))
    elseif sizeof(T)*n_elts(CV) == sizeof(CV)
        return reinterpret(T, A, tuple(n_elts(CV), size(A)...))
    end
    error("result shape not specified")
end
reinterpret{CV<:Colorant}(A::StridedArray{CV}) = reinterpret(eltype(CV), A)

# Images
reinterpret{CV1<:Colorant,CV2<:Colorant}(::Type{CV1}, img::AbstractImageDirect{CV2}) =
    shareproperties(img, reinterpret(CV1, data(img)))
function reinterpret{CV<:Colorant}(::Type{UInt32}, img::AbstractImageDirect{CV})
    CV <: Union{RGB24, ARGB32} || (CV <: AbstractRGB && sizeof(CV) == 4) || error("Can't convert $CV to UInt32")
    A = reinterpret(UInt32, data(img))
    props = copy(properties(img))
    props["colorspace"] = colorspace(img)
    Image(A, props)
end
function reinterpret{T,CV2<:Colorant}(::Type{T}, img::AbstractImageDirect{CV2})
    A = reinterpret(T, data(img))
    props = copy(properties(img))
    props["colorspace"] = colorspace(img)
    if ndims(A) > ndims(img)
        props["colordim"] = 1
    end
    Image(A, props)
end

## reinterpret: T->Color
# We have to distinguish two forms of call:
#   form 1: reinterpret(RGB, img)
#   form 2: reinterpret(RGB{UFixed8}, img)
# Arrays
reinterpret{T,CV<:Colorant}(::Type{CV}, A::Array{T,1}) = _reinterpret(CV, eltype(CV), A)
reinterpret{T,CV<:Colorant}(::Type{CV}, A::Array{T})   = _reinterpret(CV, eltype(CV), A)
_reinterpret{T,CV<:Colorant}(::Type{CV}, ::Type{Any}, A::Array{T}) =
    _reinterpret_array_cv(CV{T}, A)   # form 1 (turn into a form 2 call by filling in the element type of the array)
_reinterpret{T,CV<:Colorant}(::Type{CV}, TT::DataType, A::Array{T}) =
    _reinterpret_array_cv(CV, A)    # form 2
function _reinterpret_array_cv{T,CV<:Colorant}(::Type{CV}, A::Array{T})
    if sizeof(T) == sizeof(CV)
        return reinterpret(CV, A, size(A))
    elseif sizeof(T)*size(A,1) == sizeof(CV)
        return reinterpret(CV, A, size(A)[2:end])
    end
    error("result shape not specified")
end
# This version is used by the deserializer to convert UInt8 buffers back to their original type. Fixes #287.
_reinterpret_array_cv{CV<:Colorant}(::Type{CV}, A::Vector{UInt8}) =
    reinterpret(CV, A, (div(length(A), sizeof(CV)),))

# Images
function reinterpret{T,CV<:Colorant}(::Type{CV}, img::AbstractImageDirect{T})
    A = reinterpret(CV, data(img))
    props = copy(properties(img))
    haskey(props, "colorspace") && delete!(props, "colorspace")
    haskey(props, "colordim") && delete!(props, "colordim")
    Image(A, props)
end

# T->S
function reinterpret{T,S}(::Type{T}, img::AbstractImageDirect{S})
    if sizeof(S) != sizeof(T)
        error("result shape not specified")
    end
    shareproperties(img, reinterpret(T, data(img)))
end

"""
```
imgraw = raw(img)
```
returns a reference to the array data in raw (machine-native) storage
format. This is particularly useful when Images.jl wraps image data in
a `FixedPointNumbers` type, and raw data access is desired. For
example

```
img = load("someimage.tif")
typeof( data(img) )  # return Array{UFixed{UInt8,8},2}
typeof( raw(img) )   # returns Array{UInt8,2}
```
"""
raw(img::AbstractArray) = _raw(data(img), eltype(eltype(img)))
_raw{T<:UFixed}(A::Array, ::Type{T}) = reinterpret(FixedPointNumbers.rawtype(T), A)
_raw{T}(A::Array, ::Type{T}) = A
_raw{T}(A::AbstractArray, ::Type{T}) = _raw(convert(Array, A), T)

## convert
# Implementations safe for under-specified color types
# ambiguity resolution:
convert{T<:Colorant,n}(::Type{Array{T}}, x::Array{T,n}) = x
convert{T<:Colorant,n}(::Type{Array{T}}, x::BitArray{n}) = convert(Array{ccolor(T,Gray{U8}),n}, x)

convert{T<:Colorant,n,S}(::Type{Array{T}}, x::Array{S,n}) = convert(Array{ccolor(T,S),n}, x)
convert{T<:Colorant,n,S}(::Type{Array{T,n}}, x::Array{S,n}) = copy!(similar(x,ccolor(T,S)), x)

"""
```
img = convert(Image, A)
img = convert(Image{HSV}, img)
```
Create a 2d Image from an array, setting up default properties. The
data array is assumed to be in "vertical-major" order, and an m-by-n-by-3 array
will be assumed to encode color along its third dimension.

Optionally, you can specify the desired colorspace of the returned `img`.

See also: `Image`, `grayim`, `colorim`.
"""
convert{T}(::Type{Image{T}}, img::Image{T}) = img
convert(::Type{Image}, img::Image) = img
convert(::Type{Image}, A::AbstractArray) = Image(A, properties(A))
# Convert an indexed image (cmap) to a direct image
function convert(::Type{Image}, img::ImageCmap)
    data = reshape(img.cmap[vec(img.data)], size(img.data))
    Image(data, copy(properties(img)))
end

convert{C<:Colorant}(::Type{Image{C}}, img::Image{C}) = img
convert{Cdest<:Colorant,Csrc<:Colorant}(::Type{Image{Cdest}}, img::Image{Csrc}) = copyproperties(img, convert(Array{ccolor(Cdest,Csrc)}, data(img)))
convert{Cdest<:Colorant,Csrc<:Colorant}(::Type{Image{Cdest}}, img::AbstractArray{Csrc}) = Image(convert(Array{Cdest}, data(img)), properties(img))

# Convert an Image to an array. We convert the image into the canonical storage order convention for arrays.
# We restrict this to 2d images because for plain arrays this convention exists only for 2d.
# In other cases---or if you don't want the storage order altered---just use data(img)
"""
`A = convert(Array, img)` converts an Image `img` to an Array,
permuting dimensions (if needed) to put it in vertical-major (Matlab)
storage order.

See also `data`.
"""
convert{T<:Real,N}(::Type{Array{T}}, img::AbstractImageDirect{T,N}) = convert(Array{T,N}, img)
convert{T<:Colorant,N}(::Type{Array{T}}, img::AbstractImageDirect{T,N}) = convert(Array{T,N}, img)
convert{T}(::Type{Vector{T}}, img::AbstractImageDirect{T,1}) = convert(Vector{T}, data(img))
function convert{T,N}(::Type{Array{T,N}}, img::AbstractImageDirect{T,N})
    assert2d(img)  # only well-defined in 2d
    p = permutation_canonical(img)
    dat = convert(Array{T}, data(img))
    if issorted(p)
        return dat
    else
        return permutedims(dat, p)
    end
end
convert{T,N}(::Type{Array{T,N}}, img::AbstractArray) = copy!(Array{T}(size(img)), img)

convert{C<:Colorant}(::Type{Image{C}}, img::Image{C}) = img
convert{Cdest<:Colorant,Csrc<:Colorant}(::Type{Image{Cdest}}, img::Image{Csrc}) =
        copyproperties(img, convert(Array{ccolor(Cdest,Csrc)}, data(img)))  # FIXME when Julia issue ?? is fixed
convert{Cdest<:Colorant,Csrc<:Colorant}(::Type{Image{Cdest}}, img::AbstractImageDirect{Csrc}) =
    copyproperties(img, convert(Array{ccolor(Cdest,Csrc)}, data(img)))  # FIXME when Julia issue ?? is fixed
convert{Cdest<:Colorant,Csrc<:Colorant,N}(::Type{Array{Cdest}}, img::AbstractImageDirect{Csrc,N}) =
    convert(Array{ccolor(Cdest,Csrc)}, convert(Array{Csrc,N}, img))

convert{T<:Colorant}(::Type{Array{T}}, x) = copy!(similar(x,ccolor(T,eltype(x))), x)

"""
`imgs = separate(img)` separates the color channels of `img`, for
example returning an `m-by-n-by-3` array from an `m-by-n` array of
`RGB`.
"""
function separate{CV<:Colorant}(img::AbstractImage{CV})
    p = permutation_canonical(img)
    A = _separate(data(img), p)
    so = spatialorder(img)[p]
    props = copy(properties(img))
    props["colorspace"] = colorspace(img)
    props["colordim"] = ndims(A)
    props["spatialorder"] = so
    Image(A, props)
end
function _separate{CV}(A::Array{CV}, p)
    T = eltype(CV)
    if n_elts(CV) > 1
        permutedims(reinterpret(T, A, tuple(n_elts(CV), size(A)...)), [p+1;1])
    else
        permutedims(reinterpret(T, A, size(A)), p)
    end
end
_separate{CV}(A::AbstractArray{CV}, p) = _separate(convert(Array, A), p)
function separate{CV<:Colorant}(A::Array{CV})
    T = eltype(CV)
    if n_elts(CV) > 1
        permutedims(reinterpret(T, A, tuple(n_elts(CV), size(A)...)), [2:ndims(A)+1;1])
    else
        reinterpret(T, A, size(A))
    end
end
separate{CV<:Colorant}(A::AbstractArray{CV}) = separate(convert(Array, A))
separate(A::AbstractArray) = A

# Image{Numbers} -> Image{Colorant} (the opposite of separate)
convert{C<:Colorant,T<:Fractional}(::Type{Image{C}}, img::Union{AbstractArray{T},AbstractImageDirect{T}}) =
    _convert(Image{C}, eltype(C), img)
_convert{C<:Colorant,T<:Fractional}(::Type{Image{C}}, ::Type{Any}, img::Union{AbstractArray{T},AbstractImageDirect{T}}) =
    _convert(Image{C{T}}, img)
_convert{C<:Colorant,T<:Fractional}(::Type{Image{C}}, ::DataType, img::Union{AbstractArray{T},AbstractImageDirect{T}}) =
    _convert(Image{C}, img)
function _convert{C<:Colorant,T<:Fractional}(::Type{Image{C}}, img::Union{AbstractArray{T},AbstractImageDirect{T}})
    cd = colordim(img)
    if cd > 0
        p = [cd; setdiff(1:ndims(img), cd)]
        A = permutedims(data(img), p)
    else
        A = data(img)
    end
    CV = getcolortype(colorspace(img), T)
    ACV = convert(Array{C}, reinterpret(CV, A))
    props = copy(properties(img))
    haskey(props, "colordim") && delete!(props, "colordim")
    haskey(props, "colorspace") && delete!(props, "colorspace")
    Image(ACV, props)
end

# Indexing. In addition to conventional array indexing, support syntax like
#    img["x", 100:400, "t", 32]
# where anything not mentioned by name is taken to include the whole range

typealias RealIndex{T<:Real} Union{T, AbstractArray{T}, Colon}

# setindex!
setindex!(img::AbstractImage, X, i::Real) = setindex!(img.data, X, i)
setindex!(img::AbstractImage, X, I::RealIndex) = setindex!(img.data, X, I)
setindex!(img::AbstractImage, X, I::RealIndex, J::RealIndex) = setindex!(img.data, X, I, J)
setindex!(img::AbstractImage, X, I::RealIndex, J::RealIndex, K::RealIndex) = setindex!(img.data, X, I, J, K)
setindex!(img::AbstractImage, X, I::RealIndex, J::RealIndex,
                   K::RealIndex, L::RealIndex) = setindex!(img.data, X, I, J, K, L)
setindex!(img::AbstractImage, X, I::RealIndex...) = setindex!(img.data, X, I...)
setindex!(img::AbstractImage, X, dimname::AbstractString, ind::RealIndex, nameind...) = setindex!(img.data, X, coords(img, dimname, ind, nameind...)...)

# Adding a new property via setindex!
setindex!(img::AbstractImage, X, propname::AbstractString) = setindex!(img.properties, X, propname)

# Delete a property!
delete!(img::AbstractImage, propname::AbstractString) = delete!(img.properties, propname)

# getindex, sub, and slice return a value or AbstractArray, not an Image
getindex(img::AbstractImage, i::Real) = getindex(img.data, i)
getindex(img::AbstractImage, I::RealIndex) = getindex(img.data, I)
getindex(img::AbstractImage, I::RealIndex, J::RealIndex) = getindex(img.data, I, J)
getindex(img::AbstractImage, I::RealIndex, J::RealIndex, K::RealIndex) = getindex(img.data, I, J, K)
getindex(img::AbstractImage, I::RealIndex, J::RealIndex,
            K::RealIndex, L::RealIndex) = getindex(img.data, I, J, K, L)
getindex(img::AbstractImage, I::RealIndex...) = getindex(img.data, I...)

# getindex(img::AbstractImage, dimname::AbstractString, ind::RealIndex, nameind...) = getindex(img.data, coords(img, dimname, ind, nameind...)...)
getindex(img::AbstractImage, dimname::ASCIIString, ind, nameind...) = getindex(img.data, coords(img, dimname, ind, nameind...)...)

getindex(img::AbstractImage, propname::ASCIIString) = getindex(img.properties, propname)

typealias Indexable{T<:Real} Union{Int, AbstractVector{T}, Colon}  # for ambiguity resolution
sub(img::AbstractImage, I::Indexable...) = sub(img.data, I...)
sub(img::AbstractImage, I::RealIndex...) = sub(img.data, I...)

sub(img::AbstractImage, dimname::ASCIIString, ind::RealIndex, nameind...) = sub(img.data, coords(img, dimname, ind, nameind...)...)

slice(img::AbstractImage, I::Indexable...) = slice(img.data, I...)
slice(img::AbstractImage, I::RealIndex...) = slice(img.data, I...)

slice(img::AbstractImage, dimname::ASCIIString, ind::RealIndex, nameind...) = slice(img.data, coords(img, dimname, ind, nameind...)...)

"""
```
imgnew = getindexim(img, i, j, k,...)
imgnew = getindexim(img, "x", 100:200, "y", 400:600)
```
return a new Image `imgnew`, copying (and where necessary modifying)
the properties of `img`.  This is in contrast with `img[i, j, k...]`,
which returns an `Array`.
"""
function getindexim(img::AbstractImage, I::RealIndex...)
    ret = copyproperties(img, data(img)[I...])
    cd = colordim(img)
    nd = ndims(ret)
    if cd > nd || (cd > 0 && _length(I[cd], img, cd) < size(img, cd))
        ret["colorspace"] = "Unknown"
        if cd > nd
            ret["colordim"] = 0
        end
    end
    td = timedim(img)
    if td > nd
        ret["timedim"] = 0
    end
    sp = spatialproperties(img)
    for p in sp
        val = ret[p]
        if length(val) > nd
            ret[p] = val[1:nd]
        end
    end
    ret
end
_length(indx, A, d) = length(indx)
_length(indx::Colon, A, d) = size(A,d)

getindexim(img::AbstractImage, dimname::ASCIIString, ind::RealIndex, nameind...) = getindexim(img, coords(img, dimname, ind, nameind...)...)

"""
```
imgs = subim(img, i, j, k, ...)
imgs = subim(img, "x", 100:200, "y", 400:600)
```
returns an `Image` with `SubArray` data, with indexing semantics similar to `sub`.
"""
subim(img::AbstractImage, I::RealIndex...) = _subim(img, I)
_subim{TT}(img, I::TT) = shareproperties(img, sub(img.data, I...))  # work around #8504

subim(img::AbstractImage, dimname::ASCIIString, ind::RealIndex, nameind...) = subim(img, coords(img, dimname, ind, nameind...)...)

"""
```
imgs = sliceim(img, i, j, k, ...)
imgs = sliceim(img, "x", 100:200, "y", 400:600)
```
returns an `Image` with `SubArray` data, with indexing semantics similar to `slice`.
"""
sliceim(img::AbstractImage, I::RealIndex...) = _sliceim(img, I)
function _sliceim{IT}(img::AbstractImage, I::IT)
    dimmap = Array(Int, ndims(img))
    n = 0
    for j = 1:ndims(img)
        if !isa(I[j], Real); n += 1; end;
        dimmap[j] = n
    end
    S = slice(img.data, I...)
    ret = copyproperties(img, S)
    cd = colordim(img)
    if cd > 0
        if isa(I[cd], Real)
            ret.properties["colordim"] = 0
            ret.properties["colorspace"] = "Unknown"
        else
            ret.properties["colordim"] = dimmap[cd]
            if I[cd] != 1:size(img, cd)
                ret.properties["colorspace"] = "Unknown"
            end
        end
    end
    td = timedim(img)
    if td > 0
        ret.properties["timedim"] = isa(I[td], Real) ? 0 : dimmap[td]
    end
    sp = spatialproperties(img)
    if !isempty(sp)
        c = coords_spatial(img)
        keep = Bool[map(x -> !isa(x, Real), I[c])...]
        if !all(keep)
            for pname in sp
                p = img.properties[pname]
                if isa(p, Vector)
                    ret.properties[pname] = p[keep]
                elseif isa(p, Matrix)
                    ret.properties[pname] = p[keep, keep]
                else
                    error("Do not know how to handle property ", pname)
                end
            end
        end
    end
    ret
end

sliceim(img::AbstractImage, dimname::AbstractString, ind::RealIndex, nameind...) = sliceim(img, coords(img, dimname, ind, nameind...)...)

sliceim(img::AbstractImage, dimname::AbstractString, ind::RealIndex, nameind...) = sliceim(img, coords(img, dimname, ind, nameind...)...)

# Iteration
# Defer to the array object in case it has special iteration defined
next{T,N}(img::AbstractImage{T,N}, s::Tuple{Bool,Base.IteratorsMD.CartesianIndex{N}}) = next(data(img), s)
done{T,N}(img::AbstractImage{T,N}, s::Tuple{Bool,Base.IteratorsMD.CartesianIndex{N}}) = done(data(img), s)
start(img::AbstractImage) = start(data(img))
next(img::AbstractImage, s) = next(data(img), s)
done(img::AbstractImage, s) = done(data(img), s)

# We'll frequently want to pull out different 2d slices from the same image, so here's a type and set of functions making that easier.
# We deliberately do not require the user to specify the full list of new slicing/ranging parameters, as often we'll want to change some aspects (e.g., z-slice) but not others (e.g., color coordinates)
type SliceData
    slicedims::Tuple{Vararg{Int}}
    slicestrides::Tuple{Vararg{Int}}
    rangedims::Tuple{Vararg{Int}}

    function SliceData(A::AbstractArray, slicedims::Int...)
        keep = trues(ndims(A))
        for i = 1:length(slicedims)
            keep[slicedims[i]] = false
        end
        s = strides(A)
        new(slicedims, ntuple(i->s[slicedims[i]], length(slicedims)), tuple((1:ndims(A))[keep]...))
    end
end

SliceData(img::AbstractImage, dimname::AbstractString, dimnames::AbstractString...) = SliceData(img, dimindexes(img, dimname, dimnames...)...)

function _slice(A::AbstractArray, sd::SliceData, I::Int...)
    if length(I) != length(sd.slicedims)
        throw(BoundsError())
    end
    indexes = RangeIndex[1:size(A, i) for i = 1:ndims(A)]
    for i = 1:length(I)
        indexes[sd.slicedims[i]] = I[i]
    end
    indexes
end

function slice(A::AbstractArray, sd::SliceData, I::Int...)
    indexes = _slice(A, sd, I...)
    slice(A, indexes...)
end

function sliceim(img::AbstractImage, sd::SliceData, I::Int...)
    indexes = _slice(img, sd, I...)
    sliceim(img, indexes...)
end

function first_index(A::SubArray, sd::SliceData)
    newfirst = 1
    for i = 1:length(sd.slicedims)
        newfirst += (A.indexes[sd.slicedims[i]]-1)*sd.slicestrides[i]
    end
    strds = strides(A)
    for i = 1:length(sd.rangedims)
        newfirst += (A.indexes[sd.rangedims[i]][1]-1)*strds[i]
    end
    newfirst
end

function reslice!(A::SubArray, sd::SliceData, I::Int...)
    indexes = RangeIndex[A.indexes...]
    for i = 1:length(I)
        indexes[sd.slicedims[i]] = I[i]
    end
    A.indexes = tuple(indexes...)
    A.first_index = first_index(A, sd)
    A
end

function reslice!(img::AbstractImage, sd::SliceData, I::Int...)
    reslice!(img.data, sd, I...)
    img
end

function rerange!(A::SubArray, sd::SliceData, I::Tuple{Vararg{RangeIndex}})
    indexes = RangeIndex[A.indexes...]
    for i = 1:length(I)
        indexes[sd.rangedims[i]] = I[i]
    end
    A.indexes = tuple(indexes...)
    A.first_index = first_index(A, sd)
    A
end

function rerange!(img::AbstractImage, sd::SliceData, I::Tuple{Vararg{RangeIndex}})
    rerange!(img.data, sd, I...)
    img
end

const emptyset = Set()
function showim(io::IO, img::AbstractImageDirect)
    IT = typeof(img)
    print(io, colorspace(img), " ", IT.name, " with:\n  data: ", summary(img.data), "\n  properties:")
    showdictlines(io, img.properties, get(img, "suppress", emptyset))
end
function showim(io::IO, img::AbstractImageIndexed)
    IT = typeof(img)
    print(io, colorspace(img), " ", IT.name, " with:\n  data: ", summary(img.data), "\n  cmap: ", summary(img.cmap), "\n  properties:")
    showdictlines(io, img.properties, get(img, "suppress", emptyset))
end
show(io::IO, img::AbstractImageDirect) = showim(io, img)
writemime(io::IO, ::MIME"text/plain", img::AbstractImageDirect) = showim(io, img)
show(io::IO, img::AbstractImageIndexed) = showim(io, img)
writemime(io::IO, ::MIME"text/plain", img::AbstractImageIndexed) = showim(io, img)

"""
```
A = data(img)
```
returns a reference `A` to the array data in `img`. It allows you to
use algorithms specialized for particular `AbstractArray` types on
`Image` types. This works for both `AbstractImage`s and
`AbstractArray`s (for the latter it just returns the input), so is a
"safe" component of any algorithm.

For algorithms written to accept arbitrary `AbstractArrays`, this
function is not needed.
"""
data(img::AbstractArray) = img
data(img::AbstractImage) = img.data

minimum(img::AbstractImageDirect) = minimum(img.data)
maximum(img::AbstractImageDirect) = maximum(img.data)
# min/max deliberately not defined for AbstractImageIndexed

function _squeeze(img::AbstractImage, dims)
    imgret = copyproperties(img, squeeze(data(img), dims))
    td = timedim(img)
    if td > 0
        imgret["timedim"] = squeezedims(td, dims)
    end
    cd = colordim(img)
    if cd > 0
        imgret["colordim"] = squeezedims(cd, dims)
    end
    c = coords_spatial(img)
    keep = setdiff(c, dims)
    if length(keep) < length(c)
        sp = spatialproperties(img)
        if !isempty(sp)
            for pname in sp
                p = img.properties[pname]
                if isa(p, Vector)
                    imgret.properties[pname] = p[keep]
                elseif isa(p, Matrix)
                    imgret.properties[pname] = p[keep, keep]
                else
                    error("Do not know how to handle property ", pname)
                end
            end
        end
    end
    imgret
end
squeeze(img::AbstractImage, dims::Integer) = _squeeze(img, dims)
squeeze(img::AbstractImage, dims::Dims) = _squeeze(img, dims)
squeeze(img::AbstractImage, dims) = _squeeze(img, dims)

function squeezedims(val, dims)
    if in(val, dims)
        val = 0
    else
        dec = 0
        for d in dims
            dec += val > d
        end
        val -= dec
    end
    val
end

#### Properties ####

# Generic programming with images uses properties to obtain information. The strategy is to define a particular property name, and then write an accessor function of the same name. The accessor function provides default behavior for plain arrays and when the property is not defined. Alternatively, use get(img, "propname", default) or haskey(img, "propname") to define your own default behavior.

# You can define whatever properties you want. Here is a list of properties used
# in some algorithms:
#   colorspace: "RGB", "ARGB", "Gray", "Binary", "RGB24", "Lab", "HSV", etc.
#   colordim: the array dimension used to store color information, or 0 if there
#     is no dimension corresponding to color
#   timedim: the array dimension used for time (i.e., sequence), or 0 for single images
#   pixelspacing: the spacing between adjacent pixels along spatial dimensions
#   spacedirections: the direction of each array axis in physical space (a vector-of-vectors, one per dimension)
#   spatialorder: a string naming each spatial dimension, in the storage order of
#     the data array. Names can be arbitrary, but the choices "x" and "y" have special
#     meaning (horizontal and vertical, respectively, irrespective of storage order).
#     If supplied, you must have one entry per spatial dimension.

"""
`prop = properties(img)` returns the properties-dictionary for an
`AbstractImage`, or creates one if `img` is an `AbstractArray`.
"""
properties(A::AbstractArray) = Dict(
    "colorspace" => colorspace(A),
    "colordim" => colordim(A),
    "timedim" => timedim(A),
    "pixelspacing" => pixelspacing(A),
    "spatialorder" => spatialorder(A))
properties{C<:Colorant}(A::AbstractArray{C}) = Dict(
    "timedim" => timedim(A),
    "pixelspacing" => pixelspacing(A),
    "spatialorder" => spatialorder(A))
properties(img::AbstractImage) = img.properties

haskey(a::AbstractArray, k::AbstractString) = false
haskey(img::AbstractImage, k::AbstractString) = haskey(img.properties, k)

get(img::AbstractArray, k::AbstractString, default) = default
get(img::AbstractImage, k::AbstractString, default) = get(img.properties, k, default)

# So that defaults don't have to be evaluated unless they are needed, we also define a @get macro (thanks Toivo Hennington):
macro get(img, k, default)
    quote
        img, k = $(esc(img)), $(esc(k))
        local val
        if !isa(img, AbstractImage)
            val = $(esc(default))
        else
            index = Base.ht_keyindex(img.properties, k)
            val = (index > 0) ? img.properties.vals[index] : $(esc(default))
        end
        val
    end
end

# Using plain arrays, we have to make all sorts of guesses about colorspace and storage order. This can be a big problem for three-dimensional images, image sequences, cameras with more than 16-bits, etc. In such cases use an AbstractImage type.

# Here are the two most important assumptions (see also colorspace below):
defaultarraycolordim = 3
# defaults for plain arrays ("vertical-major")
const yx = ["y", "x"]
# order used in Cairo & most image file formats (with color as the very first dimension)
const xy = ["x", "y"]

"""
```
order = spatialorder(img)
order = spatialorder(ImageType)
```

Returns the storage order of the _spatial_ coordinates of the image, e.g.,
`["y", "x"]`. The second version works on a type, e.g., `Matrix`. See
`storageorder`, `timedim`, and `colordim` for related properties.
"""
spatialorder(::Type{Matrix}) = yx
spatialorder(img::AbstractArray) = (sdims(img) == 2) ? spatialorder(Matrix) : error("Wrong number of spatial dimensions for plain Array, use an AbstractImage type")

"""
`isdirect(img)` returns true if `img` encodes its values directly,
rather than via an indexed colormap.
"""
isdirect(img::AbstractArray) = true
isdirect(img::AbstractImageDirect) = true
isdirect(img::AbstractImageIndexed) = false

"""
`cs = colorspace(img)` returns a string specifying the colorspace
representation of the image.
"""
colorspace{C<:Colorant}(img::AbstractVector{C}) = ColorTypes.colorant_string(C)
colorspace{C<:Colorant}(img::AbstractMatrix{C}) = ColorTypes.colorant_string(C)
colorspace{C<:Colorant}(img::AbstractArray{C,3}) = ColorTypes.colorant_string(C)
colorspace{C<:Colorant}(img::AbstractImage{C}) = ColorTypes.colorant_string(C)
colorspace(img::AbstractVector{Bool}) = "Binary"
colorspace(img::AbstractMatrix{Bool}) = "Binary"
colorspace(img::AbstractArray{Bool}) = "Binary"
colorspace(img::AbstractArray{Bool,3}) = "Binary"
colorspace(img::AbstractMatrix{UInt32}) = "RGB24"
colorspace(img::AbstractVector) = "Gray"
colorspace(img::AbstractMatrix) = "Gray"
colorspace{T}(img::AbstractArray{T,3}) = (size(img, defaultarraycolordim) == 3) ? "RGB" : error("Cannot infer colorspace of Array, use an AbstractImage type")
colorspace(img::AbstractImage{Bool}) = "Binary"
colorspace{T,N,A<:AbstractArray}(img::ImageCmap{T,N,A}) = string(T.name.name)
colorspace(img::AbstractImageIndexed) = @get img "colorspace" csinfer(eltype(img.cmap))
colorspace{T}(img::AbstractImageIndexed{T,2}) = @get img "colorspace" csinfer(eltype(img.cmap))
csinfer{C<:Colorant}(::Type{C}) = ColorTypes.colorant_string(C)
csinfer(C) = "Unknown"
colorspace(img::AbstractImage) = get(img.properties, "colorspace", "Unknown")

colorspacedict = Dict{ASCIIString,Any}()
for ACV in (Color, AbstractRGB)
    for CV in subtypes(ACV)
        (length(CV.parameters) == 1 && !(CV.abstract)) || continue
        str = string(CV.name.name)
        colorspacedict[str] = CV
    end
end
function getcolortype{T}(str::ASCIIString, ::Type{T})
    if haskey(colorspacedict, str)
        CV = colorspacedict[str]
        return CV{T}
    else
        if endswith(str, "A")
            CV = colorspacedict[str[1:end-1]]
            return coloralpha(CV){T}
        elseif startswith(str, "A")
            CV = colorspacedict[str[2:end]]
            return alphacolor(CV){T}
        else
            error("colorspace $str not recognized")
        end
    end
end

"""
`dim = colordim(img)` returns the dimension used to encode color, or 0
if no dimension of the array is used for color. For example, an
`Array` of size `(m, n, 3)` would result in 3, whereas an `Array` of
`RGB` colorvalues would yield 0.

See also: `ncolorelem`, `timedim`.
"""
colordim{C<:Colorant}(img::AbstractVector{C}) = 0
colordim{C<:Colorant}(img::AbstractMatrix{C}) = 0
colordim{C<:Colorant}(img::AbstractArray{C,3}) = 0
colordim{C<:Colorant}(img::AbstractImage{C}) = 0
colordim(img::AbstractVector) = 0
colordim(img::AbstractMatrix) = 0
colordim{T}(img::AbstractImageDirect{T,3}) = get(img, "colordim", 0)::Int
colordim{T}(img::AbstractArray{T,3}) = (size(img, defaultarraycolordim) == 3) ? 3 : 0
colordim(img::AbstractImageDirect) = get(img, "colordim", 0)
colordim(img::AbstractImageIndexed) = 0

"""
`dim = timedim(img)` returns the dimension used to represent time, or
0 if this is a single image.

See also: `nimages`, `colordim`.
"""
timedim(img) = get(img, "timedim", 0)::Int

oldlimits(img::AbstractArray{Bool}) = 0,1
# limits{T<:Integer}(img::AbstractArray{T}) = typemin(T), typemax(T)  # best not to use Integers...
oldlimits{T<:AbstractFloat}(img::AbstractArray{T}) = zero(T), one(T)
oldlimits(img::AbstractImage{Bool}) = 0,1
oldlimits{T}(img::AbstractImageDirect{T}) = get(img, "limits", (zero(T), one(T)))
oldlimits(img::AbstractImageIndexed) = @get img "limits" (minimum(img.cmap), maximum(img.cmap))

"""
```
ps = pixelspacing(img)
```

Returns a vector `ps` containing the spacing between adjacent pixels along each
dimension. If this property is not available, it will be computed from
`"spacedirections"` if present; otherwise it defaults to `ones(sdims(img))`. If
desired, you can set this property in terms of physical
[units](https://github.com/Keno/SIUnits.jl).

See also: `spacedirections`.
"""
pixelspacing{T}(img::AbstractArray{T,3}) = (size(img, defaultarraycolordim) == 3) ? [1.0,1.0] : error("Cannot infer pixelspacing of Array, use an AbstractImage type")
pixelspacing(img::AbstractMatrix) = [1.0,1.0]
pixelspacing{T}(img::AbstractImage{T}) = @get img "pixelspacing" _pixelspacing(img)
function _pixelspacing(img::AbstractImage)
    if haskey(img, "spacedirections")
        sd = img["spacedirections"]
        return [maximum(abs(sd[i])) for i = 1:length(sd)]
    end
    ones(sdims(img))
end

"""
```
sd = spacedirections(img)
```

Returns a vector-of-vectors `sd`, each `sd[i]`indicating the displacement between adjacent
pixels along spatial axis `i` of the image array, relative to some external
coordinate system ("physical coordinates").  For example, you could indicate
that a photograph was taken with the camera tilted 30-degree relative to
vertical using

```
img["spacedirections"] = [[0.866025,-0.5],[0.5,0.866025]]
```

If not specified, it will be computed from `pixelspacing(img)`, placing the
spacing along the "diagonal".  If desired, you can set this property in terms of
physical [units](https://github.com/loladiro/SIUnits.jl).

See also: `pixelspacing`.
"""
spacedirections(img::AbstractArray) = @get img "spacedirections" _spacedirections(img)
function _spacedirections(img::AbstractArray)
    ps = pixelspacing(img)
    T = eltype(ps)
    nd = length(ps)
    Vector{T}[(tmp = zeros(T, nd); tmp[i] = ps[i]; tmp) for i = 1:nd]
end

"""
```
so = spatialorder(img)
so = spatialorder(ImageType)
```

Returns the storage order of the *spatial* coordinates of the image, e.g.,
`["y", "x"]`. The second version works on a type, e.g., `Matrix`.

See also: `storageorder`, `coords_spatial`, `timedim`, and `colordim`.
"""
spatialorder(img::AbstractImage) = @get img "spatialorder" _spatialorder(img)
_spatialorder(img::AbstractImage) = (sdims(img) == 2) ? spatialorder(Matrix) : error("Cannot guess default spatial order for ", sdims(img), "-dimensional images")

"""
```
so = storageorder(img)
```

Returns the complete storage order of the image array, including `"t"` for time
and `"color"` for color.

See also: `spatialorder`, `colordim`, `timedim`.
"""
function storageorder(img::AbstractArray)
    so = Array(ASCIIString, ndims(img))
    so[coords_spatial(img)] = spatialorder(img)
    cd = colordim(img)
    if cd != 0
        so[cd] = "color"
    end
    td = timedim(img)
    if td != 0
        so[td] = "t"
    end
    so
end

# number of spatial dimensions in the image
"""
`n = sdims(img)` is similar to `ndims`, but it returns just the number of *spatial* dimensions in
the image array (excluding color and time).
"""
sdims(img) = ndims(img) - (colordim(img) != 0) - (timedim(img) != 0)

# number of time slices
"""
`n = nimages(img)` returns the number of time-points in the image
array. This is safer than `size(img, "t")` because it also works for
plain `AbstractArray` types.
"""
function nimages(img)
    sd = timedim(img)
    if sd > 0
        return size(img, sd)
    else
        return 1
    end
end

"""
`n = ncolorelem(img)` returns the number of color elements/voxel, or 1 if color is not a separate dimension of the array.
"""
function ncolorelem(img)
    cd = colordim(img)
    return cd > 0 ? size(img, cd) : 1
end

"""
`c = coords_spatial(img)` returns a vector listing the spatial
dimensions of the image. For example, an `Array` of size `(m,n,3)`
would return `[1,2]`.

See also: `spatialorder`.
"""
function coords_spatial(img)
    nd = ndims(img)
    cd = colordim(img)
    td = timedim(img)
    if cd > nd || td > nd
        error("Properties are inconsistent with the array dimensionality")
    end
    ind = [1:nd;]
    if cd > td
        splice!(ind, cd)
        if td > 0
            splice!(ind, td)
        end
    elseif td > cd
        splice!(ind, td)
        if cd > 0
            splice!(ind, cd)
        end
    end
    ind
end

# size of the spatial grid
"""
```
ssz = size_spatial(img)
```

Returns a tuple listing the sizes of the spatial dimensions of the image. For
example, an `Array` of size `(m,n,3)` would return `(m,n)`.

See also: `nimages`, `width`, `height`, `widthheight`.
"""
function size_spatial(img)
    sz = size(img)
    sz[coords_spatial(img)]
end

#### Utilities for writing "simple algorithms" safely ####
# If you don't feel like supporting multiple representations, call these

"""
`assert2d(img)` triggers an error if the image has more than two spatial
dimensions or has a time dimension.
"""
function assert2d(img::AbstractArray)
    if sdims(img) != 2
        error("Only two-dimensional images are supported")
    end
    if timedim(img) != 0
        error("Image sequences are not supported")
    end
end

"""
`assert_scalar_color(img)` triggers an error if the image uses an
array dimension to encode color.
"""
function assert_scalar_color(img::AbstractArray)
    if colordim(img) != 0
        error("Only 'scalar color' is supported")
    end
end

"""
`assert_timedim_last(img)` triggers an error if the image has a time
dimension that is not the last dimension.
"""
function assert_timedim_last(img::AbstractArray)
    if 0 < timedim(img) < ndims(img)
        error("Time dimension is not last")
    end
end

"""
`tf = isyfirst(img)` tests whether the first spatial dimension is `"y"`.

See also: `isxfirst`, `assert_yfirst`.
"""
isyfirst(img::AbstractArray) = spatialorder(img)[1] == "y"
"""
`assert_yfirst(img)` triggers an error if the first spatial dimension
is not `"y"`.
"""
function assert_yfirst(img)
    if !isyfirst(img)
        error("Image must have y as its first dimension")
    end
end

"""
`tf = isxfirst(img)` tests whether the first spatial dimension is `"x"`.

See also: `isyfirst`, `assert_xfirst`.
"""
isxfirst(img::AbstractArray) = spatialorder(img)[1] == "x"
"""
`assert_xfirst(img)` triggers an error if the first spatial dimension
is not `"x"`.
"""
function assert_xfirst(img::AbstractArray)
    if !isxfirst(img)
        error("Image must have x as its first dimension")
    end
end

#### Permutations over dimensions ####

# width and height, translating "x" and "y" spatialorder into horizontal and vertical, respectively

"""
`w, h = widthheight(img)` returns the width and height of an image, regardless of storage order.

See also: `width`, `height`.
"""
function widthheight(img::AbstractArray, p)
    c = coords_spatial(img)
    size(img, c[p[1]]), size(img, c[p[2]])
end
widthheight(img::AbstractArray) = widthheight(img, spatialpermutation(xy, img))

"""
`w = width(img)` returns the horizontal size of the image, regardless
of storage order. By default horizontal corresponds to dimension
`"x"`, but see `spatialpermutation` for other options.
"""
width(img::AbstractArray) = widthheight(img)[1]
"""
`h = height(img)` returns the vertical size of the image, regardless
of storage order. By default horizontal corresponds to dimension
`"y"`, but see `spatialpermutation` for other options.
"""
height(img::AbstractArray) = widthheight(img)[2]

"""
```
p = spatialpermutation(to, img)
```

Calculates the *spatial* permutation needed to convert the spatial dimensions to
a given order. This is probably easiest to understand by examples: for an
`Array` `A` of size `(m,n,3)`, `spatialorder(A)` would yield `["y", "x"]`, so
`spatialpermutation(["y", "x"], A) = [1,2]` and `spatialpermutation(["x", "y"],
A) = [2,1]`.  For an image type, here's a demonstration:

```
julia> Aimg = convert(Image, A)
RGB Image with:
  data: 4x5x3 Array{Float64,3}
  properties:
    colordim: 3
    spatialorder:  y x
    colorspace: RGB

julia> Ap = permutedims(Aimg, [3, 1, 2])
RGB Image with:
  data: 3x4x5 Array{Float64,3}
  properties:
    colordim: 1
    spatialorder:  y x
    colorspace: RGB

julia> spatialpermutation(["x","y"], Ap)
2-element Array{Int64,1}:
 2
 1
```
"""
spatialpermutation(to, img::AbstractArray) = default_permutation(to, spatialorder(img))
function spatialpermutation(to, img::AbstractImage)
    so = spatialorder(img)
    if so != nothing
        return default_permutation(to, so)
    else
        if sdims(img) != 2
            error("Cannot guess default spatialorder when there are more than 2 spatial dimensions")
        end
        return default_permutation(to, yx)
    end
end

# Permute the dimensions of an image, also permuting the relevant properties. If you have non-default properties that are vectors or matrices relative to spatial dimensions, include their names in the list of spatialprops.
permutedims(img::AbstractImage, p::Tuple{}, spatialprops::Vector = spatialproperties(img)) = img

function permutedims(img::AbstractImage, p::Union{Vector{Int}, Tuple{Vararg{Int}}}, spatialprops::Vector = spatialproperties(img))
    if length(p) != ndims(img)
        error("The permutation must have length equal to the number of dimensions")
    end
    if issorted(p) && length(p) == ndims(img)
        return img   # should we return a copy?
    end
    ip = invperm(to_vector(p))
    cd = colordim(img)
    sd = timedim(img)
    ret = copyproperties(img, permutedims(img.data, p))
    if cd > 0
        ret.properties["colordim"] = ip[cd]
        p = setdiff(p, cd)
    end
    if sd > 0
        ret.properties["timedim"] = ip[sd]
        p = setdiff(p, sd)
    end
    if !isempty(spatialprops)
        ip = sortperm(p)
        for prop in spatialprops
            a = img.properties[prop]
            if isa(a, AbstractVector)
                ret.properties[prop] = a[ip]
            elseif isa(a, AbstractMatrix) && size(a,1) == size(a,2)
                ret.properties[prop] = a[ip,ip]
            else
                error("Do not know how to handle property ", prop)
            end
        end
    end
    ret
end

permutedims{S<:AbstractString}(img::AbstractImage, pstr::Union{Vector{S}, Tuple{Vararg{S}}}, spatialprops::Vector = spatialproperties(img)) = permutedims(img, dimindexes(img, pstr...), spatialprops)

permutedims(A::AbstractArray, p) = permutedims(convert(Array, A), p)

function permutation_canonical(img)
    assert2d(img)
    p = spatialpermutation(spatialorder(Matrix), img)
    p = coords_spatial(img)[p]
    cd = colordim(img)
    if cd > 0
        push!(p, cd)
    end
    p
end

# Define the transpose of a 2d image
function ctranspose(img::AbstractImage)
    assert2d(img)
    s = coords_spatial(img)
    p = collect(1:ndims(img))
    p[s] = s[2:-1:1]
    permutedims(img, p)
end

# Default list of spatial properties possessed by an image
"""
```
sp = spatialproperties(img)
```

Returns all properties whose values are of the form of an array or tuple, with
one entry per spatial dimension. If you have a custom type with additional
spatial properties, you can set `img["spatialproperties"] = ["property1",
"property2", ...]`. An advantage is that functions that change spatial
dimensions, like `permutedims` and `slice`, will also adjust the properties. The
default is `["spatialorder", "pixelspacing"]`; however, if you override the
setting then these are not included automatically (you'll want to do so
manually, if applicable).
"""
function spatialproperties(img::AbstractImage)
    if haskey(img, "spatialproperties")
        return img.properties["spatialproperties"]
    end
    spatialprops = ASCIIString[]
    if haskey(img, "spatialorder")
        push!(spatialprops, "spatialorder")
    end
    if haskey(img, "pixelspacing")
        push!(spatialprops, "pixelspacing")
    end
    if haskey(img, "spacedirections")
        push!(spatialprops, "spacedirections")
    end
    spatialprops
end
spatialproperties(img::AbstractVector) = ASCIIString[]  # these are not mutable

#### Low-level utilities ####
function permutation(to, from)
    n = length(to)
    nf = length(from)
    d = Dict([(from[i], i) for i = 1:length(from)])
    ind = Array(Int, max(n, nf))
    for i = 1:n
        ind[i] = get(d, to[i], 0)
    end
    ind[n+1:nf] = n+1:nf
    ind
end

function default_permutation(to, from)
    p = permutation(to, from)
    pzero = p .== 0
    if any(pzero)
        p[pzero] = setdiff(1:length(to), p)
    end
    p
end

function showdictlines(io::IO, dict::Dict, suppress::Set)
    for (k, v) in dict
        if k == "suppress"
            continue
        end
        if !in(k, suppress)
            print(io, "\n    ", k, ": ")
            printdictval(io, v)
        else
            print(io, "\n    ", k, ": <suppressed>")
        end
    end
end

printdictval(io::IO, v) = print(io, v)
function printdictval(io::IO, v::Vector)
    for i = 1:length(v)
        print(io, " ", v[i])
    end
end

# Support indexing via
#    img["t", 32, "x", 100:400]
# where anything not mentioned by name is assumed to include the whole range
function coords(img::AbstractImage, dimname::ASCIIString, ind, nameind...)
    c = Any[1:d for d in size(img)]
    so = spatialorder(img)
    c[require_dimindex(img, dimname, so)] = ind
    for i = 1:2:length(nameind)
        c[require_dimindex(img, nameind[i], so)] = nameind[i+1]
    end
    tuple(c...)
end

# Use keyword arguments
# e.g. coord(x=1:100, y=1:50)
function coords(img::AbstractImage; kwargs...)
    c = Any[1:d for d in size(img)]
    so = spatialorder(img)
    for (k, v) in kwargs
        c[require_dimindex(img, string(k), so)] = v
    end
    tuple(c...)
end

function dimindex(img::AbstractImage, dimname::ASCIIString, so = spatialorder(img))
    n::Int = 0
    if dimname == "color"
        n = colordim(img)
    elseif dimname == "t"
        n = timedim(img)
    else
        cd = colordim(img)
        td = timedim(img)
        j = 1
        tn = 1
        while j <= length(so)
            while tn == cd || tn == td
                tn += 1
            end
            if dimname == so[j]
                n = tn
                break
            end
            tn += 1
            j += 1
        end
    end
    n
end

require_dimindex(img::AbstractImage, dimname, so) = (di = dimindex(img, dimname, so); di > 0 || error("No dimension called ", dimname); di)

dimindexes(img::AbstractImage, dimnames::AbstractString...) = Int[dimindex(img, nam, spatialorder(img)) for nam in dimnames]

to_vector(v::AbstractVector) = v
to_vector(v::Tuple) = [v...]

# converts keyword argument to a dictionary
function kwargs2dict(kwargs)
    d = Dict{ASCIIString,Any}()
    for (k, v) in kwargs
        d[string(k)] = v
    end
    return d
end

n_elts{C<:Colorant}(::Type{C}) = div(sizeof(C), sizeof(eltype(C)))


include("nnet.jl")

nnet = NNet("../nnet/TestNetwork.nnet")
println("Num Inputs: "*string(num_inputs(nnet)))
println("Num Outputs: "*string(num_outputs(nnet)))
println("One evaluation:")
println(evaluate_network(nnet,[15299.0,-1.142,-1.142,600.0,500.0]))
println("\nMultiple evaluations at once:")
println(evaluate_network_multiple(nnet,[[15299.0, -1.142, -1.142, 600.0, 500.0] [15299.0,-1.142,-1.142,600.0,500.0]]))


"""
    function get_reaction_idx(rn::Catalyst.ReactionSystem)

Get the rearranged order for a reaction system, returns a matrix `m` with first column as the rearranged reactions, second column is the order in the orign `ReactionSystem`
If you know the old idx for a certain reaction in `ReactionSystem`, you can use
`m[:,2][idx]` to get the new idx.
"""
function get_reaction_idx(rn::Catalyst.ReactionSystem)
    massactionjump_order = Int64[]
    constantjump_order = Int64[]
    rxvars = []
    for (i,rx) in enumerate(ModelingToolkit.get_eqs(rn))
        empty!(rxvars)
        (rx.rate isa ModelingToolkit.Symbolic) && ModelingToolkit.get_variables!(rxvars, rx.rate)
        @inbounds for j = 1:length(rxvars)
            if isequal(rxvars[j], ModelingToolkit.get_iv(rn))
                error("Does not support VariableRateJump")
            end
        end
        if Catalyst.ismassaction(rx, rn)
            push!(massactionjump_order, i)
        else
            push!(constantjump_order, i)
        end
    end
    order = vcat(massactionjump_order, constantjump_order)
    # println("The rearranged order :")
    vcat([[rn.eqs[order[i]] order[i]] for i in eachindex(order)]...)
end

export get_reaction_idx


__precompile__(true)
module JuML
export importcsv,
       Seq,
       DataFrame,
       Factor,
       Covariate,
       BoolVariate,
       DateTimeVariate,
       factor,
       covariate,
       getname,
       getlevels,
       isordinal,
       xgblogit,
       cvxgblogit,
       predict,
       getstats,
       getgroupstats,
       getauc,
       getlogloss,
       cache,
       CovariateStats,
       FactorStats,
       LevelStats,
       tryread,
       slice,
       nextslice,
       map,
       XGBClassifier,
       fit

include("const.jl")
include("DataStructures/seq.jl")
include("DataStructures/list.jl")
include("DataStructures/tree.jl")
include("util.jl")
include("DataImport/dataimport.jl")
include("DataFrame/dataframe.jl")
include("DataFrame/groupstats.jl")

include("Covariates/groupstatscovariate.jl")

include("Factors/factor.jl")
include("Factors/constfactor.jl")
include("Factors/filefactor.jl")
include("Factors/ordinalfactor.jl")
include("Factors/cachedfactor.jl")
include("Factors/filecachedfactor.jl")
include("Factors/bincovfactor.jl")
include("Factors/boolvarfactor.jl")
include("Factors/maplevelfactor.jl")
include("Factors/permutefactor.jl")
include("Factors/widerfactor.jl")
include("Factors/bindatetimefactor.jl")

include("IntVariates/intvariate.jl")
include("IntVariates/fileintvariate.jl")

include("Covariates/covariate.jl")
include("Covariates/constcovariate.jl")
include("Covariates/filecovariate.jl")
include("Covariates/cachedcovariate.jl")
include("Covariates/filecachedcovariate.jl")
include("Covariates/transcovariate.jl")
include("Covariates/trans2covariate.jl")
include("Covariates/parsefactorcovariate.jl")
include("Covariates/ifelsecovariate.jl")
include("Covariates/unixdatecovariate.jl")
include("Covariates/transdatetimecovariate.jl")
include("Covariates/randcovariate.jl")

include("BoolVariates/boolvariate.jl")
include("BoolVariates/cachedboolvariate.jl")
include("BoolVariates/filecachedboolvariate.jl")
include("BoolVariates/transboolvariate.jl")
include("BoolVariates/trans2boolvariate.jl")
include("BoolVariates/transcovboolvariate.jl")
include("BoolVariates/trans2covboolvariate.jl")
include("BoolVariates/transdatetimeboolvariate.jl")
include("BoolVariates/trans2datetimeboolvariate.jl")
include("BoolVariates/groupstatsboolvariate.jl")

include("DateTimeVariates/datetimevariate.jl")
include("DateTimeVariates/filedatetimevariate.jl")
include("DateTimeVariates/transdatetimevariate.jl")
include("DateTimeVariates/trans2datetimevariate.jl")

include("XGBoost/tree.jl")
include("XGBoost/split.jl")
include("XGBoost/logistic.jl")

include("MLJ/MLJ_XGBoost.jl")

end


"""
    abstract type ParticleType

Type of a particle that deposits energy in a [`Semiconductor`](@ref).
Currently defined are `Alpha`, `Beta` and `Gamma`.

`ParticleType` is used to determine the radius of an [`NBodyChargeCloud`](@ref),
where the default radius for `Alpha` is 0.1mm and the default radius for `Beta`
and `Gamma` is 0.5mm.
"""
abstract type ParticleType end
abstract type Alpha <: ParticleType end
abstract type Beta <: ParticleType end
abstract type Gamma <: ParticleType end

radius_guess(charge::T, ::Type{Alpha}) where {T} = T(0.0001)
radius_guess(charge::T, ::Type{Beta}) where {T} = T(0.0005)
radius_guess(charge::T, ::Type{Gamma}) where {T} = T(0.0005)


include("iPredict.jl/src/iPredict.jl")
using .iPredict
using DataFrames
using Plots
using DelimitedFiles
using Plots.Measures

function make_hist(path, titleArg)
    Arr = readdlm(path)

    if titleArg == "3 Isoforms" && path == "figures/figure5_data/Weibull_H1_24_scnorm_counts.txt_3_isoforms_array.txt"
        #Untransformed plot
        p1 = histogram(Arr,
        title = titleArg,
        xlims = (0, 4),
        ylims = (0, 5),
        ylabel = "Probability Density",
        normalize=:pdf,
        bins = 100,
        axis = (font(150)), titlefontsize=150, yticks = 0:5:5,
        margin = 35mm,
        left_margin = 50mm,
        right_margin = 50mm)

    elseif titleArg == "Real"
        p1 = histogram(Arr,
        title = titleArg,
        xlims = (0, 4),
        ylims = (0, 5),
        normalize=:pdf,
        xlabel = "Mean No. Isoforms\nper Gene per Cell",
        bins = 100,
        axis = (font(150)), titlefontsize=150, yticks = 0:5:5,
        margin = 35mm,
        left_margin = 50mm,
        right_margin = 50mm)

    else
        #Untransformed plot
        p1 = histogram(Arr,
        title = titleArg,
        xlims = (0, 4),
        ylims = (0, 5),
        normalize=:pdf,
        bins = 100,
        axis = (font(150)), titlefontsize=150, yticks = 0:5:5,
        margin = 35mm,left_margin = 50mm,
        right_margin = 50mm)
    end
    #vline!([mean(Arr)], color = :black, linewidth = 8)
    return p1
end

#for countsMatrix in scnorm_counts
# "Weibull", "Random", "UniformObserved", "CellVariable"
Weibull_1 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_1_isoforms_array.txt", "1 Isoform")
Weibull_2 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_2_isoforms_array.txt", "2 Isoforms")
Weibull_3 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_3_isoforms_array.txt", "3 Isoforms")
Weibull_4 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_4_isoforms_array.txt", "4 Isoforms")
Weibull_real = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_real_isoforms_array.txt", "Real")

Random_1 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_1_isoforms_array.txt", "1 Isoform")
Random_2 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_2_isoforms_array.txt", "2 Isoforms")
Random_3 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_3_isoforms_array.txt", "3 Isoforms")
Random_4 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_4_isoforms_array.txt", "4 Isoforms")
Random_real = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_real_isoforms_array.txt", "Real")

UniformObserved_1 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_1_isoforms_array.txt", "1 Isoform")
UniformObserved_2 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_2_isoforms_array.txt", "2 Isoforms")
UniformObserved_3 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_3_isoforms_array.txt", "3 Isoforms")
UniformObserved_4 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_4_isoforms_array.txt", "4 Isoforms")
UniformObserved_real = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_real_isoforms_array.txt", "Real")

CellVariable_1 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_1_isoforms_array.txt", "1 Isoform")
CellVariable_2 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_2_isoforms_array.txt", "2 Isoforms")
CellVariable_3 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_3_isoforms_array.txt", "3 Isoforms")
CellVariable_4 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_4_isoforms_array.txt", "4 Isoforms")
CellVariable_real = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_real_isoforms_array.txt", "Real")

p3 = plot( Weibull_1, Random_1, UniformObserved_1, CellVariable_1,
       Weibull_2, Random_2,UniformObserved_2, CellVariable_2,
       Weibull_3, Random_3,UniformObserved_3, CellVariable_3,
       Weibull_4, Random_4,UniformObserved_4, CellVariable_4,
       Weibull_real, Random_real,UniformObserved_real, CellVariable_real,
       layout = (5,4), legend = false, size = (750 *8, 750*8), axis = (font(90)), titlefontsize=90,
       margin = 45mm)

savefig(p3, "figures/supplementary_figs/figure3.png")

###############################################################
using JLD2
using DelimitedFiles
using HypothesisTests
using DataFrames

Weibull_1 = readdlm("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_1_isoforms_array.txt")
Weibull_2 = readdlm("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_2_isoforms_array.txt")
Weibull_3 = readdlm("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_3_isoforms_array.txt")
Weibull_4 = readdlm("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_4_isoforms_array.txt")
Weibull_real = readdlm("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_real_isoforms_array.txt")

Random_1 = readdlm("figures/figure5_data/Random_H9_24_scnorm_counts.txt_1_isoforms_array.txt")
Random_2 = readdlm("figures/figure5_data/Random_H9_24_scnorm_counts.txt_2_isoforms_array.txt")
Random_3 = readdlm("figures/figure5_data/Random_H9_24_scnorm_counts.txt_3_isoforms_array.txt")
Random_4 = readdlm("figures/figure5_data/Random_H9_24_scnorm_counts.txt_4_isoforms_array.txt")
Random_real = readdlm("figures/figure5_data/Random_H9_24_scnorm_counts.txt_real_isoforms_array.txt")

UniformObserved_1 = readdlm("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_1_isoforms_array.txt")
UniformObserved_2 = readdlm("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_2_isoforms_array.txt")
UniformObserved_3 = readdlm("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_3_isoforms_array.txt")
UniformObserved_4 = readdlm("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_4_isoforms_array.txt")
UniformObserved_real = readdlm("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_real_isoforms_array.txt")

CellVariable_1 = readdlm("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_1_isoforms_array.txt")
CellVariable_2 = readdlm("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_2_isoforms_array.txt")
CellVariable_3 = readdlm("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_3_isoforms_array.txt")
CellVariable_4 = readdlm("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_4_isoforms_array.txt")
CellVariable_real = readdlm("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_real_isoforms_array.txt")

df = DataFrame(IsoformsSimulated=Int64[], Comparison = String[], pValue = Float64[])

append!(df, DataFrame(IsoformsSimulated = [1], Comparison = ["All"], pValue=[pvalue(KSampleADTest(vec(Weibull_1), vec(Random_1), vec(UniformObserved_1), vec(CellVariable_1)))]))
append!(df, DataFrame(IsoformsSimulated = [2], Comparison = "All", pValue=[pvalue(KSampleADTest(vec(Weibull_2), vec(Random_2), vec(UniformObserved_2), vec(CellVariable_2)))]))
append!(df, DataFrame(IsoformsSimulated = [3], Comparison = "All", pValue=[pvalue(KSampleADTest(vec(Weibull_3), vec(Random_3), vec(UniformObserved_3), vec(CellVariable_3)))]))
append!(df, DataFrame(IsoformsSimulated = [4], Comparison = "All", pValue=[pvalue(KSampleADTest(vec(Weibull_4), vec(Random_4), vec(UniformObserved_4), vec(CellVariable_4)))]))

append!(df, DataFrame(IsoformsSimulated = [1], Comparison = "CellVarUnif", pValue=[pvalue(KSampleADTest(vec(UniformObserved_1), vec(CellVariable_1)))]))
append!(df, DataFrame(IsoformsSimulated = [2], Comparison = "CellVarUnif", pValue=[pvalue(KSampleADTest( vec(UniformObserved_2), vec(CellVariable_2)))]))
append!(df, DataFrame(IsoformsSimulated = [3], Comparison = "CellVarUnif", pValue=[pvalue(KSampleADTest(vec(UniformObserved_3), vec(CellVariable_3)))]))
append!(df, DataFrame(IsoformsSimulated = [4], Comparison = "CellVarUnif", pValue=[pvalue(KSampleADTest(vec(UniformObserved_4), vec(CellVariable_4)))]))

jldopen("figures/figure4_data/Supfig3_AD_test_results.jld2", "w") do file
    file["df"] = df
end

###########################################
# overlap
#############################################

function make_hist(path, titleArg)

     Arr = readdlm(path)
     p1 = histogram(Arr, title = titleArg, normalize=:pdf, bins = 100,
     ylims = (0,4), color = palette(:default)[2],
     xlims = (0,1),
     axis = (font(150)), titlefontsize=150, yticks = 0:2:4,
     xticks = 0:0.5:1,
     margin = 35mm,left_margin = 50mm,
     right_margin = 50mm)
     return p1

end

Weibull_1 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_1_isoforms_overlap_array.txt", "1 Isoform")
Weibull_2 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_2_isoforms_overlap_array.txt", "2 Isoforms")
Weibull_3 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_3_isoforms_overlap_array.txt", "3 Isoforms")
Weibull_4 = make_hist("figures/figure5_data/Weibull_H9_24_scnorm_counts.txt_4_isoforms_overlap_array.txt", "4 Isoforms")

Random_1 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_1_isoforms_overlap_array.txt", "1 Isoform")
Random_2 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_2_isoforms_overlap_array.txt", "2 Isoforms")
Random_3 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_3_isoforms_overlap_array.txt", "3 Isoforms")
Random_4 = make_hist("figures/figure5_data/Random_H9_24_scnorm_counts.txt_4_isoforms_overlap_array.txt", "4 Isoforms")

UniformObserved_1 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_1_isoforms_overlap_array.txt", "1 Isoform")
UniformObserved_2 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_2_isoforms_overlap_array.txt", "2 Isoforms")
UniformObserved_3 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_3_isoforms_overlap_array.txt", "3 Isoforms")
UniformObserved_4 = make_hist("figures/figure5_data/UniformObserved_H9_24_scnorm_counts.txt_4_isoforms_overlap_array.txt", "4 Isoforms")

CellVariable_1 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_1_isoforms_overlap_array.txt", "1 Isoform")
CellVariable_2 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_2_isoforms_overlap_array.txt", "2 Isoforms")
CellVariable_3 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_3_isoforms_overlap_array.txt", "3 Isoforms")
CellVariable_4 = make_hist("figures/figure5_data/CellVariable_H9_24_scnorm_counts.txt_4_isoforms_overlap_array.txt", "4 Isoforms")

p3 = plot( Weibull_1, Random_1, UniformObserved_1, CellVariable_1,
       Weibull_2, Random_2,UniformObserved_2, CellVariable_2,
       Weibull_3, Random_3,UniformObserved_3, CellVariable_3,
       Weibull_4, Random_4,UniformObserved_4, CellVariable_4,
       layout = (4,4), legend = false, size = (750 *8, 750*8), axis = (font(90)), titlefontsize=90,
       margin = 45mm)

savefig(p3, "figures/supplementary_figs/figure3_overlap.png")


export
    Conditional,
    extend_tuple_type

using Folds
using StatsFuns

@inline tuplejoin(x, y, z...) = (x..., tuplejoin(y, z...)...)
tuplejoin(t::Tuple) = t

"""
    extend_tuple_type(T1, T2)

Given two types `T1` and `T2`, concatenate the types into a single
tuple type.

# Arguments
- `T1`: Any type
- `T2`: A tuple type

# Returns
- If `T1` is a tuple type, a tuple with the concatenation of the types in `T1` and `T2`
- If `T1` is not a tuple type, a tuple with `T1` prepended to the types in `T2`

# Examples
```
julia> extend_tuple_type(Int64, Tuple{Float64})
Tuple{Int64, Float64}

julia> extend_tuple_type(Tuple{Int64}, Tuple{Float64})
Tuple{Int64, Float64}

julia> extend_tuple_type(Tuple{Vector{Float64}}, Tuple{Symbol,Symbol})
Tuple{Vector{Float64}, Symbol, Symbol}
```
"""
function extend_tuple_type(T1, T2)
    if T1 <: Tuple
        Tuple{tuplejoin(fieldtypes(T1), fieldtypes(T2))...}
    else
        Tuple{T1, fieldtypes(T2)...}
    end
end

"""
    abstract type Conditional{I <: Tuple, J <: Tuple, K <: Tuple, O, S <: SFunc{J, O}} <: SFunc{K, O}

`Conditional` *sfuncs* represent the generation of an sfunc depending on the values of parents.  An
subtype of `Conditional` must provide a `gensf` method that takes an `I` and returns an
`SFunc{J,O}` (**important** the generated SFunc must not appear outside the Conditional.
It should not be a parent).

# Additional supported operators
- `support`
- `support_quality`
- `sample`
- `logcpdf`
- `make_factors`
- `compute_pi`
- `send_lambda`

# Type parameters
- `I`: the type of data used to generate the `Conditional`'s *sfunc*
- `J`: a tuple that represents the input types (the `I`) of the `Conditional`'s generated *sfunc*
- `K`: the input types of the `Conditional`; this is a tuple of types constructed from `I`,
and `J` using `extend_tuple_types`
- `O`: the output type(s) of both the `Conditional` and the `Conditional`'s generated *sfunc*
- `S`: the type of the `Conditional`'s generated *sfunc*

"""
abstract type Conditional{I, J <: Tuple, K <: Tuple, O, S <: SFunc{J, O}} <: SFunc{K, O}
end

function split_pars(::Conditional{I}, pars) where {I}
    if I <: Tuple
        n1 = length(fieldnames(I))
        (Tuple(pars[1:n1]), Tuple(pars[n1+1:length(pars)]))
    else
        (pars[1], Tuple(pars[2:length(pars)]))
    end
end

@impl begin
    struct ConditionalSupport end

    function support(sf::Conditional{I,J,K,O},
                    parranges::NTuple{N,Vector},
                    size::Integer,
                    curr::Vector{<:O}) where {I,J,K,O,N}
        (iranges, jranges) = split_pars(sf, parranges)
        irgs::Array{Array{Any, 1}, 1} = [r for r in iranges]
        isrange = cartesian_product(irgs)
        rng::Vector = isempty(curr) ? Vector{O}() : copy(curr)
        # inc = Int(ceil(size / length(isrange)))
        allcombranges= Dict{Any, Vector{O}}()

        # create complete range regardless of the size
        for is in isrange
            gsf = gensf(sf,tuple(is...))
            newrng = copy(support(gsf, jranges, size, curr))
            allcombranges[is] = unique(newrng)
        end

        # create range by mixing values from ranges created by each combination of parents values up to size
        allrng = collect(Iterators.flatten(values(allcombranges)))
        max_size = length(unique(allrng))
        while length(rng) < min(size, max_size)
            for is in isrange
                if(length(allcombranges[is]) > 0)
                    val = popfirst!(allcombranges[is])
                    push!(rng, val)
                end
            end
            rng = unique(rng)
        end
        return isempty(rng) ? O[] : rng
    end
end

@impl begin
    struct ConditionalSample end

    function sample(sf::Conditional{I,J,K,O}, i::K)::O where {I,J,K,O}
        (ivals, jvals) = split_pars(sf, i)
        sfg = gensf(sf,ivals)
        return sample(sfg, jvals)
    end
end

@impl begin
    struct ConditionalLogcpdf end

    function logcpdf(sf::Conditional{I,J,K,O}, i::K, o::O)::AbstractFloat where {I,J,K,O}
        (ivals, jvals) = split_pars(sf, i)
        sfgen = gensf(sf,ivals)
        return logcpdf(sfgen, jvals, o)
    end
end

@impl begin
    struct ConditionalSupportQuality end

    function support_quality(sf::Conditional{I,J,K,O,S}, parranges) where {I,J,K,O,S}
        q = support_quality_rank(:CompleteSupport)
        (iranges, jranges) = split_pars(sf, parranges)
        isrange = cartesian_product(iranges)
        for is in isrange
            gsf = gensf(sf,tuple(is...))
            imp = get_imp(MultiInterface.get_policy(), Support, gsf, jranges, 0, O[])
            q = min(q, support_quality_rank(support_quality(imp, gsf, jranges)))
        end
        return support_quality_from_rank(q)
    end
end

@impl begin
    mutable struct ConditionalMakeFactors
        numpartitions::Int64 = 1
    end

    function make_factors(sf::Conditional{I,J,K,O},
                        range::__OptVec{<:O},
                        parranges::NTuple{N,Vector},
                        id,
                        parids::Tuple)::Tuple{Vector{<:Scruff.Utils.Factor}, Vector{<:Scruff.Utils.Factor}} where {I,J,K,O,N}

        if any(isempty, parranges) # This is possible in lazy inference
            # Just return an empty factor so as not to break
            keys = (id,)
            dims = (0,)
            entries = Float64[]
            fs = [Factor(dims, keys, entries)]
            return (fs, fs)
        end

        # Here is a very general method for constructing factors for conditional sfuncs.
        # We introduce a switch variable.
        # For each of the generated sfuncs, we make its factors and extend them by saying
        # they are only relevant when the switch value corresponds to that choice.
        # We also add a factor saying that the switch variable takes on the appropriate value
        # for each value of the i inputs.
        # For this switch factor, we use another decomposition, relating the switch variable
        # to each of the i inputs separately. This avoids a quadratic blowup and brings the
        # complexity analogous to that of less general methods.
        # This can be improved by making a special case where there is only one i parent,
        # and returning a single factor.
        (iranges, jranges) = split_pars(sf, parranges)
        (iids, jids) = split_pars(sf, parids)
        icombos = cartesian_product(iranges)
        lfs = Factor[]
        ufs = Factor[]
        switchkey = nextkey()
        switchsize = length(icombos)
        for (switchval, ivals) in enumerate(icombos)
            function extend(factor)
                keys = [k for k in factor.keys]
                push!(keys, switchkey)
                keys = Tuple(keys)
                dims = [d for d in factor.dims]
                push!(dims, switchsize)
                dims = Tuple(dims)
                entries = Float64[]
                for e in factor.entries
                    for k in 1:switchsize
                        # 1.0 is the irrelevant value when factors are multiplied
                        push!(entries, k == switchval ? e : 1.0)
                    end
                end
                result = Factor(dims, keys, entries)
                return result
            end
            subsf = gensf(sf,tuple(ivals...))
            (sublfs, subufs) = make_factors(subsf, range, jranges, id, jids)
            append!(lfs, [extend(f) for f in sublfs])
            append!(ufs, [extend(f) for f in subufs])
        end

        for i in 1:length(iranges)
            switchfactorkeys = (parids[i], switchkey)
            parsize = length(parranges[i])
            switchfactordims = (parsize, switchsize)
            switchfactorentries = Float64[]
            for j in 1:parsize
                for ivals in icombos
                    # The effect of this is that the product will only be 1 for all the corresponding
                    # i values.
                    push!(switchfactorentries, j == indexin([ivals[i]], parranges[i])[1] ? 1.0 : 0.0)
                end
            end
            switchfact = Factor(switchfactordims, switchfactorkeys, switchfactorentries)
            push!(lfs, switchfact)
            push!(ufs, switchfact)
        end

        return (lfs, ufs)
    end
end

@impl begin
    struct ConditionalComputePi end

    function compute_pi(sf::Conditional{I,J,K,O},
                     range::__OptVec{<:O},
                     parranges::NTuple{N,Vector},
                     incoming_pis::Tuple)::Dist{<:O} where {N,I,J,K,O}

        (iranges, jranges) = split_pars(sf, parranges)
        # We need to correctly handle the case with duplicate parent values
        # logcpdf below will get the full incoming pi for all duplicate values
        irgs::Vector{Set{Any}} = [Set(r) for r in iranges]
        icombos = cartesian_product(irgs)
        (ipis, jpis) = split_pars(sf, incoming_pis)
        result_pieces = [Vector{Float64}() for x in range]

        for is in icombos
            gsf = gensf(sf,tuple(is...))
            ps = compute_pi(gsf, range, Tuple(jranges), jpis)
            if (!isempty(iranges))
                # ipi = sum([logcpdf(ipis[i], (), irgs[i][ind[i]]) for i in 1:length(iranges)])
                ipi = sum([logcpdf(ipis[j], (), is[j]) for j in 1:length(iranges)])
            else
                ipi = 1
            end
            for i in 1:length(range)
                push!(result_pieces[i], ipi + logcpdf(ps, (), range[i]))
            end
        end
        result = normalize([exp(StatsFuns.logsumexp(rps)) for rps in result_pieces])
        return Cat(range, result)
    end
end

@impl begin
    struct ConditionalSendLambda end

    function send_lambda(sf::Conditional{I,J,K,O},
                       lambda::Score{<:O},
                       range::__OptVec{<:O},
                       parranges::NTuple{N,Vector},
                       incoming_pis::Tuple,
                       parent_ix::Integer)::Score where {N,I,J,K,O}
        (iranges, jranges) = split_pars(sf, parranges)
        (ipis, jpis) = split_pars(sf, incoming_pis)
        iar :: Array{Array{Int, 1}} = collect(map(r -> collect(1:length(r)), iranges))
        iinds = cartesian_product(iar)
        jar :: Array{Array{Int, 1}} = collect(map(r -> collect(1:length(r)), jranges))
        jinds = cartesian_product(jar)
        result = zeros(Float64, length(parranges[parent_ix]))
        # Need to make sure the target parent range is a Vector{T} rather than a Vector{Any}
        T = typeof(parranges[parent_ix][1])
        target_parrange :: Vector{T} = parranges[parent_ix]

        if parent_ix <= length(iranges)
            rs::Array{Array{Int, 1}, 1} = [collect(1:length(rg)) for rg in iranges]
            deleteat!(rs, parent_ix)
            restranges = cartesian_product(rs)
            # We want to send a lambda message to the specific I input.
            # We can ignore the pi message from this input.
            # For each value of this input, we want to compute \sum_{i',j,o} \pi_J(j) P_ps.sf(o | j ; gen_params(ps, (i, i'))) \lambda(o).
            for i = 1:length(iranges[parent_ix])
                ival = iranges[parent_ix][i]
                ls = Vector{Float64}()
                for rest in restranges
                    ipi = 0
                    fullvals = []
                    for r = 1:parent_ix-1
                        ipi += logcpdf(ipis[r], (), parranges[r][rest[r]])
                        push!(fullvals, iranges[r][rest[r]])
                    end
                    push!(fullvals, ival)
                    for r = parent_ix+1:length(iranges)
                        ipi += logcpdf(ipis[r], (), parranges[r][rest[r-1]])
                        push!(fullvals, iranges[r][rest[r-1]])
                    end
                    gsf = gensf(sf,tuple(fullvals...))
                    for jind in jinds
                        jpi = 0
                        jval = []
                        for r = 1:length(jpis)
                            jpi += logcpdf(jpis[r], (), jind[r])
                            push!(jval, jranges[r][jind[r]])
                        end
                        jvaltypes = [typeof(j) for j in jval]
                        tjval = convert(Vector{typejoin(jvaltypes...)}, jval)
                        pi = ipi + jpi
                        for o in range
                            push!(ls, pi + logcpdf(gsf, Tuple(tjval), o) + get_log_score(lambda, o))
                        end
                    end
                end
                result[i] = StatsFuns.logsumexp(ls)
            end
        elseif parent_ix <= length(parranges)
            n = length(iranges)
            jix = parent_ix - n
                # We weight the lambda messages sent to the specific J parent by the pi message for the I parents
            result_pieces = [Vector{Float64}() for x in target_parrange]
            for iind in iinds
                ival = [iranges[r][iind[r]] for r in 1:n]
                gsf = gensf(sf,tuple(ival...))
                ipi = sum(logcpdf(ipis[r], (), iind[r]) for r in 1:n)
                lam = send_lambda(gsf, lambda, range, jranges, jpis, jix)
                for i in 1:length(target_parrange)
                    push!(result_pieces[i], ipi + get_log_score(lam, parranges[parent_ix][i]))
                end
            end
            result = [StatsFuns.logsumexp(rp) for rp in result_pieces]
        end

        return LogScore(target_parrange, result)
    end
end


@testset "WindowedAssociativeOp" begin
    @testset "empty" begin
        for T in [Float64, Bool, Array{Float64,1}]
            for op in [+, -, *]
                state = WindowedAssociativeOp{T,op}()
                @test state.previous_cumsum == T[]
                @test state.ri_previous_cumsum == 0
                @test state.values == T[]
            end
        end
    end
end


const D3=3
const POINT=15
const UNSET = 0

function GmshDiscreteModel(mshfile; renumber=true)
  @check_if_loaded
  if !isfile(mshfile)
    error("Msh file not found: $mshfile")
  end

  gmsh.initialize()
  gmsh.option.setNumber("General.Terminal", 1)
  gmsh.option.setNumber("Mesh.SaveAll", 1)
  gmsh.option.setNumber("Mesh.MedImportGroupsOfNodes", 1)
  gmsh.open(mshfile)

  renumber && gmsh.model.mesh.renumberNodes()
  renumber && gmsh.model.mesh.renumberElements()

  grid, cell_to_entity = _setup_grid(gmsh)
  grid_topology = UnstructuredGridTopology(grid)
  labeling = _setup_labeling(gmsh,grid,grid_topology,cell_to_entity)

  gmsh.finalize()

  UnstructuredDiscreteModel(grid,grid_topology,labeling)
end

function _setup_grid(gmsh)

  D = _setup_dim(gmsh)

  node_to_coords = _setup_node_coords(gmsh,D)

  cell_to_nodes, nminD = _setup_connectivity(gmsh,D)

  cell_to_type, reffes, orientation = _setup_reffes(gmsh,D)

  cell_to_entity = _setup_cell_to_entity(
    gmsh,D,length(cell_to_nodes),nminD)

  grid = UnstructuredGrid(
    node_to_coords,
    cell_to_nodes,
    reffes,
    cell_to_type,
    orientation)

  (grid, cell_to_entity)

end

function _setup_dim(gmsh)
  entities = gmsh.model.getEntities()
  D = -1
  for e in entities
    D = max(D,e[1])
  end
  if D == -1
    gmsh.finalize()
    error("No entities in the msh file.")
  end
  D
end

function _setup_node_coords(gmsh,D)
  nodeTags, coord, parametricCoord = gmsh.model.mesh.getNodes()
  nmin = minimum(nodeTags)
  nmax = maximum(nodeTags)
  nnodes = length(nodeTags)
  if !(nmax == nnodes && nmin == 1)
    gmsh.finalize()
    error("Only consecutive node tags allowed.")
  end
  node_to_coords = zeros(Point{D,Float64},nnodes)
  _fill_node_coords!(node_to_coords,nodeTags,coord,D)
  node_to_coords
end

function _fill_node_coords!(node_to_coords,nodeTags,coord,D)
  m = zero(mutable(Point{D,Float64}))
  for node in nodeTags
    for j in 1:D
      k = (node-1)*D3 + j
      xj = coord[k]
      m[j] = xj
    end
    node_to_coords[node] = m
  end
end

function _setup_connectivity(gmsh,d)

  elemTypes, elemTags, nodeTags = gmsh.model.mesh.getElements(d)

  if length(elemTypes) == 0
    ncells = 0
    ndata = 0
    nmin = 1
    cell_to_nodes_prts = zeros(Int,ncells+1)
    cell_to_nodes_data = zeros(Int32,ndata)
    cell_to_nodes = Table(cell_to_nodes_data,cell_to_nodes_prts)
    return (cell_to_nodes, nmin)
  end

  ncells, nmin, nmax = _check_cell_tags(elemTags)

  etype_to_nlnodes = _setup_etype_to_nlnodes(elemTypes,gmsh)

  ndata = sum([length(t) for t in nodeTags])

  cell_to_nodes_data = zeros(Int,ndata)
  cell_to_nodes_prts = zeros(Int32,ncells+1)

  _fill_connectivity!(
    cell_to_nodes_data,
    cell_to_nodes_prts,
    nmin-1,
    etype_to_nlnodes,
    elemTypes,
    elemTags,
    nodeTags,
    d)

  cell_to_nodes = Table(cell_to_nodes_data,cell_to_nodes_prts)

  (cell_to_nodes, nmin)

end

function _check_cell_tags(elemTags)
  nmin::Int = minimum([minimum(t) for t in elemTags])
  nmax::Int = maximum([maximum(t) for t in elemTags])
  ncells = sum([length(t) for t in elemTags])
  if !( (nmax-nmin+1) == ncells)
    gmsh.finalize()
    error("Only consecutive elem tags allowed.")
  end
  (ncells,nmin,nmax)
end

function _setup_etype_to_nlnodes(elemTypes,gmsh)
  netypes = maximum(elemTypes)
  etype_to_nlnodes = zeros(Int,netypes)
  for etype in elemTypes
    name, dim, order, numv, parv =
      gmsh.model.mesh.getElementProperties(etype)
    etype_to_nlnodes[etype] = numv
  end
  etype_to_nlnodes
end

function  _fill_connectivity!(
    cell_to_nodes_data,
    cell_to_nodes_prts,
    o,
    etype_to_nlnodes,
    elemTypes,
    elemTags,
    nodeTags,
    d)

  for (j,etype) in enumerate(elemTypes)
    nlnodes = etype_to_nlnodes[etype]
    i_to_cell = elemTags[j]
    for cell in i_to_cell
      cell_to_nodes_prts[cell+1-o] = nlnodes
    end
  end

  length_to_ptrs!(cell_to_nodes_prts)

  for (j,etype) in enumerate(elemTypes)
    nlnodes = etype_to_nlnodes[etype]
    i_to_cell = elemTags[j]
    i_lnode_to_node = nodeTags[j]
    if (nlnodes == d+1)
      _orient_simplex_connectivities!(nlnodes,i_lnode_to_node)
    elseif (nlnodes == 4)
      _sort_quad_connectivites!(nlnodes,i_lnode_to_node)
    elseif (nlnodes == 8)
      _sort_hex_connectivites!(nlnodes,i_lnode_to_node)
    end
    for (i,cell) in enumerate(i_to_cell)
      a = cell_to_nodes_prts[cell-o]-1
      for lnode in 1:nlnodes
        node = i_lnode_to_node[(i-1)*nlnodes+lnode]
        cell_to_nodes_data[a+lnode] = node
      end
    end
  end

end

function _orient_simplex_connectivities!(nlnodes,i_lnode_to_node)
  aux = zeros(eltype(i_lnode_to_node),nlnodes)
  offset = nlnodes-1
  for i in 1:nlnodes:length(i_lnode_to_node)
    aux = i_lnode_to_node[i:i+offset]
    sort!(aux)
    i_lnode_to_node[i:i+offset] = aux
  end
end

function _sort_quad_connectivites!(nlnodes,i_lnode_to_node)
  aux = zero(eltype(i_lnode_to_node))
  offset = nlnodes-1
  for i in 1:nlnodes:length(i_lnode_to_node)
    aux = i_lnode_to_node[i+offset-1]
    i_lnode_to_node[i+offset-1] = i_lnode_to_node[i+offset]
    i_lnode_to_node[i+offset] = aux
  end
end

function _sort_hex_connectivites!(nlnodes,i_lnode_to_node)
  perm = [1, 2, 4, 3, 5, 6, 8, 7]
  aux = zeros(eltype(i_lnode_to_node),nlnodes)
  offset = nlnodes-1
  for i in 1:nlnodes:length(i_lnode_to_node)
    aux = i_lnode_to_node[i:i+offset]
    i_lnode_to_node[i:i+offset] = aux[perm]
  end
end

function _setup_reffes(gmsh,d)

  elemTypes, elemTags, nodeTags = gmsh.model.mesh.getElements(d)

  if !(length(elemTypes)==1)
    gmsh.finalize()
    s = """
    Only one element type per dimension allowed for the moment.
    Dimension $d has $(length(elemTypes)) different element types
    """
    error(s)
  end

  ncells, nmin, nmax = _check_cell_tags(elemTags)
  cell_to_type = fill(Int8(1),ncells)

  etype::Int = first(elemTypes)
  name, dim, order::Int, numv, parv = gmsh.model.mesh.getElementProperties(etype)

  if order == 0 && etype == POINT
    order = 1
  end

  if order != 1
    gmsh.finalize()
    error("For the moment only for first-order elements")
  end

  reffe = _reffe_from_etype(etype)
  reffes = [reffe,]

  orientation = Val(is_simplex(get_polytope(reffe)))

  (cell_to_type, reffes, orientation)
end

function _reffe_from_etype(eltype)
  if eltype == 1
    SEG2
  elseif eltype == 2
    TRI3
  elseif eltype == 3
    QUAD4
  elseif eltype == 4
    TET4
  elseif eltype == 5
    HEX8
  elseif eltype == POINT
    VERTEX1
  else
    gmsh.finalize()
    error("Unsupported element type. elemType: $etype")
  end
end

function _setup_cell_to_entity(gmsh,d,ncells,nmin)
  cell_to_entity = fill(UNSET,ncells)
  entities = gmsh.model.getEntities(d)
  for e in entities
    _, elemTags, _ = gmsh.model.mesh.getElements(e[1], e[2])
    _fill_cell_to_entity!(cell_to_entity,elemTags,e[2],nmin-1)
  end
  cell_to_entity
end

function _fill_cell_to_entity!(cell_to_entity,elemTags,entity,o)
  for i_to_cell in elemTags
    for cell in i_to_cell
      cell_to_entity[cell-o] = entity
    end
  end
end

function _setup_labeling(gmsh,grid,grid_topology,cell_to_entity)

  D = num_cell_dims(grid)
  dim_to_gface_to_nodes, dim_gface_to_entity = _setup_faces(gmsh,D)
  push!(dim_to_gface_to_nodes,get_cell_nodes(grid))
  push!(dim_gface_to_entity,cell_to_entity)

  dim_to_group_to_entities = _setup_dim_to_group_to_entities(gmsh)
  dim_to_group_to_name = _setup_dim_to_group_to_name(gmsh)
  dim_to_offset = _setup_dim_to_offset(gmsh)

  nnodes = num_nodes(grid)
  node_to_label = _setup_node_to_label(gmsh,dim_to_offset,nnodes)

  labeling = _setup_face_labels(
    grid_topology,
    dim_to_gface_to_nodes,
    dim_gface_to_entity,
    dim_to_offset,
    dim_to_group_to_entities,
    dim_to_group_to_name,
    node_to_label)

  labeling

end

function _setup_faces(gmsh,D)
  dim_to_gface_to_nodes = []
  dim_gface_to_entity = []
  for d in 0:(D-1)
    face_to_nodes, nmin  = _setup_connectivity(gmsh,d)
    face_to_entity = _setup_cell_to_entity(gmsh,d,length(face_to_nodes),nmin)
    push!(dim_to_gface_to_nodes,face_to_nodes)
    push!(dim_gface_to_entity,face_to_entity)
  end
  (dim_to_gface_to_nodes, dim_gface_to_entity)
end

function _setup_dim_to_group_to_entities(gmsh)
  dim_to_group_to_entities = Vector{Vector{Int}}[]
  for d in 0:D3
    pgs = gmsh.model.getPhysicalGroups(d)
    n = length(pgs)
    group_to_entities = Vector{Int}[]
    for pg in pgs
      es = gmsh.model.getEntitiesForPhysicalGroup(pg...)
      entities = Int[]
      for e in es
        push!(entities,e)
      end
      push!(group_to_entities,entities)
    end
    push!(dim_to_group_to_entities,group_to_entities)
  end
  dim_to_group_to_entities
end

function _setup_dim_to_group_to_name(gmsh)
  u = 1
  dim_to_group_to_name = Vector{String}[]
  for d in 0:D3
    pgs = gmsh.model.getPhysicalGroups(d)
    n = length(pgs)
    names = String[]
    for pg in pgs
      _name = gmsh.model.getPhysicalName(pg...)
      if _name == ""
        name = "untitled$u"
        u += 1
      else
        name = _name
      end
      push!(names,name)
    end
    push!(dim_to_group_to_name,names)
  end
  dim_to_group_to_name
end

function _setup_dim_to_offset(gmsh)
  entities = gmsh.model.getEntities()
  dim_to_nentities = zeros(Int,D3+1)
  for e in entities
    d = e[1]
    id = e[2]
    _id = dim_to_nentities[d+1]
    dim_to_nentities[d+1] = max(id,_id)
  end
  dim_to_offset = zeros(Int,D3+1)
  for d=1:D3
    dim_to_offset[d+1] += dim_to_nentities[d-1+1] + dim_to_offset[d-1+1]
  end
  dim_to_offset
end

function _setup_node_to_label(gmsh,dim_to_offset,nnodes)
  node_to_label = zeros(Int,nnodes)
  entities = gmsh.model.getEntities()
  for (dim, entity) in entities
    nodes, _, _ = gmsh.model.mesh.getNodes(dim, entity)
    node_to_label[nodes] .= (entity + dim_to_offset[dim+1])
  end
  node_to_label
end

function _setup_face_labels(
  grid_topology,
  dim_to_gface_to_nodes,
  dim_gface_to_entity,
  dim_to_offset,
  dim_to_group_to_entities,
  dim_to_group_to_name,
  node_to_label)

  D = num_cell_dims(grid_topology)

  dim_to_face_to_label = [node_to_label,]

  for d in 1:D
    z = fill(UNSET,num_faces(grid_topology,d))
    push!(dim_to_face_to_label,z)
  end

  _fill_dim_to_face_to_label!(
    dim_to_face_to_label,
    grid_topology,
    dim_to_gface_to_nodes,
    dim_gface_to_entity,
    dim_to_offset)

  tag_to_name, tag_to_groups, _ = _setup_tag_to_name(dim_to_group_to_name)

  tag_to_labels = _setup_tag_to_labels(
    tag_to_groups,dim_to_group_to_entities,dim_to_offset)

  FaceLabeling(dim_to_face_to_label,tag_to_labels,tag_to_name)

end

function _fill_dim_to_face_to_label!(
  dim_to_face_to_label,
  grid_topology,
  dim_to_gface_to_nodes,
  dim_gface_to_entity,
  dim_to_offset)

  D = length(dim_to_face_to_label)-1
  d = D
  cell_to_entity =  dim_gface_to_entity[d+1]
  cell_to_label = dim_to_face_to_label[d+1]
  offset = dim_to_offset[d+1]
  _apply_offset_for_cells!(cell_to_label,cell_to_entity,offset)

  for d in 0:(D-1)

    gface_to_nodes = dim_to_gface_to_nodes[d+1]
    face_to_nodes = get_faces(grid_topology,d,0)
    node_to_faces = get_faces(grid_topology,0,d)
    gface_to_face = _setup_gface_to_face(
      face_to_nodes,
      node_to_faces,
      gface_to_nodes)
    face_to_label = dim_to_face_to_label[d+1]
    gface_to_entity = dim_gface_to_entity[d+1]
    offset = dim_to_offset[d+1]
    _apply_offset_for_faces!(face_to_label,gface_to_entity,gface_to_face,offset)

  end

  for d = 0:(D-1)
    for j in (d+1):D
      dface_to_jfaces = get_faces(grid_topology,d,j)
      dface_to_label = dim_to_face_to_label[d+1]
      jface_to_label = dim_to_face_to_label[j+1]
      _fix_dface_to_label!(dface_to_label,jface_to_label,dface_to_jfaces)
    end
  end

end

function _apply_offset_for_cells!(cell_to_label,cell_to_entity,offset)
  ncells = length(cell_to_label)
  for cell in 1:ncells
    entity = cell_to_entity[cell]
    cell_to_label[cell] = entity+offset
  end
end

function _apply_offset_for_faces!(face_to_label,gface_to_entity,gface_to_face,offset)
  ngfaces = length(gface_to_face)
  for gface in 1:ngfaces
    entity = gface_to_entity[gface]
    face = gface_to_face[gface]
    face_to_label[face] = entity+offset
  end
end

function _setup_gface_to_face(
  face_to_nodes,
  node_to_faces,
  gface_to_nodes)

  gface_to_face = _find_gface_to_face(
    face_to_nodes.data,
    face_to_nodes.ptrs,
    node_to_faces.data,
    node_to_faces.ptrs,
    gface_to_nodes.data,
    gface_to_nodes.ptrs)

  gface_to_face

end

function _find_gface_to_face(
  face_to_nodes_data,
  face_to_nodes_ptrs,
  node_to_faces_data::AbstractVector{T},
  node_to_faces_ptrs,
  gface_to_nodes_data,
  gface_to_nodes_ptrs) where T

  ngfaces = length(gface_to_nodes_ptrs) - 1
  gface_to_face = zeros(T,ngfaces)
  n = max_cells_arround_vertex(node_to_faces_ptrs)
  faces_around = fill(UNSET,n)
  faces_around_scratch = fill(UNSET,n)

  _fill_gface_to_face!(
    gface_to_face,
    face_to_nodes_data,
    face_to_nodes_ptrs,
    node_to_faces_data,
    node_to_faces_ptrs,
    gface_to_nodes_data,
    gface_to_nodes_ptrs,
    faces_around,
    faces_around_scratch)

  gface_to_face

end

function  _fill_gface_to_face!(
  gface_to_face,
  face_to_nodes_data,
  face_to_nodes_ptrs,
  node_to_faces_data,
  node_to_faces_ptrs,
  gface_to_nodes_data,
  gface_to_nodes_ptrs,
  faces_around,
  faces_around_scratch)

  ngfaces = length(gface_to_nodes_ptrs) - 1

  nfaces_around = UNSET
  nfaces_around_scratch = UNSET

  for gface in 1:ngfaces

    a = gface_to_nodes_ptrs[gface]-1
    b = gface_to_nodes_ptrs[gface+1]
    nlnodes = b-(a+1)

    for lnode in 1:nlnodes
      node = gface_to_nodes_data[lnode+a]
      if lnode == 1
        nfaces_around = _fill_cells_around_scratch!(
          faces_around,
          node,
          node_to_faces_data,
          node_to_faces_ptrs)
      else
        nfaces_around_scratch = _fill_cells_around_scratch!(
          faces_around_scratch,
          node,
          node_to_faces_data,
          node_to_faces_ptrs)
        _set_intersection!(
          faces_around,faces_around_scratch,
          nfaces_around,nfaces_around_scratch)
      end
    end

    for face in faces_around
      if face != UNSET
        gface_to_face[gface] = face
        break
      end
    end

  end

end

function _fix_dface_to_label!(dface_to_label,jface_to_label,dface_to_jfaces)

  ndfaces = length(dface_to_label)
  @assert ndfaces == length(dface_to_jfaces)

  for dface in 1:ndfaces

    dlabel = dface_to_label[dface]
    if dlabel != UNSET
      continue
    end

    jfaces = dface_to_jfaces[dface]
    for jface in jfaces
      jlabel = jface_to_label[jface]
      if jlabel != UNSET
        dface_to_label[dface] = jlabel
        break
      end
    end

  end

end

function _setup_tag_to_name(dim_to_group_to_name)
  tag_to_name = String[]
  tag_to_groups = Vector{Tuple{Int,Int}}[]
  name_to_tag = Dict{String,Int}()
  tag = 1
  for (dim, group_to_name) in enumerate(dim_to_group_to_name)
    for (id,name) in enumerate(group_to_name)
      group = (dim,id)
      if !haskey(name_to_tag,name)
        push!(tag_to_name,name)
        groups = [group,]
        push!(tag_to_groups,groups)
        name_to_tag[name] = tag
        tag += 1
      else
        _tag = name_to_tag[name]
        groups = tag_to_groups[_tag]
        push!(groups,group)
      end
    end
  end
  (tag_to_name, tag_to_groups, name_to_tag)
end

function _setup_tag_to_labels(
  tag_to_groups,dim_to_group_to_entities,dim_to_offset)

  tag_to_labels = Vector{Int}[]
  for groups in tag_to_groups

    labels = Int[]
    for group in groups
      dim, id = group
      offset = dim_to_offset[dim]
      entities = dim_to_group_to_entities[dim][id]
      for entity in entities
        label = entity + offset
        push!(labels,label)
      end
    end
    push!(tag_to_labels,labels)

  end

  tag_to_labels

end