### GENERALIZED COLUMN OPS

# We want to support arbitrary dimensional arrays.  When data comes in
# N dimensions, we assume it is an array of N-1 dimensional instances
# and the last dimension gives us the instance count.  We will refer
# to the first N-1 dimensions as generalized "columns" of the
# data. These columns are indexed by the last index of an array,
# i.e. column i corresponds to b[:,:,...,i].

# Here are some convenience functions for generalized columns:
# We consider a 1-D array a single column:

csize(a)=(ndims(a)==1 ? size(a) : size(a)[1:end-1])
csize(a,n)=tuple(csize(a)..., n) # size if you had n columns
clength(a)=(ndims(a)==1 ? length(a) : stride(a,ndims(a)))
ccount(a)=(ndims(a)==1 ? 1 : size(a,ndims(a)))
csub(a,i)=(ndims(a)==1 ? error() : sub(a, ntuple(i->(:), ndims(a)-1)..., i))
cget(a,i)=(ndims(a)==1 ? error() : getindex(a, ntuple(i->(:), ndims(a)-1)..., i))
size2(y)=(nd=ndims(y); (nd==1 ? (length(y),1) : (stride(y, nd), size(y, nd)))) # size as a matrix
size2(y,i)=size2(y)[i]

function minibatch(x, y, batchsize)
    data = Any[]
    for i=1:batchsize:ccount(x)
        j=min(i+batchsize-1,ccount(x))
        push!(data, (cget(x,i:j), cget(y,i:j)))
    end
    return data
end

# CSLICE!  Returns a slice of array b, with columns specified in range
# r, using the storage in KUarray a.  The element types need to match,
# but the size of a does not need to match, it is adjusted as
# necessary.  This is used in train and predict to get data from a raw
# array into a KUarray for minibatching.

# cslice!{A,B,T}(a::KUdense{A,T}, b::KUdense{B,T}, r::UnitRange)=cslice!(a,b.arr,r)

# function cslice!{A,T}(a::KUdense{A,T}, b::BaseArray{T}, r::UnitRange)
#     n  = clength(b) * length(r)
#     length(a.ptr) >= n || resize!(a.ptr, int(resizefactor(KUdense)*n+1))
#     b1 = 1 + clength(b) * (first(r) - 1)
#     copysync!(a.ptr, 1, b, b1, n)
#     a.arr = arr(a.ptr, csize(b, length(r)))
#     return a
# end

# # For non-contiguous columns:
# function cslice!{A,T}(a::KUdense{A,T}, b::BaseArray{T}, cols)
#     ncols = length(cols)
#     clen = clength(b)
#     n = clen * ncols
#     length(a.ptr) >= n || resize!(a.ptr, int(resizefactor(KUdense)*n+1))
#     alen = 0
#     for i=1:ncols
#         bidx = (cols[i]-1)*clen + 1
#         copysync!(a.ptr, alen+1, b, bidx, clen)
#         alen += clen
#     end
#     a.arr = arr(a.ptr, csize(b, ncols))
#     return a
# end

function cslice!{T}(a::BaseArray{T}, b::BaseArray{T}, cols)
    ncols = length(cols)
    clen = clength(b)
    n = clen * ncols
    size(a) == csize(b, ncols) || error("Size mismatch")
    alen = 0
    for i=1:ncols
        bidx = (cols[i]-1)*clen + 1
        copysync!(a, alen+1, b, bidx, clen) # t:134
        alen += clen
    end
    return a
end

function cslice!{T}(a::SparseMatrixCSC{T}, b::SparseMatrixCSC{T}, cols)
    bptr = b.colptr
    nz = 0; for i in cols; nz += bptr[i+1]-bptr[i]; end
    a.m = b.m
    a.n = length(cols)
    resize!(a.nzval, nz)
    resize!(a.rowval, nz)
    resize!(a.colptr, a.n+1)
    a.colptr[1] = a1 = aj = 1
    for bj in cols                 # copy column b[:,bj] to a[:,aj]
        b1 = bptr[bj]
        nz = bptr[bj+1]-b1
        copysync!(a.nzval, a1, b.nzval, b1, nz) # t:217
        copysync!(a.rowval, a1, b.rowval, b1, nz) # t:191
        a1 += nz
        a.colptr[aj+=1] = a1
    end
    @assert aj == a.n+1
    return a
end

# TODO: write the non-contiguous sparse version
# function cslice!{A,B,T}(a::KUsparse{A,T}, b::KUsparse{B,T}, r::UnitRange)
#     bptr = cpucopy(b.colptr)
#     nz = 0; for i in r; nz += bptr[i+1]-bptr[i]; end
#     a.m = b.m
#     a.n = length(r)
#     resize!(a.nzval, nz)
#     resize!(a.rowval, nz)
#     aptr = Array(Int32, a.n+1)
#     aptr[1] = a1 = aj = 1
#     for bj in r                 # copy column b[:,bj] to a[:,aj]
#         b1 = bptr[bj]
#         nz = bptr[bj+1]-b1
#         copysync!(a.nzval, a1, b.nzval, b1, nz)
#         copysync!(a.rowval, a1, b.rowval, b1, nz)
#         a1 += nz
#         aptr[aj+=1] = a1
#     end
#     @assert aj == a.n+1
#     a.colptr = convert(A, aptr)
#     return a
# end

# cslice!{A,T}(a::KUsparse{A,T}, b::SparseMatrixCSC{T}, r::UnitRange)=cslice!(a, convert(KUsparse, b), r)

# CCOPY! Copy n columns from src starting at column si, into dst
# starting at column di.  Used by predict to construct output.
# Don't need the sparse version, output always dense.

# ccopy!{A,T,N}(dst::BaseArray{T,N}, di, src::KUdense{A,T,N}, si=1, n=ccount(src)-si+1)=(ccopy!(dst,di,src.arr,si,n); dst)
# ccopy!{A,B,T,N}(dst::KUdense{A,T,N}, di, src::KUdense{B,T,N}, si=1, n=ccount(src)-si+1)=(ccopy!(dst.arr,di,src.arr,si,n); dst)

function ccopy!{T,N}(dst::BaseArray{T,N}, di, src::BaseArray{T,N}, si=1, n=ccount(src)-si+1)
    @assert csize(dst)==csize(src)
    clen = clength(src)
    d1 = 1 + clen * (di - 1)
    s1 = 1 + clen * (si - 1)
    copysync!(dst, d1, src, s1, clen * n)
    return dst
end

# CADD! Add n columns from src starting at column si, into dst
# starting at column di.  Used by uniq!  Don't need sparse version,
# weights always dense.

using Base.LinAlg: axpy!

# cadd!{A,T,N}(dst::BaseArray{T,N}, di, src::KUdense{A,T,N}, si=1, n=ccount(src)-si+1)=(cadd!(dst,di,src.arr,si,n); dst)
# cadd!{A,B,T,N}(dst::KUdense{A,T,N}, di, src::KUdense{B,T,N}, si=1, n=ccount(src)-si+1)=(cadd!(dst.arr,di,src.arr,si,n); dst)

function cadd!{T,N}(dst::BaseArray{T,N}, di, src::BaseArray{T,N}, si=1, n=ccount(src)-si+1)
    @assert csize(dst)==csize(src)
    @assert ccount(dst) >= di+n-1
    @assert ccount(src) >= si+n-1
    clen = clength(src)
    d1 = 1 + clen * (di - 1)
    s1 = 1 + clen * (si - 1)
    n1 = clen * n
    axpy!(n1, one(T), pointer(src, s1), 1, pointer(dst, d1), 1)
    return dst
end

# CCAT! generalizes append! to multi-dimensional arrays.  Adds the
# ability to specify particular columns to append.  Used in
# kperceptron to add support vectors.

# ccat!{A,B,T,N}(a::KUdense{A,T,N}, b::KUdense{B,T,N}, cols=(1:ccount(b)))=ccat!(a,b.arr,cols)

# function ccat!{A,T,N}(a::KUdense{A,T,N}, b::BaseArray{T,N}, cols=(1:ccount(b)))
#     @assert csize(a)==csize(b)
#     alen = length(a)
#     clen = clength(a)
#     ncols = length(cols)
#     n = alen + ncols * clen
#     length(a.ptr) >= n || resize!(a.ptr, round(Int,resizefactor(KUdense)*n+1))
#     for i=1:ncols
#         bidx = (cols[i]-1)*clen + 1
#         copysync!(a.ptr, alen+1, b, bidx, clen)
#         alen += clen
#     end
#     a.arr = arr(a.ptr, csize(a, ccount(a) + ncols))
#     return a
# end

# ccat!{A,T}(a::KUsparse{A,T}, b::SparseMatrixCSC{T}, cols=(1:ccount(b)))=ccat!(a,convert(KUsparse,b),cols)

# function ccat!{A,B,T}(a::KUsparse{A,T}, b::KUsparse{B,T}, cols=(1:ccount(b)))
#     # a: m, n, colptr, rowval, nzval
#     # colptr[i]: starting index (in rowval,nzval) of column i
#     # colptr[n+1]: nz+1
#     @assert size(a,1) == size(b,1)
#     aptr = cpucopy(a.colptr)
#     bptr = cpucopy(b.colptr)
#     na = aptr[a.n+1]-1          # count new nonzero entries in a
#     ncols = length(cols)
#     for i in cols; na += bptr[i+1]-bptr[i]; end
#     resize!(a.nzval, na)
#     resize!(a.rowval, na)
#     na = aptr[a.n+1]-1          # restart the count
#     for i=1:ncols
#         bj=cols[i]              # bj'th column of b
#         aj=a.n+i                # will become aj'th column of a
#         nz=bptr[bj+1]-bptr[bj]  # with nz nonzero values
#         @assert length(aptr) == aj
#         push!(aptr, aptr[aj]+nz) # aptr[aj+1] = aptr[aj]+nz
#         copysync!(a.nzval,na+1,b.nzval,bptr[bj],nz)
#         copysync!(a.rowval,na+1,b.rowval,bptr[bj],nz)
#         na = na+nz
#     end
#     @assert length(aptr) == a.n + ncols + 1
#     resize!(a.colptr, a.n + ncols + 1)
#     copysync!(a.colptr, a.n+2, aptr, a.n+2, ncols)
#     a.n += ncols
#     return a
# end

### UNIQ! leaves unique columns in its first argument and sums to
### corresponding columns in the remaining arguments.  Used by
### kperceptron in merging identical support vectors.

# function uniq!{A<:Array}(s::KUdense{A}, ww::KUdense...)
#     oldn = ccount(s)                                            # number of original support vectors
#     for w in ww; @assert ccount(w) == oldn; end
#     ds = Dict{Any,Int}()                                        # support vector => new index
#     newn = 0                                                    # number of new support vectors
#     for oldj=1:oldn
#         newj = get!(ds, _colkey(s,oldj), newn+1)
#         if newj <= newn                                         # s[:,oldj] already in s[:,newj]
#             @assert newj <= newn == length(ds) < oldj
#             for w in ww; cadd!(w,newj,w,oldj,1); end
#         else                                                    # s[:,oldj] to be copied to s[:,newj]
#             @assert newj == newn+1 == length(ds) <= oldj
#             newn += 1
#             if newj != oldj
#                 ccopysync!(s,newj,s,oldj,1)
#                 for w in ww; ccopy!(w,newj,w,oldj,1); end
#             end
#         end
#     end
#     @assert newn == length(ds)
#     resize!(s, csize(s, newn))
#     for w in ww; resize!(w, csize(w, newn)); end
#     return tuple(s, ww...)
# end

# function uniq!{A<:Array}(s::KUsparse{A}, ww::KUdense...)
#     oldn = ccount(s)                                            # number of original support vectors
#     for w in ww; @assert ccount(w) == oldn; end
#     ds = Dict{Any,Int}()                                        # support vector => new index
#     @assert s.colptr[1]==1
#     ncol = 0
#     nnz = 0
#     for oldj=1:oldn
#         newj = get!(ds, _colkey(s,oldj), ncol+1)
#         if newj <= ncol                                          # s[:,oldj] already in s[:,newj]
#             @assert newj <= length(ds) == ncol < oldj
#             for w in ww; cadd!(w,newj,w,oldj,1); end
#         else                                                    # s[:,oldj] to be copied to s[:,newj]
#             @assert newj == ncol+1 == length(ds) <= oldj
#             from = s.colptr[oldj]
#             nval = s.colptr[oldj+1] - from
#             to = nnz+1
#             ncol += 1
#             nnz += nval
#             if newj != oldj
#                 copysync!(s.rowval, to, s.rowval, from, nval)
#                 copysync!(s.nzval, to, s.nzval, from, nval)
#                 s.colptr[ncol+1] = nnz+1
#                 for w in ww; ccopy!(w,newj,w,oldj,1); end
#             else
#                 @assert to == from
#                 @assert s.colptr[ncol+1] == nnz+1
#             end
#         end
#     end
#     @assert length(ds) == ncol
#     s.n = ncol
#     resize!(s.colptr, ncol+1)
#     resize!(s.rowval, nnz)
#     resize!(s.nzval,  nnz)
#     for w in ww; resize!(w, csize(w, s.n)); end
#     return tuple(s, ww...)
# end

# _colkey{A<:Array}(s::KUdense{A},j)=sub(s.arr, ntuple(i->(i==ndims(s) ? (j:j) : Colon()), ndims(s))...)

# function _colkey{A<:Array}(s::KUsparse{A},j)
#     a=s.colptr[j]
#     b=s.colptr[j+1]-1
#     r=sub(s.rowval, a:b)
#     v=sub(s.nzval, a:b)
#     (r,v)
# end

# Getting columns one at a time is expensive, just copy the whole array
# CudaArray does not support sub, even if it did we would not be able to hash it
# getcol{T}(s::KUdense{CudaArray,T}, j)=(n=clength(s);copysync!(Array(T,csize(s,1)), 1, s.arr, (j-1)*n+1, n))

# we need to look at the columns, might as well copy

# function uniq!{A<:CudaArray}(s::KUdense{A}, ww::KUdense...)
#     ss = cpucopy(s)
#     uniq!(ss, ww...)
#     cslice!(s, ss, 1:ccount(ss))
#     return tuple(s, ww...)
# end

# function uniq!{A<:CudaArray}(s::KUsparse{A}, ww::KUdense...)
#     ss = cpucopy(s)
#     uniq!(ss, ww...)
#     cslice!(s, ss, 1:ccount(ss))
#     return tuple(s, ww...)
# end

# TODO: fix array types

    # ds = Dict{Any,Int}()        # dictionary of support vectors
    # ns = 0                      # number of support vectors
    # s0 = spzeros(eltype(s), Int32, size(s,1), ns) # new sv matrix
    # for j=1:size(s,2)
    #     jj = get!(ds, s[:,j], ns+1)
    #     if jj <= ns             # s[:,j] already in s0[:,jj]
    #         @assert ns == length(ds) < j
    #         u[:,jj] += u[:,j]
    #         v[:,jj] += v[:,j]
    #     else                    # s[:,j] to be added to s0
    #         @assert jj == ns+1 == length(ds) <= j
    #         ns = ns+1
    #         hcat!(s0, s, [j], 1)
    #         if jj != j
    #             u[:,jj] = u[:,j]
    #             v[:,jj] = v[:,j]
    #         end
    #     end
    # end
    # @assert ns == length(ds) == size(s0,2)
    # u = size!(u, (size(u,1),ns))
    # v = size!(v, (size(v,1),ns))
    # for f in names(s); s.(f) = s0.(f); end
    # return (s,u,v)

# function uniq!(s::SparseMatrixCSC, u::AbstractArray, v::AbstractArray)
#     ds = Dict{Any,Int}()        # dictionary of support vectors
#     ns = 0                      # number of support vectors
#     s0 = spzeros(eltype(s), Int32, size(s,1), ns) # new sv matrix
#     for j=1:size(s,2)
#         jj = get!(ds, s[:,j], ns+1)
#         if jj <= ns             # s[:,j] already in s0[:,jj]
#             @assert ns == length(ds) < j
#             u[:,jj] += u[:,j]
#             v[:,jj] += v[:,j]
#         else                    # s[:,j] to be added to s0
#             @assert jj == ns+1 == length(ds) <= j
#             ns = ns+1
#             hcat!(s0, s, [j], 1)
#             if jj != j
#                 u[:,jj] = u[:,j]
#                 v[:,jj] = v[:,j]
#             end
#         end
#     end
#     @assert ns == length(ds) == size(s0,2)
#     u = size!(u, (size(u,1),ns))
#     v = size!(v, (size(v,1),ns))
#     for f in names(s); s.(f) = s0.(f); end
#     return (s,u,v)
# end

# function uniq!(ss::CudaSparseMatrixCSC, uu::AbstractCudaArray, vv::AbstractCudaArray)
#     (s,u,v)=map(cpucopy,(ss,uu,vv))
#     (s,u,v)=uniq!(s,u,v)
#     n = size(s,2)
#     uu = size!(uu, (size(u,1),n))
#     vv = size!(vv, (size(v,1),n))
#     copysync!(uu, 1, u, 1, size(u,1)*n)
#     copysync!(vv, 1, v, 1, size(v,1)*n)
#     (ss.m, ss.n, ss.colptr, ss.rowval, ss.nzval) = (s.m, s.n, gpucopy(s.colptr), gpucopy(s.rowval), gpucopy(s.nzval))
#     return (ss,uu,vv)
# end

# function cslice!{A,T,I}(a::KUsparse{A,T,I}, b::SparseMatrixCSC{T,I}, r::UnitRange)
#     nz = 0; for i in r; nz += b.colptr[i+1]-b.colptr[i]; end
#     a.m = b.m
#     a.n = length(r)
#     resize!(a.nzval, nz)
#     resize!(a.rowval, nz)
#     aptr = Array(I, a.n+1)
#     aptr[1] = a1 = aj = 1
#     for bj in r                 # copy column b[:,bj] to a[:,aj]
#         b1 = b.colptr[bj]
#         nz = b.colptr[bj+1]-b1
#         copysync!(a.nzval.arr, a1, b.nzval, b1, nz)
#         copysync!(a.rowval.arr, a1, b.rowval, b1, nz)
#         a1 += nz
#         aptr[aj+=1] = a1
#     end
#     @assert aj == a.n+1
#     copysync!(a.colptr, aptr)
#     return a
# end


module KoalaTransforms

# Univariate:
export ChainTransformer
export ToIntTransformer
export UnivariateStandardizer, Standardizer
export OneHotEncoder
export UnivariateBoxCoxTransformer, BoxCoxTransformer
export DataFrameToArrayTransformer, RegressionTargetTransformer
export MakeCategoricalsIntTransformer
export DataFrameToStandardizedArrayTransformer
export IntegerToInt64Transformer
export UnivariateDiscretizer, Discretizer

# extended:
export transform, inverse_transform, fit # from `Koala`

# rexported from Koala:
export FeatureSelector, IdentityTransformer, Machine

# for use in this module:
import Koala: BaseType, params
import Koala: TransformerMachine, Machine
import DataFrames: names, AbstractDataFrame, DataFrame, eltypes
import Distributions
using Statistics

# to be extended:
import Koala: Transformer, transform, fit, inverse_transform
import Base: show

# so above export from Koala works:
import Koala: EmptyTransformer, FeatureSelector
import Koala: IdentityTransformer

# development only:
# using ADBUtilities

# constants:
const N_VALUES_THRESH = 16 # for BoxCoxTransformation

## COMPOSITE TRANSFORMERS

mutable struct ChainTransformer <: Transformer
    chain::Vector{Transformer} # in order applied!
end

# constructor is given individual transforms in order opposite to that
# in which they will be applied:
ChainTransformer(transformers...) = ChainTransformer(reverse(Transformer[transformers...]))

function fit(transformer::ChainTransformer, X::Union{AbstractVector, AbstractDataFrame},
             parallel, verbosity)
    schemes = Array{Any}(undef, length(transformer.chain))

    !isempty(transformer.chain) || error("Empty ChainTransformer.")

    # fit first transformer:
    schemes[1] = fit(transformer.chain[1], X, parallel, verbosity)

    # fit the rest:
    for i in 2:length(transformer.chain)
        X = transform(transformer.chain[i - 1], schemes[i - 1], X)
        schemes[i] = fit(transformer.chain[i], X, parallel, verbosity)
    end

    return schemes
end

function transform(transformer::ChainTransformer, schemes,
                   X::Union{AbstractVector, AbstractDataFrame})
    for i in eachindex(schemes)
        X = transform(transformer.chain[i], schemes[i], X)
    end
    return X
end

function inverse_transform(transformer::ChainTransformer, schemes,
                   X::Union{AbstractVector, AbstractDataFrame})
    for i in reverse(eachindex(schemes))
        X = inverse_transform(transformer.chain[i], schemes[i], X)
    end
    return X
end

function Base.getindex(machine::TransformerMachine{ChainTransformer}, i::Int)
    return TransformerMachine(machine.transformer.chain[end - i + 1],
                              machine.scheme[end - i + 1], :)
end

## FOR RELABELLING BY CONSECUTIVE INTEGERS STARTING AT 1

mutable struct ToIntTransformer <: Transformer
    sorted::Bool
    initial_label::Int # ususally 0 or 1
    map_unseen_to_minus_one::Bool # unseen inputs are transformed to -1
end

ToIntTransformer(; sorted=false, initial_label=1,
                 map_unseen_to_minus_one=false) =
                     ToIntTransformer(sorted, initial_label,
                                      map_unseen_to_minus_one)

struct ToIntScheme{T} <: BaseType
    n_levels::Int
    int_given_T::Dict{T, Int}
    T_given_int::Dict{Int, T}
end

# null scheme constructor:
ToIntScheme(S::Type{T}) where T = ToIntScheme{T}(0, Dict{T, Int}(), Dict{Int, T}())

function fit(transformer::ToIntTransformer, v::AbstractVector{T},
             parallel, verbosity) where T
    int_given_T = Dict{T, Int}()
    T_given_int = Dict{Int, T}()
    vals = collect(Set(v))
    if transformer.sorted
        sort!(vals)
    end
    n_levels = length(vals)
    if n_levels > 2^62 - 1
        error("Cannot encode with integers a vector "*
                         "having more than $(2^62 - 1) values.")
    end
    i = transformer.initial_label
    for c in vals
        int_given_T[c] = i
        T_given_int[i] = c
        i = i + 1
    end
    return ToIntScheme{T}(n_levels, int_given_T, T_given_int)
end

# scalar case:
function transform(transformer::ToIntTransformer, scheme::ToIntScheme{T}, x::T) where T
    ret = 0 # otherwise ret below stays in local scope
    try
        ret = scheme.int_given_T[x]
    catch exception
        if isa(exception, KeyError)
            if transformer.map_unseen_to_minus_one
                ret = -1
            else
                throw(exception)
            end
        end
    end
    return ret
end
inverse_transform(transformer::ToIntTransformer, scheme, y::Int) = scheme.T_given_int[y]

# vector case:
function transform(transformer::ToIntTransformer, scheme::ToIntScheme{T},
                   v::AbstractVector{T}) where T
    return Int[transform(transformer, scheme, x) for x in v]
end
inverse_transform(transformer::ToIntTransformer, scheme::ToIntScheme{T},
                  w::AbstractVector{Int}) where T = T[scheme.T_given_int[y] for y in w]

## Univariate standardization

struct UnivariateStandardizer <: Transformer end

function fit(transformer::UnivariateStandardizer, v::AbstractVector{T},
             parallel, verbosity) where T <: Real
    std(v) > eps(Float64) ||
        @warn "Extremely small standard deviation encountered in standardization."
    return  mean(v), std(v)
end

# for transforming single value:
function transform(transformer::UnivariateStandardizer, scheme, x::Real)
    mu, sigma = scheme
    return (x - mu)/sigma
end

# for transforming vector:
transform(transformer::UnivariateStandardizer, scheme,
          v::AbstractVector{T}) where T <: Real =
              [transform(transformer, scheme, x) for x in v]

# for single values:
function inverse_transform(transformer::UnivariateStandardizer, scheme, y::Real)
    mu, sigma = scheme
    return mu + y*sigma
end

# for vectors:
inverse_transform(transformer::UnivariateStandardizer, scheme,
                  w::AbstractVector{T}) where T <: Real =
                      [inverse_transform(transformer, scheme, y) for y in w]

## Standardization of ordinal features of a DataFrame

mutable struct Standardizer <: Transformer
    features::Vector{Symbol} # features to be standardized; empty means all
end

# lazy keyword constructor:
Standardizer(; features=Symbol[]) = Standardizer(features)

struct StandardizerScheme <: BaseType
    schemes::Matrix{Float64}
    features::Vector{Symbol} # all the feature labels of the data frame fitted
    is_transformed::Vector{Bool}
end

# null scheme:
StandardizerScheme() = StandardizerScheme(zeros(0,0), Symbol[], Bool[])

function fit(transformer::Standardizer, X, parallel, verbosity)

    # determine indices of features to be transformed
    features_to_try = (isempty(transformer.features) ? names(X) : transformer.features)
    is_transformed = Array{Bool}(undef, size(X, 2))
    for j in 1:size(X, 2)
        if names(X)[j] in features_to_try && eltype(X[j]) <: AbstractFloat
            is_transformed[j] = true
        else
            is_transformed[j] = false
        end
    end

    # fit each of those features
    schemes = Array{Float64}(undef, 2, size(X, 2))
    verbosity < 1 || @info "Features standarized: "
    univ_transformer = UnivariateStandardizer()
    for j in 1:size(X, 2)
        if is_transformed[j]
            schemes[:,j] = [fit(univ_transformer, collect(X[j]), true, verbosity - 1)...]
            verbosity < 1 ||
                @info "  :$(names(X)[j])    mu=$(schemes[1,j])  sigma=$(schemes[2,j])"
        else
            schemes[:,j] = Float64[0.0, 0.0]
        end
    end

    return StandardizerScheme(schemes, names(X), is_transformed)

end

function transform(transformer::Standardizer, scheme, X)

    names(X) == scheme.features ||
        error("Attempting to transform data frame with incompatible feature labels.")

    Xnew = X[1:end,:] # make a copy of X, working even for `SubDataFrames`
    univ_transformer = UnivariateStandardizer()
    for j in 1:size(X, 2)
        if scheme.is_transformed[j]
            # extract the (mu, sigma) pair:
            univ_scheme = (scheme.schemes[1,j], scheme.schemes[2,j])
            Xnew[j] = transform(univ_transformer, univ_scheme, collect(X[j]))
        end
    end
    return Xnew

end

## One-hot encoding

"""
    OneHotEncoder(drop_last=false)

Returns a transformer for one-hot encoding the categorical features of
an `AbstractDataFrame` object. Here "categorical" refers to any
feature whose eltype is not a subtype of `AbstractFloat`. All eltypes
must admit a `<` and `string` method.

"""
mutable struct OneHotEncoder <: Transformer
    drop_last::Bool
end

# lazy keyword constructor:
OneHotEncoder(; drop_last::Bool=false) = OneHotEncoder(drop_last)

struct OneHotEncoderScheme <: BaseType
    features::Vector{Symbol}         # feature labels
    spawned_features::Vector{Symbol} # feature labels after one-hot encoding
    values_given_feature::Dict{Symbol,Vector{Any}}
end

function fit(transformer::OneHotEncoder, X::AbstractDataFrame, parallel, verbosity)

    features = names(X)
    values_given_feature = Dict{Symbol, Any}()

    verbosity < 1 || @info "One-hot encoding categorical features."

    for ft in features
        if !(eltype(X[ft]) <: AbstractFloat)
            values_given_feature[ft] = sort!(unique(X[ft]))
            if transformer.drop_last
                values_given_feature[ft] = values_given_feature[ft][1:(end - 1)]
            end
            if verbosity > 1
                n_values = length(keys(values_given_feature[ft]))
                @info "Spawned $n_values columns to one-hot encode $ft."
            end
        end
    end

    spawned_features = Symbol[]

    for ft in features
        if !(eltype(X[ft]) <: AbstractFloat)
            for value in values_given_feature[ft]
                subft = Symbol(string(ft,"__",value))

                # in the (rare) case subft is not a new feature label:
                while subft in features
                    subft = Symbol(string(subft,"_"))
                end

                push!(spawned_features, subft)
            end
        else
            push!(spawned_features, ft)
        end
    end

    return OneHotEncoderScheme(features, spawned_features, values_given_feature)

end

function transform(transformer::OneHotEncoder, scheme, X::AbstractDataFrame)

    Set(names(X)) == Set(scheme.features) ||
        error("Attempting to transform DataFrame with incompatible feature labels.")

    # todo: check matching eltypes

    Xout = DataFrame()
    for ft in scheme.features
        if !(eltype(X[ft]) <: AbstractFloat)
            for value in scheme.values_given_feature[ft]
                subft = Symbol(string(ft,"__",value))

                # in case subft is not a new feature name:
                while subft in scheme.features
                    subft = Symbol(string(subft,"_"))
                end

                subft_col = map(X[ft]) do x
                    x == value ? 1.0 : 0.0
                end
                Xout[subft] = convert(Array{Float64}, subft_col)
            end
        else
            Xout[ft] = X[ft]
        end
    end
    return Xout

end

## Univariate Box-Cox transformations

function standardize(v)
    map(v) do x
        (x - mean(v))/std(v)
    end
end

function midpoints(v::AbstractVector{T}) where T <: Real
    return [0.5*(v[i] + v[i + 1]) for i in 1:(length(v) -1)]
end

function normality(v)

    n  = length(v)
    v = standardize(convert(Vector{Float64}, v))

    # sort and replace with midpoints
    v = midpoints(sort!(v))

    # find the (approximate) expected value of the size (n-1)-ordered statistics for
    # standard normal:
    d = Distributions.Normal(0,1)
    w= map(collect(1:(n-1))/n) do x
        quantile(d, x)
    end

    return cor(v, w)

end

function boxcox(lambda, c, x::Real)
    c + x >= 0 || throw(DomainError)
    if lambda == 0.0
        c + x > 0 || throw(DomainError)
        return log(c + x)
    end
    return ((c + x)^lambda - 1)/lambda
end

boxcox(lambda, c, v::AbstractVector{T}) where T <: Real =
    [boxcox(lambda, c, x) for x in v]

"""
## `struct UnivariateBoxCoxTransformer`

A type for encoding a Box-Cox transformation of a single variable
taking non-negative values, with a possible preliminary shift. Such a
transformation is of the form

    x -> ((x + c)^λ - 1)/λ for λ not 0
    x -> log(x + c) for λ = 0

###  Usage

    `trf = UnivariateBoxCoxTransformer(; n=171, shift=false)`

Returns transformer that on fitting to data (see below) will try `n`
different values of the Box-Cox exponent λ (between `-0.4` and `3`) to
find an optimal value. If `shift=true` and zero values are encountered
in the data then the transformation sought includes a preliminary
positive shift by `0.2` times the data mean. If there are no zero
values, then `s.c=0`.

See also `BoxCoxScheme` a transformer for selected ordinals in a DataFrame.

"""
mutable struct UnivariateBoxCoxTransformer <: Transformer
    n::Int      # nbr values tried in optimizing exponent lambda
    shift::Bool # whether to shift data away from zero
end

# lazy keyword constructor:
UnivariateBoxCoxTransformer(; n=171, shift=false) = UnivariateBoxCoxTransformer(n, shift)

function fit(transformer::UnivariateBoxCoxTransformer, v::AbstractVector{T},
    parallel, verbosity) where T <: Real

    m = minimum(v)
    m >= 0 || error("Cannot perform a Box-Cox transformation on negative data.")

    c = 0.0 # default
    if transformer.shift
        if m == 0
            c = 0.2*mean(v)
        end
    else
        m != 0 || error("Zero value encountered in data being Box-Cox transformed.\n"*
                        "Consider calling `fit!` with `shift=true`.")
    end

    lambdas = range(-0.4, stop=3, length=transformer.n)
    scores = Float64[normality(boxcox(l, c, v)) for l in lambdas]
    lambda = lambdas[argmax(scores)]

    return  lambda, c

end

# for X scalar or vector:
transform(transformer::UnivariateBoxCoxTransformer, scheme, X) =
    boxcox(scheme..., X)

# scalar case:
function inverse_transform(transformer::UnivariateBoxCoxTransformer,
                           scheme, x::Real)
    lambda, c = scheme
    if lambda == 0
        return exp(x) - c
    else
        return (lambda*x + 1)^(1/lambda) - c
    end
end

# vector case:
function inverse_transform(transformer::UnivariateBoxCoxTransformer,
                           scheme, w::AbstractVector{T}) where T <: Real
    return [inverse_transform(transformer, scheme, y) for y in w]
end

## `BoxCoxScheme`

"""
## `struct BoxCoxTransformer`

Transformer for Box-Cox transformations on each numerical feature of a
`DataFrame` object. Here "numerical" means of eltype `T <:
AbstractFloat`.

### Method calls

To calculate the compute Box-Cox transformations of a DataFrame `X`
and transform a new DataFrame `Y` according to the same
transformations:

    julia> transformer = BoxCoxTransformer()
    julia> transformerM = TransformerMachine(transformer, X)
    julia> transform(transformerM, Y)

### Transformer parameters

Calls to the first method above may be issued with the following
keyword arguments, with defaluts as indicated:

- `shift=true`: allow data shift in case of fields taking zero values
(otherwise no transformation will be applied).

- `n=171`: number of values of exponent `lambda` to try during optimization.

## See also

`UnivariateBoxCoxTransformer`: The single variable version of the same transformer.

"""
mutable struct BoxCoxTransformer <: Transformer
    n::Int                     # number of values considered in exponent optimizations
    shift::Bool                # whether or not to shift features taking zero as value
    features::Vector{Symbol}   # features to attempt fitting a
                               # transformation to (empty means all)
end

# lazy keyword constructor:
BoxCoxTransformer(; n=171, shift = false, features=Symbol[]) =
    BoxCoxTransformer(n, shift, features)

struct BoxCoxTransformerScheme <: BaseType
    schemes::Matrix{Float64} # each col is a [lambda, c]' pair; one col per feature
    features::Vector{Symbol} # all features in the dataframe that was fit
    feature_is_transformed::Vector{Bool} # keep track of which features are transformed
end

# null scheme:
BoxCoxTransformerScheme() = BoxCoxTransformerScheme(zeros(0,0),Symbol[],Bool[])

function fit(transformer::BoxCoxTransformer, X, parallel, verbosity)

    verbosity < 1 || @info "Computing Box-Cox transformations "*
                          "on numerical features."
    # determine indices of features to be transformed
    features_to_try = (isempty(transformer.features) ? names(X) : transformer.features)
    feature_is_transformed = Array{Bool}(undef, size(X, 2))
    for j in 1:size(X, 2)
        if names(X)[j] in features_to_try && eltype(X[j]) <:
            AbstractFloat && minimum(X[j]) >= 0
            feature_is_transformed[j] = true
        else
            feature_is_transformed[j] = false
        end
    end

    # fit each of those features with best Box Cox transformation
    schemes = Array{Float64}(undef, 2, size(X,2))
    univ_transformer = UnivariateBoxCoxTransformer(shift=transformer.shift,
                                               n=transformer.n)
    verbosity < 2 ||
        @info "Box-Cox transformations: "
    for j in 1:size(X,2)
        if feature_is_transformed[j]
            if minimum(X[j]) == 0 && !transformer.shift
                verbosity < 2 ||
                    @info "  :$(names(X)[j])    "*
                            "(*not* transformed, contains zero values)"
                feature_is_transformed[j] = false
                schemes[:,j] = [0.0, 0.0]
            else
                n_values = length(unique(X[j]))
                if n_values < N_VALUES_THRESH
                    verbosity < 2 ||
                        @info "  :$(names(X)[j])    "*
                                "(*not* transformed, less than $N_VALUES_THRESH values)"
                    feature_is_transformed[j] = false
                    schemes[:,j] = [0.0, 0.0]
                else
                    lambda, c = fit(univ_transformer, collect(X[j]), true, verbosity-1)
                    if lambda in [-0.4, 3]
                        verbosity < 2 ||
                            @info "  :$(names(X)[j])    "*
                                    "(*not* transformed, lambda too extreme)"
                        feature_is_transformed[j] = false
                        schemes[:,j] = [0.0, 0.0]
                    elseif lambda == 1.0
                        verbosity < 2 ||
                            @info "  :$(names(X)[j])    "*
                                    "(*not* transformed, not skewed)"
                        feature_is_transformed[j] = false
                        schemes[:,j] = [0.0, 0.0]
                    else
                        schemes[:,j] = [lambda, c]
                        verbosity <1 ||
                            @info "  :$(names(X)[j])    lambda=$lambda  "*
                                    "shift=$c"
                    end
                end
            end
        else
            schemes[:,j] = [0.0, 0.0]
        end
    end

    if !transformer.shift && verbosity < 1
        @info "To transform non-negative features with zero values use shift=true."
    end

    return BoxCoxTransformerScheme(schemes, names(X), feature_is_transformed)

end

function transform(transformer::BoxCoxTransformer, scheme, X::AbstractDataFrame)

    names(X) == scheme.features ||
        error("Attempting to transform a data frame with  "*
              "incompatible feature labels.")

    univ_transformer = UnivariateBoxCoxTransformer(shift=transformer.shift,
                                               n=transformer.n)

    Xnew = X[1:end,:] # make a copy of X
    for j in 1:size(X, 2)
        if scheme.feature_is_transformed[j]
            try
                # extract the (lambda, c) pair:
                univ_scheme = (scheme.schemes[1,j], scheme.schemes[2,j])

                Xnew[j] = transform(univ_transformer, univ_scheme, collect(X[j]))
            catch DomainError
                @warn "Data outside of the domain of the fitted Box-Cox"*
                      " transformation scheme encountered in feature "*
                      "$(names(df)[j]). Transformed to zero."
            end
        end
    end

    return Xnew

end

## General purpose transformer for supervised learning algorithms
## needing floating point matrix input.

"""
    DataFrameToArrayTransformer(boxcox=false, shift=true,
    standardize=false, drop_last=false)

Use this transformer to prepare dataframe inputs for use with
supervised learning algorithms requiring `Matrix{T}` inputs, for
`T<:AbstractFloat`. Here `T` is forced to be `Float64`. Includes one-hot
encoding of categoricals (any feature not of eltype T<:AbstractFloat) and an
option for Box-Cox transformations. Note that all eltypes must admit
`<` and `string` methods.

### Keyword arguments:

- `boxcox`:  whether to apply Box-Cox transformations to the columns

- `shift`: do we shift away from zero in Box-Cox transformations?

- `standardize`: whether to standardize the columns

- `drop_last`: do we drop the last slot in the one-hot-encoding?

"""
mutable struct DataFrameToArrayTransformer <: Transformer
    boxcox::Bool
    shift::Bool
    standardize::Bool
    drop_last::Bool
end

# lazy keyword constructors:
DataFrameToArrayTransformer(; boxcox=false, shift=true, standardize=false,
                            drop_last=false) =
    DataFrameToArrayTransformer(boxcox, shift, standardize, drop_last)

struct Scheme_X <: BaseType
    boxcox::BoxCoxTransformerScheme
    stand::StandardizerScheme
    hot::OneHotEncoderScheme
    features::Vector{Symbol}
    spawned_features::Vector{Symbol} # ie after one-hot encoding
end

function fit(transformer::DataFrameToArrayTransformer, X::AbstractDataFrame, parallel, verbosity)

    features = names(X)

    # fit Box-Cox transformation to numerical features:
    if transformer.boxcox
        verbosity < 1 || @info "Determining Box-Cox transformation parameters."
        boxcox_transformer = BoxCoxTransformer(shift=transformer.shift)
        boxcox = fit(boxcox_transformer, X, true, verbosity - 1)
        X = transform(boxcox_transformer, boxcox, X)
    else
        boxcox = BoxCoxTransformerScheme() # null scheme
    end

    if transformer.standardize
        verbosity < 1 || @info "Determining standardization parameters."
        standardizer = Standardizer()
        stand = fit(standardizer, X, true, verbosity - 1)
        X = transform(standardizer, stand, X)
    else
        stand = StandardizerScheme() # null scheme
    end

    verbosity < 1 || @info "Determining one-hot encodings for data frame categoricals."
    hot_transformer = OneHotEncoder(drop_last=transformer.drop_last)
    hot =  fit(hot_transformer, X, true, verbosity - 1)
    spawned_features = hot.spawned_features

    return Scheme_X(boxcox, stand, hot, features, spawned_features)

end

function transform(transformer::DataFrameToArrayTransformer, scheme_X, X::AbstractDataFrame)
    issubset(Set(scheme_X.features), Set(names(X))) ||
        error("DataFrame feature incompatibility encountered.")
    X = X[scheme_X.features]

    if transformer.boxcox
        boxcox_transformer = BoxCoxTransformer(shift=transformer.shift)
        X = transform(boxcox_transformer, scheme_X.boxcox, X)
    end

    if transformer.standardize
        X = transform(Standardizer(), scheme_X.stand, X)
    end

    hot_transformer = OneHotEncoder(drop_last=transformer.drop_last)
    X = transform(hot_transformer, scheme_X.hot, X)
    return convert(Array{Float64}, X)
end

"""
## `RegressionTargetTransformer(boxcox=false, shift=true, standardize=true)::Transformer`

A general purpose transformer for target variables in regression
problems. Standardizes by default, and includes Box-Cox
transformations as an option.

### Keyword arguments:

- `boxcox`:  whether to apply Box-Cox transformations to the input patterns

- `shift`: do we shift away from zero in Box-Cox transformations?

- `standardize`: do we standardize (after any Box-Cox transformations)?

"""
mutable struct RegressionTargetTransformer <: Transformer
    boxcox::Bool # do we apply Box-Cox transforms to target (before any standarization)?
    shift::Bool # do we shift away from zero in Box-Cox transformations?
    standardize::Bool # do we standardize targets?
end

# lazy keyword constructors:
RegressionTargetTransformer(; boxcox=false, shift=true, standardize=true) =
    RegressionTargetTransformer(boxcox, shift, standardize)

struct Scheme_y <: BaseType
    boxcox::Tuple{Float64,Float64}
    standard::Tuple{Float64,Float64}
end

function fit(transformer::RegressionTargetTransformer, y, parallel, verbosity)

    if transformer.boxcox
        verbosity < 1 || @info "Computing Box-Cox transformations for target."
        boxcox_transformer = UnivariateBoxCoxTransformer(shift=transformer.shift)
        boxcox = fit(boxcox_transformer, y, true, verbosity - 1)
        y = transform(boxcox_transformer, boxcox, y)
    else
        boxcox = (0.0, 0.0) # null scheme
    end
    if transformer.standardize
        verbosity < 1 || @info "Computing target standardization."
        standard_transformer = UnivariateStandardizer()
        standard = fit(standard_transformer, y, true, verbosity - 1)
    else
        standard = (0.0, 1.0) # null scheme
    end
    return Scheme_y(boxcox, standard)
end

function transform(transformer::RegressionTargetTransformer, scheme_y, y)
    if transformer.boxcox
        boxcox_transformer = UnivariateBoxCoxTransformer(shift=transformer.shift)
        y = transform(boxcox_transformer, scheme_y.boxcox, y)
    end
    if transformer.standardize
        standard_transformer = UnivariateStandardizer()
        y = transform(standard_transformer, scheme_y.standard, y)
    end
    return y
end

function inverse_transform(transformer::RegressionTargetTransformer, scheme_y, y)
    if transformer.standardize
        standard_transformer = UnivariateStandardizer()
        y = inverse_transform(standard_transformer, scheme_y.standard, y)
    end
    if transformer.boxcox
        boxcox_transformer = UnivariateBoxCoxTransformer(shift=transformer.shift)
        y = inverse_transform(boxcox_transformer, scheme_y.boxcox, y)
    end
    return y
end

## TRANSFORMER TO CONVERT CATEGORICALS TO INTEGER-VALUED COLUMNS

mutable struct MakeCategoricalsIntTransformer <: Transformer
    initial_label::Int
    sorted::Bool
end

MakeCategoricalsIntTransformer(; initial_label=1, sorted=false) = MakeCategoricalsIntTransformer(initial_label, sorted)

struct MakeCategoricalsIntScheme <: BaseType
    categorical_features::Vector{Symbol}
    schemes::Vector{ToIntScheme}
    to_int_transformer::ToIntTransformer
end

function fit(transformer::MakeCategoricalsIntTransformer, X::AbstractDataFrame, parallel, verbosity)
    categorical_features = Symbol[]
    schemes = ToIntScheme[]
    to_int_transformer = ToIntTransformer(sorted=transformer.sorted,
                                          initial_label=transformer.initial_label)
    types = eltypes(X)
    features = names(X)
    for j in eachindex(types)
        if !(types[j] <: AbstractFloat)
            push!(categorical_features, features[j])
            push!(schemes, fit(to_int_transformer, X[j], parallel, verbosity))
        end
    end
    verbosity < 1 || @info "Input features treated as categorical: $categorical_features"
    return MakeCategoricalsIntScheme(categorical_features, schemes, to_int_transformer)
end

function transform(transformer::MakeCategoricalsIntTransformer, scheme_X, X::AbstractDataFrame)
    Xt = X[1:end,:] # make a copy of X, working even for `SubDataFrame`s
    for j in eachindex(scheme_X.categorical_features)
        ftr = scheme_X.categorical_features[j]
        Xt[ftr] = transform(scheme_X.to_int_transformer, scheme_X.schemes[j], X[ftr])
    end
    return Xt
end

## DISCRETIZATION OF CONTINUOUS VARIABLES

"""
    UnivariateDiscretizer(n_classes=512)

Returns a `Koala.Transformer` for discretizing vectors of `Real`
eltype, where `n_classes` describes the resolution of the
discretization. Transformed vectors are of eltype `Int46`. The
transformation is chosen so that the vector on which the transformer
is fit has, in transformed form, an approximately uniform distribution
of values.

### Example

    using Koala
    using KoalaTransforms
    t = UnivariateDiscretizer(n_classes=10)
    v = randn(1000)
    tM = Machine(t, v)   # fit the transformer on `v`
    w = transform(tM, v) # transform `v` according to `tM`
    StatsBase.countmap(w)

"""
mutable struct UnivariateDiscretizer <: Transformer
    n_classes::Int
end

# lazy keyword constructor:
UnivariateDiscretizer(; n_classes=512) = UnivariateDiscretizer(n_classes)

struct UnivariateDiscretizerScheme <: BaseType
    odd_quantiles::Vector{Float64}
    even_quantiles::Vector{Float64}
end

function fit(transformer::UnivariateDiscretizer, v, parallel, verbosity)
    n_classes = transformer.n_classes
    quantiles = quantile(v, Array(range(0, stop=1, length=2*n_classes+1)))
    clipped_quantiles = quantiles[2:2*n_classes] # drop 0% and 100% quantiles

    # odd_quantiles for transforming, even_quantiles used for inverse_transforming:
    odd_quantiles = clipped_quantiles[2:2:(2*n_classes-2)]
    even_quantiles = clipped_quantiles[1:2:(2*n_classes-1)]

    return UnivariateDiscretizerScheme(odd_quantiles, even_quantiles)
end

# transforming scalars:
function transform(transformer::UnivariateDiscretizer, scheme, r::Real)
    k = 1
    for level in scheme.odd_quantiles
        if r > level
            k = k + 1
        end
    end
    return k
end

# transforming vectors:
function transform(transformer::UnivariateDiscretizer, scheme,
                   v::AbstractVector{T}) where T<:Real
    return [transform(transformer, scheme, r) for r in v]
end

# scalars:
function inverse_transform(transformer::UnivariateDiscretizer, scheme, k::Int)
    n_classes = length(scheme.even_quantiles)
    if k < 1
        return scheme.even_quantiles[1]
    elseif k > n_classes
        return scheme.even_quantiles[n_classes]
    end
    return scheme.even_quantiles[k]
end

# vectors:
function inverse_transform(transformer::UnivariateDiscretizer, scheme,
                           w::AbstractVector{T}) where T<:Integer
    return [inverse_transform(transformer, scheme, k) for k in w]
end

## CONVERTING ANY INTEGER TYPE TO INT16

mutable struct IntegerToInt64Transformer <: Transformer
end

fit(transformer::IntegerToInt64Transformer, X, parallel, verbosity) = nothing

transform(transformer::IntegerToInt64Transformer, scheme, X) = Int64[X...]

## DISCRETIZING ALL COLUMNS OF A DATAFRAME

struct NominalOrdinalIntArray <: BaseType
    A::Matrix{Int}
    features::Vector{Symbol}
    is_ordinal::Vector{Bool}
end

function show(stream::IO, ::MIME"text/plain", X::NominalOrdinalIntArray)
    features_plus = Array{String}(undef, length(X.features))
    for j in eachindex(X.features)
        kind = X.is_ordinal[j] ? "ordinal" : "nominal"
        features_plus[j] = string(X.features[j], " ($kind)  ")
    end
    for str in features_plus
        print(stream, str)
    end
    println(stream)
    show(stream, MIME("text/plain"), X.A)
end

"""
    Discretizer(n_classes=512)

Returns a `Koala.Transformer` for discretizing the columns of a
`DataFrame`. Transformed objects are of type `NominalOrdinalIntArray`,
a lightweight wrapper for an `Int64` matrix whose fields `A`,
`features` and `is_ordinal` store, respectively, the integer matrix, a
vector of feature names, and a vector recording which features are
ordinal.

Columns are discretized according to the following conventions: If the
column eltype is a subtype of `AbstractFloat`, the column is
discretized using a `UnivariateDiscretizer` transformer. All remaining
eltypes are treated as nominal, with elements relabeled with integers,
starting with 1; nominal values not encountered during fitting the
transformer will throw a `KeyError` if there is an attempt to
transform them according to the fitted scheme.

"""
mutable struct Discretizer <: Transformer
    n_classes::Int
    features::Vector{Symbol} # features to be used; empty means all
end

# lazy keyword constructor:
Discretizer(; n_classes=512, features=Symbol[]) = Discretizer(n_classes, features)

struct DiscretizerScheme
    features::Vector{Symbol} # features actually used
    transformer_machines::Vector{TransformerMachine}
    is_ordinal::Vector{Bool}
end

function fit(transformer::Discretizer, X::AbstractDataFrame, parallel, verbosity)

    to_int = ToIntTransformer(sorted=true)
    discrete = UnivariateDiscretizer()

    if isempty(transformer.features)
        features = names(X)
    else
        features = transformer.features
    end

    transformer_machines = Array{TransformerMachine}(undef, length(features))
    is_ordinal = Array{Int}(undef, length(features))

    j = 1
    for ftr in features
        if eltype(X[ftr]) <: AbstractFloat
            is_ordinal[j] = true
            transformer_machines[j] = Machine(discrete, X[ftr];
                parallel=false, verbosity=verbosity - 1)
        else
            is_ordinal[j] = false
            transformer_machines[j] = Machine(to_int, X[ftr];
            parallel=false, verbosity=verbosity - 1)
        end
        j +=1
    end

    return DiscretizerScheme(features, transformer_machines, is_ordinal)

end

function transform(transformer::Discretizer, scheme, X)

    issubset(Set(scheme.features), Set(names(X))) ||
        error("Provided DataFrame with incompatible features.")

    n_features = length(scheme.features)
    A = Array{Int}(undef, size(X, 1), n_features)

    for j in 1:n_features
        A[:,j] = transform(scheme.transformer_machines[j],
                           X[:,scheme.features[j]])
    end

    return NominalOrdinalIntArray(A, scheme.features, scheme.is_ordinal)

end

end # module


using DbgMacro

function foo()
    m = [1 2; 3 4]
    @dbg 1+2 "Hello" m
    42
end

expected_foo_err = """
Main.TestDbg:test_dbg.jl:5  1 + 2 = 3
Main.TestDbg:test_dbg.jl:5  "Hello" = "Hello"
Main.TestDbg:test_dbg.jl:5  m = [1 2; 3 4]
"""

using Suppressor: @capture_err

err = @capture_err global result = foo()
@test result == 42
@test err == expected_foo_err


"
Contains all plotting functions for displaying information collected
in module `BoltzmannMachines`. Most important function is `plotevaluation`.
"
#module BoltzmannMachinesPlots

#import scDBM

#using Compose
#using DataFrames
#using Gadfly
#using Statistics

#export plotevaluation, crossvalidationcurve, scatterhidden

function checkdata(plotdata)
   if size(plotdata, 1) == 0
      error("No data for this evaluation in monitor")
   end
end

function getvalue(monitor::Monitor, evaluation::String, epoch::Int)
   itemidx = findfirst((item -> item.evaluation == evaluation && item.epoch == epoch), monitor)
   monitor[itemidx].value
end

function extractevaluationdata(monitor::Monitor, evaluation::String)
   evaluationidxs = findall(item -> item.evaluation == evaluation, monitor)
   epochs = map(i -> monitor[i].epoch, evaluationidxs)
   values = map(i -> monitor[i].value, evaluationidxs)
   datasetnames = map(i -> monitor[i].datasetname, evaluationidxs)
   plotdata = DataFrame(epoch = epochs, value = values, datasetname = datasetnames)
end

function extractaisdata(monitor::Monitor, evaluation::String, sdrange::Float64)

   plotdata = extractevaluationdata(monitor, evaluation)

   # Now integrate the information about the precision of AIS in the dataframe.
   # It is common for all datasets.
   epochs = unique(plotdata[:epoch])

   if sdrange != 0.0

      plotdata[:ymin] = copy(plotdata[:value])
      plotdata[:ymax] = copy(plotdata[:value])

      # Use standard deviation of log partition function estimator to
      # plot ribbon around log probs.
      for epoch in epochs
         sd = getvalue(monitor, monitoraisstandarddeviation, epoch)
         logr = getvalue(monitor, monitoraislogr, epoch)

         # log(Z) is subtracted from logproblowerbound, so overstimating log(Z)
         # means underestimating the log probability
         bottom, top = aisprecision(logr, sd, sdrange)
         plotdata[:ymin][plotdata[:epoch] .== epoch] .-= top
         plotdata[:ymax][plotdata[:epoch] .== epoch] .-= bottom
      end
   end

   # avoid plotting error if NaNs were introduced when calculating ymin/ymax
   plotdata[:ymin][isnan.(plotdata[:ymin])] .= 0
   plotdata[:ymax][isnan.(plotdata[:ymax])] .= 0

   plotdata
end

"
Plots the information about the estimated lower bound of the log probability
that has been gathered while training a BMs.
"
function plotestimatedprob(monitor::Monitor, evaluationkey::String;
      sdrange::Float64 = 0.0)

   title = plottitledict[evaluationkey]
   plotdata = extractaisdata(monitor, evaluationkey, sdrange)
   checkdata(plotdata)

   # remove ribbon for values that are too uncertain and warn about it
   badribbon = false
   if sdrange != 0
      valrange = abs(maximum(plotdata[:, :value]) - minimum(plotdata[:, :value]))
      for row in eachrow(plotdata)
         if row[:ymin] < row[:value] - 5 * valrange
            row[:ymin] = row[:value]
            badribbon = true
         end
         if row[:ymax] > row[:value] + 5 * valrange
            row[:ymax] = row[:value]
            badribbon = true
         end
      end
   end
   if badribbon
      @warn "Too much uncertainty: Ribbon (partially) not displayed."
   end

   if sdrange != 0
      plot(plotdata, x = "epoch", y = "value", ymin = "ymin", ymax = "ymax",
            color = "datasetname",
            Geom.line, Geom.ribbon,
            Guide.xlabel("Epoch"), Guide.ylabel("Value"),
            Guide.colorkey(title = "Data set"),
            Guide.title(title),
            Theme(major_label_font_size=25pt, minor_label_font_size=20pt))
   else
      plot(plotdata, x = "epoch", y = "value", color = "datasetname",
            Guide.xlabel("Epoch"), Guide.ylabel("Value"),
            Guide.colorkey(title = "Data set"),
            Geom.line, Guide.title(title),
            Theme(major_label_font_size=25pt, minor_label_font_size=20pt))
   end
end

plottitledict = Dict(
      monitorreconstructionerror => "Mean reconstruction error",
      monitorlogproblowerbound => "Average lower bound of log probability",
      monitorloglikelihood => "Average log-likelihood",
      monitormeandiff => "L²-difference between means \nof generated and original data",
      monitorexactloglikelihood => "Exact log-likelihood",
      monitorweightsnorm => "L²-norm of weights",
      monitorsd => "Standard deviation parameters of visible units",
      monitorcordiff => "L²-difference between correlation matrices \nof generated and original data",
      monitorfreeenergy => "Free energy")

"""
    plotevaluation(monitor; ...)
    plotevaluation(monitor, evaluationkey; ...)
Plots a curve that shows the values of the evaluation contained in the `monitor`
and specified by the `evaluationkey` over the course of the training epochs.
If no evaluationkey is specified, the evaluation type of the first monitor
element is used.

Optional keyword argument `sdrange`:
For evaluations with keys `BoltzmannMachines.monitorloglikelihood` and
`BoltzmannMachines.monitorlogproblowerbound`,
there is additional information about the standard
deviation of the estimator. With the parameter `sdrange`, it is possible
to display this information as a ribbon around the curve. The ribbon indicates
the area around the curve that contains the values that deviate at maximum
`sdrange` times the standard deviation from the estimator.
Default value for `sdrange` is 2.0.
"""
function plotevaluation(monitor::Monitor,
      evaluationkey::String = monitor[1].evaluation;
      sdrange::Float64 = 2.0, changetitle::Function = identity)

   if evaluationkey in [monitorloglikelihood; monitorlogproblowerbound]
      return plotestimatedprob(monitor, evaluationkey; sdrange = sdrange)
   end

   # Otherwise, it is a simple line plot.
   title = changetitle(get(plottitledict, evaluationkey, evaluationkey))
   plotdata = extractevaluationdata(monitor, evaluationkey)
   checkdata(plotdata)
   plot(plotdata, x ="epoch", y = "value", color = "datasetname",
         Guide.xlabel("Epoch"), Guide.ylabel("Value"),
         Guide.colorkey(title = "Data set"),
         Geom.line, Guide.title(title),
         Theme(major_label_font_size=25pt, minor_label_font_size=20pt))
end

function plotcurvebundles(x::Matrix{Float64};
      nlabelvars::Int =
            sum(mapslices(col -> all(((col .== 1.0) .| (col .== 0.0))), x, dims = 1))
      )

   if nlabelvars == 0
      plot(x, x = Col.index, y = Col.value, color = Row.index, Geom.line,
            Guide.colorkey(title = "Sample"), Guide.xlabel("Variable index"),
            Guide.ylabel("Value"), Scale.x_discrete, Scale.color_discrete)
   else
      powersoftwo = 2 .^ (0:(nlabelvars-1))
      labels = vec(mapslices(
            row -> sum(powersoftwo[convert(Vector{Bool}, row)]) + 1,
            x[:, 1:nlabelvars], dims = 2))

      nlabels = maximum(labels)
      labelcolors = Scale.default_discrete_colors(nlabels)
      plotdata = convert(DataFrame, x[:, (nlabelvars + 1):end])
      names!(plotdata, map(Symbol, 1:ncol(plotdata)))
      plotdata[:label] = labels
      plotlayerdfs = map(i -> DataFrames.melt(plotdata[i, :], [:label]),
            1:nrow(plotdata))

      plotlayers = map(
            d -> layer(d, x = "variable", y = "value", Geom.line,
                  Theme(default_color = labelcolors[d[1,:label]])),
            plotlayerdfs)
      plot(plotlayers...,
            Guide.xlabel("Variable index"),
            Guide.ylabel("Value"))
   end
end

"""
    scatterhidden(bm, x; ...)
    scatterhidden(h; ...)
Creates a scatter plot of the logarithmized activation potential of
hidden nodes, similar to a PCA plot.
The activation is either induced by the dataset `x` in the Boltzmann machine `bm`
or it is directly specified as matrix `h`.

# Optional keyword arguments:
* `hiddennodes`: Tuple of integers, default `(1,2)`, selecting the first two
  nodes of the (last) hidden layer.
* `labels`: a vector containing string labels for each of the data points
"""
function scatterhidden(bm::AbstractBM, x::Matrix{Float64};
      hiddennodes::Tuple{Int,Int} = (1,2),
      labels = Vector{String}())

   function hiddenactivations(rbm::AbstractRBM, x)
      logit.(hiddenpotential(rbm, x))
   end

   function hiddenactivations(rbm::AbstractXBernoulliRBM, x)
      hiddeninput(rbm, x)
   end

   function hiddenactivations(dbm::MultimodalDBM, x)
      logit.(meanfield(dbm, x)[end])
   end

   hh = hiddenactivations(bm, x)
   scatterhidden(hh, hiddennodes = hiddennodes, labels = labels)
end

function scatterhidden(hh::Matrix{Float64};
   hiddennodes::Tuple{Int,Int} = (1,2),
   labels = Vector{String}())

   plottingargs = [
      Guide.xlabel("Node " * string(hiddennodes[1]));
      Guide.ylabel("Node " * string(hiddennodes[2]));
      Theme(lowlight_color = c -> Gadfly.RGBA{Float32}(c.r, c.g, c.b, 0.01));
      Geom.point
   ]

   if !isempty(labels)
      nsamples = size(hh, 1)
      plotdata = DataFrame(x = hh[:,1], y = hh[:,2])
      if length(labels) == nsamples
         plotdata[:label] = labels
      else
         error("Not enough labels ($(length(labels))) for samples ($nsamples)")
      end

      plot(plotdata, x = "x", y = "y", color = "label", plottingargs...)
   else
      plot(x = hh[:, hiddennodes[1]], y = hh[:, hiddennodes[2]],
            plottingargs...)
   end
end

function crossvalidationcurve(monitor::Monitor,
         evaluation::String = "")

   if !isempty(evaluation)
      monitor = filter(r -> r.evaluation == evaluation, monitor)
      if isempty(monitor)
         error("No data for specified evaluation")
      end
   else
      evaluations = map(r -> r.evaluation, monitor)
      if length(unique(evaluations)) > 1
         error("Please specify the evaluation via argument 'evaluation'.")
      end
   end

   boxplotdata = DataFrame(
         epoch = map(r -> r.epoch, monitor),
         score = map(r -> r.value, monitor))
   meanplotdata = aggregate(boxplotdata, :epoch, mean)
   plot(
         layer(meanplotdata, x = "epoch", y = "score_mean", Geom.line,
               Theme(default_color = parse(Compose.Colorant, "green"))),
         layer(boxplotdata, x = "epoch", y = "score", Geom.boxplot),
         Guide.xlabel("Epoch"), Guide.ylabel("Score"))
end


module RandomCircuit

using CircuitViz

g = Graph()

iter = 1 : 100
for k in iter
    add_vertex!(g)
end

prod = Iterators.product(iter, iter)
for (k, v) in prod
    if rand() > 0.99
        @info (k, v)
        add_edge!(g, k, v)
    end
end

compile!(g)

end # module


export map, mapvec, map!, map!r, map2!, mapmap, mapmapvec, mapi, mapveci, work, worki
export map2, map3, map4, map5
export mapvec2, mapvec3, mapvec4, mapvec5
export work2, work3, work4, work5
export mapprogress, mapkeys, mapvalues
export share, unshare
export shmap, shmap!, shmap!r, shmap2!, shwork
export pmap, pmap!, pmap!r, pmap2!, pwork
export pmapvec
export lmap, lmap!, lmap!r, lmap2!, lwork
export lmapvec
export hmap, hmap!, hmap!r, hmap2!, hwork
export amap, amap2, amapvec2
export table, ptable, ltable, htable, shtable, tableany, ptableany, ltableany, htableany, shtableany
export sort, sortrev, sortpermrev, uniq, filter, select, reject
export groupdict, groupby
export minelem, maxelem, extremaelem
export isany, areall
export tee
export *
export typed
export call, apply
export fold

typealias Callable Union{Function, Type}

import Base.sort
sort(a, f::Callable; kargs...) = part(a, sortperm(vec(map(a, f)); kargs...))
sort(a, key; kargs...) = part(a, sortperm(vec(extract(a, key)); kargs...))
sortrev(a) = sort(a; rev = true)
sortpermrev(a) = sortperm(a; rev = true)
sortrev(a, f) = sort(a,f; rev = true)

function minelem(a, f::Callable)
    b = map(a,f)
    at(a, findfirst(b .== minimum(b)))
end

function maxelem(a, f::Callable)
    b = map(a,f)
    at(a, findfirst(b .== maximum(b)))
end

function extremaelem(a, f::Callable)
    b = map(a,f)
    mi = minimum(b)
    ma = maximum(b)
    [at(a, findfirst(b .== mi)), at(a, findfirst(b .== ma))]
end

function uniq(a,f = id)
    d = Dict{Any,Int}()
    h = f == id ? a : @p mapvec a f
    for i = len(h):-1:1
        d[at(h,i)] = i
    end
    @p vec d | sort snd | map snd | part a _
end

#######################################
## map, pmap

mapvec(a, f::Callable) = [f(trytoview(a,i)) for i in 1:len(a)]
mapvec2(a, b, f::Callable) = [f(trytoview(a,i),trytoview(b,i)) for i in 1:len(a)]
mapvec3(a, b, c, f::Callable) = [f(trytoview(a,i),trytoview(b,i),trytoview(c,i)) for i in 1:len(a)]
mapvec4(a, b, c, d, f::Callable) = [f(trytoview(a,i),trytoview(b,i),trytoview(c,i),trytoview(d,i)) for i in 1:len(a)]
mapvec5(a, b, c, d, e, f::Callable) = [f(trytoview(a,i),trytoview(b,i),trytoview(c,i),trytoview(d,i),trytoview(e,i)) for i in 1:len(a)]

map2(a, b, f::Callable) = flatten(mapvec2(a,b,f))
map3(a, b, c, f::Callable) = flatten(mapvec3(a,b,c,f))
map4(a, b, c, d, f::Callable) = flatten(mapvec4(a,b,c,d,f))
map5(a, b, c, d, e, f::Callable) = flatten(mapvec5(a,b,c,d,e,f))

mapi(a, f::Callable ) = map2(a, 1:len(a), f)
mapveci(a, f::Callable ) = mapvec2(a, 1:len(a), f)

import Base.map
map(a, f::Callable) = map(unstack(1:len(a)), i->f(at(a,i)))
map(a::AbstractString, f::Callable) = flatten(map(unstack(a),f))
function map{T,N}(a::AbstractArray{T,N}, f::Callable)
    isempty(a) && return []

    r1 = f(fst(a))
    r = arraylike(r1, len(a), a)

    setat!(r,1,r1)
    map_(f,a,r)
    return r
end

@inline function map_{Tin,Nin,Tout,Nout}(f::Callable,a::AbstractArray{Tin,Nin},r::Array{Tout,Nout})
    for i = 2:len(a)
        b = f(at(a,i))
        setat!(r,i,b)
    end
end

function map(a::Dict, f::Callable; kargs...)
    isempty(a) && return a
    makeentry(a::Void) = []
    makeentry(a::Union{Tuple,Pair}) = [a]
    makeentry{T<:Union{Tuple,Pair}}(a::Array{T}) = a
    makeentry(a) = error("FunctionalData: map(::Dict), got entry of type $(typeof(a)), not one of Void, Tuple{Symbol,Any}, Array{Tuple}")

    r = @p vec a | map f | map makeentry | flatten
    d = Dict()
    for x in r
        d[fst(x)] = snd(x)
    end
    d
end

mapkeys(a::Dict, f) = map(a, x -> (f(fst(x)),snd(x)))
mapvalues(a::Dict, f) = map(a, x ->(fst(x),f(snd(x))))
mapmap(a::Dict, f) = [f(v) for (k,v) in a]

function mapmap(a, f)
    isempty(a) && return Any[]
    g(x) = map(x,f)
    map(a, g)
end

function mapmapvec(a, f)
    isempty(a) && return Any[]
    g(x) = mapvec(x,f)
    map(a, g)
end

function map{T<:Real}(a::DenseArray{T,1},f::Callable)
    isempty(a) && return []
    r1 = f(fst(a))
    r = arraylike(r1, len(a), a)
    if isviewable(r)
        rv = view(r,1)
        rv[:] = r1
        next!(rv)
        for i = 2:len(a)
            rv[:] = f(a[i])
            next!(rv)
        end
        r
    else
        setat!(r, 1, r1)
        map_(f,a,r)
        r
    end
end

function map{T<:Real,N}(a::DenseArray{T,N},f::Callable)
    isempty(a) && return []
    v = view(a,1)
    r1 = f(v)
    r = arraylike(r1, len(a), a)
    if isviewable(r)
        rv = view(r,1)
        rv[:] = r1
        next!(v)
        next!(rv)
        for i = 2:len(a)
            rv[:] = f(v)
            next!(v)
            next!(rv)
        end
        r
    else
        setat!(r, 1, r1)
        map_(f,a,r)
        r
    end
end

function map2!{T<:Real,N}(a::DenseArray{T,N}, f1::Callable, f2::Callable)
    isempty(a) && return []
    v = view(a,1)
    r1 = f1(v)
    r = arraylike(r1, len(a), a)
    rv = view(r,1)
    rv[:] = r1
    map2!(view(a, 2:len(a)), view(r, 2:len(a)), f2)
    r
end

function map2!{T<:Real,N,T2<:Real,M}(a::DenseArray{T,N}, r::DenseArray{T2,M}, f::Callable)
    isempty(a) && return []
    v = view(a,1)
    rv = view(r,1)
    for i = 1:len(a)
        f(rv, v)
        next!(v)
        next!(rv)
    end
    r
end

import Base.map!
function map!r{T<:Real,N}(a::DenseArray{T,N},f::Callable)
    isempty(a) && return a
    v = view(a,1)
    for i = 1:len(a)
        v[:] = f(v)
        next!(v)
    end
    a
end

function map!{T<:Real,N}(a::DenseArray{T,N},f::Callable)
    isempty(a) && return a
    v = view(a,1)
    for i = 1:len(a)
        f(v)
        next!(v)
    end
    a
end

work(a,f::Callable) = for i in 1:len(a) f(at(a,i)) end
work(a::Dict,f::Callable) = map(vec(a),(k,v)->(f;nothing))
function work{T<:Real,N}(a::DenseArray{T,N},f::Callable)
    len(a)==0 && return
    v = view(a,1)
    for i = 1:len(a)
        f(v)
        next!(v)
    end
end
lwork(a, f) = (g(x) = (f(x);nothing); lmap(a, g); nothing)
pwork(a, f) = (g(x) = (f(x);nothing); pmap(a, g); nothing)
hwork(a, f) = (g(x) = (f(x);nothing); hmap(a, g); nothing)
shwork(a, f) = (g(x) = (f(x);nothing); shmap(a, g); nothing)
workwork(a, f) = (g(x) = (f(x);nothing); mapmap(a, g); nothing)

work2(a, b, f::Callable) = [(f(at(a,i),at(b,i)); nothing) for i in 1:len(a)]
work3(a, b, c, f::Callable) = [(f(at(a,i),at(b,i),at(c,i)); nothing) for i in 1:len(a)]
work4(a, b, c, d, f::Callable) = [(f(at(a,i),at(b,i),at(c,i),at(d,i)); nothing) for i in 1:len(a)]
work5(a, b, c, d, e_, f::Callable) = [(f(at(a,i),at(b,i),at(c,i),at(d,i),at(e_,i)); nothing) for i in 1:len(a)]
worki(a, f) = mapi(a,(x,i)->(f(x,i);nothing))

share{T<:Real,N}(a::DenseArray{T,N}) = convert(SharedArray, a)
share(a::SharedArray) = a
unshare(a::SharedArray) = sdata(a)

function loopovershared(r::View, a::View, f::Callable)
    v = view(a,1)
    rv = view(r,1)
    for i = 1:len(a)
        rv[:] = f(v)
        next!(v)
        next!(rv)
    end
end

function loopovershared!(a::View, f::Callable)
    v = view(a, 1)
    for i = 1:len(a)
        f(v)
        next!(v)
    end
end

function loopovershared!r(a::View, f::Callable)
    v = view(a, 1)
    for i = 1:len(a)
        v[:] = f(v)
        next!(v)
    end
end

function loopovershared2!(r::View, a::View, f::Callable)
    v = view(a,1)
    rv = view(r,1)
    for i = 1:len(a)
        f(rv, v)
        next!(v)
        next!(rv)
    end
end

function shsetup(a::SharedArray; withfirst = false)
    pids = procs(a)
    if length(pids) > 1
        pids = pids[pids .!= 1]
    end
    ind = (withfirst ? 1 : 2):len(a)
    n = min(length(pids), len(ind))
    inds = partition(ind, n)
    pids, inds, n
end

shmap{T<:Real,N}(a::DenseArray{T,N}, f::Callable) = shmap(share(a), f)
function shmap{T<:Real,N}(a::SharedArray{T,N}, f::Callable)
    pids, inds, n = shsetup(a)

    r1 = f(view(a,1))
    r = sharraylike(r1, len(a))
    rv = view(r,1)
    rv[:] = r1

    @sync for i in 1:n
        @spawnat pids[i] loopovershared(view(r, inds[i]), view(a, inds[i]), f)
    end
    r
end

shmap!{T<:Real,N}(a::DenseArray{T,N}, f::Callable) = shmap!(share(a), f)
function shmap!{T<:Real,N}(a::SharedArray{T,N}, f::Callable)
    pids, inds, n = shsetup(a; withfirst = true)

    @sync for i in 1:n
        @spawnat pids[i] loopovershared!(view(a, inds[i]), f)
    end
    a
end

shmap!r{T<:Real,N}(a::DenseArray{T,N}, f::Callable) = shmap!r(share(a), f)
function shmap!r{T<:Real,N}(a::SharedArray{T,N}, f::Callable)
    pids, inds, n = shsetup(a; withfirst = true)

    @sync for i in 1:n
        @spawnat pids[i] loopovershared!r(view(a, inds[i]), f)
    end
    a
end

shmap2!{T<:Real, N}(a::DenseArray{T,N}, f1::Callable, f2::Callable) = shmap2!(share(a), f1, f2)
function shmap2!{T<:Real, N}(a::SharedArray{T,N}, f1::Callable, f2::Callable)
    r1 = f1(view(a,1))
    r = sharraylike(r1, len(a))
    rv = view(r,1)
    rv[:] = r1
    shmap2!(a, r, f2, withfirst = false)
    r
end

shmap2!{T<:Real,N, T2<:Real, M}(a::DenseArray{T,N}, r::SharedArray{T2,M}, f::Callable) = shmap2!(share(a), r, f)
function shmap2!{T<:Real,N, T2<:Real, M}(a::SharedArray{T,N}, r::SharedArray{T2,M}, f::Callable; withfirst = true)
    pids, inds, n = shsetup(a, withfirst = withfirst)
    @sync for i in 1:n
        @spawnat pids[i] loopovershared2!(view(r, inds[i]), view(a, inds[i]), f)
    end
    r
end

###############################
#  pmap

import Base.pmap
mapper(a, f) = map(a,f)
mappervec(a, f) = mapvec(a,f)
mapper!(a, f) = map!(a,f)
mapper!r(a, f) = map!r(a,f)
mapper2!(a, f1::Callable, f2) = map2!(a,f1,f2)
mapper2!(a, r, f) = map2!(a,r,f)
pmap(a, f::Callable; kargs...) = pmap_internal(mapper, a, f; kargs...)
pmapvec(a, f::Callable; kargs...) = pmap_internal(mappervec, a, f; kargs...)
pmap!(a, f; kargs...) = pmap_internal(mapper!, a, f; kargs...)
pmap!r(a, f; kargs...) = pmap_internal(mapper!r, a, f; kargs...)
pmap2!r(a, f1::Callable, f2::Callable; kargs...) = pmap_internal2!(mapper2!, a, f1, f2; kargs...)
pmap2!r(a, r, f::Callable; kargs...) = pmap_internal2!(mapper2!, a, r, f; kargs...)

workerpool(pids) =  VERSION < v"0.5-" ? pids : WorkerPool(pids)

function pmapsetup(a; pids = workers())
    if length(pids) > 1
        pids = pids[pids .!= 1]
    end
    n = min(length(pids)*10, len(a))
    inds = partition(1:len(a), n)
    pids, inds, n
end

function pmapparts(a, inds, n)
    if isa(a, DenseArray) && eltype(a)<:Real
        parts = [view(a, inds[i]) for i in 1:n]
    else
        parts = [part(a, inds[i]) for i in 1:n]
    end
end

function pmap_exec(g, a; nworkers = typemax(Int), vec = false, kargs...)
    pids, inds, n = pmapsetup(a; kargs...)
    parts = pmapparts(a, inds, n)
    pids = workerpool(take(pids, nworkers))
    if VERSION < v"0.5-"
        r = Base.pmap(x->(yield();g(x)), parts, pids = pids)
    else
        r = Base.pmap(pids, x->(yield();g(x)), parts)
    end
    for x in r
        isa(x,RemoteException) && rethrow(x)
    end
    flatten(r)
end

function pmap_internal(mapf::Callable, a, f::Callable; pids = workers(), kargs...)
    g(a) = mapf(a,f)
    pmap_exec(g, a; pids = pids, kargs...)
end

function pmap_internal2!(mapf::Callable, a, f1::Callable, f2::Callable; pids = workers(), kargs...)
    g(a) = mapf(a,f1,f2)
    pmap_exec(g, a; pids = pids, kargs...)
end

function pmap_internal2!(mapf::Callable, a, r, f::Callable; nworkers = typemax(Int), kargs...)
    g(a) = mapf(fst(a), snd(a), f)
    pids, inds, n = pmapsetup(a; kargs...)
    partsa = pmapparts(a, inds, n)
    partsr = pmapparts(r, inds, n)
    parts = zip(partsa, partsr)
    pids = workerpool(take(pids,nworkers))
    @show pids
    r = Base.pmap(g, parts, pids = pids)
    flatten(r)
end

export localworkers
localworkers(pid = myid()) = (r = sort(procs(pid)); len(r) > 1 ? r[2:end] : r)

export hostpids
hostpids() = @p map unstack(workers()) procs | map sort | unique | map x->x[1]==1 && len(x)>1 ? x[2] : x[1]

export hmap, hmap!, hmap!r, hmap2!r
hmap(a, f; kargs...) = pmap_internal(mapper, a, f; pids = hostpids(), kargs...)
hmap!(a, f; kargs...) = pmap_internal(mapper!, a, f; pids = hostpids(), kargs...)
hmap!r(a, f; kargs...) = pmap_internal(mapper!r, a, f; pids = hostpids(), kargs...)
hmap2!r(a, f1::Callable, f2::Callable; kargs...) = pmap_internal2!(mapper2!, a, f1, f2; pids = hostpids(), kargs...)
hmap2!r(a, r, f::Callable; kargs...) = pmap_internal2!(mapper2!, a, r, f; pids = hostpids(), kargs...)

lmap(a, f; kargs...) = pmap_internal(mapper, a, f; pids = localworkers(), kargs...)
lmapvec(a, f; kargs...) = pmap_internal(mappervec, a, f; pids = localworkers(), kargs...)
lmap!(a, f; kargs...) = pmap_internal(mapper!, a, f; pids = localworkers(), kargs...)
lmap!r(a, f; kargs...) = pmap_internal(mapper!r, a, f; pids = localworkers(), kargs...)
lmap2!r(a, f1::Callable, f2::Callable; kargs...) = pmap_internal2!(mapper2!, a, f1, f2; pids = localworkers(), kargs...)
lmap2!r(a, r, f::Callable; kargs...) = pmap_internal2!(mapper2!, a, r, f; pids = localworkers(), kargs...)

amapvec(a,f; kargs...) = amap(a,f; n = 10, mapper = mapvec)
function amap(a,f; n = 10, mapper = map)
    n = min(len(a),n)
    a = @p partition a n
    r = Array{Any}(n)
    @sync for i in 1:n
        @async r[i] = mapper(a[i], f)
    end
    flatten(r)
end
amap2(a,b,f; kargs...) = amap(czip(unstack(a),unstack(b)), x->f(fst(x),snd(x)); kargs...)
amapvec2(a,b,f; kargs...) = amapvec(czip(unstack(a),unstack(b)), x->f(fst(x),snd(x)); kargs...)

table(a...; kargs...) = table_internal(map, a[end], a[1:end-1]...; flat = true, kargs...)
ptable(a...; kargs...) = table_internal(pmap, a[end], a[1:end-1]...; flat = true, kargs...)
ltable(a...; kargs...) = table_internal(lmap, a[end], a[1:end-1]...; flat = true, kargs...)
htable(a...; kargs...) = table_internal(hmap, a[end], a[1:end-1]...; flat = true, kargs...)

table(f::Callable, a...; kargs...) = table_internal(map, f, a...; flat = true, kargs...)
ptable(f::Callable, a...; kargs...) = table_internal(pmap, f, a...; flat = true, kargs...)
ltable(f::Callable, a...; kargs...) = table_internal(lmap, f, a...; flat = true, kargs...)
htable(f::Callable, a...; kargs...) = table_internal(hmap, f, a...; flat = true, kargs...)

tableany(a...; kargs...) = table_internal(map, a[end], a[1:end-1]...; flat = false, kargs...)
ptableany(a...; kargs...) = table_internal(pmap, a[end], a[1:end-1]...; flat = false, kargs...)
ltableany(a...; kargs...) = table_internal(lmap, a[end], a[1:end-1]...; flat = false, kargs...)
htableany(a...; kargs...) = table_internal(hmap, a[end], a[1:end-1]...; flat = false, kargs...)

tableany(f::Callable, a...; kargs...) = table_internal(map, f, a...; flat = false, kargs...)
ptableany(f::Callable, a...; kargs...) = table_internal(pmap, f, a...; flat = false, kargs...)
ltableany(f::Callable, a...; kargs...) = table_internal(lmap, f, a...; flat = false, kargs...)
htableany(f::Callable, a...; kargs...) = table_internal(hmap, f, a...; flat = false, kargs...)

function table_internal(mapf, f, args...; flat = true, kargs...)
    a = [isa(x,Range) ? collect(x) : x for x in args]
    s = @p col [len(x) for x in a]
    if any(s.==0)
        error("FunctionalData.xtable: empty arguments, lengths of the $(len(args)) arguments are $(vec(s))")
    end
    S = tuple(s...)
    getarg(sub) = [at(a[i],sub[i]) for i in 1:length(a)]
    args = [getarg(ind2sub(S,x)) for x in 1:prod(s)]
    g(x) = f(x...)
    r = mapf(args,g; kargs...)

    if flat
        if length(fst(r))==1
            news = s
        else
            s1 = siz(fst(r))
            s1 = s1[end] == 1 ? s1[1:end-1] : s1
            news = vcat(s1,s)
        end
        @p flatten r | reshape news
    else
        @p reshape r s
    end
end

areall(a, f::Callable) = !any(a, not*f)

function isany(a, f::Callable)
    for i in 1:len(a)
        f(at(a,i)) && return true
    end
    return false
end

tee(a,f) = (f(a);a)

import Base.*
*(f::Callable, g::Callable) = (a...) -> f(g(a...))

typed{N}(a::Array{Any,N}) = isempty(a) ? [] : convert(Array{typeof(a[1]),N}, a)
typed(a) = a

import Base.filter
filter(r::Regex, f2::Callable) = error()
filter(f1::Callable, f2::Callable) = error()
function filter(a, f::Callable)
    ind = vec(typed(map(a,f)))
    isempty(ind) ? [] : part(a, ind)
end

import Base.select
select(a, f::Callable) = filter(a, f)
reject(a, f::Callable) = select(a, not*f)

function groupdict(a,f::Function = id)
    d = Dict()
    inds = @p map a f
    for i in 1:len(a)
        ind = @p at inds i
        d[ind] = push!(get(d,ind,[]), at(a,i))
    end
    mapvalues(d,flatten)
end

function groupby(a,f = id)
    r = groupdict(a,f)
    ks = collect(keys(r))
    try
        ks = sort(ks)
    end
    values(r,ks)
end

import Base.apply
apply(f::Callable, f2::Callable) = error("undefined")
apply(f::Callable, args...) = f(args...)
apply(args, f::Callable) = f(args)

function fold(a, f::Callable)
    if isempty(a)
        return a
    end
    if len(a) == 1
        return fst(a)
    end
    r = f(fst(a), snd(a))
    for i = 3:len(a)
        r = f(r,at(a,i))
    end
    r
end


"""
$(TYPEDEF)
$(TYPEDFIELDS)

"""
struct Float <: AbstractDelayed
    arg::AbstractDelayed
end

Base.float(arg::AbstractDelayed) = Float(arg)


@testset "Physical Constants" begin
    x = rand(keys(Fermi.PhysicalConstants.atom_num))
    @test Fermi.PhysicalConstants.atomic_number(x) == Fermi.PhysicalConstants.atom_num[x]
    @test_throws KeyError Fermi.PhysicalConstants.atomic_number("Invalid")

    @test Fermi.PhysicalConstants.num_core_electrons("Ca") == 18
end


using ImageInpainting
using ColorTypes
using TestImages
using Plots, VisualRegressionTests
using Test, Pkg

# workaround GR warnings
ENV["GKSwstype"] = "100"

# environment settings
islinux = Sys.islinux()
istravis = "TRAVIS" ∈ keys(ENV)
datadir = joinpath(@__DIR__,"data")
visualtests = !istravis || (istravis && islinux)
if !istravis
  Pkg.add("Gtk")
  using Gtk
end

# helper functions
function plot_before_after(before, mask, after)
  gr(size=(800,400),yflip=true,colorbar=false,ticks=false)
  B = copy(before); B[mask] .= NaN
  p1 = heatmap(B,title="before")
  p2 = heatmap(after,title="after")
  plot(p1, p2, aspect_ratio=:equal)
end

# list of tests
testfiles = [
  "criminisi.jl"
]

@testset "ImageInpainting.jl" begin
  for testfile in testfiles
    include(testfile)
  end
end