_add_sum(x, y) = Base.add_sum(x, y)
_add_sum(x, ::Missing) = x
_add_sum(::Missing, x) = x
_add_sum(::Missing, ::Missing) = missing
_mul_prod(x, y) = Base.mul_prod(x, y)
_mul_prod(x, ::Missing) = x
_mul_prod(::Missing, x) = x
_mul_prod(::Missing, ::Missing) = missing
_min_fun(x, y) = min(x, y)
_min_fun(x, ::Missing) = x
_min_fun(::Missing, y) = y
_min_fun(::Missing, ::Missing) = missing
_max_fun(x, y) = max(x, y)
_max_fun(x, ::Missing) = x
_max_fun(::Missing, y) = y
_max_fun(::Missing, ::Missing) = missing
_bool(f) = x->f(x)::Bool

const __NCORES = Threads.nthreads()

struct _Prehashed
    hash::UInt64
end
Base.hash(x::_Prehashed) = x.hash

Base.@propagate_inbounds function _op_for_sum!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] = _add_sum(x[i], f(y[i]))
    end
    x
end

function row_sum(ds::AbstractDataset, f::Function,  cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    T = Core.Compiler.return_type(f, (CT,))
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_sum!(x,y, f, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_sum!(x,y, f, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end

row_sum(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_sum(ds, identity, cols, threads = threads)

Base.@propagate_inbounds function _op_for_prod!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] = _mul_prod(x[i], f(y[i]))
    end
    x
end

function row_prod(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    T = Core.Compiler.return_type(f, (CT,))
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_prod!(x,y, f, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_prod!(x,y, f, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end

row_prod(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_prod(ds, identity, cols; threads = threads)

Base.@propagate_inbounds function _op_for_count!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] += f(y[i])
    end
    x
end

function row_count(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    init0 = zeros(Int32, size(ds,1))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_count!(x, y, _bool(f), lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_count!(x, y, _bool(f), 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end

row_count(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_count(ds, x->true, cols; threads = threads)

_op_bool_add(x::Bool,y::Bool) = x | y ? true : false
Base.@propagate_inbounds function op_for_any!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] = _op_bool_add(x[i], f(y[i]))
    end
    x
end

function row_any(ds::AbstractDataset, f::Union{AbstractVector{<:Function}, Function}, cols = :; threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    init0 = zeros(Bool, size(ds,1))

    multi_f = false
    if f isa AbstractVector
        @assert length(f) == length(colsidx) "number of provided functions must match the number of selected columns"
        multi_f = true
    end

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if multi_f
                mapreduce_index(f, (x, y, func) -> op_for_any!(x, y, _bool(func), lo, hi), view(_columns(ds),colsidx), init0)
            else
                mapreduce(identity, (x,y) -> op_for_any!(x,y, _bool(f), lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        if multi_f
            mapreduce_index(f, (x, y, func) -> op_for_any!(x, y, _bool(func), 1, length(x)), view(_columns(ds),colsidx), init0)
        else
            mapreduce(identity, (x,y) -> op_for_any!(x,y, _bool(f), 1, length(x)), view(_columns(ds),colsidx), init = init0)
        end
    end
    init0
end

row_any(ds::AbstractDataset, cols = :; threads = true) = row_any(ds, isequal(true), cols; threads = threads)

_op_bool_mult(x::Bool,y::Bool) = x & y ? true : false

Base.@propagate_inbounds function op_for_all!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] = _op_bool_mult(x[i], f(y[i]))
    end
    x
end

function row_all(ds::AbstractDataset, f::Union{AbstractVector{<:Function}, Function}, cols = :; threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    init0 = ones(Bool, size(ds,1))

    multi_f = false
    if f isa AbstractVector
        @assert length(f) == length(colsidx) "number of provided functions must match the number of selected columns"
        multi_f = true
    end

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if multi_f
                mapreduce_index(f, (x, y, func) -> op_for_all!(x, y, _bool(func), lo, hi), view(_columns(ds),colsidx), init0)
            else
                mapreduce(identity, (x,y) -> op_for_all!(x,y, _bool(f), lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        if multi_f
            mapreduce_index(f, (x, y, func) -> op_for_all!(x, y, _bool(func), 1, length(x)), view(_columns(ds),colsidx), init0)
        else
            mapreduce(identity, (x,y) -> op_for_all!(x,y, _bool(f), 1, length(x)), view(_columns(ds),colsidx), init = init0)
        end
    end
    init0
end
row_all(ds::AbstractDataset, cols = :; threads = true) = row_all(ds, isequal(true), cols; threads = threads)

# this is a general rule for order of arguments in isequal, isless, findfirst, ...
# if the keyword argument is `with` then eq(y, with)
# if the keyword argument is `item` then eq(item, y)

Base.@propagate_inbounds function _op_for_isequal!(x, y, x1, lo, hi)
    @simd for i in lo:hi
        x[i] &= isequal(y[i], x1[i])
    end
    x
end

function row_isequal(ds::AbstractDataset, cols = :; by::Union{AbstractVector, DatasetColumn, SubDatasetColumn, ColumnIndex, Nothing} = nothing, threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    if !(by isa ColumnIndex) && by !== nothing
        @assert length(by) == nrow(ds) "to compare values of selected columns in each row, the length of the passed vector and the number of rows must match"
    end
    if by isa SubDatasetColumn || by isa DatasetColumn
        x1 = __!(by)
    elseif by isa ColumnIndex
        x1 = _columns(ds)[index(ds)[by]]
    elseif by === nothing
        x1 = _columns(ds)[colsidx[1]]
    else
        x1 = by
    end
    init0 = ones(Bool, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_isequal!(x,y, x1, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_isequal!(x,y, x1, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end

Base.@propagate_inbounds function _op_for_isless!(x, y, vals, rev,lt, lo, hi)
    if !rev
        @simd for i in lo:hi
            x[i] &= lt(y[i], vals[i])
        end
    else
        @simd for i in lo:hi
            x[i] &= lt(vals[i], y[i])
        end
    end
    x
end

function row_isless(ds::AbstractDataset, cols, colselector::Union{AbstractVector, DatasetColumn, SubDatasetColumn, ColumnIndex}; threads = true, rev = false, lt = isless)
    if !(colselector isa ColumnIndex)
        @assert length(colselector) == nrow(ds) "to compare values of selected columns in each row, the length of the passed vector and the number of rows must match"
    end
    colsidx = multiple_getindex(index(ds), cols)
    if colselector isa SubDatasetColumn || colselector isa DatasetColumn
        colselector = __!(colselector)
    elseif colselector isa ColumnIndex
        colselector = _columns(ds)[index(ds)[colselector]]
    end
    init0 = ones(Bool, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_isless!(x, y, colselector, rev, lt, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_isless!(x, y, colselector, rev, lt, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0

end

# TODO probably we should use this approach instead of mapreduce_indexed
Base.@propagate_inbounds function _op_for_findfirst!(x, y, f, idx, missref, lo, hi)
    idx[] += 1
    @simd for i in lo:hi
        x[i] = ifelse(isequal(missref, x[i]) && isequal(true, f(y[i])), idx[], x[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_findfirst!(x, y, item, idx, missref, eq, lo, hi)
    idx[] += 1
    @simd for i in lo:hi
        x[i] = ifelse(isequal(missref, x[i]) && eq(item[i], y[i]), idx[], x[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_findlast!(x, y, f, idx, missref, lo, hi)
    idx[] += 1
    @simd for i in lo:hi
        x[i] = ifelse(isequal(true, f(y[i])), idx[], x[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_findlast!(x, y, item, idx, missref, eq, lo, hi)
    idx[] += 1
    @simd for i in lo:hi
        x[i] = ifelse(eq(item[i], y[i]), idx[], x[i])
    end
    x
end

function row_findfirst(ds::AbstractDataset, f, cols = names(ds, Union{Missing, Number});item::Union{AbstractVector, DatasetColumn, SubDatasetColumn, ColumnIndex, Nothing} = nothing, threads = true, eq = isequal)
    if !(item isa ColumnIndex) && item !== nothing
        @assert length(item) == nrow(ds) "length of item values must be the same as the number of rows"
    elseif item isa SubDatasetColumn || item isa DatasetColumn
        item = __!(item)
    elseif item isa ColumnIndex
        item = _columns(ds)[index(ds)[item]]
    end
    colsidx = multiple_getindex(index(ds), cols)

    colnames_pa = allowmissing(PooledArray(_names(ds)[colsidx]))
    push!(colnames_pa, missing)
    missref = get(colnames_pa.invpool, missing, 0)
    init0 = fill(missref, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        idx = [Ref{Int}(0) for _ in 1:__NCORES]
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if item === nothing
                mapreduce(identity, (x,y) -> _op_for_findfirst!(x, y, f, idx[i], missref, lo, hi), view(_columns(ds),colsidx), init = init0)
            else
                mapreduce(identity, (x,y) -> _op_for_findfirst!(x, y, item, idx[i], missref, eq, lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        idx = Ref{Int64}(0)
        if item === nothing
            mapreduce(identity, (x,y) -> _op_for_findfirst!(x, y, f, idx, missref, 1, length(x)), view(_columns(ds),colsidx), init = init0)
        else
            mapreduce(identity, (x,y) -> _op_for_findfirst!(x, y, item, idx, missref, eq, 1, length(x)), view(_columns(ds),colsidx), init = init0)
        end
    end
    colnames_pa.refs = init0
    colnames_pa
end

function row_findlast(ds::AbstractDataset, f, cols = names(ds, Union{Missing, Number}); item::Union{AbstractVector, DatasetColumn, SubDatasetColumn, ColumnIndex, Nothing} = nothing, threads = true, eq = isequal)
    if !(item isa ColumnIndex) && item !== nothing
        @assert length(item) == nrow(ds) "length of item values must be the same as the number of rows"
    elseif item isa SubDatasetColumn || item isa DatasetColumn
        item = __!(item)
    elseif item isa ColumnIndex
        item = _columns(ds)[index(ds)[item]]
    end
    colsidx = multiple_getindex(index(ds), cols)
    colnames_pa = allowmissing(PooledArray(_names(ds)[colsidx]))
    push!(colnames_pa, missing)
    missref = get(colnames_pa.invpool, missing, 0)
    init0 = fill(missref, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        idx = [Ref{Int}(0) for _ in 1:__NCORES]
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if item === nothing
                mapreduce(identity, (x,y) -> _op_for_findlast!(x, y, f, idx[i], missref, lo, hi), view(_columns(ds),colsidx), init = init0)
            else
                mapreduce(identity, (x,y) -> _op_for_findlast!(x, y, item, idx[i], missref, eq, lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        idx = Ref{Int}(0)
        if item === nothing
            mapreduce(identity, (x,y) -> _op_for_findlast!(x, y, f, idx, missref, 1, length(x)), view(_columns(ds),colsidx), init = init0)
        else
            mapreduce(identity, (x,y) -> _op_for_findlast!(x, y, item, idx, missref, eq, 1, length(x)), view(_columns(ds),colsidx), init = init0)
        end
    end
    colnames_pa.refs = init0
    colnames_pa
end

Base.@propagate_inbounds function _op_for_in!(x, y, x1, eq, lo, hi)
    @simd for i in lo:hi
        !x[i] ? x[i] = eq(x1[i], y[i]) : nothing
    end
    x
end

function row_in(ds::AbstractDataset, collections, items::Union{AbstractVector, DatasetColumn, SubDatasetColumn, ColumnIndex}; threads = true, eq = isequal)
    if !(items isa ColumnIndex)
        @assert length(items) == nrow(ds) "length of item values must be the same as the number of rows"
    end
    colsidx = index(ds)[collections]
    if items isa SubDatasetColumn || items isa DatasetColumn
        items = __!(items)
    elseif items isa ColumnIndex
        items = _columns(ds)[index(ds)[items]]
    end
    init0 = zeros(Bool, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_in!(x, y, items, eq, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_in!(x, y, items, eq, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end

Base.@propagate_inbounds function _op_for_select!(x, y, colselector, dsnames, idx, lo, hi)
    idx[] += 1
    @simd for i in lo:hi
        if isequal(colselector[i], dsnames[idx[]])
            x[i] = y[i]
        end
    end
    x
end

function row_select(ds::AbstractDataset, cols, colselector::Union{AbstractVector, DatasetColumn, SubDatasetColumn, ColumnIndex}; threads = true)
    if !(colselector isa ColumnIndex)
        @assert length(colselector) == nrow(ds) "to pick values of selected columns in each row, the length of the column names and the number of rows must match, i.e. the length of the vector passed as `by` must be $(nrow(ds))."
    end
    colsidx = multiple_getindex(index(ds), cols)
    CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    if colselector isa SubDatasetColumn || colselector isa DatasetColumn
        colselector = __!(colselector)
    end
    if colselector isa ColumnIndex
        colselector = _columns(ds)[index(ds)[colselector]]
    end
    if eltype(colselector) <: Union{Missing, Symbol}
        nnames = Symbol.(names(ds, colsidx))
    elseif eltype(colselector) <: Union{Missing, AbstractString}
        nnames = names(ds, colsidx)
    elseif eltype(colselector) <: Union{Missing, Integer}
        nnames = 1:length(colsidx)
    end
    init0 = _missings(CT, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        idx = [Ref{Int}(0) for _ in 1:__NCORES]
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_select!(x, y, colselector, nnames, idx[i], lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        idx = Ref{Int}(0)
        mapreduce(identity, (x,y) -> _op_for_select!(x, y, colselector, nnames, idx, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end

Base.@propagate_inbounds function _op_for_fill!(x, y, f, lo, hi)
    @simd for i in lo:hi
       y[i] = ifelse(f(y[i]), x[i], y[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_fill_roll!(x, y, f, lo, hi)
    @simd for i in lo:hi
       y[i] = ifelse(f(y[i]), x[i], y[i])
    end
    y
end

# rolling = true, means fill a column an use its value for next filling
function row_fill!(ds::AbstractDataset, cols, val::Union{AbstractVector, DatasetColumn, SubDatasetColumn, ColumnIndex}; f = ismissing, threads = true, rolling = false)
    if !(val isa ColumnIndex)
        @assert length(val) == nrow(ds) "to fill values in each row, the length of passed values and the number of rows must match."
    end
    colsidx = multiple_getindex(index(ds), cols)
    if val isa SubDatasetColumn || val isa DatasetColumn
        val = __!(val)
    end
    if val isa ColumnIndex
        val = _columns(ds)[index(ds)[val]]
    end
    init0 = val

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if rolling
                mapreduce(identity, (x,y) -> _op_for_fill_roll!(x, y, f, lo, hi), view(_columns(ds),colsidx), init = init0)
            else
                mapreduce(identity, (x,y) -> _op_for_fill!(x, y, f, lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        if rolling
            mapreduce(identity, (x,y) -> _op_for_fill_roll!(x, y, f, 1, length(x)), view(_columns(ds),colsidx), init = init0)
        else
            mapreduce(identity, (x,y) -> _op_for_fill!(x, y, f, 1, length(x)), view(_columns(ds),colsidx), init = init0)
        end
    end
    any(index(parent(ds)).sortedcols .∈ Ref(IMD.parentcols.(Ref(IMD.index(ds)), cols))) && _reset_grouping_info!(parent(ds))
    _modified(_attributes(parent(ds)))
    ds
end

Base.@propagate_inbounds function _op_for_coalesce!(x, y, lo, hi)
    if all(!ismissing, view(x, lo:hi))
        x
    else
        @simd for i in lo:hi
            x[i] = ifelse(ismissing(x[i]), y[i], x[i])
        end
    end
    x
end

function row_coalesce(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))

    init0 = _missings(CT, size(ds,1))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_coalesce!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_coalesce!(x, y, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end

function row_mean(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
    row_sum(ds, f, cols; threads = threads) ./ row_count(ds, x -> !ismissing(x), cols; threads = threads)
end
row_mean(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_mean(ds, identity, cols; threads = threads)

Base.@propagate_inbounds function _op_for_min!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] = _min_fun(x[i], f(y[i]))
    end
    x
end

Base.@propagate_inbounds function _op_for_max!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] = _max_fun(x[i], f(y[i]))
    end
    x
end

function row_minimum(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    T = Core.Compiler.return_type(f, (CT,))
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_min!(x, y, f, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_min!(x, y, f, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end
row_minimum(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_minimum(ds, identity, cols; threads = threads)

function row_maximum(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    T = Core.Compiler.return_type(f, (CT,))
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_max!(x, y, f, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_max!(x, y, f, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end
row_maximum(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_maximum(ds, identity, cols; threads = threads)

Base.@propagate_inbounds function _op_for_argminmax!(x, y, f, vals, idx, missref, lo, hi)
    idx[] += 1
    @simd for i in lo:hi
        if !ismissing(vals[i]) && isequal(vals[i], f(y[i])) && isequal(x[i], missref)
            x[i] = idx[]
        end
    end
    x
end

function row_argmin(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    minvals = row_minimum(ds, f, cols)
    colnames_pa = allowmissing(PooledArray(_names(ds)[colsidx]))
    push!(colnames_pa, missing)
    missref = get(colnames_pa.invpool, missing, missing)
    init0 = fill(missref, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        idx = [Ref{Int}(0) for _ in 1:__NCORES]
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_argminmax!(x, y, f, minvals, idx[i], missref, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        idx = Ref{Int}(0)
        mapreduce(identity, (x,y) -> _op_for_argminmax!(x, y, f, minvals, idx, missref, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end

    colnames_pa.refs = init0
    colnames_pa
end
row_argmin(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_argmin(ds, identity, cols, threads = threads)

function row_argmax(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    maxvals = row_maximum(ds, f, cols)
    colnames_pa = allowmissing(PooledArray(_names(ds)[colsidx]))
    push!(colnames_pa, missing)
    missref = get(colnames_pa.invpool, missing, missing)
    init0 = fill(missref, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        idx = [Ref{Int}(0) for _ in 1:__NCORES]
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_argminmax!(x, y, f, maxvals, idx[i], missref, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        idx = Ref{Int}(0)
        mapreduce(identity, (x,y) -> _op_for_argminmax!(x, y, f, maxvals, idx, missref, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end

    colnames_pa.refs = init0
    colnames_pa
end
row_argmax(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); threads = true) = row_argmax(ds, identity, cols, threads = threads)

# TODO better function for the first component of operator
function _row_wise_var(ss, sval, n, dof, T)
    res = Vector{T}(undef, length(ss))
    for i in 1:length(ss)
        if n[i] == 0
            res[i] = missing
        elseif n[i] == 1
            res[i] = zero(T)
        else
            res[i] = ss[i]/n[i] - (sval[i]/n[i])*(sval[i]/n[i])
            if dof
                res[i] = (n[i] * res[i])/(n[i]-1)
            end
        end
    end
    res
end

# TODO needs type stability
# TODO need abs2 for calculations
function row_var(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); dof = true, threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    CT = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    T = Core.Compiler.return_type(f, (CT,))
    _sq_(x) = x^2
    ss = row_sum(ds, _sq_ ∘ f, cols; threads = threads)
    sval = row_sum(ds, f, cols; threads = threads)
    n = row_count(ds, x -> !ismissing(x), cols; threads = threads)
    T2 = Core.Compiler.return_type(/, (eltype(ss), eltype(n)))
    res = Vector{Union{Missing, T2}}(undef, length(ss))
    res .= ss ./ n .- (sval ./ n) .^ 2
    if dof
        res .= (n .* res) ./ (n .- 1)
        res .= ifelse.(n .== 1, missing, res)
    end
    res
    # _row_wise_var(ss, sval, n, dof, T)
end
row_var(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); dof = true, threads = true) = row_var(ds, identity, cols, dof = dof, threads = threads)

function row_std(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); dof = true, threads = true)
    sqrt.(row_var(ds, f, cols, dof = dof, threads = threads))
end
row_std(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); dof = true, threads = true) = row_std(ds, identity, cols, dof = dof, threads = threads)

Base.@propagate_inbounds function _op_for_cumsum_skip!(x, y, lo, hi)
    @simd for i in lo:hi
        x[i] = _add_sum(x[i], y[i])
    end
    @simd for i in lo:hi
        y[i] = ifelse(ismissing(y[i]), missing, x[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_cumsum_ignore!(x, y, lo, hi)
    @simd for i in lo:hi
        y[i] = _add_sum(x[i], y[i])
    end
    y
end

function row_cumsum!(ds::Dataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    !(missings in (:ignore, :skip)) && throw(ArgumentError("`missings` can be either `:ignore` or `:skip`"))
    colsidx = index(ds)[cols]
    T = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    if T <: Union{Missing, INTEGERS}
        T <: Union{Missing, Base.SmallSigned}
        T = T <: Union{Missing, Base.SmallSigned, Bool} ? Union{Int, Missing} : T
        T = T <: Union{Missing, Base.SmallUnsigned} ?  Union{Missing, UInt} : T
    end
    for i in colsidx
        if eltype(ds[!, i]) >: Missing
            _columns(ds)[i] = convert(Vector{Union{Missing, T}}, _columns(ds)[i])
        else
            _columns(ds)[i] = convert(Vector{T}, _columns(ds)[i])
        end
    end
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if missings == :ignore
                mapreduce(identity, (x,y) -> _op_for_cumsum_ignore!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            else
                mapreduce(identity, (x,y) -> _op_for_cumsum_skip!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        if missings == :ignore
            mapreduce(identity, (x,y) -> _op_for_cumsum_ignore!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        else
            mapreduce(identity, (x,y) -> _op_for_cumsum_skip!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        end
    end

    removeformat!(ds, cols)
    any(index(ds).sortedcols .∈ Ref(colsidx)) && _reset_grouping_info!(ds)
    _modified(_attributes(ds))
    ds
end

function row_cumsum(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    dscopy = copy(ds)
    row_cumsum!(dscopy, cols, missings = missings, threads = threads)
    dscopy
end

Base.@propagate_inbounds function _op_for_cumprod_skip!(x, y, lo, hi)
    @simd for i in lo:hi
        x[i] = _mul_prod(x[i], y[i])
    end
    @simd for i in lo:hi
        y[i] = ifelse(ismissing(y[i]), missing, x[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_cumprod_ignore!(x, y, lo, hi)
    @simd for i in lo:hi
        y[i] = _mul_prod(x[i], y[i])
    end
    y
end

function row_cumprod!(ds::Dataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    !(missings in (:ignore, :skip)) && throw(ArgumentError("`missings` can be either `:ignore` or `:skip`"))
    colsidx = index(ds)[cols]
    T = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    for i in colsidx
        if eltype(ds[!, i]) >: Missing
            _columns(ds)[i] = convert(Vector{Union{Missing, T}}, _columns(ds)[i])
        else
            _columns(ds)[i] = convert(Vector{T}, _columns(ds)[i])
        end
    end
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if missings == :ignore
                mapreduce(identity, (x,y) -> _op_for_cumprod_ignore!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            else
                mapreduce(identity, (x,y) -> _op_for_cumprod_skip!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        if missings == :ignore
            mapreduce(identity, (x,y) -> _op_for_cumprod_ignore!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        else
            mapreduce(identity, (x,y) -> _op_for_cumprod_skip!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        end
    end

    removeformat!(ds, cols)
    any(index(ds).sortedcols .∈ Ref(colsidx)) && _reset_grouping_info!(ds)
    _modified(_attributes(ds))
    ds
end

function row_cumprod(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    dscopy = copy(ds)
    row_cumprod!(dscopy, cols; missings = missings, threads = threads)
    dscopy
end

Base.@propagate_inbounds function _op_for_cummin_skip!(x, y, lo, hi)
    @simd for i in lo:hi
        x[i] = _min_fun(x[i], y[i])
    end
    @simd for i in lo:hi
        y[i] = ifelse(ismissing(y[i]), missing, x[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_cummin_ignore!(x, y, lo, hi)
    @simd for i in lo:hi
        y[i] = _min_fun(x[i], y[i])
    end
    y
end

function row_cummin!(ds::Dataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    !(missings in (:ignore, :skip)) && throw(ArgumentError("`missings` can be either `:ignore` or `:skip`"))
    colsidx = index(ds)[cols]
    T = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    for i in colsidx
        if eltype(ds[!, i]) >: Missing
            _columns(ds)[i] = convert(Vector{Union{Missing, T}}, _columns(ds)[i])
        else
            _columns(ds)[i] = convert(Vector{T}, _columns(ds)[i])
        end
    end
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if missings == :ignore
                mapreduce(identity, (x,y) -> _op_for_cummin_ignore!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            else
                mapreduce(identity, (x,y) -> _op_for_cummin_skip!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        if missings == :ignore
            mapreduce(identity, (x,y) -> _op_for_cummin_ignore!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        else
            mapreduce(identity, (x,y) -> _op_for_cummin_skip!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        end
    end

    removeformat!(ds, cols)
    any(index(ds).sortedcols .∈ Ref(colsidx)) && _reset_grouping_info!(ds)
    _modified(_attributes(ds))
    ds
end
# row_cumsum!(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = row_cumsum!(identity, ds, cols)

function row_cummin(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    dscopy = copy(ds)
    row_cummin!(dscopy, cols, missings = missings, threads = threads)
    dscopy
end

Base.@propagate_inbounds function _op_for_cummax_skip!(x, y, lo, hi)
    @simd for i in lo:hi
        x[i] = _max_fun(x[i], y[i])
    end
    @simd for i in lo:hi
        y[i] = ifelse(ismissing(y[i]), missing, x[i])
    end
    x
end

Base.@propagate_inbounds function _op_for_cummax_ignore!(x, y, lo, hi)
    @simd for i in lo:hi
        y[i] = _max_fun(x[i], y[i])
    end
    y
end

function row_cummax!(ds::Dataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    !(missings in (:ignore, :skip)) && throw(ArgumentError("`missings` can be either `:ignore` or `:skip`"))
    colsidx = index(ds)[cols]
    T = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    for i in colsidx
        if eltype(ds[!, i]) >: Missing
            _columns(ds)[i] = convert(Vector{Union{Missing, T}}, _columns(ds)[i])
        else
            _columns(ds)[i] = convert(Vector{T}, _columns(ds)[i])
        end
    end
    init0 = _missings(T, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if missings == :ignore
                mapreduce(identity, (x,y) -> _op_for_cummax_ignore!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            else
                mapreduce(identity, (x,y) -> _op_for_cummax_skip!(x, y, lo, hi), view(_columns(ds),colsidx), init = init0)
            end
        end
    else
        if missings == :ignore
            mapreduce(identity, (x,y) -> _op_for_cummax_ignore!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        else
            mapreduce(identity, (x,y) -> _op_for_cummax_skip!(x, y, 1, length(init0)), view(_columns(ds),colsidx), init = init0)
        end
    end

    removeformat!(ds, cols)
    any(index(ds).sortedcols .∈ Ref(colsidx)) && _reset_grouping_info!(ds)
    _modified(_attributes(ds))
    ds
end
# row_cumsum!(ds::AbstractDataset, cols = names(ds, Union{Missing, Number})) = row_cumsum!(identity, ds, cols)

function row_cummax(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); missings = :ignore, threads = true)
    dscopy = copy(ds)
    row_cummax!(dscopy, cols, missings = missings, threads = threads)
    dscopy
end

function row_stdze!(ds::Dataset , cols = names(ds, Union{Missing, Number}))
    meandata = row_mean(ds, cols)
    stddata = row_std(ds, cols)
    _stdze_fun(x) = ifelse.(isequal.(stddata, 0), missing, (x .- meandata) ./ stddata)
    colsidx = index(ds)[cols]

    for i in 1:length(colsidx)
        _columns(ds)[colsidx[i]] = _stdze_fun(_columns(ds)[colsidx[i]])
    end
    removeformat!(ds, cols)
    any(index(ds).sortedcols .∈ Ref(colsidx)) && _reset_grouping_info!(ds)
    _modified(_attributes(ds))
end

function row_stdze(ds::AbstractDataset , cols = names(ds, Union{Missing, Number}))
    dscopy = copy(ds)
    row_stdze!(dscopy, cols)
    dscopy
end

function row_sort!(ds::Dataset, cols = names(ds, Union{Missing, Number}); kwargs...)
    colsidx = index(ds)[cols]
    T = mapreduce(eltype, promote_type, eachcol(ds)[colsidx])
    m = Matrix{T}(ds[!, colsidx])
    sort!(m; dims = 2, kwargs...)
    for i in 1:length(colsidx)
        _columns(ds)[colsidx[i]] = m[:, i]
    end
    removeformat!(ds, cols)
    any(index(ds).sortedcols .∈ Ref(colsidx)) && _reset_grouping_info!(ds)
    _modified(_attributes(ds))
    ds
end

function row_sort(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); kwargs...)
    dscopy = copy(ds)
    row_sort!(dscopy, cols; kwargs...)
    dscopy
end

Base.@propagate_inbounds function _op_for_issorted!(x, y, res, lt, lo, hi)
    @simd for i in lo:hi
        res[i] &= !lt(y[i], x[i])
    end
    y
end
Base.@propagate_inbounds function _op_for_issorted_rev!(x, y, res, lt, lo, hi)
    @simd for i in lo:hi
        res[i] &= !lt(x[i], y[i])
    end
    y
end

function row_issorted(ds::AbstractDataset, cols; rev = false, lt = isless, threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    init0 = ones(Bool, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            if rev
                mapreduce(identity, (x,y) -> _op_for_issorted_rev!(x, y, init0, lt, lo, hi), view(_columns(ds),colsidx))
            else
                mapreduce(identity, (x,y) -> _op_for_issorted!(x, y, init0, lt, lo, hi), view(_columns(ds),colsidx))
            end
        end
    else
        if rev
            mapreduce(identity, (x,y) -> _op_for_issorted_rev!(x, y, init0, lt, 1, length(init0)), view(_columns(ds),colsidx))
        else
            mapreduce(identity, (x,y) -> _op_for_issorted!(x, y, init0, lt, 1, length(init0)), view(_columns(ds),colsidx))
        end
    end
    init0
end

# TODO is it possible to have a faster row_count_unique??
function _fill_prehashed!(prehashed, y, f, n, j)
    @views copy!(prehashed[:, j] , _Prehashed.(hash.(f.(y))))
end

function _fill_dict_and_add!(init0, dict, prehashed, n, p)
    for i in 1:n
        for j in 1:p
            if !haskey(dict, prehashed[i, j])
                get!(dict, prehashed[i, j], nothing)
                init0[i] += 1
            end
        end
        empty!(dict)
    end
end

function row_nunique(ds::AbstractDataset, f::Function, cols = names(ds, Union{Missing, Number}); count_missing = true)
    colsidx = multiple_getindex(index(ds), cols)
    prehashed = Matrix{_Prehashed}(undef, size(ds,1), length(colsidx))
    allcols = view(_columns(ds),colsidx)

    for j in 1:size(prehashed,2)
        _fill_prehashed!(prehashed, allcols[j], f, size(ds,1), j)
    end

    init0 = zeros(Int32, size(ds,1))
    dict = Dict{_Prehashed, Nothing}()
    _fill_dict_and_add!(init0, dict, prehashed, size(ds,1), length(colsidx))
    if count_missing
        return init0
    else
        return init0 .- row_any(ds, ismissing, cols)
    end
end
row_nunique(ds::AbstractDataset, cols = names(ds, Union{Missing, Number}); count_missing = true) = row_nunique(ds, identity, cols; count_missing = count_missing)

Base.@propagate_inbounds function _op_for_hash!(x, y, f, lo, hi)
    @simd for i in lo:hi
        x[i] = hash(f(y[i]), x[i])
    end
    x
end

function row_hash(ds::AbstractDataset, f::Function, cols = :; threads = true)
    colsidx = multiple_getindex(index(ds), cols)
    init0 = zeros(UInt64, nrow(ds))

    if threads
        cz = div(length(init0), __NCORES)
        Threads.@threads for i in 1:__NCORES
            lo = (i-1)*cz+1
            i == __NCORES ? hi = length(init0) : hi = i*cz
            mapreduce(identity, (x,y) -> _op_for_hash!(x, y, f, lo, hi), view(_columns(ds),colsidx), init = init0)
        end
    else
        mapreduce(identity, (x,y) -> _op_for_hash!(x, y, f, 1, length(x)), view(_columns(ds),colsidx), init = init0)
    end
    init0
end
row_hash(ds::AbstractDataset, cols = :; threads = true) = row_hash(ds, identity, cols; threads = threads)

function _convert_uint8_to_string!(res, init0, curr_pos, ds, threads)
    if threads
        Threads.@threads for i in 1:nrow(ds)
            res[i] = String(view(init0, 1:curr_pos[i]-2, i))
        end
    else
        for i in 1:nrow(ds)
            res[i] = String(view(init0, 1:curr_pos[i]-2, i))
        end
    end
end
function _add_last_for_join!(init0, curr_pos, ds, last_uint, last_len, threads)
    if threads
        Threads.@threads for i in 1:nrow(ds)
            init0[curr_pos[i]-1:curr_pos[i]+last_len-2, i] = last_uint
            curr_pos[i] += last_len-1
        end
    else
        for i in 1:nrow(ds)
            init0[curr_pos[i]-1:curr_pos[i]+last_len-2, i] = last_uint
            curr_pos[i] += last_len-1
        end
    end
end

function row_join(ds::AbstractDataset, cols = :; threads = true, delim::AbstractString = ",", last::AbstractString = "")
    colsidx = multiple_getindex(index(ds), cols)

    max_line_size = maximum(byrow(ds, sum, colsidx, by = y->length(__STRING(y)), threads = threads))
    max_line_size += length(delim)*(length(colsidx)) + length(last)+1
    init0 = Matrix{UInt8}(undef, max_line_size, nrow(ds))
    curr_pos = ones(Int, nrow(ds))

    delimiter = Base.CodeUnits(delim)
    row_join!(init0, curr_pos, ds, repeat([identity], length(colsidx)-1), view(colsidx, 1:length(colsidx)-1); delim = delimiter, quotechar = nothing, threads = threads)
    if length(last)>0
        delimiter = Base.CodeUnits(last)
    end
    if length(colsidx) > 1
        _add_last_for_join!(init0, curr_pos, ds, delimiter, length(delimiter), threads)
    end
    row_join!(init0, curr_pos, ds, [identity], colsidx[length(colsidx)]; delim = delimiter, quotechar = nothing, threads = threads)

    res = Vector{Union{String, Missing}}(undef, nrow(ds))
    _convert_uint8_to_string!(res, init0, curr_pos, ds, threads)
    res

end

function _fill_col!(inmat, column, rows, j)
    for i in 1:length(rows)
        inmat[j, i] = column[rows[i]]
    end
end
function _fill_matrix!(inmat, all_data, rows, cols)
    for j in 1:length(cols)
        _fill_col!(inmat, all_data[j], rows, j)
    end
end

function row_generic(ds::AbstractDataset, f::Function, cols::MultiColumnIndex)
    colsidx = multiple_getindex(index(ds), cols)
    if length(colsidx) == 2
        try
            allowmissing(f.(_columns(ds)[colsidx[1]], _columns(ds)[colsidx[2]]))
        catch e
            if e isa MethodError
                _row_generic(ds, f, colsidx)
            else
                rethrow(e)
            end
        end
    else
        _row_generic(ds, f, colsidx)
    end
end
function _row_generic(ds::AbstractDataset, f::Function, colsidx)
    T = mapreduce(eltype, promote_type, view(_columns(ds),colsidx))
    inmat = Matrix{T}(undef, length(colsidx), min(1000, nrow(ds)))

    all_data = view(_columns(ds), colsidx)
    _fill_matrix!(inmat, all_data, 1:min(1000, nrow(ds)), colsidx)
    res_temp = allowmissing(f.(eachcol(inmat)))
    if !(typeof(res_temp) <:  AbstractVector)
        throw(ArgumentError("output of `f` must be a vector"))
    end

    # if length(res_temp[1]) > 1
    #     throw(ArgumentError("The matrix output is not supported"))
    #     res = similar(res_temp, nrow(ds), size(res_temp,2))
    # elseif length(res_temp[1]) == 1
    res = similar(res_temp, nrow(ds))
    # else
        # throw(ArgumentError("the result cannot be with zero dimension"))

    if nrow(ds)>1000
        if size(res, 2) == 1
            view(res, 1:1000) .= res_temp
            _row_generic_vec!(res, ds, f, colsidx, Val(T))
        else
            view(res, 1:1000, :) .= res_temp
            _row_generic_mat!(res, ds, f, colsidx)
        end
    else
        return res_temp
    end
    return res
end

function _row_generic_vec!_barrier(res, f, inmat, all_data, chunck, colsidx)

    for i in 1:chunck
        t_st = i*1000 + 1
        i == chunck ? t_en = length(res) : t_en = (i+1)*1000
        _fill_matrix!(inmat, all_data, t_st:t_en, colsidx)
        for k in t_st:t_en
            res[k] = f(view(inmat, :, k - t_st + 1))
        end
    end
end

function _row_generic_vec!(res, ds, f, colsidx, ::Val{T}) where T
    all_data = view(_columns(ds), colsidx)
    chunck = div(length(res) - 1000, 1000)
    max_cz = length(res) - 1000 - (chunck - 1)* 1000
    inmat = Matrix{T}(undef, length(colsidx), max_cz)
    # make sure that the variable inside the loop are not the same as the out of scope one
    # for i in 1:chunck
    #     t_st = i*1000 + 1
    #     i == chunck ? t_en = length(res) : t_en = (i+1)*1000
    #     _fill_matrix!(inmat, all_data, t_st:t_en, colsidx)
    #     for k in t_st:t_en
    #         res[k] = f(view(inmat, k - t_st + 1, :))
    #     end
    # end
    _row_generic_vec!_barrier(res, f, inmat, all_data, chunck, colsidx)
end


# *********************************************************************************
# REopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this list
# of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
# *********************************************************************************
const default_buildings = [
    "FastFoodRest",
    "FullServiceRest",
    "Hospital",
    "LargeHotel",
    "LargeOffice",
    "MediumOffice",
    "MidriseApartment",
    "Outpatient",
    "PrimarySchool",
    "RetailStore",
    "SecondarySchool",
    "SmallHotel",
    "SmallOffice",
    "StripMall",
    "Supermarket",
    "Warehouse",
    "FlatLoad",
]

const MMBTU_TO_KWH = 293.07107

function find_ashrae_zone_city(lat, lon)::String
    file_path = joinpath(dirname(@__FILE__), "..", "..", "data", "climate_cities.shp")
    table = Shapefile.Table(file_path)
    geoms = Shapefile.shapes(table)
    # TODO following for loop is relatively slow
    for (row, geo) in enumerate(geoms)
        g = length(geo.points)
        nodes = zeros(g, 2)
        edges = zeros(g, 2)
        for (i,p) in enumerate(geo.points)
            nodes[i,:] = [p.x, p.y]
            edges[i,:] = [i, i+1]
        end
        edges[g, :] = [g, 1]
        edges = convert(Array{Int64,2}, edges)
        # shapefiles have longitude as x, latitude as y
        if inpoly2([lon, lat], nodes, edges)[1]
            return table.city[row]
        end
        GC.gc()
    end
    @info "Could not find latitude/longitude in U.S. Using geometrically nearest city."
    cities = [
        (city="Miami", lat=25.761680, lon=-80.191790),
        (city="Houston", lat=29.760427, lon=-95.369803),
        (city="Phoenix", lat=33.448377, lon=-112.074037),
        (city="Atlanta", lat=33.748995, lon=-84.387982),
        (city="LasVegas", lat=36.1699, lon=-115.1398),
        (city="LosAngeles", lat=34.052234, lon=-118.243685),
        (city="SanFrancisco", lat=37.3382, lon=-121.8863),
        (city="Baltimore", lat=39.290385, lon=-76.612189),
        (city="Albuquerque", lat=35.085334, lon=-106.605553),
        (city="Seattle", lat=47.606209, lon=-122.332071),
        (city="Chicago", lat=41.878114, lon=-87.629798),
        (city="Boulder", lat=40.014986, lon=-105.270546),
        (city="Minneapolis", lat=44.977753, lon=-93.265011),
        (city="Helena", lat=46.588371, lon=-112.024505,),
        (city="Duluth", lat=46.786672, lon=-92.100485),
        (city="Fairbanks", lat=59.0397, lon=-158.4575),
    ]
    min_distance = 0.0
    nearest_city = ""
    for (i, c) in enumerate(cities)
        distance = sqrt((lat - c.lat)^2 + (lon - c.lon)^2)
        if i == 1
            min_distance = distance
            nearest_city = c.city
        elseif distance < min_distance
            min_distance = distance
            nearest_city = c.city
        end
    end
    return nearest_city
end

"""
    built_in_load(type::String, city::String, buildingtype::String,
        year::Int, annual_energy::Real, monthly_energies::AbstractArray{Real,1}
    )
Scale a normalized Commercial Reference Building according to inputs provided and return the 8760.
"""
function built_in_load(type::String, city::String, buildingtype::String,
    year::Int, annual_energy::Real, monthly_energies::AbstractArray{Real,1}
    )

    @assert type in ["electric", "domestic_hot_water", "space_heating"]
    monthly_scalers = ones(12)
    lib_path = joinpath(dirname(@__FILE__), "..", "..", "data", "load_profiles", type)

    profile_path = joinpath(lib_path, string("crb8760_norm_" * city * "_" * buildingtype * ".dat"))
    normalized_profile = vec(readdlm(profile_path, '\n', Float64, '\n'))

    if length(monthly_energies) == 12
        annual_energy = 1.0  # do not scale based on annual_energy
        t0 = 1
        for month in 1:12
            plus_hours = daysinmonth(Date(string(year) * "-" * string(month))) * 24
            if month == 2 && isleapyear(year)
                plus_hours -= 24
            end
            month_total = sum(normalized_profile[t0:t0+plus_hours-1])
            if month_total == 0.0  # avoid division by zero
                monthly_scalers[month] = 0.0
            else
                monthly_scalers[month] = monthly_energies[month] / month_total
            end
            t0 += plus_hours
        end
    end

    scaled_load = Float64[]
    boiler_efficiency = 1.0
    mmbtu_to_kwh = 1.0  # do not convert electric loads
    if type in ["domestic_hot_water", "space_heating"]
        # CRB thermal "loads" are in terms of energy input required (boiler fuel), not the actual energy demand.
        # So we multiply the fuel energy by the boiler_efficiency to get the actual energy demand.
        boiler_efficiency = EXISTING_BOILER_EFFICIENCY
        mmbtu_to_kwh = MMBTU_TO_KWH  # do convert thermal loads
    end
    datetime = DateTime(year, 1, 1, 1)
    for ld in normalized_profile
        month = Month(datetime).value
        push!(scaled_load, ld * annual_energy * monthly_scalers[month] * boiler_efficiency * mmbtu_to_kwh)
        datetime += Dates.Hour(1)
    end

    return scaled_load
end

"""
    blend_and_scale_doe_profiles(
        constructor,
        latitude::Float64,
        longitude::Float64,
        year::Int,
        blended_doe_reference_names::Array{String, 1},
        blended_doe_reference_percents::Array{<:Real,1},
        city::String = "",
        annual_energy::Union{Real, Nothing} = nothing,
        monthly_energies::Array{<:Real,1} = Real[],
    )

Given `blended_doe_reference_names` and `blended_doe_reference_percents` use the `constructor` function to load in DoE
    CRB profiles and create a single profile, where `constructor` is one of:
    - BuiltInElectricLoad
    - BuiltInDomesticHotWaterLoad
    - BuiltInSpaceHeatingLoad
"""
function blend_and_scale_doe_profiles(
    constructor,
    latitude::Float64,
    longitude::Float64,
    year::Int,
    blended_doe_reference_names::Array{String, 1},
    blended_doe_reference_percents::Array{<:Real,1},
    city::String = "",
    annual_energy::Union{Real, Nothing} = nothing,
    monthly_energies::Array{<:Real,1} = Real[],
    )

    @assert sum(blended_doe_reference_percents) ≈ 1 "The sum of the blended_doe_reference_percents must equal 1"
    if year != 2017
        @warn "Changing ElectricLoad.year to 2017 because DOE reference profiles start on a Sunday."
    end
    year = 2017
    if isempty(city)
        city = find_ashrae_zone_city(latitude, longitude)  # avoid redundant look-ups
    end
    profiles = Array[]  # collect the built in profiles
    for name in blended_doe_reference_names
        push!(profiles, constructor(city, name, latitude, longitude, year, annual_energy, monthly_energies))
    end
    if isnothing(annual_energy) # then annual_energy should be the sum of all the profiles' annual kwhs
        # we have to rescale the built in profiles to the total_kwh by normalizing them with their
        # own annual kwh and multiplying by the total kwh
        annual_kwhs = [sum(profile) for profile in profiles]
        total_kwh = sum(annual_kwhs)
        monthly_scaler = 1
        if length(monthly_energies) == 12
            monthly_scaler = length(blended_doe_reference_names)
        end
        for idx in 1:length(profiles)
            profiles[idx] .*= total_kwh / annual_kwhs[idx] / monthly_scaler
        end
    end
    for idx in 1:length(profiles)  # scale the profiles
        profiles[idx] .*= blended_doe_reference_percents[idx]
    end
    sum(profiles)
end


## Defines all HMAC functionality
## Pretty much everything in here learned from http://www.lysator.liu.se/~nisse/nettle/nettle.html#Keyed-hash-functions
## Also from reading the header files from libnettle source tree

import Base: show
export HMACState, update!, digest, digest!, hexdigest!, hexdigest

immutable HMACState
    hash_type::HashType
    outer::Array{UInt8,1}
    inner::Array{UInt8,1}
    state::Array{UInt8,1}
end

# Constructor for HMACState
function HMACState(name::AbstractString, key)
    hash_types = get_hash_types()
    name = uppercase(name)
    if !haskey(hash_types, name)
        throw(ArgumentError("Invalid hash type $name: call Nettle.get_hash_types() to see available list"))
    end

    # Construct HMACState object for this type and initialize using Nettle's init functions
    hash_type = hash_types[name]
    outer = Array(UInt8, hash_type.context_size)
    inner = Array(UInt8, hash_type.context_size)
    state = Array(UInt8, hash_type.context_size)
    ccall((:nettle_hmac_set_key,nettle), Void, (Ptr{Void},Ptr{Void},Ptr{Void},Ptr{Void},Csize_t,Ptr{UInt8}),
        outer, inner, state, hash_type.ptr, sizeof(key), key)
    return HMACState(hash_type, outer, inner, state)
end

function update!(state::HMACState, data)
    ccall((:nettle_hmac_update,nettle), Void, (Ptr{Void},Ptr{Void},Csize_t,Ptr{UInt8}), state.state,
        state.hash_type.ptr, sizeof(data), data)
    return state
end

function digest!(state::HMACState)
    digest = Array(UInt8,state.hash_type.digest_size)
    ccall((:nettle_hmac_digest,nettle), Void, (Ptr{Void},Ptr{Void},Ptr{Void},Ptr{Void}, Csize_t,
        Ptr{UInt8}), state.outer, state.inner, state.state, state.hash_type.ptr, sizeof(digest), digest)
    return digest
end

# Take a digest, and convert it to a printable hex representation
hexdigest!(state::HMACState) = bytes2hex(digest!(state))

# The one-shot functions that makes this whole thing so easy
digest(hmac_name::AbstractString, key, data) = digest!(update!(HMACState(hmac_name, key), data))
hexdigest(hmac_name::AbstractString, key, data) = hexdigest!(update!(HMACState(hmac_name, key), data))

show(io::IO, x::HMACState) = write(io, "$(x.hash_type.name) HMAC state")


using Test
using Forecast
using TimeSeries

@testset "stl" begin
    @test_throws AssertionError stl(rand(100),10; ns=5)

    x = stl(rand(100),10)
    @test x isa STL
    @test x.ta isa TimeArray
    @test x.call isa String
end


abstract type AbstractEvaluator end

mutable struct SameInstancesEvaluator <: AbstractEvaluator
    instances::Union{Array{CPModel}, Nothing}
    metrics::Union{Matrix{<:AbstractMetrics}, Nothing}
    evalFreq::Int64
    nbInstances::Int64
    nbHeuristics::Union{Int64, Nothing}
end

function SameInstancesEvaluator(valueSelectionArray::Array{H, 1}, generator::AbstractModelGenerator; seed=nothing, evalFreq::Int64 = 50, nbInstances::Int64 = 10, evalTimeOut::Union{Nothing,Int64} = nothing) where H<: ValueSelection
    instances = Array{CPModel}(undef, nbInstances)
    metrics = Matrix{AbstractMetrics}(undef,nbInstances, size(valueSelectionArray,1))

    for i in 1:nbInstances
        instances[i] = CPModel()
        fill_with_generator!(instances[i], generator; seed=seed)
        instances[i].limit.searchingTime = evalTimeOut
        for (j, value) in enumerate(valueSelectionArray)
            metrics[i,j]=BasicMetrics(instances[i],value;meanOver=1)   #no slidding mean on evaluation because the instance remains the same
        end
    end
    SameInstancesEvaluator(instances, metrics, evalFreq, nbInstances, size(valueSelectionArray,1))
end

function evaluate(eval::SameInstancesEvaluator, variableHeuristic::AbstractVariableSelection, strategy::S; verbose::Bool=true) where{S<:SearchStrategy}
    for j in 1:eval.nbHeuristics
        heuristic = eval.metrics[1,j].heuristic
        initsize = isa(heuristic, LearnedHeuristic) ? length(heuristic.agent.trajectory) : nothing

        testmode!(heuristic, true)
        for i in 1:eval.nbInstances
            model = eval.instances[i]
            reset_model!(model)

            dt = @elapsed search!(model, strategy, variableHeuristic, heuristic)
            eval.metrics[i,j](model,dt)

            verbose && println(typeof(heuristic), " evaluated with: ", model.statistics.numberOfNodes, " nodes, taken ", dt, "s, number of solutions found : ", model.statistics.numberOfSolutions)
        end
        testmode!(heuristic, false)
        if !isnothing(initsize)
            @assert length(heuristic.agent.trajectory) == initsize "You have leaks in your evaluation pipeline!"
        end
    end
end


using GPUArrays, CuArrays, FillArrays
using CuArrays:threadIdx, blockIdx, blockDim
import Base:minimum, maximum, findmax, findmin

import Flux.Optimise: apply!, Descent, InvDecay

#####
# patch for maximum/minimum/findmax/findmin with dims==1
# considering that discrete actions in RL are usually small,
# the implementation here may be not that slow?
# watch https://github.com/JuliaGPU/CuArrays.jl/issues/304
#####
function findminmaxcol!(f, values::GPUArray{Float32, 2}, vals::GPUArray{Float32, 2}, args::GPUArray{CartesianIndex{2}, 1})
    function kernel(state, values, vals, args)
        i = linear_index(state)
        if i <= length(args)
            for j in axes(values, 1)
                @inbounds if f(values[j, i], vals[i])
                    vals[i] = values[j, i]
                    args[i] = CartesianIndex(j, i)
                end
            end
        end
        return
    end
    gpu_call(kernel, args, (values, vals, args))
    vals, args
end

function Base._findmax(values::CuArray{Float32, 2}, dims::Int)
    if dims == 1
        findminmaxcol!(Base.isgreater, values, values[1:1, :], cu(CartesianIndex.(1, axes(values, 2))))
    else
        error("unsupported yet for dims=$dims")
    end
end

function Base._findmin(values::CuArray{Float32, 2}, dims::Int)
    if dims == 1
        findminmaxcol!(Base.isless, values, values[1:1, :], cu(CartesianIndex.(1, axes(values, 2))))
    else
        error("unsupported yet for dims=$dims")
    end
end

function Base._maximum(x::CuArray{Float32, 2}, dims::Int)
    if dims == 1
        _findmax(x, dims)[1]
    else
        error("unsupported yet for dims=$dims")
    end
end

function Base._minimum(x::CuArray{Float32, 2}, dims::Int)
    if dims == 1
        _findmin(x, dims)[1]
    else
        error("unsupported yet for dims=$dims")
    end
end

#####
# Cartesian indexing of CuArray
#####

function Base.getindex(xs::CuArray{T, N}, indices::CuArray{CartesianIndex{N}, 1}) where {T, N}
  n = length(indices)
  ys = CuArray{T}(undef, n)

  if n > 0
    num_threads = min(n, 256)
    num_blocks = ceil(Int, n / num_threads)

    function kernel(ys::CuArrays.CuDeviceArray{T}, xs::CuArrays.CuDeviceArray{T}, indices)
        i = threadIdx().x + (blockIdx().x - 1) * blockDim().x

        if i <= length(ys)
            ind = indices[i]
            ys[i] = xs[ind]
        end

        return
    end

    CuArrays.@cuda blocks=num_blocks threads=num_threads kernel(ys, xs, indices)
  end

  return ys
end

function Base.setindex!(xs::CuArray{T, N}, v::CuArray{T}, indices::CuArray{CartesianIndex{N}, 1}) where {T, N}
    # @assert length(indices) == length(v) "$xs, $(size(xs)), $v, $(size(v)), $indices, $(size(indices))"
    n = length(indices)

    if n > 0
        num_threads = min(n, 256)
        num_blocks = ceil(Int, n / num_threads)

        function kernel(xs::CuArrays.CuDeviceArray{T}, indices, v)
            i = threadIdx().x + (blockIdx().x - 1) * blockDim().x

            if i <= length(indices)
                ind = indices[i]
                xs[ind] = v[i]
            end

            return
        end

        CuArrays.@cuda blocks=num_blocks threads=num_threads kernel(xs, indices, v)
    end
    return v
end

function Base.setindex!(xs::CuArray{T, N}, v::T, indices::CuArray{CartesianIndex{N}, 1}) where {T, N}
  n = length(indices)

  if n > 0
    num_threads = min(n, 256)
    num_blocks = ceil(Int, n / num_threads)

    function kernel(xs::CuArrays.CuDeviceArray{T}, indices, v)
        i = threadIdx().x + (blockIdx().x - 1) * blockDim().x

        if i <= length(indices)
            ind = indices[i]
            xs[ind] = v
        end

        return
    end

    CuArrays.@cuda blocks=num_blocks threads=num_threads kernel(xs, indices, v)
  end
  return v
end

Base.setindex!(xs::CuArray{T, N}, v::Fill{T}, indices::CuArray{CartesianIndex{N}, 1}) where {T, N} = setindex!(xs, v.value, indices)


function init(start_time, m::JuniperProblem; incumbent = nothing)
    hash_val = string(hash(hcat(m.l_var,m.u_var)))
    node = BnBNode(1, 1, m.l_var, m.u_var, m.relaxation_solution, 0, :Branch, m.relaxation_status, m.relaxation_objval,[],hash_val)
    obj_gain_m = zeros(m.num_disc_var)
    obj_gain_p = zeros(m.num_disc_var)
    obj_gain_mc = zeros(Int64, m.num_disc_var)
    obj_gain_pc = zeros(Int64, m.num_disc_var)
    obj_gain_inf = zeros(Int64, m.num_disc_var)
    obj_gain = GainObj(obj_gain_m, obj_gain_p, obj_gain_mc, obj_gain_pc, obj_gain_inf)
    disc2var_idx = zeros(m.num_disc_var)
    var2disc_idx = zeros(m.num_var)
    disc_i = 1
    for i=1:m.num_var
        if m.var_type[i] != :Cont
            disc2var_idx[disc_i] = i
            var2disc_idx[i] = disc_i
            disc_i += 1
        end
    end
    factor = 1
    if m.obj_sense == :Min
        factor = -1
    end
    bnbTree = BnBTreeObj()
    bnbTree.m           = m
    bnbTree.obj_gain    = obj_gain
    bnbTree.disc2var_idx = disc2var_idx
    bnbTree.var2disc_idx = var2disc_idx
    bnbTree.options     = m.options
    bnbTree.obj_fac     = factor
    bnbTree.start_time  = start_time
    bnbTree.nsolutions  = 0
    bnbTree.branch_nodes = [node]
    bnbTree.best_bound  = m.relaxation_objval
    bnbTree.limit = :None
    bnbTree.global_solver = is_global_status(m.status)

    if incumbent !== nothing
        bnbTree.incumbent = incumbent
        bnbTree.nsolutions += 1
        if m.options.incumbent_constr
            add_incumbent_constr(m,bnbTree.incumbent)
            m.ncuts += 1
        end
    end

    return bnbTree
end

function new_default_node(idx, level, l_var, u_var, solution;
                            var_idx=0, state=:Solve, relaxation_state=MOI.OPTIMIZE_NOT_CALLED, best_bound=NaN, path=[])

    l_var = copy(l_var)
    u_var = copy(u_var)
    solution = copy(solution)
    hash_val = string(hash(hcat(l_var,u_var)))
    return BnBNode(idx, level, l_var, u_var, solution, var_idx, state, relaxation_state, best_bound, path, hash_val)
end

function new_left_node(node, u_var;
                       var_idx=0, state=:Solve, relaxation_state=MOI.OPTIMIZE_NOT_CALLED, best_bound=NaN, path=[])
    l_var = copy(node.l_var)
    u_var = copy(u_var)
    solution = NaN*ones(length(node.solution))
    hash_val = string(hash(hcat(l_var,u_var)))
    return BnBNode(node.idx*2, node.level+1, l_var, u_var, solution, var_idx, state, relaxation_state, best_bound, path, hash_val)
end

function new_right_node(node, l_var;
                       var_idx=0, state=:Solve, relaxation_state=MOI.OPTIMIZE_NOT_CALLED, best_bound=NaN, path=[])
    l_var = copy(l_var)
    u_var = copy(node.u_var)
    solution = NaN*ones(length(node.solution))
    hash_val = string(hash(hcat(l_var,u_var)))
    return BnBNode(node.idx*2+1, node.level+1, l_var, u_var, solution, var_idx, state, relaxation_state, best_bound, path, hash_val)
end

function init_time_obj()
    return TimeObj(0.0,0.0,0.0,0.0,0.0)
end

function new_default_step_obj(m,node)
    gains_m = zeros(m.num_disc_var)
    gains_mc = zeros(Int64, m.num_disc_var)
    gains_p = zeros(m.num_disc_var)
    gains_pc = zeros(Int64, m.num_disc_var)
    gains_inf = zeros(Int64, m.num_disc_var)
    gains = GainObj(gains_m, gains_p, gains_mc, gains_pc, gains_inf)
    idx_time = 0.0
    node_idx_time = 0.0
    upd_gains_time = 0.0
    node_branch_time = 0.0
    branch_time = 0.0
    step_obj = StepObj()
    step_obj.node             = node
    step_obj.var_idx          = 0
    step_obj.state            = :None
    step_obj.nrestarts        = 0
    step_obj.gain_gap         = 0.0
    step_obj.obj_gain         = gains
    step_obj.idx_time         = idx_time
    step_obj.node_idx_time    = node_idx_time
    step_obj.upd_gains_time   = upd_gains_time
    step_obj.node_branch_time = node_branch_time
    step_obj.branch_time      = branch_time
    step_obj.integral         = []
    step_obj.branch           = []
    step_obj.counter          = 0
    step_obj.upd_gains        = :None
    step_obj.strong_disc_vars = Int64[]
    step_obj.branch_strategy  = :None
    step_obj.strong_branching = nothing
    return step_obj
end

function init_juniper_problem!(jp::JuniperProblem, model::MOI.AbstractOptimizer)
    num_variables = length(model.variable_info)
    num_linear_le_constraints = length(model.linear_le_constraints)
    num_linear_ge_constraints = length(model.linear_ge_constraints)
    num_linear_eq_constraints = length(model.linear_eq_constraints)
    num_quadratic_le_constraints = length(model.quadratic_le_constraints)
    num_quadratic_ge_constraints = length(model.quadratic_ge_constraints)
    num_quadratic_eq_constraints = length(model.quadratic_eq_constraints)

    jp.status = MOI.OPTIMIZE_NOT_CALLED
    jp.relaxation_status = MOI.OPTIMIZE_NOT_CALLED
    jp.has_nl_objective = model.nlp_data.has_objective
    jp.nlp_evaluator = model.nlp_data.evaluator
    jp.objective = model.objective

    jp.objval = NaN
    jp.best_bound = NaN
    jp.solution = fill(NaN, num_variables)
    jp.nsolutions = 0
    jp.solutions = []
    jp.num_disc_var = 0
    jp.nintvars = 0
    jp.nbinvars = 0
    jp.nnodes = 1 # is set to one for the root node
    jp.ncuts = 0
    jp.nbranches = 0
    jp.nlevels = 1
    jp.relaxation_time = 0.0

    jp.start_time = time()

    jp.nl_solver = model.options.nl_solver
    nl_vec_opts = Vector{Pair}()
    if isa(jp.nl_solver, MOI.OptimizerWithAttributes)
        for arg in model.options.nl_solver.params
            push!(nl_vec_opts, arg)
        end
    end

    jp.nl_solver_options = nl_vec_opts

    if model.options.mip_solver !== nothing
        jp.mip_solver = model.options.mip_solver
        mip_vec_opts = Vector{Pair}()
        if isa(jp.mip_solver, MOI.OptimizerWithAttributes)
            for arg in model.options.mip_solver.params
                push!(mip_vec_opts, arg)
            end
        end

        jp.mip_solver_options = mip_vec_opts
    end
    jp.options = model.options
    if model.sense == MOI.MIN_SENSE
        jp.obj_sense = :Min
    else
        jp.obj_sense = :Max
    end
    jp.l_var = info_array_of_variables(model.variable_info, :lower_bound)
    jp.u_var = info_array_of_variables(model.variable_info, :upper_bound)
    integer_bool_arr = info_array_of_variables(model.variable_info, :is_integer)
    binary_bool_arr = info_array_of_variables(model.variable_info, :is_binary)
    primal_start_arr = info_array_of_variables(model.variable_info, :start)
    jp.primal_start = primal_start_arr
    jp.nintvars = sum(integer_bool_arr)
    jp.nbinvars = sum(binary_bool_arr)
    jp.num_disc_var = sum(integer_bool_arr)+sum(binary_bool_arr)
    jp.num_var = length(model.variable_info)
    jp.var_type = [:Cont for i in 1:jp.num_var]
    jp.var_type[integer_bool_arr .== true] .= :Int
    jp.var_type[binary_bool_arr .== true] .= :Bin
    jp.disc2var_idx = zeros(jp.num_disc_var)
    jp.var2disc_idx = zeros(jp.num_var)
    int_i = 1
    for i=1:jp.num_var
        if jp.var_type[i] != :Cont
            jp.disc2var_idx[int_i] = i
            jp.var2disc_idx[i] = int_i
            int_i += 1
        end
    end
    jp.num_l_constr = num_linear_le_constraints+num_linear_ge_constraints+num_linear_eq_constraints
    jp.num_q_constr = num_quadratic_le_constraints+num_quadratic_ge_constraints+num_quadratic_eq_constraints
    jp.num_nl_constr = length(model.nlp_data.constraint_bounds)
    jp.num_constr = jp.num_l_constr+jp.num_q_constr+jp.num_nl_constr
end

function new_default_strong_branch_step_obj(var_idx)
    strong_step_obj = StrongBranchStep()
    strong_step_obj.var_idx                 = var_idx
    strong_step_obj.l_relaxation_state      = MOI.OPTIMIZE_NOT_CALLED
    strong_step_obj.r_relaxation_state      = MOI.OPTIMIZE_NOT_CALLED
    strong_step_obj.init_restart            = false
    return strong_step_obj
end


"""
    check_feasibility(elem::T, bus::String, elemid::String, elem_formulation::Symbol, scenario::String, point::Point) where T<:AbstractNodeLabel

Evaluate the constraints defined by `elemid` of type `elem` at `bus` for `scenario` (depending on `elem_formulation`) on `point` and return a dictionary of infeasible constraints Dict(ctrname => message with information)
Return nothing if there is not violated constraints.

"""
function check_feasibility(elem::T, bus::String, elemid::String, elem_formulation::Symbol, scenario::String, point::Point,epsilon::Float64) where T<:AbstractNodeLabel
    return
end

"""
    check_feasibility(element::T, link::Link, elemid::String, elem_formulation::Symbol, link_vars::Dict{String, Variable}, scenario::String, point::Point, epsilon::Float64) where T<:AbstractLinkLabel

Evaluate the constraints defined by `elemid` of type `elem` at `link` for `scenario` (depending on `elem_formulation`) on `point` and return a dictionary of infeasible constraints Dict(ctrname => message with information)
Return nothing if there is not violated constraints.

"""

function check_feasibility(element::T, link::Link, elemid::String, elem_formulation::Symbol, link_vars::Dict{String, Variable}, scenario::String, point::Point, epsilon::Float64) where T<:AbstractLinkLabel
    return
end


# This file includes testset for sources.

@testset "ClockTestSet" begin
    @info "Running ClockTestSet ..."

    # Clock construction
    clk1 = Clock(0., 1., 10.)
    clk2 = Clock(0., 1, 10)
    clk3 = Clock(0, 1, 10)
    @test eltype(clk1.t) == Float64
    @test eltype(clk2.t) == Float64
    @test eltype(clk3.t) == Int

    # Check Clock defaults.
    clk = clk1
    @test clk.t == 0.
    @test clk.dt == 1.
    @test clk.tf == 10.
    @test typeof(clk.generator) == Channel{Float64}
    @test clk.generator.sz_max == 0
    @test !ispaused(clk)
    @test !isrunning(clk)

    # Set Clock
    set!(clk)
    @test isrunning(clk)

    # Taking values from clk
    clk = Clock(0., 1., 10.)
    set!(clk)
    @test [take!(clk) for i in 0 : 10] == collect(Float64, 0:10)
    @test isoutoftime(clk)

    # Pausing Clock
    clk = set!(Clock(0., 1, 10))
    @test take!(clk) == 0
    @test take!(clk) == 1.
    pause!(clk)
    for i = 1 : 10
        @test take!(clk) == 1.
    end

    @info "Done ClockTestSet."
end  # testset

@testset "GeneratorsTestSet" begin
    @info "Running GeneratorsTestSet ..."
    # FunctionGenerator construction
    gen = SinewaveGenerator()
    @test typeof(gen.trigger) == Inpin{Float64}
    @test typeof(gen.handshake) == Outpin{Bool}
    @test !hasfield(typeof(gen), :input)
    @test typeof(gen.output) == Outport{Outpin{Float64}}

    @def_source struct  Mygen{OP, RO} <: AbstractSource
        output::OP = Outport(2)
        readout::RO = t -> [sin(t), cos(t)]
    end
    gen = Mygen()
    @test length(gen.output) == 2

    # Driving FunctionGenerator
    gen = Mygen(readout = t -> t, output=Outport(1))
    trg = Outpin()
    hnd = Inpin{Bool}()
    ip = Inport()
    connect!(gen.output, ip)
    connect!(trg, gen.trigger)
    connect!(gen.handshake, hnd)
    task = launch(gen)
    task2 = @async while true
        all(take!(ip) .=== NaN) && break
    end
    for t in 1. : 10.
        put!(trg, t)
        take!(hnd)
    end
    @test content(ip[1].link.buffer) == collect(1:10)
    @test !istaskdone(task)
    put!(trg, NaN)
    @test istaskdone(task)
    put!(gen.output, [NaN])
    @test istaskdone(task2)

    # Construction of other generators
    sinegen = SinewaveGenerator()
    dampedsinegen = DampedSinewaveGenerator()
    sqauregen = SquarewaveGenerator()
    trigen = TriangularwaveGenerator()
    congen = ConstantGenerator()
    rampgen = RampGenerator()
    stepgen = StepGenerator()
    expgen = ExponentialGenerator()
    dampedexpgen = DampedExponentialGenerator()

    # Mutaton of generators
    @test_throws Exception sinegen.amplitude = 5.
    @test_throws Exception sqauregen.high = 10.

    # Test redefining new source types.
    @test_throws Exception @eval @def_source struct Mygen{RO,OP}
        readout::RO = t -> sin(t)
        output::OP = Outport()
    end

    @test_throws Exception @eval @def_source struct Mygen{RO,OP} <: SomeDummyType
        readout::RO = t -> sin(t)
        output::OP = Outport()
    end

    @info "Done GeneratorsTestSet ..."
end  # testset


import Automa
import Automa.RegExp: @re_str
const re = Automa.RegExp

keyword = re"break|const|continue|else|elseif|end|for|function|if|return|type|using|while"
identifier = re"[A-Za-z_][0-9A-Za-z_!]*"
operator = re"-|\+|\*|/|%|&|\||^|!|~|>|<|<<|>>|>=|<=|=>|==|==="
macrocall = re"@" * re"[A-Za-z_][0-9A-Za-z_!]*"
comment = re"#[^\r\n]*"
char = re.cat('\'', re"[ -&(-~]" | re.cat('\\', re"[ -~]"), '\'')
string = re.cat('"', re.rep(re"[ !#-~]" | re.cat("\\\"")), '"')
triplestring = re.cat("\"\"\"", (re"[ -~]*" \ re"\"\"\""), "\"\"\"")
newline = re"\r?\n"

const minijulia = Automa.compile(
    re","          => :(emit(:comma)),
    re":"          => :(emit(:colon)),
    re";"          => :(emit(:semicolon)),
    re"\."         => :(emit(:dot)),
    re"\?"         => :(emit(:question)),
    re"="          => :(emit(:equal)),
    re"\("         => :(emit(:lparen)),
    re"\)"         => :(emit(:rparen)),
    re"\["         => :(emit(:lbracket)),
    re"]"          => :(emit(:rbracket)),
    re"{"          => :(emit(:lbrace)),
    re"}"          => :(emit(:rbrace)),
    re"$"          => :(emit(:dollar)),
    re"&&"         => :(emit(:and)),
    re"\|\|"       => :(emit(:or)),
    re"::"         => :(emit(:typeannot)),
    keyword        => :(emit(:keyword)),
    identifier     => :(emit(:identifier)),
    operator       => :(emit(:operator)),
    macrocall      => :(emit(:macrocall)),
    re"[0-9]+"     => :(emit(:integer)),
    comment        => :(emit(:comment)),
    char           => :(emit(:char)),
    string         => :(emit(:string)),
    triplestring   => :(emit(:triplestring)),
    newline        => :(emit(:newline)),
    re"[\t ]+"     => :(emit(:spaces)),
)

#=
write("minijulia.dot", Automa.machine2dot(minijulia.machine))
run(`dot -Tsvg -o minijulia.svg minijulia.dot`)
=#

context = Automa.CodeGenContext()
@eval function tokenize(data)
    $(Automa.generate_init_code(context, minijulia))
    p_end = p_eof = sizeof(data)
    tokens = Tuple{Symbol,String}[]
    emit(kind) = push!(tokens, (kind, data[ts:te]))
    while p ≤ p_eof && cs > 0
        $(Automa.generate_exec_code(context, minijulia))
    end
    if cs < 0
        error("failed to tokenize")
    end
    return tokens
end

tokens = tokenize("""
quicksort(xs) = quicksort!(copy(xs))
quicksort!(xs) = quicksort!(xs, 1, length(xs))

function quicksort!(xs, lo, hi)
    if lo < hi
        p = partition(xs, lo, hi)
        quicksort!(xs, lo, p - 1)
        quicksort!(xs, p + 1, hi)
    end
    return xs
end

function partition(xs, lo, hi)
    pivot = div(lo + hi, 2)
    pvalue = xs[pivot]
    xs[pivot], xs[hi] = xs[hi], xs[pivot]
    j = lo
    @inbounds for i in lo:hi-1
        if xs[i] <= pvalue
            xs[i], xs[j] = xs[j], xs[i]
            j += 1
        end
    end
    xs[j], xs[hi] = xs[hi], xs[j]
    return j
end
""")


export SolverParameters, SolverState, Solver
export setup_coffee_lounge, add_coffee_break, solve
export load_snapshot
export make_solver_parameters
export init_solve, onestep_solve, finalize_solve
import Base.Meta: quot

@compat abstract type SolverMethod end # An enum type to identify the solver in validation functions

@compat abstract type InternalSolverState end # All the state a solver needs to update an iteration
const SolverParameters = Dict{Symbol,Any}

immutable Solver{T<:SolverMethod}
  method        :: T
  params        :: SolverParameters
  coffee_lounge :: Any # forward declaration
end

Solver{T}(method::T, params::SolverParameters) = begin
    validate_parameters(method, params)
    Solver(method, params, CoffeeLounge())
end

type SolverState{T<:InternalSolverState}
  iter               :: Int
  obj_val            :: Float64
  losses             :: Dict
  internal           :: T
end
function copy_solver_state!{T<:InternalSolverState}(dst::SolverState{T}, src::SolverState{T})
  dst.iter = src.iter
  dst.obj_val = src.obj_val
  dst.losses = src.losses
  dst.internal = src.internal
end

SolverState{T<:InternalSolverState}(internal::T) = SolverState{T}(0, Inf, Dict(), internal)

@compat abstract type SolverStateSnapshot end # Just the serializable part of the solver state, for snapshot files

function make_solver_parameters(;kwargs...)
    localDict = @compat Dict(:max_iter => Inf,
           :regu_coef => 0.0005,
           :load_from => "")
	merge(localDict,
          SolverParameters(kwargs))
end

function validate_parameters(params::SolverParameters, args...)
    if params[:regu_coef] < 0
        error("regu_coef must be non-negative")
    end
    for a in args
        if !haskey(params, a)
            error("Must provide parameter $a")
        end
    end
end

############################################################
#  API functions to be implemented by each solver instance
############################################################

function get_statistic(state::SolverState, name::AbstractString)
  if haskey(state.losses, name)
    return state.losses[name]
  else
    return NaN
  end
end

function get_statistic(state::SolverState, name::Symbol)
    if name in [:obj_val, :iter]
      @eval $(quot(state)).$name
    else
        @eval $(quot(state.internal)).$name
    end
end

function format_statistic(state::SolverState, name)
    format_statistic(state, name, get_statistic(state, name))
end

function format_statistic(state::SolverState, name, value)
    if name == :iter
        @sprintf("%s=%06d", name, value)
    else
        @sprintf("%s=%.8f", name, value)
    end
end

validate_parameters(method::SolverMethod, params::SolverParameters) = begin # should turn validate method-specific parameters
    validate_parameters(params)
    error("solver_state is not implemented for the method type $(typeof(method)), and params type $(typeof(params))")
end

function snapshot(state::SolverState) # should turn a SolverState{T} into a SolverStateSnapshot instance
    error("snapshot is not implemented for the state type $(typeof(state))")
end

function solver_state(net::Net, snapshot::SolverStateSnapshot) # should use a SolverStateSnapshot instance to create a SolverState{T}
    error("solver_state is not implemented for the net type $(typeof(net)), and snapshot type $(typeof(snapshot))")
end

function solver_state(solver::SolverMethod, net::Net, params::SolverParameters) # should create SolverState{T} from SolverParameters dictionary
    error("solver_state is not implemented for the solver type $(typeof(solver)), net type $(typeof(net)), and params type $(typeof(params))")
end

function update{T}(solver::Solver{T}, net::Net, state::SolverState) # should do one iteration of update
    error("update is not implemented for the solver type $(typeof(solver)), net type $(typeof(net)), and state type $(typeof(state))")
end
function shutdown(state::SolverState) # should shutdown the solver
    error("shutdown is not implemented for the $(typeof(state)) type")
end

############################################################
# General utilities that could be used by all solvers
############################################################
function load_snapshot(net::Net, path::AbstractString="", state=nothing)
  if isempty(path)
    return state
  end

  if endswith(path, ".hdf5") || endswith(path, ".h5")
    # load from HDF5 file, possibly exported from caffe, but training
    # from the beginning (iteration 0) as the solver state is not saved
    # in a HDF5 file
    if isfile(path)
      @info("Loading existing model from $path")
      h5open(path) do file
        load_network(file, net)
      end
    end
    return state
  else
    if endswith(path, ".jld")
      # load from some specific JLD sanpshot, the solver state is also
      # recovered
      filename = path
    else
      # automatically load from the latest snapshot in a directory
      filename = ""
      if isdir(path)
        # load the latest snapshot from the directory
        snapshots = glob(path, r"^snapshot-[0-9]+\.jld", sort_by=:mtime)
        if length(snapshots) > 0
          filename = joinpath(path, snapshots[end])
        end
      end
    end

    if !isempty(filename) && isfile(filename)
      @info("Loading existing model from $filename")
      jldopen(filename) do file
        load_network(file, net)
        return solver_state(net, read(file, SOLVER_STATE_KEY))
      end
    else
      return state
    end
  end
end

function stop_condition_satisfied(solver::Solver, state::SolverState, net::Net)
  if state.iter >= solver.params[:max_iter]
    return true
  end
  return false
end

############################################################
# General Solver Loop
############################################################

function init_solve(solver::Solver, net::Net)
    @debug("#DEBUG Checking network topology for back-propagation")
    check_bp_topology(net)

    state = solver_state(solver.method, net, solver.params)
    state = load_snapshot(net, solver.params[:load_from], state)

    # we init network AFTER loading. If the parameters are loaded from file, the
    # initializers will be automatically set to NullInitializer
    init(net)
    state.obj_val = forward(net, solver.params[:regu_coef])

    @debug("#DEBUG Initializing coffee breaks")
    setup(solver.coffee_lounge, state, net)

    # coffee break for iteration 0, before everything starts
    check_coffee_break(solver.coffee_lounge, state, net)

    return state
end

function finalize_solve(solver::Solver, net::Net, state::SolverState)
    shutdown(solver.coffee_lounge, net)
    shutdown(state)
end

function onestep_solve(solver::Solver, net::Net, state::SolverState)
    state.iter += 1
    layer_states = updatable_layer_states(net)

    backward(net, solver.params[:regu_coef])
    update(solver, net, state)

    # apply weight constraints
    for layer_state in layer_states
      for param in layer_state.parameters
        cons_every = param.constraint.every_n_iter
        if cons_every > 0 && state.iter % cons_every == 0
          constrain!(net.backend, param.constraint, param.blob)
        end
      end
    end

    state.obj_val = forward(net, solver.params[:regu_coef])

    state.losses = Dict()
    for i = 1:length(net.layers)
      if has_loss(net.layers[i])
        state.losses[net.layers[i].name] = net.states[i].loss
      end
    end
    check_coffee_break(solver.coffee_lounge, state, net)
end

function solve(solver::Solver, net::Net)
    state = init_solve(solver, net)
    do_solve_loop(solver, net, state)
    finalize_solve(solver, net, state)
end

function do_solve_loop(solver::Solver, net::Net, state::SolverState)
  @debug("#DEBUG Entering solver loop")
  while !stop_condition_satisfied(solver, state, net)
    onestep_solve(solver,net,state)

    if stop_condition_satisfied(solver, state, net)
      break
    end
  end
  return state
end

trainable_layers(net::Net) = filter(i -> has_param(net.layers[i]) && !is_frozen(net.states[i]),
                                    1:length(net.layers))

updatable_layer_states(net::Net) = [net.states[i] for i in trainable_layers(net)]