const Height = Int8
const TreeNode{T, I, D} = Tuple{T, I, I, I, Height, D}

mutable struct Node{T, D}
    value::T
    left::Union{Nothing, Node{T, D}}
    right::Union{Nothing, Node{T, D}}
    parent::Union{Nothing, Node{T, D}}
    height::Height
    data::D
end
const Tree{T, D} = Union{Nothing, Node{T, D}}

@inline nil() = nothing
@inline node(t::T, parent, data::D) where {T, D} =
    Node{T, D}(t, nothing, nothing, parent, Height(1), data)
@inline node(t::T, left, right, parent, height, data::D) where {T, D} =
    Node{T, D}(t, left, right, parent, height, data)

@inline nil(alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}} = 0
@inline node(t::T, parent, data::D, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}} =
    allocateend!(alloc, (t, 0, 0, parent, Height(1), data))
@inline node(t::T, left, right, parent, height, data::D, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}} =
    allocateend!(alloc, (t, left, right, parent, height, data))

@inline deallocate(i::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}} =
    deallocate!(alloc, i)

@inline value(tree::Node{T, D}) where {T, D} = tree.value

@inline left(tree::Node{T, D}) where {T, D} = tree.left

@inline right(tree::Node{T, D}) where {T, D} = tree.right

@inline parent_(tree::Node{T, D}) where {T, D} = tree.parent

@inline height(tree::Nothing) = Height(0)
@inline height(tree::Node{T, D}) where {T, D} = tree.height

@inline data(tree::Node{T, D}) where {T, D} = tree.data

@inline function value!(tree::Node{T, D}, value::T) where {T, D}
    tree.value = value
end

@inline function left!(tree::Node{T, D}, left::Tree{T}) where {T, D}
    tree.left = left
end

@inline function right!(tree::Node{T, D}, right::Tree{T}) where {T, D}
    tree.right = right
end

@inline function parent!(tree::Node{T, D}, parent::Tree{T}) where {T, D}
    tree.parent = parent
end

@inline function height!(tree::Node{T, D}, height::Height) where {T, D}
    tree.height = height
end

@inline function height!(tree::Node{T, D}, height_left::Height, height_right::Height) where {T, D}
    tree.height = max(height_left, height_right) + Height(1)
end

@inline function height!(tree::Node{T, D}) where {T, D}
    _left = left(tree)
    _right = right(tree)
    height!(tree, height(_left), height(_right))
end

@inline function data!(tree::Node{T, D}, data::D) where {T, D}
    tree.data = data
end

@inline function value(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    alloc[tree][1]
end
@inline function value(tree::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 1]
end

@inline function left(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    alloc[tree][2]
end
@inline function left(tree::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 2]
end

@inline function right(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    alloc[tree][3]
end
@inline function right(tree::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 3]
end

@inline function parent_(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    alloc[tree][4]
end
@inline function parent_(tree::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 4]
end

@inline function height(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    tree === 0 ? Height(0) : alloc[tree][5]
end
@inline function height(tree::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    tree === 0 ? Height(0) : alloc[tree, 5]
end

@inline function data(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    alloc[tree][6]
end
@inline function data(tree::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 6]
end

@inline function value!(tree::I, _value::T, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    node = alloc[tree]
    alloc[tree] = (_value, node[2], node[3],
                   node[4], node[5], node[6])
end
@inline function value!(tree::I, _value::T, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 1] = _value
end

@inline function left!(tree::I, _left::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    node = alloc[tree]
    alloc[tree] = (node[1], _left, node[3],
                   node[4], node[5], node[6])
end
@inline function left!(tree::I, _left::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 2] = _left
end

@inline function right!(tree::I, _right::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    node = alloc[tree]
    alloc[tree] = (node[1], node[2], _right,
                   node[4], node[5], node[6])
end
@inline function right!(tree::I, _right::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 3] = _right
end

@inline function parent!(tree::I, _parent::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    node = alloc[tree]
    alloc[tree] = (node[1], node[2], node[3],
                   _parent, node[5], node[6])
end
@inline function parent!(tree::I, _parent::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 4] = _parent
end

@inline function height!(tree::I, _height::Height, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    node = alloc[tree]
    alloc[tree] = (node[1], node[2], node[3],
                   node[4], _height, node[6])
end
@inline function height!(tree::I, _height::I, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 5] = _height
end

@inline function height!(tree::I, height_left::Height, height_right::Height, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _height = max(height_left, height_right) + Height(1)
    height!(tree, _height, alloc)
end

@inline function height!(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _left = left(tree, alloc)
    _right = right(tree, alloc)
    height_left = height(_left, alloc)
    height_right = height(_right, alloc)
    height!(tree, height_left, height_right, alloc)
end

@inline function data!(tree::I, _data::D, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    node = alloc[tree]
    alloc[tree] = (node[1], node[2], node[3],
                   node[4], node[5], _data)
end
@inline function data!(tree::I, _data::D, alloc::A) where {T, I, D, S, A <: AbstractFreeListSOAllocator{TreeNode{T, I, D}, I, S}}
    alloc[tree, 6] = _data
end

invariantvalue(tree::Nothing) = true
function invariantvalue(tree::Node{T, D}) where {T, D}
    invariant = true
    _value = value(tree)
    _left = left(tree)
    invariant &= _left === nothing || value(_left) < _value
    _right = right(tree)
    invariant &= _right === nothing || value(_right) > _value
    @assert invariant
    invariant
end

invariantparent(tree::Nothing) = true
function invariantparent(tree::Node{T, D}) where {T, D}
    invariant = true
    _left = left(tree)
    invariant &= _left === nothing || parent_(_left) === tree
    _right = right(tree)
    invariant &= _right === nothing || parent_(_right) === tree
    @assert invariant
    invariant
end

invariantheight(tree::Nothing) = true
function invariantheight(tree::Node{T, D}) where {T, D}
    _left = left(tree)
    _right = right(tree)
    _height = height(tree)
    height_left = height(_left)
    height_right = height(_right)
    @assert _height == max(height_left, height_right) + 1
    invariant = abs(height_left - height_right) < 2
    @assert invariant
    invariant
end

function invariantvalue(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    if tree === 0
        return true
    else
        invariant = true
        _value = value(tree, alloc)
        _left = left(tree, alloc)
        invariant &= _left === 0 || value(_left, alloc) < _value
        _right = right(tree, alloc)
        invariant &= _right === 0 || value(_right, alloc) > _value
        @assert invariant
        invariant
    end
end

function invariantparent(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    if tree === 0
        return true
    else
        invariant = true
        _left = left(tree, alloc)
        invariant &= _left === 0 || parent_(_left, alloc) === tree
        _right = right(tree, alloc)
        invariant &= _right === 0 || parent_(_right, alloc) === tree
        @assert invariant
        invariant
    end
end

function invariantheight(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    if tree === 0
        return true
    else
        _left = left(tree, alloc)
        _right = right(tree, alloc)
        _height = height(tree, alloc)
        height_left = height(_left, alloc)
        height_right = height(_right, alloc)
        @assert _height == max(height_left, height_right) + 1
        invariant = abs(height_left - height_right) < 2
        @assert invariant
        invariant
    end
end

@inline function rotateright(tree::Node{T, D}) where {T, D}
    _left = left(tree)
    _parent = parent_(tree)
    left_right = right(_left)
    right!(_left, tree)
    parent!(tree, _left)
    left!(tree, left_right)
    if left_right !== nothing
        parent!(left_right, tree)
    end
    # height!(tree)
    _height = height!(tree, height(left_right), height(right(tree)))
    parent!(_left, _parent)
    # height!(_left)
    height!(_left, height(left(_left)), _height)
    _left
end
@inline function rotateleft(tree::Node{T, D}) where {T, D}
    _right = right(tree)
    _parent = parent_(tree)
    right_left = left(_right)
    left!(_right, tree)
    parent!(tree, _right)
    right!(tree, right_left)
    if right_left !== nothing
        parent!(right_left, tree)
    end
    # height!(tree)
    _height = height!(tree, height(left(tree)), height(right_left))
    parent!(_right, _parent)
    # height!(_right)
    height!(_right, _height, height(right(_right)))
    _right
end

@inline function rotateright(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _left = left(tree, alloc)
    _parent = parent_(tree, alloc)
    left_right = right(_left, alloc)
    right!(_left, tree, alloc)
    parent!(tree, _left, alloc)
    left!(tree, left_right, alloc)
    if left_right !== 0
        parent!(left_right, tree, alloc)
    end
    # height!(tree, alloc)
    _right = right(tree, alloc)
    height!(tree, height(left_right, alloc), height(_right, alloc), alloc)
    parent!(_left, _parent, alloc)
    # height!(_left, alloc)
    left_left = left(_left, alloc)
    height!(_left, height(left_left, alloc), height(tree, alloc), alloc)
    _left
end
@inline function rotateleft(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _right = right(tree, alloc)
    _parent = parent_(tree, alloc)
    right_left = left(_right, alloc)
    left!(_right, tree, alloc)
    parent!(tree, _right, alloc)
    right!(tree, right_left, alloc)
    if right_left !== 0
        parent!(right_left, tree, alloc)
    end
    # height!(tree, alloc)
    _left = left(tree, alloc)
    height!(tree, height(_left, alloc), height(right_left, alloc), alloc)
    parent!(_right, _parent, alloc)
    # height!(_right, alloc)
    right_right = right(_right, alloc)
    height!(_right, height(tree, alloc), height(right_right, alloc), alloc)
    _right
end

@inline function balance(tree::Node{T, D}) where {T, D}
    _left = left(tree)
    _right = right(tree)
    height_left = height(_left)
    height_right = height(_right)
    if height_left < height_right - 1
        right_left = left(_right)
        right_right = right(_right)
        height_right_left = height(right_left)
        height_right_right = height(right_right)
        if height_right_left > height_right_right
            _right = rotateright(_right)
            right!(tree, _right)
        end
        tree = rotateleft(tree)
    elseif height_left > height_right + 1
        left_left = left(_left)
        left_right = right(_left)
        height_left_left = height(left_left)
        height_left_right = height(left_right)
        if height_left_left < height_left_right
            _left = rotateleft(_left)
            left!(tree, _left)
        end
        tree = rotateright(tree)
    else
        height!(tree, height_left, height_right)
    end
    tree
end

@inline function balance(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _left = left(tree, alloc)
    _right = right(tree, alloc)
    height_left = height(_left, alloc)
    height_right = height(_right, alloc)
    if height_left < height_right - 1
        right_left = left(_right, alloc)
        right_right = right(_right, alloc)
        height_right_left = height(right_left, alloc)
        height_right_right = height(right_right, alloc)
        if height_right_left > height_right_right
            _right = rotateright(_right, alloc)
            right!(tree, _right, alloc)
        end
        tree = rotateleft(tree, alloc)
    elseif height_left > height_right + 1
        left_left = left(_left, alloc)
        left_right = right(_left, alloc)
        height_left_left = height(left_left, alloc)
        height_left_right = height(left_right, alloc)
        if height_left_left < height_left_right
            _left = rotateleft(_left, alloc)
            left!(tree, _left, alloc)
        end
        tree = rotateright(tree, alloc)
    else
        height!(tree, height_left, height_right, alloc)
    end
    tree
end

@noinline function insert(tree::Tree{T}, t::T, data::D) where {T, D}
    _root = tree
    _parent = nothing
    while tree !== nothing
        _value = value(tree)
        if t < _value
            _parent = tree
            tree = left(tree)
        elseif t > _value
            _parent = tree
            tree = right(tree)
        else
            break
        end
    end
    _balance = false
    _update = false
    if tree === nothing
        _balance = true
        _update = true
        tree = node(t, _parent, data)
    else
        data!(tree, data)
    end
    while _parent !== nothing
        if _update
            _value = value(_parent)
            if t < _value
                left!(_parent, tree)
            elseif t > _value
                right!(_parent, tree)
            end
            if _balance
                _height = height(_parent)
                _parent = balance(_parent)
                _balance = height(_parent) !== _height
            else
                _update = false
            end
        else
            return _root
        end
        tree = _parent
        _parent = parent_(tree)
    end
    return tree
end

@noinline function insert(tree::I, t::T, data::D, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _root = tree
    _parent = 0
    while tree !== 0
        _value = value(tree, alloc)
        if t < _value
            _parent = tree
            tree = left(tree, alloc)
        elseif t > _value
            _parent = tree
            tree = right(tree, alloc)
        else
            break
        end
    end
    _balance = false
    _update = false
    if tree === 0
        _balance = true
        _update = true
        tree = node(t, _parent, data, alloc)
    else
        data!(tree, data, alloc)
    end
    while _parent !== 0
        if _update
            _value = value(_parent, alloc)
            if t < _value
                left!(_parent, tree, alloc)
            elseif t > _value
                right!(_parent, tree, alloc)
            end
            if _balance
                _height = height(_parent, alloc)
                _parent = balance(_parent, alloc)
                _balance = height(_parent, alloc) !== _height
            else
                _update = false
            end
        else
            return _root
        end
        tree = _parent
        _parent = parent_(tree, alloc)
    end
    return tree
end

@inline function Base.haskey(tree::Tree{T, D}, t::T) where {T, D}
    while tree !== nothing
        _value = value(tree)
        if t < _value
            tree = left(tree)
        elseif t > _value
            tree = right(tree)
        else
            return true
        end
    end
    return false
end

@inline function Base.haskey(tree::I, t::T, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    while tree !== 0
        _value = value(tree, alloc)
        if t < _value
            tree = left(tree, alloc)
        elseif t > _value
            tree = right(tree, alloc)
        else
            return true
        end
    end
    return false
end

@inline function Base.getindex(tree::Tree{T, D}, t::T) where {T, D}
    while tree !== nothing
        _value = value(tree)
        if t < _value
            tree = left(tree)
        elseif t > _value
            tree = right(tree)
        else
            return data(tree)
        end
    end
    @assert false
end

@inline function Base.haskey(tree::I, t::T, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    while tree !== 0
        _value = value(tree, alloc)
        if t < _value
            tree = left(tree, alloc)
        elseif t > _value
            tree = right(tree, alloc)
        else
            return true
        end
    end
    return false
end

@inline function Base.getindex(tree::I, t::T, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    while tree !== 0
        _value = value(tree, alloc)
        if t < _value
            tree = left(tree, alloc)
        elseif t > _value
            tree = right(tree, alloc)
        else
            return data(tree, alloc)
        end
    end
    @assert false
end

@inline function getleftmost(tree::Node{T, D}) where {T, D}
    _left = left(tree)
    while _left !== nothing
        tree = _left
        _left = left(tree)
    end
    value(tree)
end

@inline function getrightmost(tree::Node{T, D}) where {T, D}
    _right = right(tree)
    while _right !== nothing
        tree = _right
        _right = right(tree)
    end
    value(tree)
end

@inline function getleftmost(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _left = left(tree, alloc)
    while _left !== 0
        tree = _left
        _left = left(tree, alloc)
    end
    value(tree, alloc)
end

@inline function getrightmost(tree::I, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _right = right(tree, alloc)
    while _right !== 0
        tree = _right
        _right = right(tree, alloc)
    end
    value(tree, alloc)
end

@noinline function delete(tree::Tree{T, D}, t::T) where {T, D}
    _root = tree
    _parent = nothing
    while tree !== nothing
        _value = value(tree)
        if t < _value
            _parent = tree
            tree = left(tree)
        elseif t > _value
            _parent = tree
            tree = right(tree)
        else
            break
        end
    end
    _balance = false
    _update = false
    if tree !== nothing
        _balance = true
        _update = true
        _left = left(tree)
        _right = right(tree)
        if _left === nothing && _right === nothing
            tree = nothing
        elseif _left === nothing
            value!(tree, value(_right))
            left!(tree, left(_right))
            right!(tree, right(_right))
            # height!(tree)
            height!(tree, height(_right))
            data!(tree, data(_right))
            if _left !== nothing
                parent!(_left, tree)
            end
            _right = right(tree)
            if _right !== nothing
                parent!(_right, tree)
            end
        elseif _right === nothing
            value!(tree, value(_left))
            left!(tree, left(_left))
            right!(tree, right(_left))
            # height!(tree)
            height!(tree, height(_left))
            data!(tree, data(_left))
            _left = left(tree)
            if _left !== nothing
                parent!(_left, tree)
            end
            _right = right(tree)
            if _right !== nothing
                parent!(_right, tree)
            end
        else
            height_left = height(_left)
            height_right = height(_right)
            if height_left >= height_right
                _value = getrightmost(_left)
                value!(tree, _value)
                parent!(_left, nothing)
                _left = delete(_left, _value)
                if _left !== nothing
                    parent!(_left, tree)
                end
                left!(tree, _left)
                # height!(tree)
                height!(tree, height(_left), height_right)
            else
                _value = getleftmost(_right)
                value!(tree, _value)
                parent!(_right, nothing)
                _right = delete(_right, _value)
                if _right !== nothing
                    parent!(_right, tree)
                end
                right!(tree, _right)
                # height!(tree)
                height!(tree, height_left, height(_right))
            end
        end
    end
    while _parent !== nothing
        if _update
            _value = value(_parent)
            if t < _value
                left!(_parent, tree)
            elseif t > _value
                right!(_parent, tree)
            end
            if _balance
                _height = height(_parent)
                _parent = balance(_parent)
                _balance = height(_parent) !== _height
            else
                _update = false
            end
        else
            return _root
        end
        tree = _parent
        _parent = parent_(tree)
    end
    return tree
end

@noinline function delete(tree::I, t::T, alloc::A) where {T, I, D, A <: AbstractFreeListAllocator{TreeNode{T, I, D}, I}}
    _root = tree
    _parent = 0
    while tree !== 0
        _value = value(tree, alloc)
        if t < _value
            _parent = tree
            tree = left(tree, alloc)
        elseif t > _value
            _parent = tree
            tree = right(tree, alloc)
        else
            break
        end
    end
    _balance = false
    _update = false
    if tree !== 0
        _balance = true
        _update = true
        _left = left(tree, alloc)
        _right = right(tree, alloc)
        if _left === 0 && _right === 0
            deallocate(tree, alloc)
            tree = 0
        elseif _left === 0
            alloc[tree] = alloc[_right]
            parent!(tree, _parent, alloc)
            deallocate(_right, alloc)
            _left = left(tree, alloc)
            if _left !== 0
                parent!(_left, tree, alloc)
            end
            _right = right(tree, alloc)
            if _right !== 0
                parent!(_right, tree, alloc)
            end
        elseif _right === 0
            alloc[tree] = alloc[_left]
            parent!(tree, _parent, alloc)
            deallocate(_left, alloc)
            _left = left(tree, alloc)
            if _left !== 0
                parent!(_left, tree, alloc)
            end
            _right = right(tree, alloc)
            if _right !== 0
                parent!(_right, tree, alloc)
            end
        else
            height_left = height(_left, alloc)
            height_right = height(_right, alloc)
            if height_left >= height_right
                _value = getrightmost(_left, alloc)
                value!(tree, _value, alloc)
                parent!(_left, 0, alloc)
                _left = delete(_left, _value, alloc)
                if _left !== 0
                    parent!(_left, tree, alloc)
                end
                left!(tree, _left, alloc)
                # height!(tree, alloc)
                height!(tree, height(_left, alloc), height_right, alloc)
            else
                _value = getleftmost(_right, alloc)
                value!(tree, _value, alloc)
                parent!(_right, 0, alloc)
                _right = delete(_right, _value, alloc)
                if _right !== 0
                    parent!(_right, tree, alloc)
                end
                right!(tree, _right, alloc)
                # height!(tree, alloc)
                height!(tree, height_left, height(_right, alloc), alloc)
            end
        end
    end
    while _parent !== 0
        if _update
            _value = value(_parent, alloc)
            if t < _value
                left!(_parent, tree, alloc)
            elseif t > _value
                right!(_parent, tree, alloc)
            end
            if _balance
                _height = height(_parent, alloc)
                _parent = balance(_parent, alloc)
                _balance = height(_parent, alloc) !== _height
            else
                _update = false
            end
        else
            return _root
        end
        tree = _parent
        _parent = parent_(_parent, alloc)
    end
    return tree
end


__precompile__()

module UnitfulAstro

import Unitful
using Unitful: @unit, @logscale, @logunit

macro import_from_unitful(args...)
    expr = Expr(:block)
    for arg in args
        use_SI_prefixes, sym = should_we_use_SI_prefixes(arg)
        if use_SI_prefixes
            for prefix in Unitful.si_prefixes
                sym′ = Symbol(prefix, sym)
                push!(expr.args, :(import Unitful.$sym′))
            end
        else
            push!(expr.args, :(import Unitful.$sym))
        end
    end
    expr
end

function should_we_use_SI_prefixes(arg::Expr)
    if arg.head == :(call)
        if arg.args[1] == :(~)
            return true, arg.args[2]
        else
            error("incorrect first argument")
        end
    else
        error("incorrect expression head")
    end
end
should_we_use_SI_prefixes(arg::Symbol) = false, arg

@import_from_unitful ~m ~s ~A ~K ~cd ~yr
@import_from_unitful ~L ~Hz ~N ~Pa ~J ~W ~C ~V ~Ω ~S ~F ~H ~T ~Wb ~lm ~lx ~Bq ~Gy ~Sv ~kat ~eV
@import_from_unitful sr rad ° °C °F Ra minute hr d wk ~bar atm Torr
@import_from_unitful q c0 c μ0 µ0 ε0 ϵ0 Z0 G gn ge h ħ Φ0 me mn mp μB µB Na R k σ
@import_from_unitful inch ft yd mi ac lb oz dr gr lbf

# CGS units have been moved to default Unitful.jl now
@import_from_unitful dyn erg

import UnitfulAngles: arcminute, arcsecond

@unit AU         "AU"       AstronomicalUnit          149_597_870_700.0*m       false  # cf IAU 2012
@unit ly         "ly"       LightYear                 1*c*yr                    false
@unit pc         "pc"       Parsec                    1*AU/arcsecond            true   # cf IAU 2015
# Note that IAU 2015 defines the small angle approximation as exact in the definition of the parsec.
@unit Jy         "Jy"       Jansky                    1e-23erg*s^-1*cm^-2*Hz^-1 true
# Note that Jy uses a Float64 conversion factor because 10^23 overflows Int64.

# SOLAR CONVERSION CONSTANTS (IAU 2015)
@unit Rsun       "R⊙"       SolarRadius               6.957e8*m                 false
@unit Ssun       "S⊙"       SolarIrradiance           1361.0*W*m^-2             false
@unit Lsun       "L⊙"       SolarLuminosity           3.828e26*W                false
# TODO solar effective temperature
@unit GMsun      "GM⊙"      GSolarMass                1.327_124_4e20*m^3*s^-2   false
@unit Msun       "M⊙"       SolarMass                 1*GMsun/G                 false

# PLANETARY CONVERSION CONSTANTS (IAU 2015)
@unit Rearth_e   "R⊕ₑ"      EarthEquatorialRadius     6.3781e6*m                false
@unit Rearth_p   "R⊕ₚ"      EarthPolarRadius          6.3568e6*m                false
@unit Rjup_e     "Rjupₑ"    JupiterEquatorialRadius   7.1492e7*m                false
@unit Rjup_p     "Rjupₚ"    JupiterPolarRadius        6.6854e7*m                false
@unit GMearth    "GM⊕"      GEarthMass                3.986_004e14*m^3*s^-2     false
@unit GMjup      "GMjup"    GJupiterMass              1.266_865_3e17*m^3*s^-2   false
@unit Mearth     "M⊕"       EarthMass                 1*GMearth/G               false
@unit Mjup       "Mjup"     JupiterMass               1*GMjup/G                 false
@unit Rearth     "R⊕"       EarthRadius               1*Rearth_e                false
@unit Rjup       "Rjup"     JupiterRadius             1*Rjup_e                  false
# Note that IAU 2015 states that when the radius is not specified as polar or equatorial, the
# equatorial radius should be used.

# "ten to the Fifty-One Ergs" (approximate energy of a supernova explosition)
@unit foe        "foe"      FiftyOneErgs              1e51*erg                  false

# Solar flux unit
# https://en.wikipedia.org/wiki/Solar_flux_unit
@unit SFU        "SFU"      SolarFluxUnit             10*kJy                    false

# Total electron content unit (used for ionospheric physics and low-frequency radio astronomy)
@unit TECU       "TECU"     TotalElectronContentUnit  1e16*m^-2                 false

@logscale mag "mag" Magnitude 10 -2.5 false

#https://en.wikipedia.org/wiki/AB_magnitude
@logunit    AB_mag  "AB mag"        Magnitude   3631Jy

# Bolometric magnitudes (cf IAU 2015)
# https://en.wikipedia.org/wiki/Absolute_magnitude#Bolometric_magnitude
@logunit    bol_Mag "bol Mag"       Magnitude   3.0128e28*W
@logunit    bol_mag "bol mag"       Magnitude   2.518_021_002e-8*W*m^-2

#https://en.wikipedia.org/wiki/Apparent_magnitude#Standard_reference_values
#https://www.cfa.harvard.edu/~dfabricant/huchra/ay145/mags.html
@logunit    U_mag   "Johnson U mag" Magnitude   1810Jy
@logunit    B_mag   "Johnson B mag" Magnitude   4260Jy
@logunit    V_mag   "Johnson V mag" Magnitude   3640Jy
@logunit    R_mag   "Johnson R mag" Magnitude   3080Jy
@logunit    I_mag   "Johnson I mag" Magnitude   2550Jy
@logunit    J_mag   "Johnson J mag" Magnitude   1600Jy
@logunit    H_mag   "Johnson H mag" Magnitude   1080Jy
@logunit    K_mag   "Johnson K mag" Magnitude    670Jy
@logunit    g_mag   "Gunn g mag"    Magnitude   3730Jy
@logunit    r_mag   "Gunn r mag"    Magnitude   4490Jy
@logunit    i_mag   "Gunn i mag"    Magnitude   4760Jy
@logunit    z_mag   "Gunn z mag"    Magnitude   4810Jy

struct MagnitudeError <: Exception
    x
end
Base.showerror(io::IO, e::MagnitudeError) = print(io, "MagnitudeError: ", e.x)

# https://en.wikipedia.org/wiki/Color_index
# for color indices (subtracting magnitudes of different passbands)
import Base.-
function -(x::Unitful.Level{L,S,T},y::Unitful.Level{L,S,T}) where {L<:Magnitude,S,T<:Unitful.Quantity}
    return Unitful.Level{L,S}(-(x.val, y.val))
end
import Base.-
function -(x::Unitful.Level{L,S,T},y::Unitful.Level{L,R,T}) where {L<:Magnitude,S,R,T<:Unitful.Quantity}
    return Unitful.ustrip(x) - Unitful.ustrip(y)
end
import Base.+
function +(x::Unitful.Level{L,S,T},y::Unitful.Level{L,S,T}) where {L<:Magnitude,S,T<:Unitful.Quantity}
    return Unitful.Level{L,S}(+(x.val, y.val))
end
import Base.+
function +(x::Unitful.Level{L,S,T},y::Unitful.Level{L,R,T}) where {L<:Magnitude,S,R,T<:Unitful.Quantity}
    throw(MagnitudeError("an invalid operation was attempted with magnitudes: $x, $y"))
end
import Base.*
function *(x::Unitful.Level{L,S,T},y::Unitful.Level{L,R,T}) where {L<:Magnitude,S,R,T<:Unitful.Quantity}
    throw(MagnitudeError("an invalid operation was attempted with magnitudes: $x, $y"))
end
import Base./
function /(x::Unitful.Level{L,S,T},y::Unitful.Level{L,R,T}) where {L<:Magnitude,S,R,T<:Unitful.Quantity}
    throw(MagnitudeError("an invalid operation was attempted with magnitudes: $x, $y"))
end
import Base.//
function //(x::Unitful.Level{L,S,T},y::Unitful.Level{L,R,T}) where {L<:Magnitude,S,R,T<:Unitful.Quantity}
    throw(MagnitudeError("an invalid operation was attempted with magnitudes: $x, $y"))
end

const localunits = Unitful.basefactors
function __init__()
    merge!(Unitful.basefactors, localunits)
    Unitful.register(UnitfulAstro)
end

end


module HSL

using LinearAlgebra
using SparseArrays

if isfile(joinpath(dirname(@__FILE__), "..", "deps", "deps.jl"))
  include("../deps/deps.jl")
else
  error("HSL library not properly installed. Please run Pkg.build(\"HSL\")")
end

# definitions applicable to all packages
const data_map = Dict{Type, Type}(Float32 => Cfloat,
                                  Float64 => Cdouble,
                                  ComplexF32 => Cfloat,
                                  ComplexF64 => Cdouble)

# package-specific definitions
if @isdefined libhsl_ma97
  include("hsl_ma97.jl")
end

end


@testset "Loop chunk distribution" begin
using DiskArrays: DiskArrays, GridChunks, RegularChunks, IrregularChunks, AbstractDiskArray
using YAXArrayBase: YAXArrayBase
struct LargeDiskArray{N,CT<:GridChunks{N}} <: AbstractDiskArray{Float64,N}
    size::NTuple{N,Int}
    chunks::CT
    compressed::Bool
end
Base.size(a::LargeDiskArray) = a.size
DiskArrays.eachchunk(a::LargeDiskArray) = a.chunks
DiskArrays.haschunks(::LargeDiskArray) = DiskArrays.Chunked()
YAXArrayBase.iscompressed(a::LargeDiskArray) = a.compressed
s = (4000,2000,1500)
cs = (100,100,700)
YAXArrays.YAXDefaults.max_cache[] = 1.0e8
a1 = YAXArray(LargeDiskArray(s, GridChunks(s,cs),true))

#Test case where chunk has to be split
dc = mapslices(sum, a1, dims="Dim_1", debug = true)
ch = YAXArrays.DAT.getloopchunks(dc)

@test length(ch) == 2
@test ch[1] == RegularChunks(4,0,2000)
@test ch[2] == RegularChunks(700,0,1500)
dc.outcubes[1].cube
# Test that the allocated buffer is close to what the prescribes size
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0;

#Test subsets and offset
a2 = a1[Dim_3=200..1300.5]
dc = mapslices(sum, a2, dims="Dim_1", debug = true)
ch = YAXArrays.DAT.getloopchunks(dc)
@test ch == (RegularChunks(4,0,2000), RegularChunks(700,200,1100))
# Test that the allocated buffer is close to what the prescribes size
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0;

#Test with different max_cache
YAXArrays.YAXDefaults.max_cache[] = 2.0e8
dc = mapslices(sum, a2, dims="Dim_1", debug = true)
ch = YAXArrays.DAT.getloopchunks(dc)
#Test loop chunk sizes
@test ch == (RegularChunks(5,0,2000), RegularChunks(700,200,1100))
# Test that the allocated buffer is close to what the prescribes size
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size

# Test case that is chunk-friendly
YAXArrays.YAXDefaults.max_cache[] = 1.5e8
dc = mapslices(sum, a1, dims="Dim_3", debug = true)
ch = YAXArrays.DAT.getloopchunks(dc)
@test ch == (RegularChunks(100,0,4000), RegularChunks(100,0,2000))
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc)
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size

#With offset
a2 = a1[Dim_1=50..3050.5]
dc = mapslices(sum, a2, dims="Dim_3", debug = true)
ch = YAXArrays.DAT.getloopchunks(dc)
@test ch == (RegularChunks(100,50,3000), RegularChunks(100,0,2000))
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc)
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size

#With more working memory
YAXArrays.YAXDefaults.max_cache[] = 4.5e8
a2 = a1[Dim_1=50..3050.5]
dc = mapslices(sum, a1, dims="Dim_3", debug = true);
ch = YAXArrays.DAT.getloopchunks(dc)
@test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000))
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
@test 0.5 < (sum(sizeof,incubes) + sum(sizeof,outcubes))/YAXArrays.YAXDefaults.max_cache[] <= 1.0# Test that the allocated buffer is close to what the prescribes size

#Now a completely different test, this is more for DataFrame functionality
# and tests if we can proagate irregular axes into the cubes
using Dates
yearchunks = map(y->isleapyear(y) ? 366 : 365, 2001:2008)
c = GridChunks(RegularChunks(100,0,4000), RegularChunks(100,0,2000), IrregularChunks(chunksizes=yearchunks))
s = last.(last(c))
YAXArrays.YAXDefaults.max_cache[] = 1.0e8
a1 = YAXArray(LargeDiskArray(s, c,true))

dc = mapCube(identity, a1, indims = InDims(), outdims = (), debug=true)
ch = YAXArrays.DAT.getloopchunks(dc)
@test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000), RegularChunks(365,0,2922))
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
@test 0.5 < sum(sizeof,incubes)/YAXArrays.YAXDefaults.max_cache[] <= 1.0#

dc = mapCube(identity, a1, indims = InDims(), outdims = (), debug=true,irregular_loopranges = true)
ch = YAXArrays.DAT.getloopchunks(dc)
@test ch == (RegularChunks(300,0,4000), RegularChunks(100,0,2000), IrregularChunks(chunksizes=yearchunks))
incubes, outcubes = YAXArrays.DAT.getCubeCache(dc);
@test 0.5 < sum(sizeof,incubes)/YAXArrays.YAXDefaults.max_cache[] <= 1.0#
end


mutable struct qp_controller_input_t <: LCMType
    be_silent::Bool
    timestamp::Int64
    zmp_data::zmp_data_t

    num_support_data::Int32
    support_data::Vector{support_data_t}

    num_tracked_bodies::Int32
    body_motion_data::Vector{body_motion_data_t}

    num_external_wrenches::Int32
    body_wrench_data::Vector{body_wrench_data_t}

    whole_body_data::whole_body_data_t

    num_joint_pd_overrides::Int32
    joint_pd_override::Vector{joint_pd_override_t}

    param_set_name::String
    torque_alpha_filter::Float64
end

@lcmtypesetup(qp_controller_input_t,
    support_data => (num_support_data,),
    body_motion_data => (num_tracked_bodies,),
    body_wrench_data => (num_external_wrenches,),
    joint_pd_override => (num_joint_pd_overrides,),
)


############################################################################################
"""
$(TYPEDEF)
Abstract super type for Supported Automatic Differentiation backends.
"""
abstract type AutomaticDifferentiationMethod end
struct ADForward <: AutomaticDifferentiationMethod end
struct ADZygote <: AutomaticDifferentiationMethod end
struct ADReverse <: AutomaticDifferentiationMethod end
struct ADReverseUntaped <: AutomaticDifferentiationMethod end

############################################################################################
#Helper function for AD packages
function checkfinite(θ::ReverseDiff.TrackedArray{T}, max_val::R=max_val) where {T,R<:Real}
    @inbounds @simd for iter in eachindex(θ)
        if !checkfinite(θ[iter], max_val=max_val)
            return false
        end
    end
    return true
end

"""
$(SIGNATURES)
Initiate DiffResults.MutableDiffResult struct. Not exported.

# Examples
```julia
```

"""
function _diffresults_buffer(θᵤ::AbstractVector{T}) where {T<:Real}
    #NOTE: Adjusted from: https://github.com/tpapp/LogDensityProblems.jl/blob/master/src/DiffResults_helpers.jl
    S = T <: Real ? T : Float64
    return DiffResults.MutableDiffResult(zero(S), (Vector{S}(undef, size(θᵤ, 1)),))
end
############################################################################################
struct AutomaticDiffTune{M<:AutomaticDifferentiationMethod,C} <: AbstractDifferentiableTune
    "Automatic Differentiation (AD) backend."
    backend::M
    "Chunck size configuration for AD backend."
    config::C
    function AutomaticDiffTune(backend::M, config::C
    ) where {M<:AutomaticDifferentiationMethod,C}
        return new{M,C}(backend, config)
    end
end
function AutomaticDiffTune(backend::M, objective::Objective) where {M<:AutomaticDifferentiationMethod}
    return AutomaticDiffTune(
    backend, _config(backend, objective, unconstrain_flatten(objective.model, objective.tagged))
    )
end

############################################################################################
function update(
    tune::AutomaticDiffTune,
    objective::Objective)
    return AutomaticDiffTune(tune.backend, objective)
end

############################################################################################
function _log_density(
    objective::Objective, tune::AutomaticDiffTune, θᵤ::AbstractVector{T}
) where {T<:Real}
    return objective(θᵤ)
end

############################################################################################
#Make Symbol intialization easier
function AutomaticDiffTune(backend::Symbol, objective::Objective)
    return AutomaticDiffTune(Val(backend), objective)
end

############################################################################################
#!NOTE: All those functions need to be dispatched if new AD framework is included
"""
$(SIGNATURES)
Write config file for AD wrapper.

# Examples
```julia
```

"""
function _config(
    differentiation::ADForward, objective::Objective, θᵤ::AbstractVector{R}
) where {R<:Real}
    return ForwardDiff.GradientConfig(objective, θᵤ)
end
function _config(
    differentiation::ADZygote, objective::Objective, θᵤ::AbstractVector{R}
) where {R<:Real}
    return nothing
end
function _config(
    differentiation::ADReverse, objective::Objective, θᵤ::AbstractVector{R}
) where {R<:Real}
    return ReverseDiff.compile(ReverseDiff.GradientTape(objective, θᵤ))
end
function _config(
    differentiation::ADReverseUntaped, objective::Objective, θᵤ::AbstractVector{R}
) where {R<:Real}
    return nothing
end

############################################################################################
function AutomaticDiffTune(
    backend::Val{:ForwardDiff},
    objective::Objective,
    config::C=_config(ADForward(), objective, unconstrain_flatten(objective.model, objective.tagged)),
) where {C}
    return AutomaticDiffTune(ADForward(), config)
end

function AutomaticDiffTune(
    backend::Val{:Zygote},
    objective::Objective,
    config::C=_config(ADZygote(), objective, unconstrain_flatten(objective.model, objective.tagged)),
) where {C}
    return AutomaticDiffTune(ADZygote(), config)
end

function AutomaticDiffTune(
    backend::Val{:ReverseDiff},
    objective::Objective,
    config::C=_config(ADReverse(), objective, unconstrain_flatten(objective.model, objective.tagged)),
) where {C}
    return AutomaticDiffTune(ADReverse(), config)
end

function AutomaticDiffTune(
    backend::Val{:ReverseDiffUntaped},
    objective::Objective,
    config::C=_config(ADReverseUntaped(), objective, unconstrain_flatten(objective.model, objective.tagged)),
) where {C}
    return AutomaticDiffTune(ADReverseUntaped(), config)
end

############################################################################################
function _log_density_and_gradient(
    objective::Objective, tune::AutomaticDiffTune{ADForward}, θᵤ::AbstractVector{T}
) where {T<:Real}
    buffer = _diffresults_buffer(θᵤ)
    result = ForwardDiff.gradient!(buffer, objective, θᵤ, tune.config)
    return DiffResults.value(result), DiffResults.gradient(result)
end
function _log_density_and_gradient(
    objective::Objective, tune::AutomaticDiffTune{ADZygote}, θᵤ::AbstractVector{T}
) where {T<:Real}
    _val, back = Zygote.pullback(objective, θᵤ)
    return T(_val), first(back(Zygote.sensitivity(_val)))
end
function _log_density_and_gradient(
    objective::Objective, tune::AutomaticDiffTune{ADReverse}, θᵤ::AbstractVector{T}
) where {T<:Real}
    buffer = _diffresults_buffer(θᵤ)
    result = ReverseDiff.gradient!(buffer, tune.config, θᵤ)
    return DiffResults.value(result), DiffResults.gradient(result)
end
function _log_density_and_gradient(
    objective::Objective, tune::AutomaticDiffTune{ADReverseUntaped}, θᵤ::AbstractVector{T}
) where {T<:Real}
    buffer = _diffresults_buffer(θᵤ)
    result = ReverseDiff.gradient!(buffer, objective, θᵤ)
    return DiffResults.value(result), DiffResults.gradient(result)
end

############################################################################################
# Export
export AutomaticDiffTune


module DB
  using Catlab: @present
  using Catlab.Present
  using Catlab.CategoricalAlgebra.CSets
  using Catlab.CSetDataStructures: struct_acset
  export TheorySQL, generate_schema_sql, @present, get_fields, TypeToSQL, typeToSQL, @db_schema, AbstractSQL

  TypeToSQL = Dict("String" => "text",
                   "Int" => "int",
                   "Int64" => "int",
                   "Real" => "real",
                   "Bool" => "boolean")

  typeToSQL(x) = TypeToSQL[string(x)]
  @present TheorySQL(FreeSchema) begin
    Int::AttrType
    Int64::AttrType
    Real::AttrType
    String::AttrType
    Bool::AttrType
  end;

  @abstract_acset_type AbstractSQL

  # TODO: This should be replacable with a cleaner method
  macro db_schema(head)
    struct_name = gensym()
    quote
      $(esc(:eval))(struct_acset($(Meta.quot(struct_name)), AbstractSQL, $(esc(head.args[2]))))
      $(esc(head.args[1]))() = $(esc(struct_name)){$(esc(Int)), $(esc(Int)), $(esc(Real)), $(esc(String)), $(esc(Bool))}()
    end
  end

  function generate_schema_sql(schema::AbstractSQL)
    queries = map(collect(get_fields(schema))) do (name, col)
      cols = ["$n $(typeToSQL(t))" for (n,t) in col]
      "CREATE TABLE $name ($(join(cols, ", ")))"
    end
    string(join(queries, ";\n"), ";")
  end

  function get_fields(schema::AbstractSQL)
    fields = Dict{Symbol, Array{Tuple{Symbol, Type},1}}()
    for (name, table) in pairs(tables(schema))
      table_name = name

      # Get the column names and types
      col_names = propertynames(table)
      types = eltype.([schema[c] for c in col_names])
      col_names = map(x -> Symbol(split(string(x), r"_\d+_")[end]), col_names)
      fields[table_name] = map(zip(col_names,types)) do (n,t)
        (n, t)
      end
    end
    fields
  end

  # TODO: Maybe add in a way to get_fields from a Presentation?
  #       Allows more convenience when working with workflows,
  #       but doesn't allow you to take advantage of the CSet
  #       structure
end


# TableLogger

mutable struct TableCol
    id::Symbol
    name::String
    type::DataType
    width::Int
    alignment::Symbol # :left, :center, :right
    b_format::Bool
end

mutable struct TableEntry{T}
    col_id::Symbol
    value::T
end

mutable struct TableSetup
    cols::Vector{TableCol}
    col_idx::Dict{Symbol,Int}
    new_row_criteria::Bool
    diff_criteria::Dict{Symbol,Any}
    last_row::Vector{TableEntry}
end

# SolverOptions
mutable struct ActivityOptions
    decay::Float64
    max_probes::Int
    max_confidence_deviation::Float64
end

mutable struct SolverOptions
    logging::Vector{Symbol}
    table::TableSetup
    time_limit::Float64 # time limit in backtracking in seconds
    seed::Int
    traverse_strategy::Symbol
    branch_strategy::Symbol
    branch_split::Symbol # defines splitting in the middle, or takes smallest, biggest value
    backtrack::Bool
    max_bt_steps::Int
    backtrack_sorting::Bool
    keep_logs::Bool
    rtol::Float64
    atol::Float64
    solution_type::Type
    all_solutions::Bool
    all_optimal_solutions::Bool
    lp_optimizer::Any
    no_prune::Bool
    activity::ActivityOptions
    simplify::Bool
end

# General
"""
Save the changes for a variable with two vectors
- `indices` saves a list of indices that can be extracted by the step_nr
    - i.e [1,1,3,5] would mean that step_nr 1 did no changes to the variable and steps 2 and 3 have two changes each
- `changes`:
    Explanation of the tuple
    - [1] :fix, :rm, :rm_below, :rm_above
    - [2] To which value got it fixed, which value was removed, which value was the upper/lower bound
    - [3] Only if fixed it saves the last ptr to revert the changes otherwise 0
    - [4] How many values got removed (0 for fix)
"""
mutable struct VariableChanges
    indices :: Vector{Int}
    changes :: Vector{Tuple{Symbol, Int, Int, Int}}
    VariableChanges() = new(Int[1,1], Tuple{Symbol, Int, Int, Int}[])
end

mutable struct Variable
    idx::Int
    lower_bound::Int # initial lower and
    upper_bound::Int # upper bound of the variable see min, max otherwise
    first_ptr::Int
    last_ptr::Int
    values::Vector{Int}
    indices::Vector{Int}
    init_vals::Vector{Int} # saves all initial values
    init_val_to_index::Vector{Int} # saves the index in which val appears in init_vals
    offset::Int
    min::Int # the minimum value during the solving process
    max::Int # for initial see lower/upper_bound
    changes::VariableChanges
    has_upper_bound::Bool # must be true to work
    has_lower_bound::Bool # must be true to work
    is_fixed::Bool
    is_integer::Bool # must be true to work
    # branching strategies
    activity::Float64 #  + 1 if variable was used in node, * activity.decay if it wasn't
    in_all_different::Vector{Bool} # saves in which all different constraint this variable is used
                                   # can be used to check if two or more variables are in the same alldifferent constraint
end

mutable struct NumberConstraintTypes
    equality::Int
    inequality::Int
    notequal::Int
    alldifferent::Int
    table::Int
    indicator::Int
    reified::Int
end

mutable struct CSInfo
    pre_backtrack_calls::Int
    backtracked::Bool
    backtrack_fixes::Int
    in_backtrack_calls::Int
    backtrack_reverses::Int
    n_constraint_types::NumberConstraintTypes
end

#====================================================================================
=================== SETS FOR VARIABLES AND CONSTRAINTS ==============================
====================================================================================#

struct Integers <: MOI.AbstractScalarSet
    values::Vector{Int}
end
Integers(vals::Union{UnitRange{Int},StepRange{Int,Int}}) = Integers(collect(vals))
Base.copy(I::Integers) = Integers(I.values)

struct AllDifferent <: JuMP.AbstractVectorSet end
JuMP.moi_set(::AllDifferent, dim) = CPE.AllDifferent(dim)

struct TableSetInternal <: MOI.AbstractVectorSet
    dimension::Int
    table::Array{Int,2}
end
Base.copy(T::TableSetInternal) = TableSetInternal(T.dimension, T.table)

struct TableSet <: JuMP.AbstractVectorSet
    table::Array{Int,2}
end
function JuMP.moi_set(ts::TableSet, dim)
    if size(ts.table, 2) != dim
        throw(ArgumentError("The table provided has $(size(ts.table,2)) columns but the variable vector has $dim elements"))
    end
    TableSetInternal(dim, ts.table)
end

struct GeqSetInternal <: MOI.AbstractVectorSet
    dimension::Int
end
Base.copy(G::GeqSetInternal) = GeqSetInternal(G.dimension)

struct GeqSet <: JuMP.AbstractVectorSet end
JuMP.moi_set(::GeqSet, dim) = GeqSetInternal(dim)

struct AllEqual <: JuMP.AbstractVectorSet end
JuMP.moi_set(::AllEqual, dim) = CPE.AllEqual(dim)

#====================================================================================
====================== TYPES FOR TRAVERSING ========================================
====================================================================================#

abstract type Priority end

struct PriorityDFS{T<:Real} <: Priority
    depth::Int
    bound::T
    neg_idx::Int # the negative backtrack index (negative because of maximizing)
end

function Base.isless(p1::PriorityDFS, p2::PriorityDFS)
    if p1.depth < p2.depth
        return true
    elseif p1.depth == p2.depth
        if p1.bound < p2.bound
            return true
        elseif p1.bound == p2.bound
            return p1.neg_idx < p2.neg_idx
        else
            return false
        end
    end
    return false
end

struct PriorityBFS{T<:Real} <: Priority
    bound::T
    depth::Int
    neg_idx::Int # the negative backtrack index (negative because of maximizing)
end

function Base.isless(p1::PriorityBFS, p2::PriorityBFS)
    if p1.bound < p2.bound
        return true
    elseif p1.bound == p2.bound
        if p1.depth < p2.depth
            return true
        elseif p1.depth == p2.depth
            return p1.neg_idx < p2.neg_idx
        else
            return false
        end
    end
    return false
end

#====================================================================================
====================== TYPES FOR CONSTRAINTS ========================================
====================================================================================#

abstract type Constraint end

"""
    BoundRhsVariable
idx - variable index in the lp Model
lb  - lower bound of that variable
ub  - upper bound of that variable
"""
mutable struct BoundRhsVariable
    idx::Int
    lb::Int
    ub::Int
end

mutable struct MatchingInit
    l_in_len::Int
    matching_l::Vector{Int}
    matching_r::Vector{Int}
    index_l::Vector{Int}
    process_nodes::Vector{Int}
    depths::Vector{Int}
    parents::Vector{Int}
    used_l::Vector{Bool}
    used_r::Vector{Bool}
end

mutable struct SCCInit
    index_ei::Vector{Int}
    ids::Vector{Int}
    low::Vector{Int}
    on_stack::Vector{Bool}
    group_id::Vector{Int}
end

"""
    RSparseBitSet

See https://arxiv.org/pdf/1604.06641.pdf
words[x] will save 64 possibilities in a TableConstraint a `1` at position y will mean that the
words in row (x-1)*64+y of the table are possible a 0 means that they aren't
Similar to `Variable` a whole block of 64 rows can be removed by changing the `indices` and `last_ptr`.
The `mask` saves the current mask to change words
"""
mutable struct RSparseBitSet
    words::Vector{UInt64}
    indices::Vector{Int}
    last_ptr::Int
    mask::Vector{UInt64}
    RSparseBitSet() = new([], [], 1, [])
end

mutable struct TableSupport
    # defines the range for each variable
    # i.e [1,3,7,10] means that the first variable has 2 values, the second 4
    var_start::Vector{Int}
    values::Array{UInt64,2}
    TableSupport() = new([], Array{UInt64,2}(undef, (0, 0)))
end

mutable struct TableResidues
    # defines the range for each variable
    # i.e [1,3,7,10] means that the first variable has 2 values, the second 4
    var_start::Vector{Int}
    values::Vector{Int}
    TableResidues() = new([], [])
end

mutable struct TableBacktrackInfo
    words::Vector{UInt64}
    last_ptr::Int
    indices::Vector{Int}
end

struct IndicatorSet{A,F,S<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet}} <: MOI.AbstractVectorSet
    set::S
    dimension::Int
end
IndicatorSet{A,F}(set::S) where {A,F,S} = IndicatorSet{A,F,S}(set, 1+MOI.dimension(set))
Base.copy(I::IndicatorSet{A,F,S}) where {A,F,S} = IndicatorSet{A,F,S}(I.set, I.dimension)

struct ReifiedSet{A,F,S<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet}} <:
       MOI.AbstractVectorSet
    set::S
    dimension::Int
end
Base.copy(R::ReifiedSet{A,F,S}) where {A,F,S} = ReifiedSet{A,F,S}(R.set, R.dimension)

abstract type AbstractBoolSet{
    F1<:Union{SAF,VAF,MOI.VectorOfVariables},
    F2<:Union{SAF,VAF,MOI.VectorOfVariables},
    F1dim<:Val,
    F2dim<:Val,
    S1<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet},
    S2<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet},
} <: MOI.AbstractVectorSet end

struct BoolSetInternals{S1,S2}
    lhs_set::S1
    rhs_set::S2
    lhs_dimension::Int
    rhs_dimension::Int
    dimension::Int
end

const BOOL_SET_TO_CONSTRAINT = (
    :AndSet => (constraint = :AndConstraint, op = :(&&)),
    :OrSet  => (constraint = :OrConstraint, op = :(||)),
    :XorSet => (constraint = :XorConstraint, op = :(⊻), needs_call = true, specific_constraint = true),
    :XNorSet => (constraint = :XNorConstraint, res_op = :(!), op = :(⊻), needs_call = true, specific_constraint = true)
)

for (set, bool_data) in BOOL_SET_TO_CONSTRAINT
    @eval begin
        struct $set{
            F1<:Union{SAF,VAF,MOI.VectorOfVariables},
            F2<:Union{SAF,VAF,MOI.VectorOfVariables},
            F1dim<:Val,
            F2dim<:Val,
            S1<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet},
            S2<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet}
        } <: AbstractBoolSet{F1,F2,F1dim,F2dim,S1,S2}
            bsi::BoolSetInternals{S1,S2}
        end
    end
end

function (::Type{BS})(lhs_set::S1, rhs_set::S2) where {S1,S2,F1,F2,BS<:CS.AbstractBoolSet{F1,F2,F1dim,F2dim,S1_,S2_} where {F1dim,F2dim,S1_,S2_}}
    lhs_dim = MOI.dimension(lhs_set)
    rhs_dim = MOI.dimension(rhs_set)
    F1dim = Val{lhs_dim}
    F2dim = Val{rhs_dim}
    internals = BoolSetInternals(lhs_set, rhs_set, lhs_dim, rhs_dim, lhs_dim + rhs_dim)
    return typeof_without_params(BS){F1,F2,F1dim,F2dim,S1,S2}(internals)
end

function Base.copy(A::AbstractBoolSet{F1,F2,F1dim,F2dim,S1,S2}) where {F1,F2,F1dim,F2dim,S1,S2}
    typeof_without_params(A){F1,F2,F1dim,F2dim,S1,S2}(A.bsi)
end

struct ComplementSet{F<:MOI.AbstractFunction, S<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet}} <: MOI.AbstractVectorSet
    set::S
    dimension::Int
end
ComplementSet{F}(set::S) where {F,S} = ComplementSet{F,S}(set, MOI.dimension(set))
Base.copy(A::ComplementSet{F,S}) where {F,S} = ComplementSet{F,S}(A.set, A.dimension)

#====================================================================================
====================================================================================#

mutable struct ImplementedConstraintFunctions
    init::Bool
    activate::Bool
    update_init::Bool
    finished_pruning::Bool
    restore_pruning::Bool
    single_reverse_pruning::Bool
    reverse_pruning::Bool
    update_best_bound::Bool
end

mutable struct ConstraintInternals{
    FCT<:Union{MOI.AbstractScalarFunction,MOI.AbstractVectorFunction},
    SET<:Union{MOI.AbstractScalarSet,MOI.AbstractVectorSet},
}
    idx::Int
    fct::FCT
    set::SET
    indices::Vector{Int}
    pvals::Vector{Int}
    is_initialized::Bool
    is_activated::Bool
    is_deactivated::Bool # can be deactivated if it's absorbed by other constraints
    bound_rhs::Vector{BoundRhsVariable}# should be set if `update_best_bound` is true
end

#====================================================================================
====================== CONSTRAINTS ==================================================
====================================================================================#

mutable struct BasicConstraint <: Constraint
    std::ConstraintInternals
end

mutable struct EqualConstraint <: Constraint
    std::ConstraintInternals
    first_ptrs::Vector{Int} # for faster apply_changes!
end

mutable struct AllDifferentConstraint <: Constraint
    std::ConstraintInternals
    pval_mapping::Vector{Int}
    vertex_mapping::Vector{Int}
    vertex_mapping_bw::Vector{Int}
    di_ei::Vector{Int}
    di_ej::Vector{Int}
    matching_init::MatchingInit
    scc_init::SCCInit
    # corresponds to `in_all_different`: Saves the constraint idxs where all variables are part of this alldifferent constraint
    sub_constraint_idxs::Vector{Int}
end

abstract type BoolConstraint{C1<:Constraint,C2<:Constraint} <: Constraint end

mutable struct BoolConstraintInternals{C1<:Constraint,C2<:Constraint}
    lhs_activated::Bool
    lhs_activated_in_backtrack_idx::Int
    rhs_activated::Bool
    rhs_activated_in_backtrack_idx::Int
    lhs::C1
    rhs::C2
end

struct XorConstraint{C1,C2} <: BoolConstraint{C1,C2}
    std::ConstraintInternals
    bool_std::BoolConstraintInternals{C1,C2}
    complement_lhs::Union{Nothing, Constraint}
    complement_rhs::Union{Nothing, Constraint}
end

struct XNorConstraint{C1,C2} <: BoolConstraint{C1,C2}
    std::ConstraintInternals
    bool_std::BoolConstraintInternals{C1,C2}
    complement_lhs::Union{Nothing, Constraint}
    complement_rhs::Union{Nothing, Constraint}
end

for (set, bool_data) in BOOL_SET_TO_CONSTRAINT
    get(bool_data, :specific_constraint, false) && continue
    @eval begin
        struct $(bool_data.constraint){C1,C2} <: BoolConstraint{C1,C2}
            std::ConstraintInternals
            bool_std::BoolConstraintInternals{C1,C2}
        end
    end
end

# support for a <= b constraint
mutable struct SingleVariableConstraint <: Constraint
    std::ConstraintInternals
    lhs::Int
    rhs::Int
end

mutable struct LinearConstraint{T<:Real} <: Constraint
    std::ConstraintInternals
    in_all_different::Bool
    is_strict::Bool # for differentiate between < and <=
    is_equal::Bool # for ==
    is_rhs_strong::Bool # saves whether the rhs is already a strong rhs
    rhs::T # combines value - constant
    # same as rhs but for `<` it saves the original value - constant
    # (rhs is then computed to be usable as <=)
    strict_rhs::T
    coeffs::Vector{T}
    mins::Vector{T}
    maxs::Vector{T}
    pre_mins::Vector{T}
    pre_maxs::Vector{T}
end

mutable struct TableConstraint <: Constraint
    std::ConstraintInternals
    current::RSparseBitSet
    supports::TableSupport
    last_sizes::Vector{Int}
    residues::TableResidues
    # holds current, last_ptr and indices from each node
    # maybe it's better to compute some of them to save some space...
    # This is the easy implementation first
    backtrack::Vector{TableBacktrackInfo}
    changed_vars::Vector{Int}
    unfixed_vars::Vector{Int}
    sum_min::Vector{Int}
    sum_max::Vector{Int}
end

mutable struct GeqSetConstraint <: Constraint
    std::ConstraintInternals
    vidx::Int
    greater_than::Vector{Int}
    # saves all constraints where all indices are part of the greater_than
    sub_constraint_idxs::Vector{Int}
end

abstract type ActivatorConstraint <: Constraint end

mutable struct ActivatorConstraintInternals
    activate_on::MOI.ActivationCondition
    activator_in_inner::Bool
    inner_activated::Bool
    inner_activated_in_backtrack_idx::Int
end

mutable struct IndicatorConstraint{C<:Constraint} <: ActivatorConstraint
    std::ConstraintInternals
    act_std::ActivatorConstraintInternals
    inner_constraint::C
end

mutable struct ReifiedConstraint{C<:Constraint, AC<:Union{Constraint,Nothing}} <: ActivatorConstraint
    std::ConstraintInternals
    act_std::ActivatorConstraintInternals
    inner_constraint::C
    complement_constraint::AC
    complement_inner_constraint::Bool
    complement_inner_constraint_in_backtrack_idx::Int
end

#====================================================================================
====================== OBJECTIVES ==================================================
====================================================================================#

abstract type ObjectiveFunction end

mutable struct SingleVariableObjective <: ObjectiveFunction
    fct::MOI.SingleVariable
    vidx::Int # index of the variable
    indices::Vector{Int}
end

# used to designate a feasibility sense
struct NoObjective <: ObjectiveFunction end

mutable struct LinearCombination{T<:Real}
    indices::Vector{Int}
    coeffs::Vector{T}
end

mutable struct LinearCombinationObjective{T<:Real} <: ObjectiveFunction
    fct::MOI.ScalarAffineFunction
    lc::LinearCombination{T}
    constant::T
    indices::Vector{Int} # must exist to update the objective only if one of these changed
end

mutable struct BacktrackObj{T<:Real}
    idx::Int
    step_nr::Int
    parent_idx::Int
    depth::Int
    status::Symbol
    is_feasible::Bool
    vidx::Int
    lb::Int # lb <= var[vidx] <= ub
    ub::Int
    best_bound::T
    primal_start::Vector{Float64}
    solution::Vector{Float64} # holds the solution values of the bound computation
end

function Base.convert(::Type{B}, obj::BacktrackObj{T2}) where {T1,T2,B<:BacktrackObj{T1}}
    return BacktrackObj{T1}(
        obj.idx,
        obj.step_nr,
        obj.parent_idx,
        obj.depth,
        obj.status,
        obj.is_feasible,
        obj.vidx,
        obj.lb,
        obj.ub,
        convert(T1, obj.best_bound),
        obj.primal_start,
        obj.solution,
    )
end

mutable struct TreeLogNode{T<:Real}
    id::Int
    status::Symbol
    feasible::Bool
    best_bound::T
    step_nr::Int
    vidx::Int
    lb::Int
    ub::Int
    var_states::Dict{Int,Vector{Int}}
    var_changes::Dict{Int,Vector{Tuple{Symbol,Int,Int,Int}}}
    activity::Dict{Int,Float64}
    children::Vector{TreeLogNode{T}}
end

mutable struct Solution{T<:Real}
    incumbent::T
    values::Vector{Int}
    backtrack_id::Int # save where the solution was found
    hash::UInt64
end
Solution(incumbent, values, backtrack_id) =
    Solution(incumbent, values, backtrack_id, hash(values))

mutable struct ActivityObj
    nprobes::Int
    is_free::Vector{Bool}
    ActivityObj() = new(0, [false]) # will be overwritten later
end

"""
    BranchVarObj

Determines the next branch variable and stores if still feasible and if solution was found
"""
mutable struct BranchVarObj
    is_feasible::Bool
    is_solution::Bool
    vidx::Int # only relevant if is_feasible && !is_solution
end

struct VarAndVal
    vidx::Int
    lb::Int
    ub::Int
end

mutable struct ConstraintSolverModel{T<:Real}
    lp_model::Union{Nothing,Model} # only used if lp_optimizer is set
    lp_x::Vector{VariableRef}
    init_search_space::Vector{Variable}
    search_space::Vector{Variable}
    init_fixes::Vector{Tuple{Int,Int}}
    subscription::Vector{Vector{Int}}
    constraints::Vector{Constraint}
    root_infeasible_vars::Vector{VarAndVal}
    bt_infeasible::Vector{Int}
    c_backtrack_idx::Int
    c_step_nr::Int
    backtrack_vec::Vector{BacktrackObj{T}}
    backtrack_pq::PriorityQueue
    sense::MOI.OptimizationSense
    objective::ObjectiveFunction
    var_in_obj::Vector{Bool} # saves whether a variable is part of the objective function
    traverse_strategy::Val
    branch_strategy::Val
    branch_split::Val
    in_probing_phase::Bool
    activity_vars::ActivityObj
    best_sol::T # Objective of the best solution
    best_bound::T # Overall best bound
    solutions::Vector{Solution}
    info::CSInfo
    input::Dict{Symbol,Any}
    logs::Vector{TreeLogNode{T}}
    options::SolverOptions
    start_time::Float64
    solve_time::Float64 # seconds spend in solve
end
parametric_type(::ConstraintSolverModel{T}) where {T} = T


# audio.jl

@testset "audio.jl" begin
    # Example audio
    audio_one_channel = SampleBuf(rand(1000), 10)
    audio_two_channel = SampleBuf(rand(1000,2), 10)
    audio_multi_channel = SampleBuf(rand(1000,2), 10)

    @testset "mono" begin
        @test size(mono(audio_two_channel)) == (1000,)
        @test size(mono(audio_multi_channel)) == (1000,)

    end

    @testset "resample.jl" begin
        @test size(resample(audio_one_channel, 5))[1] == 500
        @test size(resample(audio_two_channel, 5))[1] == 500
        @test size(resample(audio_multi_channel, 5))[1] == 500
    end

    @testset "duration" begin
        @test duration(audio_one_channel) == 100
        @test duration(audio_two_channel) == 100
        @test duration(audio_multi_channel) == 100
    end

    # @testset "pitchshift" begin
    #     @test duration(pitchshift(audio_one_channel)) == duration(audio_one_channel)
    #     @test duration(pitchshift(audio_two_channel)) == duration(audio_two_channel)
    #     @test duration(pitchshift(audio_multi_channel)) == duration(audio_multi_channel)
    # end

    # @testset "speedup" begin
    #     @test duration(speedup(audio_one_channel, 2)) == duration(audio_one_channel) // 2
    #     @test duration(speedup(audio_two_channel, 2)) == duration(audio_two_channel) // 2
    #     @test duration(speedup(audio_multi_channel, 2)) == duration(audio_multi_channel) // 2
    # end

    # @testset "slowdown" begin
    #     @test duration(slowdown(audio_one_channel, 2)) == duration(audio_one_channel) * 2
    #     @test duration(slowdown(audio_two_channel, 2)) == duration(audio_two_channel) * 2
    #     @test duration(slowdown(audio_multi_channel, 2)) == duration(audio_multi_channel) * 2
    # end

end


export BitStr, @bit_str, @lbit_str, BitStr64, LongBitStr, bit_literal
export bcat, onehot, onehot_batch, buffer

"""
    BitStr{N,T} <: Integer

struct for bit string with fixed length `N`, the storage type is `T`.

    BitStr{N,T}(value)
    BitStr64{N}(value)
    LongBitStr{N}(value)

Returns a `BitStr`.

## Example

`BitStr` supports some basic arithmetic operations. It acts like an integer, but supports
some frequently used methods for binary basis.

```julia
julia> bit"0101" * 2
1010 ₍₂₎

julia> bcat(bit"101" for i in 1:10)
101101101101101101101101101101 (766958445)

julia> repeat(bit"101", 2)
101101 ₍₂₎

julia> bit"1101"[2]
0
```
"""
struct BitStr{N,T} <: Integer
    buf::T
end

const BitStr64{N} = BitStr{N,Int64}
const LongBitStr{N} = BitStr{N,BigInt}

BitStr{N,T}(val::BitStr{N,T}) where {N,T} = val
Base.zero(::Type{BitStr{N,T}}) where {N,T} = BitStr{N,T}(zero(T))
Base.zero(::BitStr{N,T}) where {N,T} = BitStr{N,T}(zero(T))

buffer(b::BitStr) = b.buf
Base.reinterpret(::Type{BitStr{N,T}}, x::Integer) where {N,T} = BitStr{N,T}(reinterpret(T,x))
Base.reinterpret(::Type{T}, x::BitStr) where {T} = reinterpret(T, buffer(x))
Base.convert(::Type{T}, b::BitStr) where T<:Integer = convert(T, buffer(b))
Base.convert(::Type{T}, b::Integer) where T<:BitStr = T(b)
Base.convert(::Type{T1}, b::BitStr{N2,T2}) where {T1<:BitStr,N2,T2} = convert(T1, buffer(b))
#Base.promote_rule(::Type{BitStr{N,T1}}, ::Type{BitStr{N,T2}}) where {N,T1,T2} = BitStr{N,promote_rule(T1,T2)}
for IT in [:BigInt, :Int128, :UInt128, :Int64,:UInt64, :Int32, :UInt32, :Int16, :UInt16, :Int8, :UInt8, :Bool]
    @eval Base.$IT(b::BitStr) = $IT(buffer(b))
end
for op in [:+, :-, :*, :÷, :|, :⊻, :&, :%, :mod, :mod1]
    @eval Base.$op(a::T, b::Integer) where T<:BitStr = T($op(buffer(a),b))
    @eval Base.$op(a::Integer, b::T) where T<:BitStr = T($op(a,buffer(b)))
    @eval Base.$op(a::BitStr{N,T}, b::BitStr{N,T}) where {N,T} = BitStr{N,T}($op(buffer(a), buffer(b)))
    @eval Base.$op(a::BitStr, b::BitStr) = error("type mismatch: $(typeof(a)), $(typeof(b))")
end
Base.:-(x::BitStr{N,T}) where {N,T} = BitStr{N,T}(-buffer(x))

for op in [:(>>), :(<<)]
    @eval Base.$op(a::BitStr{N,T}, b::Int) where {N,T} = BitStr{N,T}(Base.$op(buffer(a),b))
    #@eval Base.$op(a::T, b::T) where T<:BitStr = T(Base.$op(buffer(a),buffer(b)))
end

for op in [:<, :>, :(<=), :(>=)]
    @eval Base.$op(a::T, b::T) where T<:BitStr = Base.$op(buffer(a),buffer(b))
end

for op in [:(==)]
    @eval Base.$op(a::T, b::Number) where T<:BitStr = Base.$op(buffer(a),b)
    @eval Base.$op(a::Number, b::T) where T<:BitStr = Base.$op(a,buffer(b))
    @eval Base.$op(a::BitStr{N}, b::BitStr{N}) where N = Base.$op(buffer(a),buffer(b))
end
for op in [:count_ones, :count_zeros, :leading_ones, :leading_zeros]
    @eval Base.$op(a::BitStr) = Base.$op(buffer(a))
end

# Note: the transitivity of == is not satisfied here.
Base.:(==)(lhs::BitStr, rhs::BitStr) = false
Base.isapprox(a::BitStr, b::Number; kwargs...) = Base.isapprox(buffer(a),b; kwargs...)
Base.isapprox(a::Number, b::BitStr; kwargs...) = Base.isapprox(a,buffer(b); kwargs...)
Base.isapprox(lhs::BitStr, rhs::BitStr; kwargs...) = false
Base.isapprox(a::T, b::T; kwargs...) where T<:BitStr = Base.isapprox(buffer(a),buffer(b); kwargs...)

# Note: it is a bit confusing, with x::BitStr == y::Integer,
# they behave different when used for indexing.
Base.to_index(x::BitStr) = error("please do not use bit string for indexing, you may want to use `buffer(x)+1` for indexing to avoid ambiguity.")
Base.to_index(x::UnitRange{<:BitStr}) = error("please do not use bit string for indexing, you may want to use `buffer(x)+1` for indexing to avoid ambiguity.")

# use system interface
Base.checkindex(::Type{Bool}, inds::AbstractUnitRange, i::BitStr) =
    checkindex(Bool, inds, Base.to_index(i))
Base.length(bits::BitStr{N}) where N = N
Base.lastindex(bits::BitStr) = length(bits)

Base.typemax(::Type{BitStr{N,T}}) where {N,T} = BitStr{N,T}(1<<N-1)
Base.typemin(::Type{BitStr{N,T}}) where {N,T} = BitStr{N,T}(0)
Base.typemax(::BitStr{N,T}) where {N,T} = BitStr{N,T}(1<<N-1)
Base.typemin(::BitStr{N,T}) where {N,T} = BitStr{N,T}(0)

"""
    @bit_str -> BitStr64

Construct a bit string. such as `bit"0000"`. The bit strings also supports string `bcat`. Just use
it like normal strings.

## Example

```jldoctest
julia> bit"10001"
10001 ₍₂₎

julia> bit"100_111_101"
100111101 ₍₂₎

julia> bcat(bit"1001", bit"11", bit"1110")
1001111110 ₍₂₎

julia> onehot(bit"1001")
16-element Array{Float64,1}:
 0.0
 0.0
 0.0
 0.0
 0.0
 0.0
 0.0
 0.0
 0.0
 1.0
 0.0
 0.0
 0.0
 0.0
 0.0
 0.0

```
"""
macro bit_str(str)
    return parse_bit(Int64, str)
end

"""
    @bit_str -> LongBitStr

Long bit string version of `@bit_str` macro.
"""
macro lbit_str(str)
    return parse_bit(BigInt, str)
end

function parse_bit(::Type{T}, str::String) where {T <: Integer}
    val = zero(T); k = 1
    for each in reverse(filter(x->x!='_', str))
        if each == '1'
            val += one(T) << (k - 1)
            k += 1
        elseif each == '0'
            k += 1
        elseif each == '_'
            continue
        else
            error("expect 0 or 1, got $each at $k-th bit")
        end
        (isbitstype(T) && k>bsizeof(T)) && error("string length is larger than $(bsizeof(T)), use @lbit_str instead")
    end
    return BitStr{k-1,T}(val)
end

sum_length(a::BitStr, bits::BitStr...) = length(a) + sum_length(bits...)
sum_length(a::BitStr) = length(a)

function bcat(bits::(BitStr{N,T} where N)...) where T
    total_bits = sum_length(bits...)
    val, len = zero(T), 0

    for k in length(bits):-1:1
        val += buffer(bits[k]) << len
        len += length(bits[k])
    end
    return BitStr{total_bits,T}(val)
end

# expand iterator to tuple
bcat(bits) = bcat(bits...)

Base.@propagate_inbounds function Base.getindex(bit::BitStr{N}, index::Int) where N
    @boundscheck 1 <= index <= N || throw(BoundsError(bit, index))
    return buffer(readbit(bit, index))
end

Base.@propagate_inbounds function Base.getindex(bit::BitStr{N}, itr::Union{AbstractVector, AbstractRange}) where N
    @boundscheck all(x->1<=x<=N, itr) || throw(BoundsError(bit, itr))
    return map(x->buffer(readbit(bit, x)), itr)
end

# TODO: support AbstractArray, should return its corresponding shape

Base.@propagate_inbounds function Base.getindex(bit::BitStr{N,T}, mask::Union{Vector{Bool}, BitArray}) where {N,T}
    @boundscheck N == length(mask) || error("length of bits and mask does not match.")

    out = T[]
    for k in eachindex(mask)
        if mask[k]
            push!(out, bit[k])
        end
    end
    return out
end

Base.eltype(::BitStr{N,T}) where {N,T} = T

function Base.iterate(bit::BitStr, state::Integer=1)
    if state > length(bit)
        return nothing
    else
        return bit[state], state + 1
    end
end
Base.IteratorSize(::BitStr) = Base.HasLength()

Base.repeat(s::BitStr, n::Integer) = bcat(s for i in 1:n)
Base.show(io::IO, bitstr::BitStr64{N}) where N = print(io, string(buffer(bitstr), base=2, pad=N), " ₍₂₎")
Base.show(io::IO, bitstr::LongBitStr{N}) where N = print(io, join(map(string, [bitstr[end:-1:1]...])), " ₍₂₎")

"""
    onehot([T=Float64], bit_str[, nbatch])

Returns an onehot vector in type `Vector{T}`, or a batch of onehot
vector in type `Matrix{T}`, where the `bit_str`-th element is one.
"""
onehot(::Type{T}, n::BitStr{N}) where {T,N} = onehot(T, N, buffer(n))
onehot(n::BitStr) = onehot(Float64, n)

onehot(::Type{T}, n::BitStr{N}, nbatch::Int) where {T,N} = onehot(T, N, buffer(n), nbatch)
onehot(n::BitStr, nbatch::Int) = onehot(Float64, n, nbatch)

# operations
"""
    breflect(bit_str[, masks])

Return left-right reflected bit string.
"""
breflect(b::BitStr{N}) where N = breflect(b; nbits=N)
breflect(b::BitStr{N,T}, masks::Vector{<:BitStr{N,T}}) where {N,T} = BitStr{N,T}(breflect(buffer(b), reinterpret(T,masks); nbits=N))

"""
    neg(b::BitStr) -> BitStr
"""
neg(b::BitStr{N}) where N = neg(b, N)

"""
    bfloat(b::BitStr) -> Float64
"""
bfloat(b::BitStr{N}) where N = bfloat(buffer(b); nbits=N)

"""
    bfloat_r(b::BitStr) -> Float64
"""
bfloat_r(b::BitStr{N}) where N = bfloat_r(buffer(b); nbits=N)

"""
    bint_r(b::BitStr) -> Integer
"""
bint_r(b::BitStr{N}) where N = buffer(breflect(b))

"""
    bint(b::BitStr) -> Integer
"""
bint(b::BitStr) = buffer(b)

"""
    bit_literal(xs...)

Create a [`BitStr`](@ref) by input bits `xs`.

# Example

```jldoctest
julia> bit_literal(1, 0, 1, 0, 1, 1)
110101 ₍₂₎
```
"""
bit_literal(xs...) = bit_literal(xs)
function bit_literal(xs::NTuple{N, T}) where {N, T<:Integer}
    val = T(0)
    for k in 1:N
        xs[k] == 0 || xs[k] == 1 || error("expect 0 or 1, got $(xs[k])")
        val += xs[k] << (k - 1)
    end
    return BitStr64{N}(val)
end

basis(b::BitStr) = typemin(b):typemax(b)
basis(::Type{BT}) where BT<:BitStr = typemin(BT):typemax(BT)


mutable struct Rake
    stop_words::Vector{String}
    min_char_length::Int
    max_words_length::Int
    min_keyword_frequency::Int
    min_words_length_adj::Int
    max_words_length_adj::Int
    min_phrase_freq_adj::Int
end

struct Score
    rank::Float64
    frequency::Int
end

function Rake(stop_words; min_char_length=3, max_words_length=3, min_keyword_frequency=10,
              min_words_length_adj=1, max_words_length_adj=1, min_phrase_freq_adj=2)
    Rake(stop_words, min_char_length, max_words_length, min_keyword_frequency,
         min_words_length_adj, max_words_length_adj, min_phrase_freq_adj)
end

import Base.run

"Take vector of strings and split each string into all possible keyphrases"
function find_all_possible_keyphrases(phrases, max_phrase_length::Int)
    all_possible_keyphrases = Array(String,0)
    for sentence in phrases
        split_sentence = split.(sentence, ' ')
        for j = 0: max_phrase_length-1
            for i = 1:(length(split_sentence)-j)
                push!(all_possible_keyphrases, join(split_sentence[i:i+j], " "))
            end
        end
    end
    return all_possible_keyphrases
end

function run(text, stop_words, max_phrase_length)

    punct = ['.', '?', '!', '\n', ';', ',', ':', '\u2019', '\u2013']
    whitespace = false
    buffer = ""
    current_word = ""

    phrases = SubString{String}[]
    for i in text
        if i in punct # punctuation
            append!(phrases, find_all_possible_keyphrases([buffer], max_phrase_length))
            buffer = ""
            current_word = ""
            whitespace = false

        elseif (i == ' ') && (!whitespace) # hit whitespace, last character wasnt whitespace
            if current_word in stop_words # stopword
                append!(phrases, find_all_possible_keyphrases([buffer], max_phrase_length))
                buffer = ""
                current_word = ""
                whitespace = false
            else
                if buffer != ""
                    buffer = buffer*" "*string(current_word)
                else
                    buffer = buffer*string(current_word)
                end
                current_word = ""
                whitespace = true
            end

        elseif (i == ' ') && (whitespace) # hit whitspace, last character was whitespace
            continue

        else
            current_word = current_word*string(lowercase(i))
            whitespace = false
        end
    end

    if buffer != ""
        buffer = buffer*" "*string(current_word)
    else
        buffer = buffer*string(current_word)
    end
    print(buffer)
    append!(phrases, find_all_possible_keyphrases([buffer], max_phrase_length))

    keyphrases = unique(phrases)
    d = Dict([(i, count(x->x==i,phrases)) for i in phrases])
    return d
end


module TestBasic

using Test
using DelimitedFiles
using LightGBM

LGBM_PATH = ENV["LIGHTGBM_EXAMPLES_PATH"]
LGBM_PATH = if isabspath(LGBM_PATH) LGBM_PATH else abspath(joinpath(pwd(), "..", LGBM_PATH)) end

@testset "basic_tests.jl  -- binary" begin

    # Use binary example for generic tests.
    binary_test = readdlm(joinpath(LGBM_PATH, "examples", "binary_classification", "binary.test"), '\t')
    binary_train = readdlm(joinpath(LGBM_PATH, "examples", "binary_classification", "binary.train"), '\t')
    X_train = binary_train[:, 2:end]
    y_train = binary_train[:, 1]
    X_test = binary_test[:, 2:end]
    y_test = binary_test[:, 1]

    # Test wrapper functions.
    train_ds = LightGBM.LGBM_DatasetCreateFromMat(X_train, "objective=binary")
    @test LightGBM.LGBM_DatasetGetNumData(train_ds) == 7000
    @test LightGBM.LGBM_DatasetGetNumFeature(train_ds) == 28
    @test LightGBM.LGBM_DatasetSetField(train_ds, "label", y_train) == nothing
    @test LightGBM.LGBM_DatasetGetField(train_ds, "label") == y_train
    bst = LightGBM.LGBM_BoosterCreate(train_ds, "lambda_l1=10. metric=auc, verbosity=-1")

    test_ds = LightGBM.LGBM_DatasetCreateFromMat(X_test, "objective=binary", train_ds);
    @test LightGBM.LGBM_DatasetSetField(test_ds, "label", y_test) == nothing
    @test LightGBM.LGBM_BoosterAddValidData(bst, test_ds) == nothing
    @test LightGBM.LGBM_BoosterUpdateOneIter(bst) == 0
    @test LightGBM.LGBM_BoosterGetEvalCounts(bst) == 1
    @test LightGBM.LGBM_BoosterGetEvalNames(bst) |> first == "auc"

    # Test binary estimator.
    estimator = LightGBM.LGBMClassification(
        objective = "binary",
        num_class = 1,
        num_iterations = 20,
        learning_rate = .1,
        early_stopping_round = 1,
        feature_fraction = .8,
        bagging_fraction = .9,
        bagging_freq = 1,
        num_leaves = 1000,
        metric = ["auc", "binary_logloss"],
        is_training_metric = true,
        max_bin = 255,
        min_sum_hessian_in_leaf = 0.,
        min_data_in_leaf = 1,
    )

    estimator_equiv = LightGBM.LGBMClassification(
        objective = "multiclassova",
        num_class = 2,
        num_iterations = 20,
        learning_rate = .1,
        early_stopping_round = 1,
        feature_fraction = .8,
        bagging_fraction = .9,
        bagging_freq = 1,
        num_leaves = 1000,
        metric = ["multi_logloss"],
        is_training_metric = true,
        max_bin = 255,
        min_sum_hessian_in_leaf = 0.,
        min_data_in_leaf = 1,
    )

    # Test fitting.
    LightGBM.fit!(estimator, X_train, y_train, verbosity = -1);
    LightGBM.fit!(estimator, X_train, y_train, (X_test, y_test), verbosity = -1)
    LightGBM.fit!(estimator_equiv, X_train, y_train, (X_test, y_test), verbosity = -1)

    p_binary = LightGBM.predict(estimator, X_train, verbosity = -1)
    p_multi = LightGBM.predict(estimator_equiv, X_train, verbosity = -1)
    binary_classes = LightGBM.predict_classes(estimator, X_train, verbosity = -1)
    multi_classes = LightGBM.predict_classes(estimator_equiv, X_train, verbosity = -1)

    # check also that binary outputs and classes are invariant in prediction output shapes, i.e. they're matrices
    @test length(size(p_binary)) == 2
    @test length(size(p_multi)) == 2
    @test length(size(binary_classes)) == 2
    @test length(size(multi_classes)) == 2

    # probabilities can't match for multiclassova but predictions do for binary and multiclass (with 2 classes)
    @test binary_classes == multi_classes

    @testset "feature importances" begin
    # This is kinda lazy .... tests start to need more refactoring

        # Check the feature importances seems to work
        gains = LightGBM.gain_importance(estimator)
        splits = LightGBM.split_importance(estimator)

        # splits should always be ints, gains can be anything but what we know is that they should be different than splits
        @test all(isinteger.(splits))
        @test splits != gains

        sub_gains = LightGBM.gain_importance(estimator, 1)
        sub_splits = LightGBM.split_importance(estimator, 1)

        @test all(isinteger.(splits))
        @test splits != gains

        @test sub_splits != splits
        @test sub_gains != gains

        # check it voms appropriately when fed an untrained thingy
        empty_estimator = LightGBM.LGBMClassification()
        @test_throws ErrorException LightGBM.gain_importance(empty_estimator)
        @test_throws ErrorException LightGBM.split_importance(empty_estimator)

    end

    # Test setting feature names
    jl_feature_names = ["testname_$i" for i in 1:28]
    LightGBM.LGBM_DatasetSetFeatureNames(estimator.booster.datasets |> first, jl_feature_names)
    lgbm_feature_names = LightGBM.LGBM_DatasetGetFeatureNames(estimator.booster.datasets |> first)
    @test jl_feature_names == lgbm_feature_names

    # Test prediction, and loading and saving models.
    test_filename, filehandle = mktemp()
    close(filehandle)
    LightGBM.savemodel(estimator, test_filename)

    pre = LightGBM.predict(estimator, X_train, verbosity = -1)
    LightGBM.loadmodel!(estimator, test_filename);
    post = LightGBM.predict(estimator, X_train, verbosity = -1)

    rm(test_filename)

    @test isapprox(pre, post, rtol=1e-16)

    # test making copy of estimator
    copy_estimator = deepcopy(estimator)
    @test copy_estimator.booster.handle != estimator.booster.handle

    # check that deepcopy predictions are equivalent
    copy_pre_preds = LightGBM.predict(copy_estimator, X_train, verbosity = -1)
    @test isapprox(copy_pre_preds, pre, rtol=1e-16)

    # test reload upon predict works if we have a null handle
    LightGBM.LGBM_BoosterFree(copy_estimator.booster)
    copy_preds = LightGBM.predict(copy_estimator, X_train, verbosity = -1)

    @test isapprox(copy_pre_preds, copy_preds, rtol=1e-16)
    @test isapprox(pre, copy_preds, rtol=1e-16)

    # Test cross-validation.
    splits = (collect(1:3500), collect(3501:7000))
    LightGBM.cv(estimator, X_train, y_train, splits; verbosity = -1)

    # Test exhaustive search.
    params = [
        Dict(:num_iterations => num_iterations, :num_leaves => num_leaves)
        for num_iterations in (1, 2), num_leaves in (5, 10)
    ]
    LightGBM.search_cv(estimator, X_train, y_train, splits, params; verbosity = -1)

end

@testset "basic_tests.jl -- regression" begin

    # Test regression estimator.
    regression_test = readdlm(joinpath(LGBM_PATH, "examples", "regression", "regression.test"), '\t')
    regression_train = readdlm(joinpath(LGBM_PATH, "examples", "regression", "regression.train"), '\t')
    X_train = regression_train[:, 2:end]
    y_train = regression_train[:, 1]
    X_test = regression_test[:, 2:end]
    y_test = regression_test[:, 1]

    estimator = LightGBM.LGBMRegression(
        num_iterations = 100,
        learning_rate = .05,
        feature_fraction = .9,
        bagging_fraction = .8,
        bagging_freq = 5,
        num_leaves = 31,
        metric = ["l2"],
        metric_freq = 1,
        is_training_metric = true,
        max_bin = 255,
        min_sum_hessian_in_leaf = 5.,
        min_data_in_leaf = 100,
        max_depth = -1,
    )

    scores = LightGBM.fit!(estimator, X_train, y_train, (X_test, y_test), verbosity = -1)
    @test scores["metrics"]["test_1"]["l2"][end] < .5

end

@testset "basic_tests.jl -- multiclass" begin

    # Test multiclass estimator.
    multiclass_test = readdlm(joinpath(LGBM_PATH, "examples", "multiclass_classification", "multiclass.test"), '\t')
    multiclass_train = readdlm(joinpath(LGBM_PATH, "examples", "multiclass_classification", "multiclass.train"), '\t')
    X_train = Matrix(multiclass_train[:, 2:end])
    y_train = Array(multiclass_train[:, 1])
    X_test = Matrix(multiclass_test[:, 2:end])
    y_test = Array(multiclass_test[:, 1])

    estimator = LightGBM.LGBMClassification(
        num_iterations = 100,
        learning_rate = .05,
        feature_fraction = .9,
        bagging_fraction = .8,
        bagging_freq = 5,
        num_leaves = 31,
        metric = ["multi_logloss"],
        metric_freq = 1,
        is_training_metric = true,
        max_bin = 255,
        min_sum_hessian_in_leaf = 5.,
        min_data_in_leaf = 100,
        num_class = 5,
        early_stopping_round = 10,
    )

    scores = LightGBM.fit!(estimator, X_train, y_train, (X_test, y_test), verbosity = -1)
    @test scores["metrics"]["test_1"]["multi_logloss"][end] < 1.4

    # Test row major multiclass
    X_train = Matrix(multiclass_train[:, 2:end]')
    X_test = Matrix(multiclass_test[:, 2:end]')

    estimator = LightGBM.LGBMClassification(
        num_iterations = 100,
        learning_rate = .05,
        feature_fraction = .9,
        bagging_fraction = .8,
        bagging_freq = 5,
        num_leaves = 31,
        metric = ["multi_logloss"],
        metric_freq = 1,
        is_training_metric = true,
        max_bin = 255,
        min_sum_hessian_in_leaf = 5.,
        min_data_in_leaf = 100,
        num_class = 5,
        early_stopping_round = 10,
    )

    scores = LightGBM.fit!(estimator, X_train, y_train, (X_test, y_test), verbosity = -1, is_row_major = true)
    @test scores["metrics"]["test_1"]["multi_logloss"][end] < 1.4

end

end # module


function DSSTTesseral(arg0::Frame, arg1::jdouble, arg2::UnnormalizedSphericalHarmonicsProvider)
    return DSSTTesseral((Frame, jdouble, UnnormalizedSphericalHarmonicsProvider), arg0, arg1, arg2)
end

function DSSTTesseral(arg0::Frame, arg1::jdouble, arg2::UnnormalizedSphericalHarmonicsProvider, arg3::jint, arg4::jint, arg5::jint, arg6::jint, arg7::jint, arg8::jint, arg9::jint)
    return DSSTTesseral((Frame, jdouble, UnnormalizedSphericalHarmonicsProvider, jint, jint, jint, jint, jint, jint, jint), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9)
end

function get_events_detectors(obj::DSSTTesseral)
    return jcall(obj, "getEventsDetectors", Vector{EventDetector}, ())
end

function get_field_events_detectors(obj::DSSTTesseral, arg0::Field)
    return jcall(obj, "getFieldEventsDetectors", Vector{FieldEventDetector}, (Field,), arg0)
end

function get_mean_element_rate(obj::DSSTTesseral, arg0::FieldSpacecraftState, arg1::FieldAuxiliaryElements, arg2::Vector{RealFieldElement})
    return jcall(obj, "getMeanElementRate", Vector{RealFieldElement}, (FieldSpacecraftState, FieldAuxiliaryElements, Vector{RealFieldElement}), arg0, arg1, arg2)
end

function get_mean_element_rate(obj::DSSTTesseral, arg0::SpacecraftState, arg1::AuxiliaryElements, arg2::Vector{jdouble})
    return jcall(obj, "getMeanElementRate", Vector{jdouble}, (SpacecraftState, AuxiliaryElements, Vector{jdouble}), arg0, arg1, arg2)
end

function get_parameters_drivers(obj::DSSTTesseral)
    return jcall(obj, "getParametersDrivers", Vector{ParameterDriver}, ())
end

function initialize(obj::DSSTTesseral, arg0::AuxiliaryElements, arg1::PropagationType, arg2::Vector{jdouble})
    return jcall(obj, "initialize", List, (AuxiliaryElements, PropagationType, Vector{jdouble}), arg0, arg1, arg2)
end

function initialize(obj::DSSTTesseral, arg0::FieldAuxiliaryElements, arg1::PropagationType, arg2::Vector{RealFieldElement})
    return jcall(obj, "initialize", List, (FieldAuxiliaryElements, PropagationType, Vector{RealFieldElement}), arg0, arg1, arg2)
end

function register_attitude_provider(obj::DSSTTesseral, arg0::AttitudeProvider)
    return jcall(obj, "registerAttitudeProvider", void, (AttitudeProvider,), arg0)
end

function update_short_period_terms(obj::DSSTTesseral, arg0::Vector{RealFieldElement}, arg1::Vector{FieldSpacecraftState})
    return jcall(obj, "updateShortPeriodTerms", void, (Vector{RealFieldElement}, Vector{FieldSpacecraftState}), arg0, arg1)
end

function update_short_period_terms(obj::DSSTTesseral, arg0::Vector{jdouble}, arg1::Vector{SpacecraftState})
    return jcall(obj, "updateShortPeriodTerms", void, (Vector{jdouble}, Vector{SpacecraftState}), arg0, arg1)
end


using StatsFuns

@testset "Reasonable return from varprop function." begin
    sigma = Matrix{Int}(I, 3, 3)
    recAbund = ones(5)

    retval = runCkineVarProp(tps, rxntfR, sigma, recAbund)

    @test length(retval) == length(tps)
    @test all(retval .>= 0.0)
end

@testset "Run fitting." begin
    p = gcSolver.getUnkVec()
    outtG = similar(p)

    outt = gcSolver.resids(p)
    ForwardDiff.gradient!(outtG, gcSolver.resids, p)

    @time ForwardDiff.gradient!(outtG, gcSolver.resids, p)

    @test isfinite(outt)
    @test all(isfinite.(outtG))
end

@testset "Test that residuals can be calculated using Fit." begin
    fitVec = gcSolver.importFit()
    fitVec = convert(Vector{Float64}, fitVec[!, :Fit])
    fitVec = softplus.(fitVec)
    resids = gcSolver.resids(fitVec)
    @test isfinite(resids)
end


"""
Data types and functions support `Configuration`s, which describe the participants
in a Paxos cluster.
"""
module Configurations

export isquorum,
  quorumSize,
  NoQuorumAvailableException,
  Configuration,
  configuration,
  memberAddress,
  memberAddresses,
  addMember,
  removeMember

using ..Nodes

struct OrdinaryConfiguration
  """
  Dictionary mapping IDs of members to their addresses
  """
  members::Dict{NodeID,Any}
end

"""
A transition configuration (inspired by Raft) supports rollout of a
new configuration, with potentially differences in membership
"""
struct TransitionConfiguration
  old::OrdinaryConfiguration
  new::OrdinaryConfiguration
end

"""
A configuration is the general type for maintaining a list of menbers
and addreses where they can be reached over transport
"""
const Configuration = Union{OrdinaryConfiguration,TransitionConfiguration}

"""
Return the address for the specfied node, or error if no such address
"""
function memberAddress(cfg::OrdinaryConfiguration, memberID::NodeID)
  cfg.members[memberID]
end

function memberAddress(cfg::TransitionConfiguration, memberID::NodeID)
  if haskey(cfg.old, memberID)
    memberAddress(cfg.old, memberID)
  else
    memberAddress(cfg.new, memberID)
  end
end

"""
Return the addresses of all membes in the configuration
"""
function memberAddresses(cfg::OrdinaryConfiguration)
  Set(keys(cfg.members))
end

function memberAddresses(cfg::TransitionConfiguration)
  union(memberAddresses(cfg.old), memberAddresses(cfg.new))
end

"""
Return the minimum size for a majority of members in the configuration
"""
function quorumSize(cfg::OrdinaryConfiguration)
  quorumSize(length(cfg.members))
end

"""
For `Integer` values, return the required number of participants for a majority
"""
function quorumSize(len::Integer)
  convert(Integer, if iseven(len)
    (len / 2) + 1
  else
    ceil(len / 2)
  end)
end

"""
Exception thrown when no quorum is available to respond to Paxos messages
"""
struct NoQuorumAvailableException <: Exception end

"""
Return true if the list of members represents a quorum for the configuration
"""
function isquorum(cfg::OrdinaryConfiguration, members::Set{NodeID})
  count = 0
  for member in members
    if haskey(cfg.members, member)
      count += 1
    end
  end
  count >= quorumSize(cfg)
end

"""
During transitions, eturn true if the list of members represents a quorum in
both old and new configurations
"""
function isquorum(cfg::TransitionConfiguration, members::Set{NodeID})
  isquorum(cfg.old, members) && isquorum(cfg.new, members)
end

function Base.isempty(cfg::OrdinaryConfiguration)
  isempty(cfg.members)
end

function Base.isempty(cfg::TransitionConfiguration)
  isempty(cfg.old) && isempty(cfg.new)
end

"""
Create a new, empty `OrdinaryConfiguration`
"""
function configuration(members::Dict{NodeID,Any} = Dict{NodeID,Any}())
  OrdinaryConfiguration(members)
end

function addMember(cfg::OrdinaryConfiguration, id::NodeID, address)
  TransitionConfiguration(cfg, configuration(merge(cfg.members, Dict(id => address))))
end

function addMember(cfg::TransitionConfiguration, id::NodeID, address)
  TransitionConfiguration(
    cfg.old,
    configuration(merge(cfg.new.members, Dict(id => address))),
  )
end

function removeMember(cfg::OrdinaryConfiguration, id::NodeID)
  TransitionConfiguration(cfg, configuration(delete!(Dict(cfg.members), id)))
end

function removeMember(cfg::TransitionConfiguration, id::NodeID)
  TransitionConfiguration(cfg, configuration(delete!(Dict(cfg.new.members), id)))
end

end


# Features are peaks the may occur in a spectrum. They include characteristic X-rays,
# Compton shifted characteristic X-rays, and escape peaks.

struct EscapeArtifact
    xray::CharXRay
    escape::CharXRay
    function EscapeArtifact(xray::CharXRay, escape::CharXRay = n"Si K-L3")
        @assert energy(xray) > energy(escape) "The energy($xray) must be larger than energy($escape)"
        return new(xray, escape)
    end
end

NeXLCore.exists(::Type{EscapeArtifact}, cxr::CharXRay, esc::CharXRay = n"Si K-L3") =
    energy(cxr) - energy(esc) > 100.0

NeXLCore.element(esc::EscapeArtifact) = element(esc.xray)
NeXLCore.brightest(escs::AbstractVector{EscapeArtifact}) =
    brightest(map(ea -> ea.xray, escs))

Base.show(io::IO, esc::EscapeArtifact) = print(io, name(esc))
Base.show(io::IO, escs::AbstractVector{EscapeArtifact}) = print(io, name(escs))
NeXLCore.name(esc::EscapeArtifact) = "Esc[$(esc.xray)]"
NeXLCore.name(escs::AbstractVector{EscapeArtifact}) =
    "Esc[$(join(map(esc->"$(esc.xray)", escs),","))]"

struct ComptonArtifact
    xray::CharXRay
    angle::Float64
    function ComptonArtifact(prim::CharXRay, θ::Float64)
        @assert (θ >= 0.0) && (θ <= π) "The Compton scatter angle must be between 0.0 and π"
        return new(prim, θ)
    end
end

Base.show(io::IO, ca::ComptonArtifact) = print(io, name(ca))
NeXLCore.name(ca::ComptonArtifact) = "Compton[$(ca.xray), $(rad2deg(ca.angle))°]"
NeXLCore.name(cas::AbstractVector{ComptonArtifact}) =
    "Compton[$(name(map(ca->ca.xray, cas)))]"
NeXLCore.element(ca::ComptonArtifact) = element(xray)

NeXLCore.energy(ca::ComptonArtifact) =
    energy(ca.xray) * NeXLCore.comptonShift(ca.angle, energy(ca.xray))
NeXLCore.weight(esc::ComptonArtifact) = weight(NormalizeToUnity, esc.xray)

NeXLCore.energy(esc::EscapeArtifact) = energy(esc.xray) - energy(esc.escape)

"""
    weight(esc::EscapeArtifact, factor=0.01)

The weight of an EscapeArtifact which is factor * weight(esc.xray).
"""
NeXLCore.weight(esc::EscapeArtifact, factor = 0.01) = factor * weight(NormalizeToUnity, esc.xray)

"""
    SpectrumFeature

A union representing the different type of peak-like features (helpful and harmful) that can appear in a spectrum.
"""
SpectrumFeature = Union{CharXRay,EscapeArtifact,ComptonArtifact}

Base.show(io::IO, sfs::AbstractVector{<:SpectrumFeature}) =
    map(sf -> println(io, name(sf)), sfs)

Base.convert(
    ::AbstractVector{SpectrumFeature},
    x::AbstractVector{CharXRay},
)::AbstractVector{SpectrumFeature} = x

charFeature(
    elm::Element,
    tr::Transition...;
    minweight = 1.0e-3,
    maxE = 1.0e6,
)::AbstractVector{SpectrumFeature} = characteristic(elm, tr, minweight, maxE)

escapeFeature(
    elm::Element,
    trs::Transition...;
    minweight = 0.1,
    maxE = 1.0e6,
    escape = n"Si K-L3",
)::AbstractVector{SpectrumFeature} =
    map(tr -> EscapeArtifact(tr, escape), characteristic(elm, trs, minweight, maxE))

comptonFeature(
    elm::Element,
    trs::Transition...;
    minweight = 1.0e-3,
)::AbstractVector{SpectrumFeature} =
    map(tr -> ComptonArtifact(tr, escape), characteristic(elm, trs, minweight, maxE))