function _attach_ports!(component::PSY.DynamicComponent)
    PSY.get_ext(component)[PORTS] = Ports(component)
    return
end

function index_local_states(component::PSY.DynamicComponent, device_states::Vector{Symbol})
    component_state_index = Vector{Int}(undef, PSY.get_n_states(component))
    component_states = PSY.get_states(component)
    for (ix, s) in enumerate(component_states)
        component_state_index[ix] = findfirst(x -> x == s, device_states)
    end
    return component_state_index
end

function index_port_mapping!(component::PSY.DynamicComponent, device_states::Vector{Symbol})
    _attach_ports!(component)
    index_component_inputs = Vector{Int}()
    for i in PSY.get_ext(component)[PORTS][:states]
        tmp = [(ix, var) for (ix, var) in enumerate(device_states) if var == i]
        isempty(tmp) && continue
        push!(index_component_inputs, tmp[1][1])
    end
    return index_component_inputs
end

function _add_dynamic_bus_states!(
    DAE_vector::Vector{Bool},
    voltage_buses_ix::Vector{Int},
    bus_ix::Int,
    n_buses::Int,
)
    push!(voltage_buses_ix, bus_ix)
    DAE_vector[bus_ix] = DAE_vector[bus_ix + n_buses] = true
    return
end

function index_dynamic_lines!(
    inputs::SimulationInputs,
    branch::PSY.DynamicBranch,
    n_buses::Int,
)
    DAE_vector = get_DAE_vector(inputs)
    voltage_buses_ix = get_voltage_buses_ix(inputs)
    arc = PSY.get_arc(branch)
    from_bus_number = PSY.get_number(arc.from)
    to_bus_number = PSY.get_number(arc.to)
    bus_ix_from = get_lookup(inputs)[from_bus_number]
    bus_ix_to = get_lookup(inputs)[to_bus_number]
    b_from = PSY.get_b(branch).from
    b_to = PSY.get_b(branch).to
    total_shunts = get_total_shunts(inputs)
    total_shunts[bus_ix_from, bus_ix_from] += 1im * b_from
    total_shunts[bus_ix_to, bus_ix_to] += 1im * b_to
    b_from > 0.0 &&
        _add_dynamic_bus_states!(DAE_vector, voltage_buses_ix, bus_ix_from, n_buses)
    b_to > 0.0 && _add_dynamic_bus_states!(DAE_vector, voltage_buses_ix, bus_ix_to, n_buses)
    return
end

function index_static_injection(inputs::SimulationInputs, ::PSY.StaticInjection)
    return
end

function add_states_to_global!(
    global_state_index::MAPPING_DICT,
    state_space_ix::Vector{Int},
    device::PSY.Device,
)
    global_state_index[PSY.get_name(device)] = Dict{Symbol, Int}()
    for s in PSY.get_states(device)
        state_space_ix[1] += 1
        global_state_index[PSY.get_name(device)][s] = state_space_ix[1]
    end

    return
end

function index_dynamic_injection(
    inputs::SimulationInputs,
    dynamic_device::PSY.DynamicInjection,
    state_space_ix::Vector{Int},
)
    DAE_vector = get_DAE_vector(inputs)
    make_device_index!(dynamic_device, DAE_vector)
    add_states_to_global!(inputs.global_index, state_space_ix, dynamic_device)
    return
end

var_count(::PSY.DynamicGenerator) = GEN_INNER_VARS_SIZE
var_count(::PSY.DynamicInverter) = INV_INNER_VARS_SIZE

function attach_inner_vars!(
    dynamic_device::T,
    ::Type{U} = Real,
) where {T <: PSY.DynamicInjection, U <: Real}
    dynamic_device.ext[INNER_VARS] = zeros(U, var_count(dynamic_device))
    return
end

function _attach_control_refs!(device::PSY.StaticInjection)
    dynamic_device = PSY.get_dynamic_injector(device)
    dynamic_device.ext[CONTROL_REFS] = [
        PSY.get_V_ref(dynamic_device),
        PSY.get_ω_ref(dynamic_device),
        PSY.get_P_ref(dynamic_device),
        PSY.get_reactive_power(device),
    ]
    return
end

function _attach_control_refs!(device::PSY.Source)
    device.ext[CONTROL_REFS] =
        [PSY.get_internal_voltage(device), PSY.get_internal_angle(device)]
    return
end

function _attach_control_refs!(device::PSY.ElectricLoad)
    return
end

function _get_internal_mapping(
    dynamic_device::PSY.DynamicInjection,
    key::AbstractString,
    ty::Type{T},
) where {T <: PSY.DynamicComponent}
    device_index = PSY.get_ext(dynamic_device)[key]
    val = get(device_index, ty, nothing)
    @assert !isnothing(val)
    return val
end

function get_local_state_ix(
    dynamic_device::PSY.DynamicInjection,
    ty::Type{T},
) where {T <: PSY.DynamicComponent}
    return _get_internal_mapping(dynamic_device, LOCAL_STATE_MAPPING, ty)
end

function get_input_port_ix(
    dynamic_device::PSY.DynamicInjection,
    ty::Type{T},
) where {T <: PSY.DynamicComponent}
    return _get_internal_mapping(dynamic_device, INPUT_PORT_MAPPING, ty)
end

"""
Default implementation to index the devices states and maps to the ports
"""
function make_device_index!(dynamic_device::PSY.DynamicInjection, ::Vector{Bool})
    device_states = PSY.get_states(dynamic_device)
    device_state_mapping = DEVICE_INTERNAL_MAPPING()
    input_port_mapping = DEVICE_INTERNAL_MAPPING()
    attach_inner_vars!(dynamic_device)
    for c in PSY.get_dynamic_components(dynamic_device)
        device_state_mapping[typeof(c)] = index_local_states(c, device_states)
        input_port_mapping[typeof(c)] = index_port_mapping!(c, device_states)
    end
    dynamic_device.ext[LOCAL_STATE_MAPPING] = device_state_mapping
    dynamic_device.ext[INPUT_PORT_MAPPING] = input_port_mapping
    return
end


using Zarr: AbstractStore, Zarr, zopen, zgroup, zcreate
export DiskArrayShuffleStore
struct DiskArrayShuffleStore{S<:DiskArrayShuffler} <: AbstractStore
    s::S
    name::String
    metadata::Dict{String,Vector{UInt8}}
end
Base.show(io::IO,s::DiskArrayShuffleStore) = print(io,"Shuffled Store")
function DiskArrayShuffleStore(s::DiskArrayShuffler; name = "batches", attrs = Dict(),nbatches=100)
    onear = s.currentcoll[].chunkviews |> first |> first
    soutput = size(onear)
    cs = (soutput..., s.sampler.batchsize)
    soutput = (cs...,nbatches)
    #Create a mock Zarr array to contain the metadata
    g = zgroup(Zarr.DictStore())
    zcreate(eltype(onear),g,name,soutput..., compressor=Zarr.NoCompressor(), attrs=attrs)
    Zarr.consolidate_metadata(g)
    DiskArrayShuffleStore(s,name,g.storage.a)
end

function Base.getindex(s::DiskArrayShuffleStore,i::AbstractString)
    if i in (s.name * "/.zarray", s.name*"/.zattrs",".zgroup",".zmetadata")
        return get(s.metadata,i,nothing)
    else
        schunk = split(i,'/')[end]
        chunk = parse.(Int,split(schunk,'.'))
        all(iszero,chunk[2:end]) || return nothing
        r = readbatch(s.s)
        nd = ndims(first(r))
        rone = reduce((i,j)->cat(i,j,dims=nd+1),r)
        return collect(vec(reinterpret(UInt8,rone)))
    end
end


using Drawing

p = 1.02
q = 0.9375

with(
     PNG("ndy.png", 300, 300),
#     TK(300, 300),
     Axes(negative=true)) do
         paint(Ink("darkgreen")) do
             square(100, align=5)
         end
         paint(Ink("red")) do
             square(2, align=5)
         end
         paint(Ink("black"), Font("Arial Black, Bold"; size=0.58)) do
             move(-q, p)
             text("NOT", align=1)
             move(0.0, 0.0)
             text("DEAD", align=5)
             move(q, -p)
             text("YET", align=9)
         end
     end


import JuMP

@testset "shapetrait and nin/nout" begin
    import NaiveNASlib: shapetrait, NoShapeTraitError, noutnin_errmsg
    struct NoutNinFailDummy end

    @test shapetrait(NoutNinFailDummy()) == NoutNinFailDummy

    @test_throws NoShapeTraitError nout(NoutNinFailDummy())
    @test_throws NoShapeTraitError nin(NoutNinFailDummy())

    @test noutnin_errmsg(NoutNinFailDummy(), nout) == "nout not defined for NoutNinFailDummy()! Either implement NaiveNASlib.nout(::NoutNinFailDummy) or implement NaiveNASlib.nout(::ST, ::NoutNinFailDummy) where ST is the return type of NaiveNASlib.shapetrait(::NoutNinFailDummy)"
    @test noutnin_errmsg(NoutNinFailDummy(), nin) == "nin not defined for NoutNinFailDummy()! Either implement NaiveNASlib.nin(::NoutNinFailDummy) or implement NaiveNASlib.nin(::ST, ::NoutNinFailDummy) where ST is the return type of NaiveNASlib.shapetrait(::NoutNinFailDummy)"

    struct NinNoutSuccessDummy end
    NaiveNASlib.nout(::NinNoutSuccessDummy, args...) = 3
    NaiveNASlib.nin(::NinNoutSuccessDummy, args...) = [3]

    @test nout(NinNoutSuccessDummy()) == 3
    @test nin(NinNoutSuccessDummy()) == [3]

    # This is not really how the traits are supposed to be used, but it saves us from defining a new type and associated nout and nin methods here
    struct NinNoutTraitSuccessDummy end
    NaiveNASlib.shapetrait(::NinNoutTraitSuccessDummy) = NinNoutSuccessDummy()

    @test nout(NinNoutTraitSuccessDummy()) == 3
    @test nin(NinNoutTraitSuccessDummy()) == [3]
end

@testset "Size mutations" begin

    inpt(size, id="in") = inputvertex(id, size)
    nt(name) = named(name)
    tf(name) = t -> nt(name)(AfterΔSizeTrait(validateafterΔsize(base=ΔSizeFailNoOp()), t))
    cc(in...; name="cc") = conc(in..., dims=1, traitdecoration = tf(name))
    ea(ins...; name="ea") = +(traitconf(tf(name)) >> ins[1], ins[2:end]...)

    @testset "Basic tests" begin
        using NaiveNASlib: Input, Output, Both, neighbours

        av(in, outsize, name="av") = absorbvertex(SizeDummy(nout(in), outsize), in; traitdecoration=tf(name))

        @testset "AbsorbVertex" begin
            v0 = av(inpt(2), 2, "iv")
            v1 = av(v0, 3, "v1")

            @test nin(v1) == [2]
            @test nout(v1) == 3

            @test outputs.(inputs(v1)) == [[v1]]

            Δnin!(v1, 2)
            @test nin(v1) == [4]

            Δnin!(v1, -3)
            @test nin(v1) == [1]

            Δnout!(v1, -2)
            @test nout(v1) == 1

            # Add one vertex and see that change propagates
            v2 = av(v1, 4, "v2")
            @test nout(v1) == nin(v2)[1]
            @test outputs(v1) == [v2]
            @test inputs(v2) == [v1]

            Δnin!(v2, 4)
            @test nout(v1) == nin(v2)[1] == 5

            Δnout!(v1, -2)
            @test nout(v1) == nin(v2)[1] == 3

            # Fork of v1 into a new vertex
            v3 = av(v1, 2, "v3")
            @test outputs(v1) == [v2, v3]
            @test inputs(v3) == inputs(v2) == [v1]

            Δnout!(v1, -2)
            @test nout(v1) == nin(v2)[1] == nin(v3)[1] == 1

            Δnin!(v3, 3)
            @test nout(v1) == nin(v2)[1] == nin(v3)[1] == 4

            Δnin!(v2, -2)
            @test nout(v1) == nin(v2)[1] == nin(v3)[1] == 2
        end

        @testset "StackingVertex" begin

            @testset "StackingVertex 1 to 1" begin

                v0 = av(inpt(2), 2, "v0")
                v1 = cc(v0)
                v2 = av(v1, 2,"v2")

                @test outputs.(inputs(v2)) == [[v2]]
                @test outputs.(inputs(v1)) == [[v1]]
                @test outputs(v0) == [v1]
                @test neighbours(Both(), v1) == [v0, v2]

                Δnout!(v0, 3)
                @test [nout(v0)] == nin(v1) == [nout(v1)] == nin(v2) == [5]

                Δnin!(v2, -2)
                @test [nout(v0)] == nin(v1) == [nout(v1)] == nin(v2) == [3]

                Δnin!(v1, +1)
                @test [nout(v0)] == nin(v1) == [nout(v1)] == nin(v2) == [4]

                Δnout!(v1, -1)
                @test [nout(v0)] == nin(v1) == [nout(v1)] == nin(v2) == [3]
            end

            @testset "StackingVertex 2 inputs" begin
                # Try with two inputs to StackingVertex
                v1 = av(inpt(2,"in1"), 2, "v1")
                v2 = av(inpt(3,"in2"), 3, "v2")
                v3 = cc(v1, v2)
                v4 = av(v3, 5, "v4")

                @test inputs(v3) == [v1, v2]
                @test outputs.(inputs(v3)) == [[v3], [v3]]
                @test outputs(v1) == [v3]
                @test outputs(v2) == [v3]
                @test neighbours(Both(), v3) == [v1, v2, v4]

                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == [5]

                Δnout!(v2, -2)
                @test nin(v4) == [nout(v3)] == [sum(nin(v3))] == [nout(v1) + nout(v2)]  ==  [3]

                Δnin!(v4, 3)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == [6]

                Δnout!(v1, 4)
                Δnin!(v4, -8)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == [2]
                @test nout(v1) == nout(v2) == 1

                #Add another output
                v5 = av(v3, 2, "v5")

                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == nin(v5) == [2]

                Δnout!(v1, 3)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == nin(v5) == [5]

                Δnin!(v5, -2)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == nin(v5) == [3]

                Δnin!(v4, 3)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == nin(v5) == [6]

                Δnin!(v3, -1, missing)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == nin(v5) == [5]

                Δnin!(v3, missing, 2)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == nin(v5) == [7]

                Δnout!(v3, -1)
                @test [nout(v1) + nout(v2)] == [sum(nin(v3))] == [nout(v3)] == nin(v4) == nin(v5) == [6]
            end

        end
        @testset "InvariantVertex" begin

            @testset "InvariantVertex 1 to 1" begin
                #Behaviour is identical to StackingVertex in this case
                v1 = av(inpt(2), 2, "v1")
                v2 = ea(v1)
                v3 = av(v2, 2, "v3")

                @test outputs.(inputs(v3)) == [[v3]]
                @test outputs.(inputs(v2)) == [[v2]]
                @test outputs(v1) == [v2]

                Δnout!(v1, 3)
                @test [nout(v1)] == nin(v2) == [nout(v2)] == nin(v3) == [5]

                Δnin!(v3, -2)
                @test [nout(v1)] == nin(v2) == [nout(v2)] == nin(v3) == [3]
            end

            @testset "invariantvertex 2 inputs" begin
                # Try with two inputs a v3ariantVertex
                v1 = av(inpt(2, "in1"), 3, "v1")
                v2 = av(inpt(2, "in2"), 3, "v2")
                v3 = ea(v1, v2)
                v4 = av(v3, 5, "v4")

                @test inputs(v3) == [v1, v2]
                @test outputs.(inputs(v3)) == [[v3], [v3]]
                @test outputs(v1) == [v3]
                @test outputs(v2) == [v3]

                @test nout(v1) == nout(v2) == nin(v3)[1] == nin(v3)[2] == nout(v3) == nin(v4)[] == 3

                Δnout!(v2, -2)
                @test nin(v4)[] == nout(v3) == nin(v3)[1] == nin(v3)[2]  == nout(v1) == nout(v2)  ==  1

                Δnin!(v4, 3)
                @test nout(v1) == nout(v2) == nin(v3)[1] == nin(v3)[2] == nout(v3) == nin(v4)[] == 4

                #Add another output
                v5 = av(v3, 4, "v5")

                @test nout(v1) == nout(v2) == nin(v3)[1] == nin(v3)[2] == nout(v3) == nin(v4)[] == nin(v5)[] == 4

                Δnout!(v1, -3)
                @test nout(v1) == nout(v2) == nin(v3)[1] == nin(v3)[2] == nout(v3) == nin(v4)[] == nin(v5)[] == 1

                Δnin!(v5, 2)
                @test nout(v1) == nout(v2) == nin(v3)[1] == nin(v3)[2] == nout(v3) == nin(v4)[] == nin(v5)[] == 3

                Δnout!(v2, 3)
                @test nout(v1) == nout(v2) == nin(v3)[1] == nin(v3)[2] == nout(v3) == nin(v4)[] == nin(v5)[] == 6
            end

            @testset "invariantvertex with params and size" begin
                struct InvariantWithSize
                    s::Int
                end
                NaiveNASlib.nin(i::InvariantWithSize, ::SizeInvariant, v::AbstractVertex) = [i.s]

                v = invariantvertex(InvariantWithSize(7), inpt(3, "in"); traitdecoration=tf("v"))

                @test nin(v) == [nout(v)] == [7]
            end
        end

        @testset "validate after Δsize" begin
            import NaiveNASlib: ΔSizeFailError

            # Mock for creating invalid size changes
            struct Ignore{T}
                f::T
            end
            NaiveNASlib.nout(i::Ignore) = nout(i.f)
            NaiveNASlib.nin(i::Ignore) = nin(i.f)
            NaiveNASlib.Δsize!(::Ignore, ::AbstractVector{<:Integer}, ::Integer) = nothing
            NaiveNASlib.Δsizetype(i::Ignore) = NaiveNASlib.Δsizetype(i.f)

            @testset "With AfterΔSizeTrait" begin

                v1 = inpt(3, "v1")
                v2 = absorbvertex(Ignore(SizeDummy(nout(v1), 5)), v1; traitdecoration=tf("v2"))
                v3 = av(v2, 7, "v3")
                v4 = absorbvertex(Ignore(SizeDummy(nout(v3), 5)), v3; traitdecoration=tf("v4"))

                @test_throws ΔSizeFailError Δnout!(v2 => 2)
                @test_throws ΔSizeFailError Δnin!(v3 => -3)
                @test_throws ΔSizeFailError Δnin!(v4 => 5)
                @test_throws ΔSizeFailError Δnout!(v3 => -7)
            end

            @testset "As strategy" begin
                v1 = inpt(3, "v1")
                v2 = absorbvertex(Ignore(SizeDummy(nout(v1), 5)), v1; traitdecoration=nt("v2"))
                v3 = absorbvertex(SizeDummy(nout(v2), 7), v2; traitdecoration=nt("v3"))
                v4 = absorbvertex(Ignore(SizeDummy(nout(v3), 5)), v3; traitdecoration=nt("v4"))

                @test_throws ΔSizeFailError Δsize!(validateafterΔsize(base=ΔNout(v2 => 2)))
                @test_throws ΔSizeFailError Δsize!(validateafterΔsize(base=ΔNin(v3 =>-3)))
                @test_throws ΔSizeFailError Δsize!(validateafterΔsize(base=ΔNin(v4, 5)))
                @test_throws ΔSizeFailError Δsize!(validateafterΔsize(base=ΔNout(v3 => -7)))
            end
        end

        @testset "Fail mutate immutable" begin
            import NaiveNASlib: ΔSizeFailError

            v0 = inpt(3)
            v1 = av(v0, 5, "v1")

            @test_throws ΔSizeFailError Δsize!(ΔNoutExact(v0, 2;fallback=ThrowΔSizeFailError("Success?!")), [v0, v1])
            @test [nout(v0)] == nin(v1) == [3]

            @test_throws ΔSizeFailError Δsize!(ΔNinExact(v1, -2; fallback=ThrowΔSizeFailError("Success?!")), [v1, v0])
            @test [nout(v0)] == nin(v1) == [3]
        end
    end

    @testset "Time limit" begin
        import NaiveNASlib: TimeLimitΔSizeStrategy, sizemodel
        @test JuMP.time_limit_sec(sizemodel(TimeLimitΔSizeStrategy(123), AbstractVertex[])) == 123
    end

    @testset "Mutate tricky structures" begin

        ## Helper functions
        iv(in; name="iv") = invariantvertex(identity, in; traitdecoration=tf(name))
        concd1(paths...;name="conc") = conc(paths..., dims=1, traitdecoration=tf(name))
        av(in, outsize, name="comp") = absorbvertex(SizeDummy(nout(in), outsize), in, traitdecoration = tf(name))
        function stack(start, nouts...; bname = "stack")
            # Can be done on one line with mapfoldl, but it is not pretty...
            next = start
            for i in 1:length(nouts)
                next = av(next, nouts[i], "$(bname)_$i")
            end
            return next
        end

        @testset "Residual fork block" begin
            start = av(inputvertex("in", 3), 9, "start")
            p1 = stack(start, 3,4, bname = "p1")
            p2 = stack(start, 4,5, bname = "p2")
            resout = "add" >> start + concd1(p1, p2)
            out = av(resout, 9, "out")

            @test nout(resout) == 9

            # Propagates to out, start outputs of start and also to p1 as objective is minimized by increasing p1 by 1
            @test Δnout!(p2, -2)
            @test nout(p2) == 3
            @test nin(out) == [nout(start)] == [8]
            @test nout(p2) + nout(p1) == 8
            #outputs(start) = first vertex in p1, p2 and resout (which has two inputs)
            @test reduce(vcat, nin.(outputs(start))) == [8, 8, 8, 8]

            @test Δnin!(out, 2)
            @test nin(out) == [nout(start)] == [10]
            @test reduce(vcat, nin.(outputs(start))) == [10, 10, 10, 10]
        end

        @testset "Half transparent residual fork block" begin
            start = av(inputvertex("in", 3), 8, "start")
            split = av(start, 4, "split")
            p1 = iv(split, name="p1") #Just an identity vertex
            p2 = stack(split, 3,2,4, bname="p2")
            resout = "add" >> start + concd1(p1, p2)
            out = av(resout, 3, "out")

            @test nout(resout) == 8

            # Propagates to input of first vertex of p2, input of out and start
            # via p1 and resout as well as to input of split
            @test Δnout!(split, -1)
            @test nin(out) == [nout(start)] == nin(split) == [8]
            @test reduce(vcat, nin.(outputs(split))) == [3, 3]

            # Should basically undo the previous mutation
            @test Δnin!(out, +1)
            @test nin(out) == [nout(start)] == nin(split) == [9]
        end

        @testset "Transparent fork block" begin
            start = av(inputvertex("in", 3), 4, "start")
            p1 = iv(start, name="p1")
            p2 = iv(start, name="p2")
            joined = concd1(p1, p2, name="join")
            out = av(joined, 3, "out")

            @test nout(joined) == 8

            # Evil action: This will propagate to both p1 and p2 which are in
            # turn both input to the conc before resout.
            @test Δnout!(start, -1)
            @test nin(out) == [2nout(start)] == [6]

            # Should basically undo the previous mutation
            @test Δnin!(out, 2)
            @test nin(out) == [2nout(start)] == [8]
        end

        @testset "Transparent residual fork block" begin
            start = av(inputvertex("in", 3), 8, "start")
            split = av(start, 4, "split")
            p1 = iv(split, name="p1")
            p2 = iv(split, name="p2")
            resout = "add" >> start + concd1(p1, p2, name="join")
            out = av(resout, 3, "out")

            @test nout(resout) == 8

            # Evil action: This will propagate to both p1 and p2 which are in
            # turn both input to the conc before resout.
            @test Δnout!(split, -1)
            @test nin(out) == [nout(start)] == nin(split) == [6]

            # Should basically undo the previous mutation
            @test Δnin!(out, +2)
            @test nin(out) == [nout(start)] == nin(split) == [8]

            @test Δnout!(start, -2)
            @test nin(out) == [nout(start)] == [6]
            @test nout(split) == 3
        end

        @testset "Transparent residual fork block with single absorbing path" begin
            start = av(inputvertex("in", 3), 8, "start")
            split = av(start, 3, "split")
            p1 = iv(split, name="p1")
            p2 = iv(split, name="p2")
            p3 = av(split, 2, "p3")
            resout = "add" >> start + concd1(p1, p2, p3, name="join")
            out = av(resout, 3, "out")

            @test nout(resout) == 8

            # Evil action: This will propagate to both p1 and p2 which are in
            # turn both input to the conc before resout.
            @test Δnout!(split, -1)
            @test nin(out) == [nout(start)] == nin(split) == [6]

            # Should basically undo the previous mutation
            @test Δnin!(out, 2)
            @test nin(out) == [nout(start)] == nin(split) == [8]

            @test Δnout!(start, -2)
            @test nin(out) == [nout(start)] == [2*nout(split) + nout(p3)] == [6]
            @test nout(split) == 2
            @test nout(p3) == 2
        end
    end

    @testset "Size Mutation possibilities" begin
        using NaiveNASlib: newsizes

        # Helpers
        struct SizeConstraint{T}
            constraint::Int
            w::T
        end

        function NaiveNASlib.compconstraint!(::NaiveNASlib.ScalarSize, s, c::SizeConstraint, data)
            fv_out = JuMP.@variable(data.model, integer=true)
            JuMP.@constraint(data.model, c.constraint * fv_out ==  nout(data.vertex) - data.noutdict[data.vertex])

            ins = filter(vin -> vin in keys(data.noutdict), inputs(data.vertex))
            fv_in = JuMP.@variable(data.model, [1:length(ins)], integer=true)
            JuMP.@constraint(data.model, [i=1:length(ins)], c.constraint * fv_in[i] ==  nout(ins[i]) - data.noutdict[ins[i]])
        end
        NaiveNASlib.Δsizetype(c::SizeConstraint) = NaiveNASlib.Δsizetype(c.w)
        NaiveNASlib.nout(c::SizeConstraint) = nout(c.w)
        NaiveNASlib.nin(c::SizeConstraint) = nin(c.w)
        NaiveNASlib.Δsize!(c::SizeConstraint, insize::AbstractVector{<:Integer}, outsize::Integer) = Δsize!(c.w, insize, outsize)
        av(size, csize, in... ;name = "av") = absorbvertex(SizeConstraint(csize, SizeDummy(collect(nout.(in)), size)), in..., traitdecoration=tf(name))

        @testset "SizeStack multi inputs" begin
            v1 = av(6,3, inpt(3), name="v1")
            v2 = av(7,2, inpt(3), name="v2")
            v3 = av(8,2, inpt(3), name="v3")

            cc1 = cc(v1, v2, name="cc1")
            cc2 = cc(v3, v2, v1, v2, name="cc2")

            # Expect only v1 to change as size change is not compatible with v2
            @test Δnout!(cc1, -3)
            @test nin(cc1) == [nout(v1), nout(v2)] == [3, 7]
            @test nin(cc2) == [nout(v3), nout(v2), nout(v1), nout(v2)] == [8, 7, 3, 7]
            @test nout(cc1) == sum(nin(cc1))
            @test nout(cc2) == sum(nin(cc2))

            # v1 can't change as it is too small already
            @test Δnout!(cc2, -6)
            @test nin(cc1) == [nout(v1), nout(v2)] == [3, 5]
            @test nin(cc2) == [nout(v3), nout(v2), nout(v1), nout(v2)] == [6, 5, 3, 5]
            @test nout(cc1) == sum(nin(cc1))
            @test nout(cc2) == sum(nin(cc2))

            # Evil action! Must have understanding that change will propagate to cc1 from
            # both v1 and v2
            @test Δnout!(cc2, 9)
            @test nin(cc1) == [nout(v1), nout(v2)] == [6, 7]
            @test nin(cc2) == [nout(v3), nout(v2), nout(v1), nout(v2)] == [8, 7, 6, 7]
            @test nout(cc1) == sum(nin(cc1))
            @test nout(cc2) == sum(nin(cc2))

        end

        @testset "Stacked SizeStacks" begin
            v1 = av(100,1, inpt(3), name="v1")
            v2 = av(100,2, inpt(3), name="v2")
            v3 = av(100,3, inpt(3), name="v3")
            v4 = av(100,5, inpt(3), name="v4")

            cc1 = cc(v2, v3, name="cc1")
            cc2 = cc(cc1, v1, name="cc2")
            cc3 = cc(cc2, v4, v2, name="cc3")

            @test Δnout!(cc3, -60)
            @test nin(cc1) == nout.(inputs(cc1)) == [86, 91]
            @test nin(cc2) == nout.(inputs(cc2)) == [177, 87]
            @test nin(cc3) == nout.(inputs(cc3)) == [264, 90, 86]
            @test nout(cc1) == nout(v2) + nout(v3) == 177
            @test nout(cc2) == nout(cc1) + nout(v1) == 264
            @test nout(cc3) == nout(cc2) + nout(v4) + nout(v2) == 440

            v5 = av(10, 3, cc3, name="v5")

            @test Δnout!(v1, 3)
            @test nin(cc1) == nout.(inputs(cc1)) == [86, 91]
            @test nin(cc2) == nout.(inputs(cc2)) == [177, 90]
            @test nin(cc3) == nout.(inputs(cc3)) == [267, 90, 86]
            @test nout(cc1) == nout(v2) + nout(v3) == 177
            @test nout(cc2) == nout(cc1) + nout(v1) == 267
            @test [nout(cc3)] == [(nout(cc2) + nout(v4) + nout(v2))] == nin(v5) == [443]

            # Evil action! Must have understanding that the change in v2 will propagate
            # to cc3 input 1 through cc1 and cc2
            @test Δnout!(v2, -12)
            @test nin(cc1) == nout.(inputs(cc1)) == [74, 103]
            @test nin(cc2) == nout.(inputs(cc2)) == [177, 90]
            @test nin(cc3) == nout.(inputs(cc3)) == [267, 90, 74]
            @test nout(cc1) == nout(v2) + nout(v3) == 177
            @test nout(cc2) == nout(cc1) + nout(v1) == 267
            @test [nout(cc3)] == [(nout(cc2) + nout(v4) + nout(v2))] == nin(v5) == [431]
        end

        @testset "SizeInvariant multi input" begin
            v1 = av(10,1, inpt(3), name="v1")

            ea1 = ea(v1, name="ea1")
            ea2 = ea(ea1, v1, name="ea2")

            @test Δnout!(ea2, -2)

            @test nout(ea2) == nout(ea1) == nout(v1) == 8
            @test nin(ea2) == [nout(ea1), nout(v1)] == [8, 8]
            @test nin(ea1) == [nout(ea1)] == [8]
        end

        @testset "SizeInvariant multi SizeStack input" begin
            v1 = av(7,1, inpt(3), name="v1")
            v2 = av(13,1, inpt(3), name="v2")

            cc1 = cc(v1, v2, name="cc1")
            cc2 = cc(v1, v2, name="cc2")

            ea1 = ea(cc1, cc2, name="ea1")
            ea2 = ea(cc1, cc2, name="ea2")
            ea3 = ea(ea1, ea2, name="ea3")

            @test Δnout!(ea3, -4)

            @test nout(ea3) == nout(ea2) == nout(ea1) == nout(cc2) == nout(cc1) == nout(v2) + nout(v1) == 16
            @test nin(ea3) == nin(ea2) == nin(ea3) == [nout(cc1), nout(cc2)] == [16, 16]
            @test nin(cc2) == nin(cc1) == [nout(v1), nout(v2)] == [6, 10]
        end

        @testset "Stacked SizeInvariant" begin
            v1 = av(100,1, inpt(3), name="v1")
            v2 = av(100,2, inpt(3), name="v2")
            v3 = av(100,3, inpt(3), name="v3")
            v4 = av(100,5, inpt(3), name="v4")

            ea1 = ea(v2, v3, name="ea1")
            ea2 = ea(ea1, v1, name="ea2")
            ea3 = ea(ea2, v4, v2, name="ea3")

            # Everything touches everything in this setup
            @test Δnout!(ea3, -30)
            @test nout(v1) == nout(v2) == nout(v3) == nout(v4) == 70
            @test nout(ea1) == nout(ea2) == nout(ea3) == 70

            v5 = av(10, 3, ea3, name="v5")

            @test Δnout!(v1, 60)
            @test nout(v1) == nout(v2) == nout(v3) == nout(v4) == 130
            @test [nout(ea1)] == [nout(ea2)] == [nout(ea3)] == nin(v5) == [130]

            # Evil action! Must have understanding that the change in v2 will propagate
            # to ea3 input 1 through ea1 and ea2 and hold off updating it through input 3
            @test Δnout!(v2, -30)
            @test nout(v1) == nout(v2) == nout(v3) == nout(v4) == 100
            @test [nout(ea1)] == [nout(ea2)] == [nout(ea3)] == nin(v5) == [100]
        end

        @testset "Stacked input to SizeInvariant" begin
            v0 = inpt(3, "in")
            v1 = av(20, 2, v0, name="v1")
            v2 = av(30, 3, v0, name="v2")
            v3 = cc(v1,v2,v1, name="v3")
            v4 = av(70, 5, v0, name="v4")
            v5 = ea(v4,v3, name="v5")
            v6 = av(10, 7, v5, name="v6")

            @test Δnout!(v5, 2*2*3*5*7)

            @test nout(v5) == nout(v4) == nout(v3) == 490
            @test nin(v5) == [nout(v4), nout(v3)] == [490, 490]
            @test nin(v3) == [nout(v1), nout(v2), nout(v1)] == [140, 210, 140]
        end

        @testset "Invariant input to SizeInvariant" begin
            v0 = inpt(3, "in")
            v1 = av(20, 2, v0, name="v1")
            v2 = av(20, 3, v0, name="v2")
            v3 = ea(v1,v2, name="v3")
            v4 = av(20, 5, v0, name="v4")
            v5 = ea(v4,v3, name="v5")
            v6 = av(10, 7, v5, name="v6")

            @test Δnout!(v5, 2*3*5*7)

            @test nout(v5) == nout(v4) == nout(v3) == 230
            @test nin(v5) == [nout(v4), nout(v3)] == nin(v3) == [nout(v1), nout(v2)] == [230, 230]
        end

        @testset "SizeStack duplicate through SizeInvariant" begin
            v0 = inpt(5, "in")
            v1 = av(2, 2, v0, name="v1")
            v2 = ea(v1, name="v2")
            v3 = ea(v1, name="v3")
            v4 = cc(v2,v3, name="v4")
            v5 = ea(v4, name="v5")
            v6 = cc(v5, v2, name="v6")

            @test Δnout!(v6, 12)

            @test nout(v6) == sum(nin(v6)) == 18
            @test nout(v5) == nin(v5)[] == 12
            @test nout(v4) == sum(nin(v4)) == 12
            @test nout(v1) == 6
        end

        @testset "SizeInvariant zig-zag" begin
                v0 = inpt(5, "in")
                v1 = av(2, 2, v0, name="v1")
                v2 = av(3, 3, v0, name="v2")
                v3 = cc(v1,v2, name="v3")
                function zigzag(vin1, sc, vin2=v0;name="zig")
                    vnew = av(nout(vin1), sc, vin2, name=name*"_new")
                    vout = ea(vnew, name=name*"_ivB")
                    vcon = ea(vout, vin1, name=name*"_ivA")
                    return vout
                end
                v4 = zigzag(v3, 5, name="z1")
                v5 = zigzag(v4, 7, name="z2")
                v6 = av(11, 11, v5, name="v6")

                expectedΔf = 2*3*5*7*11

                @test Δnout!(v2, expectedΔf)
                @test nout(v3) == nout(v4) == nout(v5) == expectedΔf + 2+3
                @test nin(v4) == nout.(inputs(v4)) == [expectedΔf + 2+3]
                @test nin(v5) == nout.(inputs(v5)) == [expectedΔf + 2+3]
        end

        @testset "SizeStack duplicate SizeInvariant mini-zig-zag" begin
            v0 = inpt(5, "in")
            v1 = av(2, 2, v0, name="v1")
            v2 = av(2, 3, v0, name="v2")
            v3 = ea(v1, name="v3")
            v4 = ea(v3, v2, name="v4")
            v5 = av(3, 5, v3, name="v5")
            v6 = cc(v4,v4, name="v6")
            v7 = ea(v6, v6, name="v7")
            v8 = av(4, 7, v7, name="v8")

            expectedΔf = 2*2*3*5*7

            @test Δnout!(v6, expectedΔf)
            @test nin(v8) == [nout(v7)] == [nout(v6)] == [4+expectedΔf]

            @test nin(v6) == [2 + expectedΔf ÷ 2, 2 + expectedΔf ÷ 2]
            @test nout(v1) == nout(v2) == 2 + expectedΔf ÷ 2
            @test nin(v5) == [2 + expectedΔf ÷ 2]
        end

        @testset "Deep SizeStack" begin
            v0 = inpt(3, "in")
            v1 = av(8, 1, v0, name="v1")
            v2 = av(4, 1, v0, name="v2")
            v3 = cc(v1,v2, name= "v3")
            pa1 = ea(v3, name="pa1")
            pb1 = ea(v3, name="pb1")
            pc1 = ea(v3, name="pc1")
            pd1 = av(5, 1, v3, name="pd1")
            pa1pa1 = ea(pa1, name="pa1pa1")
            pa1pb1 = ea(pa1, name="pa1pb1")
            pa2 = cc(pa1pa1, pa1pb1, name = "pa2")
            v4 = cc(pa2, pb1, pc1, pd1, name = "v4")

            @test Δnout!(v4, 8)

            @test nout(v4) == 61
            @test nin(v4) == nout.(inputs(v4)) == [28, 14, 14, 5]
            @test nin(pa2) == nout.(inputs(pa2)) == [14, 14]
            @test nin(pa1pa1) == nin(pa1pb1) == [14]
            @test nin(pa1) == nin(pb1) == nin(pc1) == [nout(v3)] == [14]
            @test nin(v3) == [10, 4]
        end

        @testset "Fail invalid size change" begin
            import NaiveNASlib: ΔSizeFailError
            v0 = inpt(3)
            v1 = av(3,3, v0, name="v1")
            v2 = av(5,2, v1, name="v2")

            @test_throws ΔSizeFailError newsizes(ΔNoutExact(v1, 2; fallback=ThrowΔSizeFailError("")), all_in_graph(v1))
            @test_throws ΔSizeFailError newsizes(ΔNoutExact(v1, 3; fallback=ThrowΔSizeFailError("")), all_in_graph(v1))

            @test_throws ΔSizeFailError newsizes(ΔNinExact(v2, 2; fallback=ThrowΔSizeFailError("")), all_in_graph(v1))
            @test_throws ΔSizeFailError newsizes(ΔNinExact(v2, 3; fallback=ThrowΔSizeFailError("")), all_in_graph(v1))

            @test newsizes(ΔNoutExact(v1, 2; fallback=ΔSizeFailNoOp()), all_in_graph(v1))[1] == false
            @test newsizes(ΔNinExact(v2, 2; fallback=ΔSizeFailNoOp()), all_in_graph(v2))[1] == false

            @test (@test_logs (:info, "Giving up") newsizes(ΔNoutExact(v1, 2; fallback=LogΔSizeExec("Giving up")), all_in_graph(v1)))[1] == false

            @test_logs (:warn, r"Could not change nout of .* by 2! Relaxing constraints...") Δnout!(v1, 2)
            @test [nout(v1)] == nin(v2) == [9]

            @test_logs (:warn, r"Could not change nin of .* by -2! Relaxing constraints...") Δnin!(v2, -2)
            @test [nout(v1)] == nin(v2) == [3]

            v3 = cc(v1,v2, name="v3")

            @test_logs (:warn, r"Could not change nin of .* by 2 and 6! Relaxing constraints...") Δnin!(v3, 2, 6)
            @test [nout(v1)] == nin(v2) == [9]
            @test nin(v3) == [nout(v1), nout(v2)] == [9, 11]

            @test_logs (:warn, r"Could not change nout of .* by -3! Relaxing constraints...") Δnout!(v3, -3)
            @test [nout(v1)] == nin(v2) == [9]
            @test nin(v3) == [nout(v1), nout(v2)] == [9, 9]
        end

    end

    @testset "Log size change" begin
        import Logging
        using Logging: Info
        using NaiveNASlib: NeuronIndices

        @testset "With strategy" begin
            av(in, outsize, name="av") = absorbvertex(SizeDummy(nout(in), outsize), in; traitdecoration=tf(name))

            v1 = inpt(3, "v1")
            v2 = av(v1, 10, "v2")
            v3 = av(v2, 4, "v3")

            Δninlog!(args...) = Δsize!(logafterΔsize(;level=Info, base=ΔNin(args...)))
            Δnoutlog!(args...) = Δsize!(logafterΔsize(;level=Info, base=ΔNout(args...)))

            Δnoutloginds!(args...) = Δsize!(NeuronIndices(), logafterΔsize(;level=Info, base=ΔNout(args...)))

            @test @test_logs (:info, "Change nin of v3 by 3") (:info, "Change nout of v2 by 3")  match_mode=:any Δninlog!(v3 => 3)

            @test @test_logs (:info, "Change nout of v2 by -3") (:info, "Change nin of v3 by -3") Δnoutlog!(v2 => -3)

            v4 = av(v1, nout(v3), "v4")
            v5 = "v5" >> v3 + v4

            @test @test_logs (:info, "Change nin of v5 by 30 and 30") (:info, "Change nout of v5 by 30") (:info, "Change nout of v3 by 30") (:info, "Change nout of v4 by 30") Δnoutlog!(v5 => 30)

            @test @test_logs (:info, "Change nin of v5 by [1,…, 34, -1×10] and [1,…, 34, -1×10]") (:info, "Change nout of v5 by [1,…, 34, -1×10]") (:info, "Change nout of v3 by [1,…, 34, -1×10]") (:info, "Change nout of v4 by [1,…, 34, -1×10]") Δnoutloginds!(v5 => 10)
        end

        @testset "With trait" begin
            traitfun(name) = t -> AfterΔSizeTrait(logafterΔsize(;level=Logging.Info), NamedTrait(name, t))
            avlogged(in, size ;name = "av") = absorbvertex(SizeDummy(nout(in), size), in; traitdecoration=traitfun(name))

            v1 = inpt(3, "v1")
            v2 = avlogged(v1, 10, name="v2")
            v3 = avlogged(v2, 4, name="v3")

            @test @test_logs (:info, "Change nin of v3 by 3") (:info, "Change nout of v2 by 3")  match_mode=:any Δnin!(v3, 3)

            @test @test_logs (:info, "Change nout of v2 by -3") (:info, "Change nin of v3 by -3") Δnout!(v2, -3)

            v4 = avlogged(v1, nout(v3), name="v4")
            v5 = traitconf(traitfun("v5")) >> v3 + v4

            @test @test_logs (:info, "Change nin of v5 by 30 and 30") (:info, "Change nout of v5 by 30") (:info, "Change nout of v3 by 30") (:info, "Change nout of v4 by 30") Δnout!(v5, 30)

            @test @test_logs (:info, "Change nin of v5 by [1,…, 34, -1×10] and [1,…, 34, -1×10]") (:info, "Change nout of v5 by [1,…, 34, -1×10]") (:info, "Change nout of v3 by [1,…, 34, -1×10]") (:info, "Change nout of v4 by [1,…, 34, -1×10]") Δsize!(NeuronIndices(), ΔNoutExact(v5, 10))
        end
    end

    @testset "WithKwargs" begin
        mutable struct WantKwargsSize
            insize::Int
            outsize::Int
        end
        NaiveNASlib.nout(w::WantKwargsSize) = w.outsize
        NaiveNASlib.nin(w::WantKwargsSize) = [w.insize]
        NaiveNASlib.Δsizetype(::WantKwargsSize) = NaiveNASlib.ScalarSize()

        seenkwargs = Dict()
        function NaiveNASlib.Δsize!(w::WantKwargsSize, ins::AbstractVector{<:Integer}, outs::Integer; kwargs...)
            if ins[] !== missing
                w.insize = ins[]
            end
            w.outsize =outs
            merge!(seenkwargs, kwargs)
        end

        iv = inpt(3)
        v1 = absorbvertex(WantKwargsSize(3, 5), iv; traitdecoration=tf("v1"))

        @test Δsize!(WithKwargs(ΔNout(v1 => 2); a=3, b=4)) == true

        @test nout(v1) == 7
        @test seenkwargs == Dict(:a => 3, :b => 4)

        @test Δsize!(validateafterΔsize(base=WithKwargs(ΔNout(v1 =>-2); c = 5))) == true
        @test nout(v1) == 5
        @test seenkwargs == Dict(:a => 3, :b => 4, :c=>5)
    end
end


"""
Main module for `FourierFlows.jl` -- an ecosystem for solving partial differential equations
on periodic domains using Fourier-based pseudospectral methods."

# Exports
$(EXPORTS)
"""
module FourierFlows

export
  Device,
  CPU,
  GPU,
  ArrayType,

  cxtype,
  fltype,
  innereltype,

  AbstractVars,
  AbstractParams,

  AbstractGrid,
  ZeroDGrid,
  OneDGrid,
  TwoDGrid,
  ThreeDGrid,
  dealias!,
  gridpoints,

  Diagnostic,
  resize!,
  update!,
  increment!,

  Output,
  saveoutput,
  saveproblem,
  groupsize,
  savediagnostic,

  @zeros,
  @devzeros,
  @createarrays,
  @superzeros,
  devzeros,
  superzeros,
  supersize,

  AbstractTimeStepper,
  TimeStepper,
  ForwardEulerTimeStepper,
  FilteredForwardEulerTimeStepper,
  RK4TimeStepper,
  FilteredRK4TimeStepper,
  ETDRK4TimeStepper,
  FilteredETDRK4TimeStepper,
  AB3TimeStepper,
  FilteredAB3TimeStepper,
  stepforward!,
  step_until!

using
  FFTW,
  JLD2,
  Statistics,
  Interpolations,
  Reexport,
  DocStringExtensions

@reexport using FFTW: fft, ifft, rfft, irfft
@reexport using CUDA

import Base: resize!, getindex, setindex!, push!, append!, show, summary

using Base: fieldnames
using FFTW: fftfreq, rfftfreq

"Abstract supertype for grids."
abstract type AbstractGrid{T, A} end

"Abstract supertype for timesteppers."
abstract type AbstractTimeStepper{T} end

"Abstract supertype for parameters."
abstract type AbstractParams end

"Abstract supertype for variables."
abstract type AbstractVars end

"Abstract supertype for diagnostics."
abstract type AbstractDiagnostic end

"Abstract supertype for device."
abstract type Device end

"CPU device."
struct CPU <: Device end

"GPU device."
struct GPU <: Device end

# The main show
include("problem.jl")
include("domains.jl")
include("utils.jl")
include("diagnostics.jl")
include("output.jl")
include("timesteppers.jl")

# Physics
include("diffusion.jl")

# CUDA functionality
if CUDA.has_cuda()
  include("CuFourierFlows.jl")
end

function __init__()
  threads = Threads.nthreads()
  if threads > 1
    @info "FourierFlows will use $threads threads"
    FFTW.set_num_threads(threads)
  end

  if CUDA.has_cuda()
    @debug "CUDA-enabled GPU(s) detected:"
    for (gpu, dev) in enumerate(CUDA.devices())
      @debug "$dev: $(CUDA.name(dev))"
    end

    CUDA.allowscalar(false)
  end
end

function show(io::IO, vars::AbstractVars)
  names = propertynames(vars)
  showstring = ""
  for name in names[1:end-1]
    field = getproperty(vars, name)
    showstring = string(showstring, "  ├───── variable: " * string(name) * " -> ", summary(field), "\n")
  end
  name = names[end]
  field = getproperty(vars, name)
  showstring = string(showstring, "  └───── variable: " * string(name) * " -> ", summary(field), "\n")

  return print(io, "Variables\n", showstring)
end

summary(::Function) = "Function"

function show(io::IO, params::AbstractParams)
  names = propertynames(params)
  showstring = ""
  for name in names[1:end-1]
    field = getproperty(params, name)
    showstring = string(showstring, "  ├───── parameter: " * string(name) * " -> ", summary(field), "\n")
  end
  name = names[end]
  field = getproperty(params, name)
  showstring = string(showstring, "  └───── parameter: " * string(name) * " -> ", summary(field), "\n")

  return print(io, "Parameters\n", showstring)
end

end # module


"""
    mutable struct Scenarios <: Forecast
        name::String
        resolution::Dates.Period
        scenario_count::Int64
        data::Union{
            SortedDict{Dates.DateTime, Matrix{CONSTANT}},
            SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}},
            SortedDict{Dates.DateTime, Matrix{PWL}},
        }
        scaling_factor_multiplier::Union{Nothing, Function}
        internal::InfrastructureSystemsInternal
    end

A Discrete Scenario Based time series for a particular data field in a Component.

# Arguments
- `name::String`: user-defined name
- `resolution::Dates.Period`: forecast resolution
- `scenario_count::Int64`: Number of scenarios
- `data::Union{SortedDict{Dates.DateTime, Matrix{CONSTANT}}, SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}}, SortedDict{Dates.DateTime, Matrix{PWL}}}`: timestamp - scalingfactor
- `scaling_factor_multiplier::Union{Nothing, Function}`: Applicable when the time series
  data are scaling factors. Called on the associated component to convert the values.
- `internal::InfrastructureSystemsInternal`
"""
mutable struct Scenarios <: Forecast
    "user-defined name"
    name::String
    "timestamp - scalingfactor"
    data::Union{
        SortedDict{Dates.DateTime, Matrix{CONSTANT}},
        SortedDict{Dates.DateTime, Matrix{POLYNOMIAL}},
        SortedDict{Dates.DateTime, Matrix{PWL}},
    }
    "Number of scenarios"
    scenario_count::Int64
    "forecast resolution"
    resolution::Dates.Period
    "Applicable when the time series data are scaling factors. Called on the associated component to convert the values."
    scaling_factor_multiplier::Union{Nothing, Function}
    internal::InfrastructureSystemsInternal
end

function Scenarios(;
    name,
    data,
    scenario_count,
    resolution,
    scaling_factor_multiplier = nothing,
    normalization_factor = 1.0,
    internal = InfrastructureSystemsInternal(),
)
    data = handle_normalization_factor(convert_data(data), normalization_factor)
    Scenarios(name, data, scenario_count, resolution, scaling_factor_multiplier, internal)
end

"""
Construct Scenarios from a SortedDict of Arrays.

# Arguments
- `name::AbstractString`: user-defined name
- `input_data::AbstractDict{Dates.DateTime, Matrix{Float64}}`: time series data.
- `resolution::Dates.Period`: The resolution of the forecast in Dates.Period`
- `normalization_factor::NormalizationFactor = 1.0`: optional normalization factor to apply
  to each data entry
- `scaling_factor_multiplier::Union{Nothing, Function} = nothing`: If the data are scaling
  factors then this function will be called on the component and applied to the data when
  [`get_time_series_array`](@ref) is called.
"""
function Scenarios(
    name::AbstractString,
    input_data::AbstractDict,
    resolution::Dates.Period;
    normalization_factor::NormalizationFactor = 1.0,
    scaling_factor_multiplier::Union{Nothing, Function} = nothing,
)
    scenario_count = size(first(values(input_data)))[2]

    return Scenarios(
        name = name,
        data = input_data,
        scenario_count = scenario_count,
        resolution = resolution,
        scaling_factor_multiplier = scaling_factor_multiplier,
        normalization_factor = normalization_factor,
        internal = InfrastructureSystemsInternal(),
    )
end

"""
Construct Scenarios from a Dict of TimeArrays.

# Arguments
- `name::AbstractString`: user-defined name
- `input_data::AbstractDict{Dates.DateTime, TimeSeries.TimeArray}`: time series data.
- `normalization_factor::NormalizationFactor = 1.0`: optional normalization factor to apply
  to each data entry
- `scaling_factor_multiplier::Union{Nothing, Function} = nothing`: If the data are scaling
  factors then this function will be called on the component and applied to the data when
  [`get_time_series_array`](@ref) is called.
- `timestamp = :timestamp`: If the values are DataFrames is passed then this must be the column name that
  contains timestamps.
"""
function Scenarios(
    name::AbstractString,
    input_data::AbstractDict{Dates.DateTime, <:TimeSeries.TimeArray};
    normalization_factor::NormalizationFactor = 1.0,
    scaling_factor_multiplier::Union{Nothing, Function} = nothing,
)
    data = SortedDict{Dates.DateTime, Matrix{Float64}}()
    resolution =
        TimeSeries.timestamp(first(values(input_data)))[2] -
        TimeSeries.timestamp(first(values(input_data)))[1]
    for (k, v) in input_data
        data[k] = TimeSeries.values(v)
    end

    return Scenarios(
        name,
        data,
        resolution;
        normalization_factor = normalization_factor,
        scaling_factor_multiplier = scaling_factor_multiplier,
    )
end

function Scenarios(ts_metadata::ScenariosMetadata, data::SortedDict)
    return Scenarios(
        name = get_name(ts_metadata),
        scenario_count = get_scenario_count(ts_metadata),
        resolution = get_resolution(ts_metadata),
        data = data,
        scaling_factor_multiplier = get_scaling_factor_multiplier(ts_metadata),
        internal = InfrastructureSystemsInternal(get_time_series_uuid(ts_metadata)),
    )
end

function Scenarios(info::TimeSeriesParsedInfo)
    return Scenarios(
        info.name,
        info.data,
        info.resolution;
        normalization_factor = info.normalization_factor,
        scaling_factor_multiplier = info.scaling_factor_multiplier,
    )
end

function ScenariosMetadata(time_series::Scenarios)
    return ScenariosMetadata(
        get_name(time_series),
        get_resolution(time_series),
        get_initial_timestamp(time_series),
        get_interval(time_series),
        get_scenario_count(time_series),
        get_count(time_series),
        get_uuid(time_series),
        get_horizon(time_series),
        get_scaling_factor_multiplier(time_series),
    )
end

function get_array_for_hdf(forecast::Scenarios)
    interval_count = get_count(forecast)
    scenario_count = get_scenario_count(forecast)
    horizon = get_horizon(forecast)
    data = get_data(forecast)

    data_for_hdf = Array{Float64, 3}(undef, scenario_count, horizon, interval_count)
    for (ix, f) in enumerate(values(data))
        data_for_hdf[:, :, ix] = transpose(f)
    end
    return data_for_hdf
end

function get_horizon(forecast::Scenarios)
    return size(first(values(get_data(forecast))))[1]
end

"""Get [`Scenarios`](@ref) `name`."""
get_name(value::Scenarios) = value.name
"""Get [`Scenarios`](@ref) `resolution`."""
get_resolution(value::Scenarios) = value.resolution
"""Get [`Scenarios`](@ref) `scenario_count`."""
get_scenario_count(value::Scenarios) = value.scenario_count
"""Get [`Scenarios`](@ref) `data`."""
get_data(value::Scenarios) = value.data
"""Get [`Scenarios`](@ref) `scaling_factor_multiplier`."""
get_scaling_factor_multiplier(value::Scenarios) = value.scaling_factor_multiplier
"""Get [`Scenarios`](@ref) `internal`."""
get_internal(value::Scenarios) = value.internal

"""Set [`Scenarios`](@ref) `name`."""
set_name!(value::Scenarios, val) = value.name = val
"""Set [`Scenarios`](@ref) `resolution`."""
set_resolution!(value::Scenarios, val) = value.resolution = val
"""Set [`Scenarios`](@ref) `scenario_count`."""
set_scenario_count!(value::Scenarios, val) = value.scenario_count = val
"""Set [`Scenarios`](@ref) `data`."""
set_data!(value::Scenarios, val) = value.data = val
"""Set [`Scenarios`](@ref) `scaling_factor_multiplier`."""
set_scaling_factor_multiplier!(value::Scenarios, val) =
    value.scaling_factor_multiplier = val
"""Set [`Scenarios`](@ref) `internal`."""
set_internal!(value::Scenarios, val) = value.internal = val

eltype_data(forecast::Scenarios) = eltype_data_common(forecast)
get_count(forecast::Scenarios) = get_count_common(forecast)
get_initial_times(forecast::Scenarios) = get_initial_times_common(forecast)
get_initial_timestamp(forecast::Scenarios) = get_initial_timestamp_common(forecast)
get_interval(forecast::Scenarios) = get_interval_common(forecast)
get_window(f::Scenarios, initial_time::Dates.DateTime; len = nothing) =
    get_window_common(f, initial_time; len = len)
iterate_windows(forecast::Scenarios) = iterate_windows_common(forecast)


using SampledSignals

#####
##### `SynthesizerSource`
#####

struct SynthesizerSource{T<:AbstractSynthComponent} <: SampleSource
    output::T
end

function SampledSignals.unsafe_read!(source::SynthesizerSource, buf::Array, frameoffset, framecount)
    for i in (1:framecount) .+ frameoffset
        buf[i] = next!(source.output)
    end
    return framecount
end

SampledSignals.samplerate(::SynthesizerSource) = SAMPLE_RATE[]
SampledSignals.nchannels(::SynthesizerSource) = 1
SampledSignals.eltype(::SynthesizerSource{T}) where {T} = output_type(T)


# struct ParentsBeforeChildrenIterator
#     ops::Vector{Operation}
#     visited::Vector{Bool}
# end
# function iterate(pbci::ParentsBeforeChildrenIterator)
#     for (i,op) ∈ enumerate(pbci.ops)
#         if iszero(length(parents(op)))
#             pbci.visited[i] = true
#             return op, pbci
#         end
#     end
#     nothing
# end
# function iterate()

# end

function dependent_outer_reducts(ls::LoopSet, op)
    for i ∈ ls.outer_reductions
        search_tree(parents(operations(ls)[i]), name(op)) && return true
    end
    false
end

function isnopidentity(ls::LoopSet, op::Operation, u₁loop::Symbol, u₂loop::Symbol, vectorized::Symbol, suffix)
    parents_op = parents(op)
    if iscompute(op) && instruction(op).instr === :identity && name(first(parents_op)) === name(op) && isone(length(parents_op))
        mvar, u₁unrolledsym, u₂unrolledsym = variable_name_and_unrolled(op, u₁loop, u₂loop, suffix)
        parents_u₁syms, parents_u₂syms = parent_unroll_status(op, u₁loop, u₂loop, suffix)
        if (u₁unrolledsym == first(parents_u₁syms)) && ((!isnothing(suffix)) == parents_u₂syms[1])
            #TODO: identifer(first(parents_op)) ∉ ls.outer_reductions is going to miss a lot of cases
            #Should probably replace that with `DVec` (demoting Vec) types, that demote to scalar.
            if (isvectorized(first(parents_op)) && !isvectorized(op)) && !dependent_outer_reducts(ls, op)
                op.instruction = reduction_to_scalar(instruction(first(parents_op)))
                op.mangledvariable = gensym(op.mangledvariable)
                false
            else
                true
            end
        else
            false
        end
    else
        false
    end
end

function set_upstream_family!(adal::Vector{T}, op::Operation, val::T, ld::Vector{Symbol}, id::Int) where {T}
    adal[identifier(op)] == val && return # must already have been set
    if ld != loopdependencies(op) || id == identifier(op)
        (adal[identifier(op)] = val)
    end
    for opp ∈ parents(op)
        identifier(opp) == identifier(op) && continue
        set_upstream_family!(adal, opp, val, ld, id)
    end
end

function addoptoorder!(
    ls::LoopSet, included_vars::Vector{Bool}, place_after_loop::Vector{Bool}, op::Operation,
    loopsym::Symbol, _n::Int, u₁loop::Symbol, u₂loop::Symbol, vectorized::Symbol, loopistiled::Bool
)
    lo = ls.loop_order
    id = identifier(op)
    included_vars[id] && return nothing
    loopsym ∈ loopdependencies(op) || return nothing
    for opp ∈ parents(op) # ensure parents are added first
        addoptoorder!(ls, included_vars, place_after_loop, opp, loopsym, _n, u₁loop, u₂loop, vectorized, loopistiled)
    end
    included_vars[id] && return nothing
    included_vars[id] = true
    isunrolled = (isu₁unrolled(op)) + 1
    istiled = isu₂unrolled(op)
    # optype = Int(op.node_type) + 1
    after_loop = place_after_loop[id] + 1
    if !isloopvalue(op)
        if istiled
            isnopidentity(ls, op, u₁loop, u₂loop, vectorized, 0) || push!(lo[isunrolled,2,after_loop,_n], op)
        else
            isnopidentity(ls, op, u₁loop, u₂loop, vectorized, nothing) || push!(lo[isunrolled,1,after_loop,_n], op)
        end
    end
    # @show op, after_loop
    # isloopvalue(op) || push!(lo[isunrolled,istiled,after_loop,_n], op)
    # all(opp -> iszero(length(reduceddependencies(opp))), parents(op)) &&
    set_upstream_family!(place_after_loop, op, false, loopdependencies(op), identifier(op)) # parents that have already been included are not moved, so no need to check included_vars to filter
    nothing
end

function fillorder!(ls::LoopSet, order::Vector{Symbol}, u₁loop::Symbol, u₂loop::Symbol, loopistiled::Bool, vectorized::Symbol)
    lo = ls.loop_order
    resize!(lo, length(ls.loopsymbols))
    ro = lo.loopnames # reverse order; will have same order as lo
    nloops = length(order)
    ops = operations(ls)
    nops = length(ops)
    included_vars = fill!(resize!(ls.included_vars, nops), false)
    place_after_loop = fill!(resize!(ls.place_after_loop, nops), true)
    # to go inside out, we just have to include all those not-yet included depending on the current sym
    empty!(lo)
    for _n ∈ 1:nloops
        n = 1 + nloops - _n
        ro[_n] = loopsym = order[n]
        #loopsym = order[n]
        for op ∈ ops
            addoptoorder!( ls, included_vars, place_after_loop, op, loopsym, _n, u₁loop, u₂loop, vectorized, loopistiled )
        end
    end
end


using Dates

@testset "util.jl" begin
    p1 = DDR2import.util.Point(1.0, 2.0)
    @test p1.lat == 1.0
    @test p1.lon == 2.0

    @test DDR2import.util.extract_lat("S123456") ≈ -12.58222222 atol = 0.0001
    @test DDR2import.util.extract_lat("N123456.0") ≈ 12.58222222 atol = 0.0001
    @test DDR2import.util.extract_lat("123456N") ≈ 12.58222222 atol = 0.0001
    @test DDR2import.util.extract_lat("S1234") ≈ -12.56666666 atol = 0.0001
    @test DDR2import.util.extract_lat("N12") ≈ 12.0 atol = 0.0001
    @test DDR2import.util.extract_lat("1234S") ≈ -12.56666666 atol = 0.0001
    @test DDR2import.util.extract_lat("12N") ≈ 12.0 atol = 0.0001
    @test DDR2import.util.extract_lat("") === NaN

    @test DDR2import.util.extract_lon("W1123456") ≈ -112.58222222 atol = 0.0001
    @test DDR2import.util.extract_lon("E1123456.0") ≈ 112.58222222 atol = 0.0001
    @test DDR2import.util.extract_lon("0123456E") ≈ 12.58222222 atol = 0.0001
    @test DDR2import.util.extract_lon("W01234") ≈ -12.56666666 atol = 0.0001
    @test DDR2import.util.extract_lon("E112") ≈ 112.0 atol = 0.0001
    @test DDR2import.util.extract_lon("11234W") ≈ -112.56666666 atol = 0.0001
    @test DDR2import.util.extract_lon("012E") ≈ 12.0 atol = 0.0001
    @test DDR2import.util.extract_lon("E1") === NaN

    # @test DDR2import.util.latlon("S123456", "012E").lat ≈ -12.58222222 atol = 0.0001
    # @test DDR2import.util.latlon("S123456", "012E").lon ≈ 12.0 atol = 0.0001
    # @test DDR2import.util.latlon("S123456", "E1") === NaN
    #
    # @test DDR2import.util.latlon("S123456E1123456.0").lat ≈ -12.58222222 atol = 0.0001
    # @test DDR2import.util.latlon("S123456E1123456.0").lon ≈ 112.58222222 atol = 0.0001
    # @test DDR2import.util.latlon("S123456W1123456.0").lon ≈ -112.58222222 atol = 0.0001

    @test DDR2import.util.latlon("S123456", "012E").ϕ ≈ -12.58222222 atol = 0.0001
    @test DDR2import.util.latlon("S123456", "012E").λ ≈ 12.0 atol = 0.0001
    @test DDR2import.util.latlon("S123456", "E1") === NaN

    @test DDR2import.util.latlon("123456S1123456.0E").ϕ ≈ -12.58222222 atol = 0.0001
    @test DDR2import.util.latlon("123456S1123456.0E").λ ≈ 112.58222222 atol = 0.0001
    @test DDR2import.util.latlon("123456S1123456.0W").λ ≈ -112.58222222 atol = 0.0001

    yymmdd = DateFormat("YYmmdd")
    testdate = DDR2import.util.format_date("200110", yymmdd, addyear=Year(2000))
    @test Dates.year(testdate) == 2020
    @test Dates.month(testdate) == 1
    @test Dates.day(testdate) == 10

    hhmmss = DateFormat("HHMMSS")
    testtime = DDR2import.util.format_time("102030", hhmmss)
    @test Dates.hour(testtime) == 10
    @test Dates.minute(testtime) == 20
    @test Dates.second(testtime) == 30
    hhmm = DateFormat("HHMM")
    testtime2 = DDR2import.util.format_time("2400", hhmm)
    @test Dates.hour(testtime2) == 23
    @test Dates.minute(testtime2) == 59
    @test Dates.second(testtime2) == 59

    yyyymmddhhmmss = DateFormat("YYYYmmddHHMMSS")
    testdatetime = DDR2import.util.format_datetime("20200110102030", yyyymmddhhmmss)
    @test Dates.year(testdatetime) == 2020
    @test Dates.month(testdatetime) == 1
    @test Dates.day(testdatetime) == 10
    @test Dates.hour(testdatetime) == 10
    @test Dates.minute(testdatetime) == 20
    @test Dates.second(testdatetime) == 30
end


# Geometry which displays points at given (x, y) positions.
immutable PointGeometry <: Gadfly.GeometryElement
    tag::Symbol

    function PointGeometry(; tag::Symbol=empty_tag)
        new(tag)
    end
end

const point = PointGeometry

function element_aesthetics(::PointGeometry)
    [:x, :y, :size, :color, :shape]
end

# Generate a form for a point geometry.
#
# Args:
#   geom: point geometry.
#   theme: the plot's theme.
#   aes: aesthetics.
#
# Returns:
#   A compose Form.
#
function render(geom::PointGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
    Gadfly.assert_aesthetics_defined("Geom.point", aes, :x, :y)
    Gadfly.assert_aesthetics_equal_length("Geom.point", aes,
                                          element_aesthetics(geom)...)

    default_aes = Gadfly.Aesthetics()
    default_aes.color = PooledDataArray(RGBA{Float32}[theme.default_color])
    default_aes.size = Measure[theme.default_point_size]
    aes = inherit(aes, default_aes)

    lw_hover_scale = 10
    lw_ratio = theme.line_width / aes.size[1]

    aes_x, aes_y = concretize(aes.x, aes.y)

    ctx = context()
    if aes.shape != nothing
        xs = Array(eltype(aes_x), 0)
        ys = Array(eltype(aes_y), 0)
        cs = Array(eltype(aes.color), 0)
        size = Array(eltype(aes.size), 0)
        shape_max = maximum(aes.shape)
        if shape_max > length(theme.shapes)
            error("Too many values for the shape aesthetic. Define more shapes in Theme.shapes")
        end

        for shape in 1:maximum(aes.shape)
            for (x, y, c, sz, sh) in Compose.cyclezip(aes.x, aes.y, aes.color,
                                                      aes.size, aes.shape)
                if sh == shape
                    push!(xs, x)
                    push!(ys, y)
                    push!(cs, c)
                    push!(size, sz)
                end
            end
            compose!(ctx, (context(), theme.shapes[shape](xs, ys, size), fill(cs)))
            empty!(xs)
            empty!(ys)
            empty!(cs)
            empty!(size)
        end
    else
        compose!(ctx,
            circle(aes.x, aes.y, aes.size, geom.tag),
            fill(aes.color))
    end
    compose!(ctx, linewidth(theme.highlight_width))

    if aes.color_key_continuous != nothing && aes.color_key_continuous
        compose!(ctx,
            stroke(map(theme.continuous_highlight_color, aes.color)))
    else
        stroke_colors =
            Gadfly.pooled_map(RGBA{Float32}, theme.discrete_highlight_color, aes.color)
        classes =
            Gadfly.pooled_map(Compat.ASCIIString,
                c -> svg_color_class_from_label(escape_id(aes.color_label([c])[1])),
                aes.color)

        compose!(ctx, stroke(stroke_colors), svgclass(classes))
    end

    return compose!(context(order=4), svgclass("geometry"), ctx)
end


using SignedDistanceFunction
using Test
using DelimitedFiles
include("../src/utils/utils.jl")
using .Utils

@testset "SignedDistanceFunction.jl" begin
    # @test signedDistance2D("mock_csv_data/interface.csv", 100) == readdlm("result/interface_result_n100.csv", ',', Float64)
    @test signedDistance2D("mock_csv_data/interface.csv", 300) == readdlm("result/interface_result_n300.csv", ',', Float64)
    # @test signedDistance2D("mock_csv_data/interface.csv", 300, "multi") == readdlm("result/interface_floodfill_result_n300.csv", ',', Float64)
    # @test signedDistance2D("mock_csv_data/multiple_curves.csv", 100, "multi") == readdlm("result/multiple_curves_result_n100.csv", ',', Float64)
    # @test signedDistance2D("mock_csv_data/multiple_curves.csv", 500, "multi") == readdlm("result/multiple_curves_result_n500.csv", ',', Float64)
end

@testset "utils.jl" begin
    @test remove_same_point([1 2; 2 3; 3 2; 3 2; 3 2; 1 2; 1 2]) == [1 2; 2 3; 3 2; 1 2]
end


# This file is a part of Julia. License is MIT: https://julialang.org/license

module Sort

import ..@__MODULE__, ..parentmodule
const Base = parentmodule(@__MODULE__)
using .Base.Order
using .Base: copymutable, LinearIndices, length, (:),
    eachindex, axes, first, last, similar, zip, OrdinalRange,
    AbstractVector, @inbounds, AbstractRange, @eval, @inline, Vector, @noinline,
    AbstractMatrix, AbstractUnitRange, isless, identity, eltype, >, <, <=, >=, |, +, -, *, !,
    extrema, sub_with_overflow, add_with_overflow, oneunit, div, getindex, setindex!,
    length, resize!, fill, Missing, require_one_based_indexing, keytype

using .Base: >>>, !==

import .Base:
    sort,
    sort!,
    issorted,
    sortperm,
    to_indices

export # also exported by Base
    # order-only:
    issorted,
    searchsorted,
    searchsortedfirst,
    searchsortedlast,
    # order & algorithm:
    sort,
    sort!,
    sortperm,
    sortperm!,
    partialsort,
    partialsort!,
    partialsortperm,
    partialsortperm!,
    # algorithms:
    InsertionSort,
    QuickSort,
    MergeSort,
    PartialQuickSort

export # not exported by Base
    Algorithm,
    DEFAULT_UNSTABLE,
    DEFAULT_STABLE,
    SMALL_ALGORITHM,
    SMALL_THRESHOLD

## functions requiring only ordering ##

function issorted(itr, order::Ordering)
    y = iterate(itr)
    y === nothing && return true
    prev, state = y
    y = iterate(itr, state)
    while y !== nothing
        this, state = y
        lt(order, this, prev) && return false
        prev = this
        y = iterate(itr, state)
    end
    return true
end

"""
    issorted(v, lt=isless, by=identity, rev:Bool=false, order::Ordering=Forward)

Test whether a vector is in sorted order. The `lt`, `by` and `rev` keywords modify what
order is considered to be sorted just as they do for [`sort`](@ref).

# Examples
```jldoctest
julia> issorted([1, 2, 3])
true

julia> issorted([(1, "b"), (2, "a")], by = x -> x[1])
true

julia> issorted([(1, "b"), (2, "a")], by = x -> x[2])
false

julia> issorted([(1, "b"), (2, "a")], by = x -> x[2], rev=true)
true
```
"""
issorted(itr;
    lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) =
    issorted(itr, ord(lt,by,rev,order))

function partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}, o::Ordering)
    inds = axes(v, 1)
    sort!(v, first(inds), last(inds), PartialQuickSort(k), o)
    maybeview(v, k)
end

maybeview(v, k) = view(v, k)
maybeview(v, k::Integer) = v[k]

"""
    partialsort!(v, k; by=<transform>, lt=<comparison>, rev=false)

Partially sort the vector `v` in place, according to the order specified by `by`, `lt` and
`rev` so that the value at index `k` (or range of adjacent values if `k` is a range) occurs
at the position where it would appear if the array were fully sorted via a non-stable
algorithm. If `k` is a single index, that value is returned; if `k` is a range, an array of
values at those indices is returned. Note that `partialsort!` does not fully sort the input
array.

# Examples
```jldoctest
julia> a = [1, 2, 4, 3, 4]
5-element Array{Int64,1}:
 1
 2
 4
 3
 4

julia> partialsort!(a, 4)
4

julia> a
5-element Array{Int64,1}:
 1
 2
 3
 4
 4

julia> a = [1, 2, 4, 3, 4]
5-element Array{Int64,1}:
 1
 2
 4
 3
 4

julia> partialsort!(a, 4, rev=true)
2

julia> a
5-element Array{Int64,1}:
 4
 4
 3
 2
 1
```
"""
partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange};
             lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) =
    partialsort!(v, k, ord(lt,by,rev,order))

"""
    partialsort(v, k, by=<transform>, lt=<comparison>, rev=false)

Variant of [`partialsort!`](@ref) which copies `v` before partially sorting it, thereby returning the
same thing as `partialsort!` but leaving `v` unmodified.
"""
partialsort(v::AbstractVector, k::Union{Integer,OrdinalRange}; kws...) =
    partialsort!(copymutable(v), k; kws...)

# This implementation of `midpoint` is performance-optimized but safe
# only if `lo <= hi`.
midpoint(lo::T, hi::T) where T<:Integer = lo + ((hi - lo) >>> 0x01)
midpoint(lo::Integer, hi::Integer) = midpoint(promote(lo, hi)...)

# reference on sorted binary search:
#   http://www.tbray.org/ongoing/When/200x/2003/03/22/Binary

# index of the first value of vector a that is greater than or equal to x;
# returns length(v)+1 if x is greater than all values in v.
function searchsortedfirst(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer
    u = T(1)
    lo = lo - u
    hi = hi + u
    @inbounds while lo < hi - u
        m = midpoint(lo, hi)
        if lt(o, v[m], x)
            lo = m
        else
            hi = m
        end
    end
    return hi
end

# index of the last value of vector a that is less than or equal to x;
# returns 0 if x is less than all values of v.
function searchsortedlast(v::AbstractVector, x, lo::T, hi::T, o::Ordering)::keytype(v) where T<:Integer
    u = T(1)
    lo = lo - u
    hi = hi + u
    @inbounds while lo < hi - u
        m = midpoint(lo, hi)
        if lt(o, x, v[m])
            hi = m
        else
            lo = m
        end
    end
    return lo
end

# returns the range of indices of v equal to x
# if v does not contain x, returns a 0-length range
# indicating the insertion point of x
function searchsorted(v::AbstractVector, x, ilo::T, ihi::T, o::Ordering)::UnitRange{keytype(v)} where T<:Integer
    u = T(1)
    lo = ilo - u
    hi = ihi + u
    @inbounds while lo < hi - u
        m = midpoint(lo, hi)
        if lt(o, v[m], x)
            lo = m
        elseif lt(o, x, v[m])
            hi = m
        else
            a = searchsortedfirst(v, x, max(lo,ilo), m, o)
            b = searchsortedlast(v, x, m, min(hi,ihi), o)
            return a : b
        end
    end
    return (lo + 1) : (hi - 1)
end

function searchsortedlast(a::AbstractRange{<:Real}, x::Real, o::DirectOrdering)::keytype(a)
    require_one_based_indexing(a)
    if step(a) == 0
        lt(o, x, first(a)) ? 0 : length(a)
    else
        n = round(Integer, clamp((x - first(a)) / step(a) + 1, 1, length(a)))
        lt(o, x, a[n]) ? n - 1 : n
    end
end

function searchsortedfirst(a::AbstractRange{<:Real}, x::Real, o::DirectOrdering)::keytype(a)
    require_one_based_indexing(a)
    if step(a) == 0
        lt(o, first(a), x) ? length(a) + 1 : 1
    else
        n = round(Integer, clamp((x - first(a)) / step(a) + 1, 1, length(a)))
        lt(o, a[n] ,x) ? n + 1 : n
    end
end

function searchsortedlast(a::AbstractRange{<:Integer}, x::Real, o::DirectOrdering)::keytype(a)
    require_one_based_indexing(a)
    h = step(a)
    if h == 0
        lt(o, x, first(a)) ? 0 : length(a)
    elseif h > 0 && x < first(a)
        firstindex(a) - 1
    elseif h > 0 && x >= last(a)
        lastindex(a)
    elseif h < 0 && x > first(a)
        firstindex(a) - 1
    elseif h < 0 && x <= last(a)
        lastindex(a)
    else
        if o isa ForwardOrdering
            fld(floor(Integer, x) - first(a), h) + 1
        else
            fld(ceil(Integer, x) - first(a), h) + 1
        end
    end
end

function searchsortedfirst(a::AbstractRange{<:Integer}, x::Real, o::DirectOrdering)::keytype(a)
    require_one_based_indexing(a)
    h = step(a)
    if h == 0
        lt(o, first(a), x) ? length(a)+1 : 1
    elseif h > 0 && x <= first(a)
        firstindex(a)
    elseif h > 0 && x > last(a)
        lastindex(a) + 1
    elseif h < 0 && x >= first(a)
        firstindex(a)
    elseif h < 0 && x < last(a)
        lastindex(a) + 1
    else
        if o isa ForwardOrdering
            -fld(floor(Integer, -x) + Signed(first(a)), h) + 1
        else
            -fld(ceil(Integer, -x) + Signed(first(a)), h) + 1
        end
    end
end

function searchsortedfirst(a::AbstractRange{<:Integer}, x::Unsigned, o::DirectOrdering)::keytype(a)
    require_one_based_indexing(a)
    if lt(o, first(a), x)
        if step(a) == 0
            length(a) + 1
        else
            min(cld(x - first(a), step(a)), length(a)) + 1
        end
    else
        1
    end
end

function searchsortedlast(a::AbstractRange{<:Integer}, x::Unsigned, o::DirectOrdering)::keytype(a)
    require_one_based_indexing(a)
    if lt(o, x, first(a))
        0
    elseif step(a) == 0
        length(a)
    else
        min(fld(x - first(a), step(a)) + 1, length(a))
    end
end

searchsorted(a::AbstractRange{<:Real}, x::Real, o::DirectOrdering) =
    searchsortedfirst(a, x, o) : searchsortedlast(a, x, o)

for s in [:searchsortedfirst, :searchsortedlast, :searchsorted]
    @eval begin
        $s(v::AbstractVector, x, o::Ordering) = (inds = axes(v, 1); $s(v,x,first(inds),last(inds),o))
        $s(v::AbstractVector, x;
           lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) =
            $s(v,x,ord(lt,by,rev,order))
    end
end

"""
    searchsorted(a, x; by=<transform>, lt=<comparison>, rev=false)

Return the range of indices of `a` which compare as equal to `x` (using binary search)
according to the order specified by the `by`, `lt` and `rev` keywords, assuming that `a`
is already sorted in that order. Return an empty range located at the insertion point
if `a` does not contain values equal to `x`.

# Examples
```jldoctest
julia> searchsorted([1, 2, 4, 5, 5, 7], 4) # single match
3:3

julia> searchsorted([1, 2, 4, 5, 5, 7], 5) # multiple matches
4:5

julia> searchsorted([1, 2, 4, 5, 5, 7], 3) # no match, insert in the middle
3:2

julia> searchsorted([1, 2, 4, 5, 5, 7], 9) # no match, insert at end
7:6

julia> searchsorted([1, 2, 4, 5, 5, 7], 0) # no match, insert at start
1:0
```
""" searchsorted

"""
    searchsortedfirst(a, x; by=<transform>, lt=<comparison>, rev=false)

Return the index of the first value in `a` greater than or equal to `x`, according to the
specified order. Return `length(a) + 1` if `x` is greater than all values in `a`.
`a` is assumed to be sorted.

# Examples
```jldoctest
julia> searchsortedfirst([1, 2, 4, 5, 5, 7], 4) # single match
3

julia> searchsortedfirst([1, 2, 4, 5, 5, 7], 5) # multiple matches
4

julia> searchsortedfirst([1, 2, 4, 5, 5, 7], 3) # no match, insert in the middle
3

julia> searchsortedfirst([1, 2, 4, 5, 5, 7], 9) # no match, insert at end
7

julia> searchsortedfirst([1, 2, 4, 5, 5, 7], 0) # no match, insert at start
1
```
""" searchsortedfirst

"""
    searchsortedlast(a, x; by=<transform>, lt=<comparison>, rev=false)

Return the index of the last value in `a` less than or equal to `x`, according to the
specified order. Return `0` if `x` is less than all values in `a`. `a` is assumed to
be sorted.

# Examples
```jldoctest
julia> searchsortedlast([1, 2, 4, 5, 5, 7], 4) # single match
3

julia> searchsortedlast([1, 2, 4, 5, 5, 7], 5) # multiple matches
5

julia> searchsortedlast([1, 2, 4, 5, 5, 7], 3) # no match, insert in the middle
2

julia> searchsortedlast([1, 2, 4, 5, 5, 7], 9) # no match, insert at end
6

julia> searchsortedlast([1, 2, 4, 5, 5, 7], 0) # no match, insert at start
0
```
""" searchsortedlast

## sorting algorithms ##

abstract type Algorithm end

struct InsertionSortAlg <: Algorithm end
struct QuickSortAlg     <: Algorithm end
struct MergeSortAlg     <: Algorithm end

"""
    PartialQuickSort{T <: Union{Integer,OrdinalRange}}

Indicate that a sorting function should use the partial quick sort
algorithm. Partial quick sort returns the smallest `k` elements sorted from smallest
to largest, finding them and sorting them using [`QuickSort`](@ref).

Characteristics:
  * *not stable*: does not preserve the ordering of elements which
    compare equal (e.g. "a" and "A" in a sort of letters which
    ignores case).
  * *in-place* in memory.
  * *divide-and-conquer*: sort strategy similar to [`MergeSort`](@ref).
"""
struct PartialQuickSort{T <: Union{Integer,OrdinalRange}} <: Algorithm
    k::T
end

"""
    InsertionSort

Indicate that a sorting function should use the insertion sort
algorithm. Insertion sort traverses the collection one element
at a time, inserting each element into its correct, sorted position in
the output list.

Characteristics:
  * *stable*: preserves the ordering of elements which
    compare equal (e.g. "a" and "A" in a sort of letters
    which ignores case).
  * *in-place* in memory.
  * *quadratic performance* in the number of elements to be sorted:
    it is well-suited to small collections but should not be used for large ones.
"""
const InsertionSort = InsertionSortAlg()
"""
    QuickSort

Indicate that a sorting function should use the quick sort
algorithm, which is *not* stable.

Characteristics:
  * *not stable*: does not preserve the ordering of elements which
    compare equal (e.g. "a" and "A" in a sort of letters which
    ignores case).
  * *in-place* in memory.
  * *divide-and-conquer*: sort strategy similar to [`MergeSort`](@ref).
  * *good performance* for large collections.
"""
const QuickSort     = QuickSortAlg()
"""
    MergeSort

Indicate that a sorting function should use the merge sort
algorithm. Merge sort divides the collection into
subcollections and repeatedly merges them, sorting each
subcollection at each step, until the entire
collection has been recombined in sorted form.

Characteristics:
  * *stable*: preserves the ordering of elements which compare
    equal (e.g. "a" and "A" in a sort of letters which ignores
    case).
  * *not in-place* in memory.
  * *divide-and-conquer* sort strategy.
"""
const MergeSort     = MergeSortAlg()

const DEFAULT_UNSTABLE = QuickSort
const DEFAULT_STABLE   = MergeSort
const SMALL_ALGORITHM  = InsertionSort
const SMALL_THRESHOLD  = 20

function sort!(v::AbstractVector, lo::Integer, hi::Integer, ::InsertionSortAlg, o::Ordering)
    @inbounds for i = lo+1:hi
        j = i
        x = v[i]
        while j > lo
            if lt(o, x, v[j-1])
                v[j] = v[j-1]
                j -= 1
                continue
            end
            break
        end
        v[j] = x
    end
    return v
end

# selectpivot!
#
# Given 3 locations in an array (lo, mi, and hi), sort v[lo], v[mi], v[hi]) and
# choose the middle value as a pivot
#
# Upon return, the pivot is in v[lo], and v[hi] is guaranteed to be
# greater than the pivot

@inline function selectpivot!(v::AbstractVector, lo::Integer, hi::Integer, o::Ordering)
    @inbounds begin
        mi = midpoint(lo, hi)

        # sort v[mi] <= v[lo] <= v[hi] such that the pivot is immediately in place
        if lt(o, v[lo], v[mi])
            v[mi], v[lo] = v[lo], v[mi]
        end

        if lt(o, v[hi], v[lo])
            if lt(o, v[hi], v[mi])
                v[hi], v[lo], v[mi] = v[lo], v[mi], v[hi]
            else
                v[hi], v[lo] = v[lo], v[hi]
            end
        end

        # return the pivot
        return v[lo]
    end
end

# partition!
#
# select a pivot, and partition v according to the pivot

function partition!(v::AbstractVector, lo::Integer, hi::Integer, o::Ordering)
    pivot = selectpivot!(v, lo, hi, o)
    # pivot == v[lo], v[hi] > pivot
    i, j = lo, hi
    @inbounds while true
        i += 1; j -= 1
        while lt(o, v[i], pivot); i += 1; end;
        while lt(o, pivot, v[j]); j -= 1; end;
        i >= j && break
        v[i], v[j] = v[j], v[i]
    end
    v[j], v[lo] = pivot, v[j]

    # v[j] == pivot
    # v[k] >= pivot for k > j
    # v[i] <= pivot for i < j
    return j
end

function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::QuickSortAlg, o::Ordering)
    @inbounds while lo < hi
        hi-lo <= SMALL_THRESHOLD && return sort!(v, lo, hi, SMALL_ALGORITHM, o)
        j = partition!(v, lo, hi, o)
        if j-lo < hi-j
            # recurse on the smaller chunk
            # this is necessary to preserve O(log(n))
            # stack space in the worst case (rather than O(n))
            lo < (j-1) && sort!(v, lo, j-1, a, o)
            lo = j+1
        else
            j+1 < hi && sort!(v, j+1, hi, a, o)
            hi = j-1
        end
    end
    return v
end

function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::MergeSortAlg, o::Ordering, t=similar(v,0))
    @inbounds if lo < hi
        hi-lo <= SMALL_THRESHOLD && return sort!(v, lo, hi, SMALL_ALGORITHM, o)

        m = midpoint(lo, hi)
        (length(t) < m-lo+1) && resize!(t, m-lo+1)

        sort!(v, lo,  m,  a, o, t)
        sort!(v, m+1, hi, a, o, t)

        i, j = 1, lo
        while j <= m
            t[i] = v[j]
            i += 1
            j += 1
        end

        i, k = 1, lo
        while k < j <= hi
            if lt(o, v[j], t[i])
                v[k] = v[j]
                j += 1
            else
                v[k] = t[i]
                i += 1
            end
            k += 1
        end
        while k < j
            v[k] = t[i]
            k += 1
            i += 1
        end
    end

    return v
end

function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::PartialQuickSort{<:Integer},
               o::Ordering)
    @inbounds while lo < hi
        hi-lo <= SMALL_THRESHOLD && return sort!(v, lo, hi, SMALL_ALGORITHM, o)
        j = partition!(v, lo, hi, o)
        if j >= a.k
            # we don't need to sort anything bigger than j
            hi = j-1
        elseif j-lo < hi-j
            # recurse on the smaller chunk
            # this is necessary to preserve O(log(n))
            # stack space in the worst case (rather than O(n))
            lo < (j-1) && sort!(v, lo, j-1, a, o)
            lo = j+1
        else
            (j+1) < hi && sort!(v, j+1, hi, a, o)
            hi = j-1
        end
    end
    return v
end

function sort!(v::AbstractVector, lo::Integer, hi::Integer, a::PartialQuickSort{T},
               o::Ordering) where T<:OrdinalRange
    @inbounds while lo < hi
        hi-lo <= SMALL_THRESHOLD && return sort!(v, lo, hi, SMALL_ALGORITHM, o)
        j = partition!(v, lo, hi, o)

        if j <= first(a.k)
            lo = j+1
        elseif j >= last(a.k)
            hi = j-1
        else
            if j-lo < hi-j
                lo < (j-1) && sort!(v, lo, j-1, a, o)
                lo = j+1
            else
                hi > (j+1) && sort!(v, j+1, hi, a, o)
                hi = j-1
            end
        end
    end
    return v
end

## generic sorting methods ##

defalg(v::AbstractArray) = DEFAULT_STABLE
defalg(v::AbstractArray{<:Union{Number, Missing}}) = DEFAULT_UNSTABLE

function sort!(v::AbstractVector, alg::Algorithm, order::Ordering)
    inds = axes(v,1)
    sort!(v,first(inds),last(inds),alg,order)
end

"""
    sort!(v; alg::Algorithm=defalg(v), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward)

Sort the vector `v` in place. [`QuickSort`](@ref) is used by default for numeric arrays while
[`MergeSort`](@ref) is used for other arrays. You can specify an algorithm to use via the `alg`
keyword (see [Sorting Algorithms](@ref) for available algorithms). The `by` keyword lets you provide
a function that will be applied to each element before comparison; the `lt` keyword allows
providing a custom "less than" function; use `rev=true` to reverse the sorting order. These
options are independent and can be used together in all possible combinations: if both `by`
and `lt` are specified, the `lt` function is applied to the result of the `by` function;
`rev=true` reverses whatever ordering specified via the `by` and `lt` keywords.

# Examples
```jldoctest
julia> v = [3, 1, 2]; sort!(v); v
3-element Array{Int64,1}:
 1
 2
 3

julia> v = [3, 1, 2]; sort!(v, rev = true); v
3-element Array{Int64,1}:
 3
 2
 1

julia> v = [(1, "c"), (3, "a"), (2, "b")]; sort!(v, by = x -> x[1]); v
3-element Array{Tuple{Int64,String},1}:
 (1, "c")
 (2, "b")
 (3, "a")

julia> v = [(1, "c"), (3, "a"), (2, "b")]; sort!(v, by = x -> x[2]); v
3-element Array{Tuple{Int64,String},1}:
 (3, "a")
 (2, "b")
 (1, "c")
```
"""
function sort!(v::AbstractVector;
               alg::Algorithm=defalg(v),
               lt=isless,
               by=identity,
               rev::Union{Bool,Nothing}=nothing,
               order::Ordering=Forward)
    ordr = ord(lt,by,rev,order)
    if (ordr === Forward || ordr === Reverse) && eltype(v)<:Integer
        n = length(v)
        if n > 1
            min, max = extrema(v)
            (diff, o1) = sub_with_overflow(max, min)
            (rangelen, o2) = add_with_overflow(diff, oneunit(diff))
            if !o1 && !o2 && rangelen < div(n,2)
                return sort_int_range!(v, rangelen, min, ordr === Reverse ? reverse : identity)
            end
        end
    end
    sort!(v, alg, ordr)
end

# sort! for vectors of few unique integers
function sort_int_range!(x::AbstractVector{<:Integer}, rangelen, minval, maybereverse)
    offs = 1 - minval

    where = fill(0, rangelen)
    @inbounds for i = eachindex(x)
        where[x[i] + offs] += 1
    end

    idx = firstindex(x)
    @inbounds for i = maybereverse(1:rangelen)
        lastidx = idx + where[i] - 1
        val = i-offs
        for j = idx:lastidx
            x[j] = val
        end
        idx = lastidx + 1
    end

    return x
end

"""
    sort(v; alg::Algorithm=defalg(v), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward)

Variant of [`sort!`](@ref) that returns a sorted copy of `v` leaving `v` itself unmodified.

# Examples
```jldoctest
julia> v = [3, 1, 2];

julia> sort(v)
3-element Array{Int64,1}:
 1
 2
 3

julia> v
3-element Array{Int64,1}:
 3
 1
 2
```
"""
sort(v::AbstractVector; kws...) = sort!(copymutable(v); kws...)

## partialsortperm: the permutation to sort the first k elements of an array ##

"""
    partialsortperm(v, k; by=<transform>, lt=<comparison>, rev=false)

Return a partial permutation `I` of the vector `v`, so that `v[I]` returns values of a fully
sorted version of `v` at index `k`. If `k` is a range, a vector of indices is returned; if
`k` is an integer, a single index is returned. The order is specified using the same
keywords as `sort!`. The permutation is stable, meaning that indices of equal elements
appear in ascending order.

Note that this function is equivalent to, but more efficient than, calling `sortperm(...)[k]`.

# Examples
```jldoctest
julia> v = [3, 1, 2, 1];

julia> v[partialsortperm(v, 1)]
1

julia> p = partialsortperm(v, 1:3)
3-element view(::Array{Int64,1}, 1:3) with eltype Int64:
 2
 4
 3

julia> v[p]
3-element Array{Int64,1}:
 1
 1
 2
```
"""
partialsortperm(v::AbstractVector, k::Union{Integer,OrdinalRange}; kwargs...) =
    partialsortperm!(similar(Vector{eltype(k)}, axes(v,1)), v, k; kwargs..., initialized=false)

"""
    partialsortperm!(ix, v, k; by=<transform>, lt=<comparison>, rev=false, initialized=false)

Like [`partialsortperm`](@ref), but accepts a preallocated index vector `ix` the same size as
`v`, which is used to store (a permutation of) the indices of `v`.

If the index vector `ix` is initialized with the indices of `v` (or a permutation thereof), `initialized` should be set to
`true`.

If `initialized` is `false` (the default), then `ix` is initialized to contain the indices of `v`.

If `initialized` is `true`, but `ix` does not contain (a permutation of) the indices of `v`, the behavior of
`partialsortperm!` is undefined.

(Typically, the indices of `v` will be `1:length(v)`, although if `v` has an alternative array type
with non-one-based indices, such as an `OffsetArray`, `ix` must also be an `OffsetArray` with the same
indices, and must contain as values (a permutation of) these same indices.)

Upon return, `ix` is guaranteed to have the indices `k` in their sorted positions, such that

```julia
partialsortperm!(ix, v, k);
v[ix[k]] == partialsort(v, k)
```

The return value is the `k`th element of `ix` if `k` is an integer, or view into `ix` if `k` is
a range.

# Examples
```jldoctest
julia> v = [3, 1, 2, 1];

julia> ix = Vector{Int}(undef, 4);

julia> partialsortperm!(ix, v, 1)
2

julia> ix = [1:4;];

julia> partialsortperm!(ix, v, 2:3, initialized=true)
2-element view(::Array{Int64,1}, 2:3) with eltype Int64:
 4
 3
```
 """
function partialsortperm!(ix::AbstractVector{<:Integer}, v::AbstractVector,
                          k::Union{Integer, OrdinalRange};
                          lt::Function=isless,
                          by::Function=identity,
                          rev::Union{Bool,Nothing}=nothing,
                          order::Ordering=Forward,
                          initialized::Bool=false)
    if axes(ix,1) != axes(v,1)
        throw(ArgumentError("The index vector is used as a workspace and must have the " *
                            "same length/indices as the source vector, $(axes(ix,1)) != $(axes(v,1))"))
    end
    if !initialized
        @inbounds for i = axes(ix,1)
            ix[i] = i
        end
    end

    # do partial quicksort
    sort!(ix, PartialQuickSort(k), Perm(ord(lt, by, rev, order), v))

    maybeview(ix, k)
end

## sortperm: the permutation to sort an array ##

"""
    sortperm(v; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward)

Return a permutation vector `I` that puts `v[I]` in sorted order. The order is specified
using the same keywords as [`sort!`](@ref). The permutation is guaranteed to be stable even
if the sorting algorithm is unstable, meaning that indices of equal elements appear in
ascending order.

See also [`sortperm!`](@ref).

# Examples
```jldoctest
julia> v = [3, 1, 2];

julia> p = sortperm(v)
3-element Array{Int64,1}:
 2
 3
 1

julia> v[p]
3-element Array{Int64,1}:
 1
 2
 3
```
"""
function sortperm(v::AbstractVector;
                  alg::Algorithm=DEFAULT_UNSTABLE,
                  lt=isless,
                  by=identity,
                  rev::Union{Bool,Nothing}=nothing,
                  order::Ordering=Forward)
    ordr = ord(lt,by,rev,order)
    if ordr === Forward && isa(v,Vector) && eltype(v)<:Integer
        n = length(v)
        if n > 1
            min, max = extrema(v)
            (diff, o1) = sub_with_overflow(max, min)
            (rangelen, o2) = add_with_overflow(diff, oneunit(diff))
            if !o1 && !o2 && rangelen < div(n,2)
                return sortperm_int_range(v, rangelen, min)
            end
        end
    end
    ax = axes(v, 1)
    p = similar(Vector{eltype(ax)}, ax)
    for (i,ind) in zip(eachindex(p), ax)
        p[i] = ind
    end
    sort!(p, alg, Perm(ordr,v))
end

"""
    sortperm!(ix, v; alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward, initialized::Bool=false)

Like [`sortperm`](@ref), but accepts a preallocated index vector `ix`.  If `initialized` is `false`
(the default), `ix` is initialized to contain the values `1:length(v)`.

# Examples
```jldoctest
julia> v = [3, 1, 2]; p = zeros(Int, 3);

julia> sortperm!(p, v); p
3-element Array{Int64,1}:
 2
 3
 1

julia> v[p]
3-element Array{Int64,1}:
 1
 2
 3
```
"""
function sortperm!(x::AbstractVector{<:Integer}, v::AbstractVector;
                   alg::Algorithm=DEFAULT_UNSTABLE,
                   lt=isless,
                   by=identity,
                   rev::Union{Bool,Nothing}=nothing,
                   order::Ordering=Forward,
                   initialized::Bool=false)
    if axes(x,1) != axes(v,1)
        throw(ArgumentError("index vector must have the same length/indices as the source vector, $(axes(x,1)) != $(axes(v,1))"))
    end
    if !initialized
        @inbounds for i = axes(v,1)
            x[i] = i
        end
    end
    sort!(x, alg, Perm(ord(lt,by,rev,order),v))
end

# sortperm for vectors of few unique integers
function sortperm_int_range(x::Vector{<:Integer}, rangelen, minval)
    offs = 1 - minval
    n = length(x)

    where = fill(0, rangelen+1)
    where[1] = 1
    @inbounds for i = 1:n
        where[x[i] + offs + 1] += 1
    end

    #cumsum!(where, where)
    @inbounds for i = 2:length(where)
        where[i] += where[i-1]
    end

    P = Vector{Int}(undef, n)
    @inbounds for i = 1:n
        label = x[i] + offs
        P[where[label]] = i
        where[label] += 1
    end

    return P
end

## sorting multi-dimensional arrays ##

"""
    sort(A; dims::Integer, alg::Algorithm=DEFAULT_UNSTABLE, lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward)

Sort a multidimensional array `A` along the given dimension.
See [`sort!`](@ref) for a description of possible
keyword arguments.

To sort slices of an array, refer to [`sortslices`](@ref).

# Examples
```jldoctest
julia> A = [4 3; 1 2]
2×2 Array{Int64,2}:
 4  3
 1  2

julia> sort(A, dims = 1)
2×2 Array{Int64,2}:
 1  2
 4  3

julia> sort(A, dims = 2)
2×2 Array{Int64,2}:
 3  4
 1  2
```
"""
function sort(A::AbstractArray;
              dims::Integer,
              alg::Algorithm=DEFAULT_UNSTABLE,
              lt=isless,
              by=identity,
              rev::Union{Bool,Nothing}=nothing,
              order::Ordering=Forward)
    dim = dims
    order = ord(lt,by,rev,order)
    n = length(axes(A, dim))
    if dim != 1
        pdims = (dim, setdiff(1:ndims(A), dim)...)  # put the selected dimension first
        Ap = permutedims(A, pdims)
        Av = vec(Ap)
        sort_chunks!(Av, n, alg, order)
        permutedims(Ap, invperm(pdims))
    else
        Av = A[:]
        sort_chunks!(Av, n, alg, order)
        reshape(Av, axes(A))
    end
end

@noinline function sort_chunks!(Av, n, alg, order)
    inds = LinearIndices(Av)
    for s = first(inds):n:last(inds)
        sort!(Av, s, s+n-1, alg, order)
    end
    Av
end

"""
    sort!(A; dims::Integer, alg::Algorithm=defalg(A), lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward)

Sort the multidimensional array `A` along dimension `dims`.
See [`sort!`](@ref) for a description of possible keyword arguments.

To sort slices of an array, refer to [`sortslices`](@ref).

!!! compat "Julia 1.1"
    This function requires at least Julia 1.1.

# Examples
```jldoctest
julia> A = [4 3; 1 2]
2×2 Array{Int64,2}:
 4  3
 1  2

julia> sort!(A, dims = 1); A
2×2 Array{Int64,2}:
 1  2
 4  3

julia> sort!(A, dims = 2); A
2×2 Array{Int64,2}:
 1  2
 3  4
```
"""
function sort!(A::AbstractArray;
               dims::Integer,
               alg::Algorithm=defalg(A),
               lt=isless,
               by=identity,
               rev::Union{Bool,Nothing}=nothing,
               order::Ordering=Forward)
    ordr = ord(lt, by, rev, order)
    nd = ndims(A)
    k = dims

    1 <= k <= nd || throw(ArgumentError("dimension out of range"))

    remdims = ntuple(i -> i == k ? 1 : size(A, i), nd)
    for idx in CartesianIndices(remdims)
        Av = view(A, ntuple(i -> i == k ? Colon() : idx[i], nd)...)
        sort!(Av, alg, ordr)
    end
    A
end

## fast clever sorting for floats ##

module Float
using ..Sort
using ...Order
using ..Base: @inbounds, AbstractVector, Vector, last, axes

import Core.Intrinsics: slt_int
import ..Sort: sort!
import ...Order: lt, DirectOrdering

const Floats = Union{Float32,Float64}

struct Left <: Ordering end
struct Right <: Ordering end

left(::DirectOrdering) = Left()
right(::DirectOrdering) = Right()

left(o::Perm) = Perm(left(o.order), o.data)
right(o::Perm) = Perm(right(o.order), o.data)

lt(::Left, x::T, y::T) where {T<:Floats} = slt_int(y, x)
lt(::Right, x::T, y::T) where {T<:Floats} = slt_int(x, y)

isnan(o::DirectOrdering, x::Floats) = (x!=x)
isnan(o::Perm, i::Integer) = isnan(o.order,o.data[i])

function nans2left!(v::AbstractVector, o::Ordering, lo::Integer=first(axes(v,1)), hi::Integer=last(axes(v,1)))
    i = lo
    @inbounds while i <= hi && isnan(o,v[i])
        i += 1
    end
    j = i + 1
    @inbounds while j <= hi
        if isnan(o,v[j])
            v[i], v[j] = v[j], v[i]
            i += 1
        end
        j += 1
    end
    return i, hi
end
function nans2right!(v::AbstractVector, o::Ordering, lo::Integer=first(axes(v,1)), hi::Integer=last(axes(v,1)))
    i = hi
    @inbounds while lo <= i && isnan(o,v[i])
        i -= 1
    end
    j = i - 1
    @inbounds while lo <= j
        if isnan(o,v[j])
            v[i], v[j] = v[j], v[i]
            i -= 1
        end
        j -= 1
    end
    return lo, i
end

nans2end!(v::AbstractVector, o::ForwardOrdering) = nans2right!(v,o)
nans2end!(v::AbstractVector, o::ReverseOrdering) = nans2left!(v,o)
nans2end!(v::AbstractVector{<:Integer}, o::Perm{<:ForwardOrdering}) = nans2right!(v,o)
nans2end!(v::AbstractVector{<:Integer}, o::Perm{<:ReverseOrdering}) = nans2left!(v,o)

issignleft(o::ForwardOrdering, x::Floats) = lt(o, x, zero(x))
issignleft(o::ReverseOrdering, x::Floats) = lt(o, x, -zero(x))
issignleft(o::Perm, i::Integer) = issignleft(o.order, o.data[i])

function fpsort!(v::AbstractVector, a::Algorithm, o::Ordering)
    i, j = lo, hi = nans2end!(v,o)
    @inbounds while true
        while i <= j &&  issignleft(o,v[i]); i += 1; end
        while i <= j && !issignleft(o,v[j]); j -= 1; end
        i <= j || break
        v[i], v[j] = v[j], v[i]
        i += 1; j -= 1
    end
    sort!(v, lo, j,  a, left(o))
    sort!(v, i,  hi, a, right(o))
    return v
end

fpsort!(v::AbstractVector, a::Sort.PartialQuickSort, o::Ordering) =
    sort!(v, first(axes(v,1)), last(axes(v,1)), a, o)

sort!(v::AbstractVector{<:Floats}, a::Algorithm, o::DirectOrdering) = fpsort!(v,a,o)
sort!(v::Vector{Int}, a::Algorithm, o::Perm{<:DirectOrdering,<:Vector{<:Floats}}) = fpsort!(v,a,o)

end # module Sort.Float

end # module Sort