"""
    function addUpdateMedium_!(p::Program_,platform::String)

Generate the functions related with Medium Update.
"""
function addIntegratorMediumFTCS_!(p::Program_,platform::String)

    if "UpdateMedium" in keys(p.agent.declaredUpdates)

        #Construct functions

            #Make function to compute everything inside the volume

        f = p.agent.declaredUpdates["UpdateMedium"]
        for (i,j) in enumerate(p.agent.declaredSymbols["Medium"]) # Change symbol for update
            f = postwalk(x -> @capture(x,g_(s_)=v_) && s == j && g == DIFFMEDIUMSYMBOL ? :($j = $j + $v*dt) : x, f)
        end
        f = postwalk(x->@capture(x,s_(v_)) ? :($(adaptOperatorsMediumFTCS_(v,s,p))) : x, f) # Adapt
        f = postwalk(x->@capture(x,s_) ? adaptSymbolsMediumFTCS_(s,p) : x, f) # Adapt

        f = vectorizeMedium_(p.agent,f,p)

        f = simpleGridLoopWrapInFunction_(platform,:mediumInnerStep_!, f, p.agent.dims)

        #Remove count over boundaries
        if platform == "cpu"
            f = postwalk(x->@capture(x,1:Nx_) && Nx == :Nx_ ? :(2:Nx_-1) : x, f)
            f = postwalk(x->@capture(x,1:Ny_) && Ny == :Ny_ ? :(2:Ny_-1) : x, f)
            f = postwalk(x->@capture(x,1:Nz_) && Nz == :Nz_ ? :(2:Nz_-1) : x, f)
        elseif platform == "gpu"
            f = postwalk(x->@capture(x,indexX_:strideX_:Nx_) && Nx == :Nx_ ? :(indexX_+1:strideX_:Nx_-1) : x, f)
            f = postwalk(x->@capture(x,indexY_:strideY_:Ny_) && Ny == :Ny_ ? :(indexY_+1:strideY_:Ny_-1) : x, f)
            f = postwalk(x->@capture(x,indexZ_:strideZ_:Nz_) && Nz == :Nz_ ? :(indexZ_+1:strideZ_:Nz_-1) : x, f)
        end
        push!(p.declareF.args, ## Add it to code
                f
            )

        #Add boundary computation
        boundariesFunctionDefinition(p.agent.boundary, p, platform)

        if "UpdateMediumInteraction" in keys(p.agent.declaredUpdates)
            fWrap = wrapInFunction_(:mediumStep_!,
                                    :(begin
                                        @platformAdapt mediumInnerStep_!(ARGS_)
                                        @platformAdapt mediumBoundaryStep_!(ARGS_)
                                        @platformAdapt mediumInteractionStep_!(ARGS_)
                                    end)
                                    )
        else
            fWrap = wrapInFunction_(:mediumStep_!,
                                    :(begin
                                        @platformAdapt mediumInnerStep_!(ARGS_)
                                        @platformAdapt mediumBoundaryStep_!(ARGS_)
                                    end)
                                    )
        end
        push!(p.declareF.args, ## Add it to code
            fWrap
            )
        # println(prettify(code))

        push!(p.execInloop.args,
                :(mediumStep_!(ARGS_))
            )
    end

    return nothing
end

function adaptSymbolsMediumFTCS_(s,p)

    if s == :xₘ

        return :(ic1_*dxₘ_+box[1,1])

    elseif s == :yₘ

        return :(ic2_*dyₘ_+box[2,1])

    elseif s == :zₘ

        return :(ic1_*dzₘ_+box[3,1])

    end

    return s

end

function adaptOperatorsMediumFTCS_(f,op,p)

    f = vectorizeMedium_(p.agent,f,p)

    if op == :δx

        f = :(δMedium_(round(Int,($f-simulationBox[1,1])/dxₘ_)+1,ic1_))

    elseif op == :δy

        if p.agent.dims < 2
            error("Operator δy cannot be used in Agent with dimension 1.")
        end

        f = :(δMedium_(round(Int,($f-simulationBox[2,1])/dyₘ_)+1,ic2_))

    elseif op == :δz

        if p.agent.dims < 3
            error("Operator δz cannot be used in Agent with dimension 1 or 2.")
        end

        f = :(δMedium_(round(Int,($f-simulationBox[3,1])/dzₘ_)+1,ic3_))

    elseif op == :Δ
        if p.agent.dims == 1
            add = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,$j]) : x,f)

            f = :(($add+$subs-2*$f)/dxₘ_^2)
        elseif p.agent.dims == 2
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,$j]) : x,f)
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,$j]) : x,f)

            f = :(($add+$subs-2*$f)/dxₘ_^2+($addy+$subsy-2*$f)/dyₘ_^2)
        elseif p.agent.dims == 3
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,ic3_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,ic3_,$j]) : x,f)
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,ic3_,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,ic3_,$j]) : x,f)
            addz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_+1,$j]) : x,f)
            subsz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_-1,$j]) : x,f)

            f = :(($add+$subs-2*$f)/dxₘ_^2+($addy+$subsy-2*$f)/dyₘ_^2+($addz+$subsz-2*$f)/dzₘ_^2)

        end
    elseif op == :Δx
        if p.agent.dims == 1
            add = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,$j]) : x,f)

            f = :(($add+$subs-2*$f)/dxₘ_^2)
        elseif p.agent.dims == 2
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,$j]) : x,f)

            f = :(($add+$subs-2*$f)/dxₘ_^2)
        elseif p.agent.dims == 3
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,ic3_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,ic3_,$j]) : x,f)

            f = :(($add+$subs-2*$f)/dxₘ_^2)
        end
    elseif op == :Δy
        if p.agent.dims == 1
            error("Operator Δy cannot exist in Agent with dimension 1.")
        elseif p.agent.dims == 2
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,$j]) : x,f)

            f = :(($addy+$subsy-2*$f)/dyₘ_^2)
        elseif p.agent.dims == 3
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,ic3_,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,ic3_,$j]) : x,f)

            f = :(($addy+$subsy-2*$f)/dyₘ_^2)
        end
    elseif op == :Δz
        if p.agent.dims == 1
            error("Operator Δz cannot exist in Agent with dimension 1.")
        elseif p.agent.dims == 2
            error("Operator Δz cannot exist in Agent with dimension 2.")
        elseif p.agent.dims == 3
            addz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_+1,$j]) : x,f)
            subsz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_-1,$j]) : x,f)

            f = :(($addz+$subsz-2*$f)/dzₘ_^2)
        end
    elseif op == :∇
        if p.agent.dims == 1
            add = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,$j]) : x,f)

            f = :(($add-$subs)/(2*dxₘ_))
        elseif p.agent.dims == 2
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,$j]) : x,f)
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,$j]) : x,f)

            f = :(($add-$subs)/(2*dxₘ_)+($addy-$subsy)/(2*dyₘ_))
        elseif p.agent.dims == 3
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,ic3_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,ic3_,$j]) : x,f)
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,ic3_,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,ic3_,$j]) : x,f)
            addz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_+1,$j]) : x,f)
            subsz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_-1,$j]) : x,f)

            f = :(($add-$subs)/(2*dxₘ_)+($addy-$subsy)/(2*dyₘ_)+($addz-$subsz)/(2*dzₘ_))
        end
    elseif op == :∇x
        if p.agent.dims == 1
            add = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,$j]) : x,f)

            f = :(($add-$subs)/(2*dxₘ_))
        elseif p.agent.dims == 2
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,$j]) : x,f)

            f = :(($add-$subs)/(2*dxₘ_))
        elseif p.agent.dims == 3
            add = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_+1,ic2_,ic3_,$j]) : x,f)
            subs = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_-1,ic2_,ic3_,$j]) : x,f)

            f = :(($add-$subs)/(2*dxₘ_))
        end
    elseif op == :∇y
        if p.agent.dims == 1
            error("Operator Δy cannot exist in Agent with dimension 1.")
        elseif p.agent.dims == 2
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,$j]) : x,f)

            f = :(($addy-$subsy)/(2*dyₘ_))
        elseif p.agent.dims == 3
            addy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_+1,ic3_,$j]) : x,f)
            subsy = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_-1,ic3_,$j]) : x,f)

            f = :(($addy-$subsy)/(2*dyₘ_))
        end
    elseif op == :∇z
        if p.agent.dims == 1
            error("Operator Δz cannot exist in Agent with dimension 1.")
        elseif p.agent.dims == 2
            error("Operator Δy cannot exist in Agent with dimension 2.")
        elseif p.agent.dims == 3
            addz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_+1,$j]) : x,f)
            subsz = postwalk(x->@capture(x,s_[ic1_,ic2_,ic3_,j_]) && s == :mediumV ? :(mediumV[ic1_,ic2_,ic3_-1,$j]) : x,f)

            f = :(($addz-$subsz)/(2*dzₘ_))
        end
    end

    return f

end


function calculate_tgrad(sys::AbstractODESystem;
                         simplify=false)
  isempty(get_tgrad(sys)[]) || return get_tgrad(sys)[]  # use cached tgrad, if possible

  # We need to remove explicit time dependence on the state because when we
  # have `u(t) * t` we want to have the tgrad to be `u(t)` instead of `u'(t) *
  # t + u(t)`.
  rhs = [detime_dvs(eq.rhs) for eq ∈ equations(sys)]
  iv = get_iv(sys)
  xs = states(sys)
  rule = Dict(map((x, xt) -> xt=>x, detime_dvs.(xs), xs))
  rhs = substitute.(rhs, Ref(rule))
  tgrad = [expand_derivatives(ModelingToolkit.Differential(iv)(r), simplify) for r in rhs]
  reverse_rule = Dict(map((x, xt) -> x=>xt, detime_dvs.(xs), xs))
  tgrad = Num.(substitute.(tgrad, Ref(reverse_rule)))
  get_tgrad(sys)[] = tgrad
  return tgrad
end

function calculate_jacobian(sys::AbstractODESystem;
                            sparse=false, simplify=false)
    isempty(get_jac(sys)[]) || return get_jac(sys)[]  # use cached Jacobian, if possible
    rhs = [eq.rhs for eq ∈ equations(sys)]

    iv = get_iv(sys)
    dvs = states(sys)

    if sparse
        jac = sparsejacobian(rhs, dvs, simplify=simplify)
    else
        jac = jacobian(rhs, dvs, simplify=simplify)
    end

    get_jac(sys)[] = jac  # cache Jacobian
    return jac
end

function generate_tgrad(sys::AbstractODESystem, dvs = states(sys), ps = parameters(sys);
                        simplify=false, kwargs...)
    tgrad = calculate_tgrad(sys,simplify=simplify)
    return build_function(tgrad, dvs, ps, get_iv(sys); kwargs...)
end

function generate_jacobian(sys::AbstractODESystem, dvs = states(sys), ps = parameters(sys);
                           simplify=false, sparse = false, kwargs...)
    jac = calculate_jacobian(sys;simplify=simplify,sparse=sparse)
    return build_function(jac, dvs, ps, get_iv(sys); kwargs...)
end

@noinline function throw_invalid_derivative(dervar, eq)
    msg = "The derivative variable must be isolated to the left-hand " *
    "side of the equation like `$dervar ~ ...`.\n Got $eq."
    throw(InvalidSystemException(msg))
end

function check_derivative_variables(eq, expr=eq.rhs)
    istree(expr) || return nothing
    if operation(expr) isa Differential
        throw_invalid_derivative(expr, eq)
    end
    foreach(Base.Fix1(check_derivative_variables, eq), arguments(expr))
end

function generate_function(sys::AbstractODESystem, dvs = states(sys), ps = parameters(sys); kwargs...)
    # optimization
    #obsvars = map(eq->eq.lhs, observed(sys))
    #fulldvs = [dvs; obsvars]

    eqs = equations(sys)
    foreach(check_derivative_variables, eqs)
    # substitute x(t) by just x
    rhss = [deq.rhs for deq in eqs]
    #obss = [makesym(value(eq.lhs)) ~ substitute(eq.rhs, sub) for eq ∈ observed(sys)]
    #rhss = Let(obss, rhss)

    # TODO: add an optional check on the ordering of observed equations
    return build_function(rhss,
                          map(x->time_varying_as_func(value(x), sys), dvs),
                          map(x->time_varying_as_func(value(x), sys), ps),
                          get_iv(sys); kwargs...)
end

function time_varying_as_func(x, sys)
    # if something is not x(t) (the current state)
    # but is `x(t-1)` or something like that, pass in `x` as a callable function rather
    # than pass in a value in place of x(t).
    #
    # This is done by just making `x` the argument of the function.
    if istree(x) &&
        operation(x) isa Sym &&
        !(length(arguments(x)) == 1 && isequal(arguments(x)[1], independent_variable(sys)))
        return operation(x)
    end
    return x
end

function calculate_massmatrix(sys::AbstractODESystem; simplify=false)
    eqs = equations(sys)
    dvs = states(sys)
    M = zeros(length(eqs),length(eqs))
    for (i,eq) in enumerate(eqs)
        if eq.lhs isa Term && operation(eq.lhs) isa Differential
            j = findfirst(x->isequal(tosymbol(x),tosymbol(var_from_nested_derivative(eq.lhs)[1])),dvs)
            M[i,j] = 1
        else
            _iszero(eq.lhs) || error("Only semi-explicit constant mass matrices are currently supported. Faulty equation: $eq.")
        end
    end
    M = simplify ? ModelingToolkit.simplify.(M) : M
    # M should only contain concrete numbers
    M == I ? I : M
end

jacobian_sparsity(sys::AbstractODESystem) =
    jacobian_sparsity([eq.rhs for eq ∈ equations(sys)],
                      [dv for dv in states(sys)])

function isautonomous(sys::AbstractODESystem)
    tgrad = calculate_tgrad(sys;simplify=true)
    all(iszero,tgrad)
end

function DiffEqBase.ODEFunction(sys::AbstractODESystem, args...; kwargs...)
    ODEFunction{true}(sys, args...; kwargs...)
end

"""
```julia
function DiffEqBase.ODEFunction{iip}(sys::AbstractODESystem, dvs = states(sys),
                                     ps = parameters(sys);
                                     version = nothing, tgrad=false,
                                     jac = false,
                                     sparse = false,
                                     kwargs...) where {iip}
```

Create an `ODEFunction` from the [`ODESystem`](@ref). The arguments `dvs` and `ps`
are used to set the order of the dependent variable and parameter vectors,
respectively.
"""
function DiffEqBase.ODEFunction{iip}(sys::AbstractODESystem, dvs = states(sys),
                                     ps = parameters(sys), u0 = nothing;
                                     version = nothing, tgrad=false,
                                     jac = false,
                                     eval_expression = true,
                                     sparse = false, simplify=false,
                                     eval_module = @__MODULE__,
                                     kwargs...) where {iip}

    f_gen = generate_function(sys, dvs, ps; expression=Val{eval_expression}, expression_module=eval_module, kwargs...)
    f_oop,f_iip = eval_expression ? (@RuntimeGeneratedFunction(eval_module, ex) for ex in f_gen) : f_gen
    f(u,p,t) = f_oop(u,p,t)
    f(du,u,p,t) = f_iip(du,u,p,t)

    if tgrad
        tgrad_gen = generate_tgrad(sys, dvs, ps;
                                   simplify=simplify,
                                   expression=Val{eval_expression}, expression_module=eval_module, kwargs...)
        tgrad_oop,tgrad_iip = eval_expression ? (@RuntimeGeneratedFunction(eval_module, ex) for ex in tgrad_gen) : tgrad_gen
        _tgrad(u,p,t) = tgrad_oop(u,p,t)
        _tgrad(J,u,p,t) = tgrad_iip(J,u,p,t)
    else
        _tgrad = nothing
    end

    if jac
        jac_gen = generate_jacobian(sys, dvs, ps;
                                    simplify=simplify, sparse = sparse,
                                    expression=Val{eval_expression}, expression_module=eval_module, kwargs...)
        jac_oop,jac_iip = eval_expression ? (@RuntimeGeneratedFunction(eval_module, ex) for ex in jac_gen) : jac_gen
        _jac(u,p,t) = jac_oop(u,p,t)
        _jac(J,u,p,t) = jac_iip(J,u,p,t)
    else
        _jac = nothing
    end

    M = calculate_massmatrix(sys)

    _M = (u0 === nothing || M == I) ? M : ArrayInterface.restructure(u0 .* u0',M)

    observedfun = let sys = sys, dict = Dict()
        function generated_observed(obsvar, u, p, t)
            obs = get!(dict, value(obsvar)) do
                build_explicit_observed_function(sys, obsvar)
            end
            obs(u, p, t)
        end
    end

    ODEFunction{iip}(
                     f,
                     jac = _jac === nothing ? nothing : _jac,
                     tgrad = _tgrad === nothing ? nothing : _tgrad,
                     mass_matrix = _M,
                     jac_prototype = sparse ? similar(get_jac(sys)[],Float64) : nothing,
                     syms = Symbol.(states(sys)),
                     indepsym = Symbol(independent_variable(sys)),
                     observed = observedfun,
                    )
end

"""
```julia
function DiffEqBase.ODEFunctionExpr{iip}(sys::AbstractODESystem, dvs = states(sys),
                                     ps = parameters(sys);
                                     version = nothing, tgrad=false,
                                     jac = false,
                                     sparse = false,
                                     kwargs...) where {iip}
```

Create a Julia expression for an `ODEFunction` from the [`ODESystem`](@ref).
The arguments `dvs` and `ps` are used to set the order of the dependent
variable and parameter vectors, respectively.
"""
struct ODEFunctionExpr{iip} end

struct ODEFunctionClosure{O, I} <: Function
    f_oop::O
    f_iip::I
end
(f::ODEFunctionClosure)(u, p, t) = f.f_oop(u, p, t)
(f::ODEFunctionClosure)(du, u, p, t) = f.f_iip(du, u, p, t)

function ODEFunctionExpr{iip}(sys::AbstractODESystem, dvs = states(sys),
                                     ps = parameters(sys), u0 = nothing;
                                     version = nothing, tgrad=false,
                                     jac = false,
                                     linenumbers = false,
                                     sparse = false, simplify=false,
                                     kwargs...) where {iip}

    f_oop, f_iip = generate_function(sys, dvs, ps; expression=Val{true}, kwargs...)
    fsym = gensym(:f)
    _f = :($fsym = ModelingToolkit.ODEFunctionClosure($f_oop, $f_iip))
    tgradsym = gensym(:tgrad)
    if tgrad
        tgrad_oop, tgrad_iip = generate_tgrad(sys, dvs, ps;
                                simplify=simplify,
                                expression=Val{true}, kwargs...)
        _tgrad = :($tgradsym = ModelingToolkit.ODEFunctionClosure($tgrad_oop, $tgrad_iip))
    else
        _tgrad = :($tgradsym = nothing)
    end

    jacsym = gensym(:jac)
    if jac
        jac_oop,jac_iip = generate_jacobian(sys, dvs, ps;
                                 sparse=sparse, simplify=simplify,
                                 expression=Val{true}, kwargs...)
        _jac = :($jacsym = ModelingToolkit.ODEFunctionClosure($jac_oop, $jac_iip))
    else
        _jac = :($jacsym = nothing)
    end

    M = calculate_massmatrix(sys)

    _M = (u0 === nothing || M == I) ? M : ArrayInterface.restructure(u0 .* u0',M)

    jp_expr = sparse ? :(similar($(get_jac(sys)[]),Float64)) : :nothing
    ex = quote
        $_f
        $_tgrad
        $_jac
        M = $_M
        ODEFunction{$iip}(
                          $fsym,
                          jac = $jacsym,
                          tgrad = $tgradsym,
                          mass_matrix = M,
                          jac_prototype = $jp_expr,
                          syms = $(Symbol.(states(sys))),
                          indepsym = $(QuoteNode(Symbol(independent_variable(sys)))),
                         )
    end
    !linenumbers ? striplines(ex) : ex
end

function process_DEProblem(constructor, sys::AbstractODESystem,u0map,parammap;
                           version = nothing, tgrad=false,
                           jac = false,
                           checkbounds = false, sparse = false,
                           simplify=false,
                           linenumbers = true, parallel=SerialForm(),
                           eval_expression = true,
                           kwargs...)
    dvs = states(sys)
    ps = parameters(sys)
    defs = defaults(sys)

    u0 = varmap_to_vars(u0map,dvs; defaults=defs)
    p = varmap_to_vars(parammap,ps; defaults=defs)

    if u0 !== nothing
        length(dvs) == length(u0) || throw(ArgumentError("States ($(length(dvs))) and initial conditions ($(length(u0))) are of different lengths."))
    end

    f = constructor(sys,dvs,ps,u0;tgrad=tgrad,jac=jac,checkbounds=checkbounds,
                    linenumbers=linenumbers,parallel=parallel,simplify=simplify,
                    sparse=sparse,eval_expression=eval_expression,kwargs...)
    return f, u0, p
end

function ODEFunctionExpr(sys::AbstractODESystem, args...; kwargs...)
    ODEFunctionExpr{true}(sys, args...; kwargs...)
end

function DiffEqBase.ODEProblem(sys::AbstractODESystem, args...; kwargs...)
    ODEProblem{true}(sys, args...; kwargs...)
end

"""
```julia
function DiffEqBase.ODEProblem{iip}(sys::AbstractODESystem,u0map,tspan,
                                    parammap=DiffEqBase.NullParameters();
                                    version = nothing, tgrad=false,
                                    jac = false,
                                    checkbounds = false, sparse = false,
                                    simplify=false,
                                    linenumbers = true, parallel=SerialForm(),
                                    kwargs...) where iip
```

Generates an ODEProblem from an ODESystem and allows for automatically
symbolically calculating numerical enhancements.
"""
function DiffEqBase.ODEProblem{iip}(sys::AbstractODESystem,u0map,tspan,
                                    parammap=DiffEqBase.NullParameters();kwargs...) where iip
    f, u0, p = process_DEProblem(ODEFunction{iip}, sys, u0map, parammap; kwargs...)
    ODEProblem{iip}(f,u0,tspan,p;kwargs...)
end

"""
```julia
function DiffEqBase.ODEProblemExpr{iip}(sys::AbstractODESystem,u0map,tspan,
                                    parammap=DiffEqBase.NullParameters();
                                    version = nothing, tgrad=false,
                                    jac = false,
                                    checkbounds = false, sparse = false,
                                    linenumbers = true, parallel=SerialForm(),
                                    skipzeros=true, fillzeros=true,
                                    simplify=false,
                                    kwargs...) where iip
```

Generates a Julia expression for constructing an ODEProblem from an
ODESystem and allows for automatically symbolically calculating
numerical enhancements.
"""
struct ODEProblemExpr{iip} end

function ODEProblemExpr{iip}(sys::AbstractODESystem,u0map,tspan,
                             parammap=DiffEqBase.NullParameters();
                             kwargs...) where iip

    f, u0, p = process_DEProblem(ODEFunctionExpr{iip}, sys, u0map, parammap; kwargs...)
    linenumbers = get(kwargs, :linenumbers, true)

    ex = quote
        f = $f
        u0 = $u0
        tspan = $tspan
        p = $p
        ODEProblem(f,u0,tspan,p;$(kwargs...))
    end
    !linenumbers ? striplines(ex) : ex
end

function ODEProblemExpr(sys::AbstractODESystem, args...; kwargs...)
    ODEProblemExpr{true}(sys, args...; kwargs...)
end

### Enables Steady State Problems ###
function DiffEqBase.SteadyStateProblem(sys::AbstractODESystem, args...; kwargs...)
    SteadyStateProblem{true}(sys, args...; kwargs...)
end

"""
```julia
function DiffEqBase.SteadyStateProblem(sys::AbstractODESystem,u0map,tspan,
                                    parammap=DiffEqBase.NullParameters();
                                    version = nothing, tgrad=false,
                                    jac = false,
                                    checkbounds = false, sparse = false,
                                    linenumbers = true, parallel=SerialForm(),
                                    kwargs...) where iip
```
Generates an SteadyStateProblem from an ODESystem and allows for automatically
symbolically calculating numerical enhancements.
"""
function DiffEqBase.SteadyStateProblem{iip}(sys::AbstractODESystem,u0map,
                                            parammap=DiffEqBase.NullParameters();
                                            kwargs...) where iip
    f, u0, p = process_DEProblem(ODEFunction{iip}, sys, u0map, parammap; kwargs...)
    SteadyStateProblem(f,u0,p;kwargs...)
end

"""
```julia
function DiffEqBase.SteadyStateProblemExpr(sys::AbstractODESystem,u0map,tspan,
                                    parammap=DiffEqBase.NullParameters();
                                    version = nothing, tgrad=false,
                                    jac = false,
                                    checkbounds = false, sparse = false,
                                    skipzeros=true, fillzeros=true,
                                    linenumbers = true, parallel=SerialForm(),
                                    kwargs...) where iip
```
Generates a Julia expression for building a SteadyStateProblem from
an ODESystem and allows for automatically symbolically calculating
numerical enhancements.
"""
struct SteadyStateProblemExpr{iip} end

function SteadyStateProblemExpr{iip}(sys::AbstractODESystem,u0map,
                                    parammap=DiffEqBase.NullParameters();
                                    kwargs...) where iip
    f, u0, p = process_DEProblem(ODEFunctionExpr{iip}, sys, u0map, parammap; kwargs...)
    linenumbers = get(kwargs, :linenumbers, true)
    ex = quote
        f = $f
        u0 = $u0
        p = $p
        SteadyStateProblem(f,u0,p;$(kwargs...))
    end
    !linenumbers ? striplines(ex) : ex
end

function SteadyStateProblemExpr(sys::AbstractODESystem, args...; kwargs...)
    SteadyStateProblemExpr{true}(sys, args...; kwargs...)
end

isdifferential(expr) = istree(expr) && operation(expr) isa Differential
isdiffeq(eq) = isdifferential(eq.lhs)


# *********************************************************************************
# REopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this list
# of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
# *********************************************************************************
"""
    REoptInputs

The data structure for all the inputs necessary to construct the JuMP model.
```julia
struct REoptInputs <: AbstractInputs
    s::AbstractScenario
    techs::Techs
    min_sizes::Dict{String, Float64}  # (techs)
    max_sizes::Dict{String, Float64}  # (techs)
    existing_sizes::Dict{String, Float64}  # (techs)
    cap_cost_slope::Dict{String, Any}  # (techs)
    om_cost_per_kw::Dict{String, Float64}  # (techs)
    time_steps::UnitRange
    time_steps_with_grid::Array{Int, 1}
    time_steps_without_grid::Array{Int, 1}
    hours_per_timestep::Float64
    months::UnitRange
    production_factor::DenseAxisArray{Float64, 2}  # (techs, time_steps)
    levelization_factor::Dict{String, Float64}  # (techs)
    value_of_lost_load_per_kwh::Array{R, 1} where R<:Real #default set to 1 US dollar per kwh
    pwf_e::Float64
    pwf_om::Float64
    third_party_factor::Float64
    pvlocations::Array{Symbol, 1}
    maxsize_pv_locations::DenseAxisArray{Float64, 1}  # indexed on pvlocations
    pv_to_location::Dict{String, Dict{Symbol, Int64}}  # (techs.pv, pvlocations)
    ratchets::UnitRange
    techs_by_exportbin::Dict{Symbol, AbstractArray}  # keys can include [:NEM, :WHL, :CUR]
    export_bins_by_tech::Dict
    n_segs_by_tech::Dict{String, Int}
    seg_min_size::Dict{String, Dict{Int, Float64}}
    seg_max_size::Dict{String, Dict{Int, Float64}}
    seg_yint::Dict{String, Dict{Int, Float64}}
    pbi_pwf::Dict{String, Any}  # (pbi_techs)
    pbi_max_benefit::Dict{String, Any}  # (pbi_techs)
    pbi_max_kw::Dict{String, Any}  # (pbi_techs)
    pbi_benefit_per_kwh::Dict{String, Any}  # (pbi_techs)
    boiler_efficiency::Dict{String, Float64}
end
```
"""
struct REoptInputs <: AbstractInputs
    s::AbstractScenario
    techs::Techs
    min_sizes::Dict{String, Float64}  # (techs)
    max_sizes::Dict{String, Float64}  # (techs)
    existing_sizes::Dict{String, Float64}  # (techs)
    cap_cost_slope::Dict{String, Any}  # (techs)
    om_cost_per_kw::Dict{String, Float64}  # (techs)
    time_steps::UnitRange
    time_steps_with_grid::Array{Int, 1}
    time_steps_without_grid::Array{Int, 1}
    hours_per_timestep::Float64
    months::UnitRange
    production_factor::DenseAxisArray{Float64, 2}  # (techs, time_steps)
    levelization_factor::Dict{String, Float64}  # (techs)
    value_of_lost_load_per_kwh::Array{R, 1} where R<:Real #default set to 1 US dollar per kwh
    pwf_e::Float64
    pwf_om::Float64
    pwf_fuel::Dict{String, Float64}
    third_party_factor::Float64
    pvlocations::Array{Symbol, 1}
    maxsize_pv_locations::DenseAxisArray{Float64, 1}  # indexed on pvlocations
    pv_to_location::Dict{String, Dict{Symbol, Int64}}  # (techs.pv, pvlocations)
    ratchets::UnitRange
    techs_by_exportbin::Dict{Symbol, AbstractArray}  # keys can include [:NEM, :WHL, :CUR]
    export_bins_by_tech::Dict
    n_segs_by_tech::Dict{String, Int}
    seg_min_size::Dict{String, Dict{Int, Float64}}
    seg_max_size::Dict{String, Dict{Int, Float64}}
    seg_yint::Dict{String, Dict{Int, Float64}}
    pbi_pwf::Dict{String, Any}  # (pbi_techs)
    pbi_max_benefit::Dict{String, Any}  # (pbi_techs)
    pbi_max_kw::Dict{String, Any}  # (pbi_techs)
    pbi_benefit_per_kwh::Dict{String, Any}  # (pbi_techs)
    boiler_efficiency::Dict{String, Float64}
end

"""
    REoptInputs(fp::String)

Use `fp` to load in JSON scenario:
```
function REoptInputs(fp::String)
    s = Scenario(JSON.parsefile(fp))
    REoptInputs(s)
end
```
Useful if you want to manually modify REoptInputs before solving the model.
"""
function REoptInputs(fp::String)
    s = Scenario(JSON.parsefile(fp))
    REoptInputs(s)
end

"""
    REoptInputs(s::AbstractScenario)

Constructor for REoptInputs. Translates the `Scenario` into all the data necessary for building the JuMP model.
"""
function REoptInputs(s::AbstractScenario)

    time_steps = 1:length(s.electric_load.loads_kw)
    hours_per_timestep = 1 / s.settings.time_steps_per_hour
    techs, pv_to_location, maxsize_pv_locations, pvlocations,
        production_factor, max_sizes, min_sizes, existing_sizes, cap_cost_slope, om_cost_per_kw, n_segs_by_tech,
        seg_min_size, seg_max_size, seg_yint, techs_by_exportbin, export_bins_by_tech, boiler_efficiency =
        setup_tech_inputs(s)

    pbi_pwf, pbi_max_benefit, pbi_max_kw, pbi_benefit_per_kwh = setup_pbi_inputs(s, techs)

    months = 1:12

    levelization_factor, pwf_e, pwf_om, pwf_fuel, third_party_factor = setup_present_worth_factors(s, techs)
    # the following hardcoded values for levelization_factor matches the public REopt API value
    # and makes the test values match.
    # the REopt code herein uses the Desktop method for levelization_factor, which is more accurate
    # (Desktop has non-linear degradation vs. linear degradation in API)
    # levelization_factor = Dict("PV" => 0.9539)
    # levelization_factor = Dict("ground" => 0.942238, "roof_east" => 0.942238, "roof_west" => 0.942238)
    # levelization_factor = Dict("PV" => 0.9539, "Generator" => 1.0)
    time_steps_with_grid, time_steps_without_grid, = setup_electric_utility_inputs(s)

    if any(pv.existing_kw > 0 for pv in s.pvs)
        adjust_load_profile(s, production_factor)
    end

    REoptInputs(
        s,
        techs,
        min_sizes,
        max_sizes,
        existing_sizes,
        cap_cost_slope,
        om_cost_per_kw,
        time_steps,
        time_steps_with_grid,
        time_steps_without_grid,
        hours_per_timestep,
        months,
        production_factor,
        levelization_factor,
        typeof(s.financial.value_of_lost_load_per_kwh) <: Array{<:Real, 1} ? s.financial.value_of_lost_load_per_kwh : fill(s.financial.value_of_lost_load_per_kwh, length(time_steps)),
        pwf_e,
        pwf_om,
        pwf_fuel,
        third_party_factor,
        pvlocations,
        maxsize_pv_locations,
        pv_to_location,
        1:length(s.electric_tariff.tou_demand_ratchet_timesteps),  # ratchets
        techs_by_exportbin,
        export_bins_by_tech,
        n_segs_by_tech,
        seg_min_size,
        seg_max_size,
        seg_yint,
        pbi_pwf,
        pbi_max_benefit,
        pbi_max_kw,
        pbi_benefit_per_kwh,
        boiler_efficiency
    )
end

"""
    function setup_tech_inputs(s::AbstractScenario)

Create data arrays associated with techs necessary to build the JuMP model.
"""
function setup_tech_inputs(s::AbstractScenario)

    techs = Techs(s)

    boiler_efficiency = Dict{String, Float64}()
    production_factor = DenseAxisArray{Float64}(undef, techs.all, 1:length(s.electric_load.loads_kw))

    # REoptInputs indexed on techs:
    max_sizes = Dict(t => 0.0 for t in techs.all)
    min_sizes = Dict(t => 0.0 for t in techs.all)
    existing_sizes = Dict(t => 0.0 for t in techs.all)
    cap_cost_slope = Dict{String, Any}()
    om_cost_per_kw = Dict(t => 0.0 for t in techs.all)

    # export related inputs
    techs_by_exportbin = Dict(k => [] for k in s.electric_tariff.export_bins)
    export_bins_by_tech = Dict{String, Array{Symbol, 1}}()

    # REoptInputs indexed on techs.segmented
    n_segs_by_tech = Dict{String, Int}()
    seg_min_size = Dict{String, Any}()
    seg_max_size = Dict{String, Any}()
    seg_yint = Dict{String, Any}()

    # PV specific arrays
    pvlocations = [:roof, :ground, :both]
    pv_to_location = Dict(t => Dict(loc => 0) for (t, loc) in zip(techs.pv, pvlocations))
    maxsize_pv_locations = DenseAxisArray([1.0e5, 1.0e5, 1.0e5], pvlocations)
    # default to large max size per location. Max size by roof, ground, both

    if !isempty(techs.pv)
        setup_pv_inputs(s, max_sizes, min_sizes, existing_sizes, cap_cost_slope, om_cost_per_kw, production_factor,
                        pvlocations, pv_to_location, maxsize_pv_locations, techs.segmented, n_segs_by_tech,
                        seg_min_size, seg_max_size, seg_yint, techs_by_exportbin, techs.no_curtail)
    end

    if "Wind" in techs.all
        setup_wind_inputs(s, max_sizes, min_sizes, existing_sizes, cap_cost_slope, om_cost_per_kw, production_factor,
            techs_by_exportbin, techs.segmented, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint, techs.no_curtail)
    end

    if "Generator" in techs.all
        setup_gen_inputs(s, max_sizes, min_sizes, existing_sizes, cap_cost_slope, om_cost_per_kw, production_factor,
            techs_by_exportbin, techs.segmented, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint, techs.no_curtail)
    end

    if "ExistingBoiler" in techs.all
        setup_existing_boiler_inputs(s, max_sizes, min_sizes, existing_sizes, cap_cost_slope, boiler_efficiency)
    end

    # filling export_bins_by_tech MUST be done after techs_by_exportbin has been filled in
    for t in techs.elec
        export_bins_by_tech[t] = [bin for (bin, ts) in techs_by_exportbin if t in ts]
    end

    return techs, pv_to_location, maxsize_pv_locations, pvlocations,
    production_factor, max_sizes, min_sizes, existing_sizes, cap_cost_slope, om_cost_per_kw, n_segs_by_tech,
    seg_min_size, seg_max_size, seg_yint, techs_by_exportbin, export_bins_by_tech, boiler_efficiency
end

"""
    setup_pbi_inputs(s::AbstractScenario, techs::Techs)

Create data arrays for production based incentives.
All arrays can be empty if no techs have production_incentive_per_kwh > 0.
"""
function setup_pbi_inputs(s::AbstractScenario, techs::Techs)

    pbi_pwf = Dict{String, Any}()
    pbi_max_benefit = Dict{String, Any}()
    pbi_max_kw = Dict{String, Any}()
    pbi_benefit_per_kwh = Dict{String, Any}()

    for tech in techs.all
        if !(tech in techs.pv)
            T = typeof(eval(Meta.parse(tech)))
            if :production_incentive_per_kwh in fieldnames(T)
                if eval(Meta.parse("s."*tech*".production_incentive_per_kwh")) > 0
                    push!(techs.pbi, tech)
                    pbi_pwf[tech], pbi_max_benefit[tech], pbi_max_kw[tech], pbi_benefit_per_kwh[tech] =
                        production_incentives(eval(Meta.parse("s."*tech)), s.financial)
                end
            end
        else
            pv = get_pv_by_name(tech, s.pvs)
            if pv.production_incentive_per_kwh > 0
                push!(techs.pbi, tech)
                pbi_pwf[tech], pbi_max_benefit[tech], pbi_max_kw[tech], pbi_benefit_per_kwh[tech] =
                    production_incentives(pv, s.financial)
            end
        end

    end
    return pbi_pwf, pbi_max_benefit, pbi_max_kw, pbi_benefit_per_kwh
end

"""
    update_cost_curve!(tech::AbstractTech, tech_name::String, financial::Financial,
        cap_cost_slope, segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint
    )

Modifies cap_cost_slope, segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint based on tech attributes.
In the simplest case (zero incentives, no existing_kw) the cap_cost_slope is updated with:
```julia
    cap_cost_slope[tech_name] = tech.installed_cost_per_kw
```
However, if there are non-zero incentives or `existing_kw` then there will be more than one cost curve segment typically
and all of the other arguments will be updated as well.
"""
function update_cost_curve!(tech::AbstractTech, tech_name::String, financial::Financial,
    cap_cost_slope, segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint
    )
    cost_slope, cost_curve_bp_x, cost_yint, n_segments = cost_curve(tech, financial)
    cap_cost_slope[tech_name] = cost_slope[1]
    if n_segments > 1
        cap_cost_slope[tech_name] = cost_slope
        push!(segmented_techs, tech_name)
        seg_max_size[tech_name] = Dict{Int,Float64}()
        seg_min_size[tech_name] = Dict{Int,Float64}()
        n_segs_by_tech[tech_name] = n_segments
        seg_yint[tech_name] = Dict{Int,Float64}()
        for s in 1:n_segments
            seg_min_size[tech_name][s] = cost_curve_bp_x[s]
            seg_max_size[tech_name][s] = cost_curve_bp_x[s+1]
            seg_yint[tech_name][s] = cost_yint[s]
        end
    end
    nothing
end

function setup_pv_inputs(s::AbstractScenario, max_sizes, min_sizes,
    existing_sizes, cap_cost_slope, om_cost_per_kw, production_factor,
    pvlocations, pv_to_location, maxsize_pv_locations,
    segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint, techs_by_exportbin, techs_no_curtail)

    pv_roof_limited, pv_ground_limited, pv_space_limited = false, false, false
    roof_existing_pv_kw, ground_existing_pv_kw, both_existing_pv_kw = 0.0, 0.0, 0.0
    roof_max_kw, land_max_kw = 1.0e5, 1.0e5

    for pv in s.pvs
        production_factor[pv.name, :] = prodfactor(pv, s.site.latitude, s.site.longitude)
        for location in pvlocations
            if pv.location == location
                pv_to_location[pv.name][location] = 1
            else
                pv_to_location[pv.name][location] = 0
            end
        end

        beyond_existing_kw = pv.max_kw
        if pv.location == "both"
            both_existing_pv_kw += pv.existing_kw
            if !(s.site.roof_squarefeet === nothing) && !(s.site.land_acres === nothing)
                # don"t restrict unless both land_area and roof_area specified,
                # otherwise one of them is "unlimited"
                roof_max_kw = s.site.roof_squarefeet * pv.kw_per_square_foot
                land_max_kw = s.site.land_acres / pv.acres_per_kw
                beyond_existing_kw = min(roof_max_kw + land_max_kw, beyond_existing_kw)
                pv_space_limited = true
            end
        elseif pv.location == "roof"
            roof_existing_pv_kw += pv.existing_kw
            if !(s.site.roof_squarefeet === nothing)
                roof_max_kw = s.site.roof_squarefeet * pv.kw_per_square_foot
                beyond_existing_kw = min(roof_max_kw, beyond_existing_kw)
                pv_roof_limited = true
            end

        elseif pv.location == "ground"
            ground_existing_pv_kw += pv.existing_kw
            if !(s.site.land_acres === nothing)
                land_max_kw = s.site.land_acres / pv.acres_per_kw
                beyond_existing_kw = min(land_max_kw, beyond_existing_kw)
                pv_ground_limited = true
            end
        end

        existing_sizes[pv.name] = pv.existing_kw
        min_sizes[pv.name] = pv.existing_kw + pv.min_kw
        max_sizes[pv.name] = pv.existing_kw + beyond_existing_kw

        update_cost_curve!(pv, pv.name, s.financial,
            cap_cost_slope, segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint
        )

        om_cost_per_kw[pv.name] = pv.om_cost_per_kw
        fillin_techs_by_exportbin(techs_by_exportbin, pv, pv.name)

        if !pv.can_curtail
            push!(techs_no_curtail, pv.name)
        end
    end

    if pv_roof_limited
        maxsize_pv_locations[:roof] = float(roof_existing_pv_kw + roof_max_kw)
    end
    if pv_ground_limited
        maxsize_pv_locations[:ground] = float(ground_existing_pv_kw + land_max_kw)
    end
    if pv_space_limited
        maxsize_pv_locations[:both] = float(both_existing_pv_kw + roof_max_kw + land_max_kw)
    end

    return nothing
end

function setup_wind_inputs(s::AbstractScenario, max_sizes, min_sizes, existing_sizes,
    cap_cost_slope, om_cost_per_kw, production_factor, techs_by_exportbin,
    segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint, techs_no_curtail
    )
    max_sizes["Wind"] = s.wind.max_kw
    min_sizes["Wind"] = s.wind.min_kw
    existing_sizes["Wind"] = 0.0
    update_cost_curve!(s.wind, "Wind", s.financial,
        cap_cost_slope, segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint
    )
    om_cost_per_kw["Wind"] = s.wind.om_cost_per_kw
    production_factor["Wind", :] = prodfactor(s.wind, s.site.latitude, s.site.longitude, s.settings.time_steps_per_hour)
    fillin_techs_by_exportbin(techs_by_exportbin, s.wind, "Wind")
    if !s.wind.can_curtail
        push!(techs_no_curtail, "Wind")
    end
    return nothing
end

function setup_gen_inputs(s::AbstractScenario, max_sizes, min_sizes, existing_sizes,
    cap_cost_slope, om_cost_per_kw, production_factor, techs_by_exportbin,
    segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint, techs_no_curtail
    )
    max_sizes["Generator"] = s.generator.max_kw
    min_sizes["Generator"] = s.generator.existing_kw + s.generator.min_kw
    existing_sizes["Generator"] = s.generator.existing_kw
    update_cost_curve!(s.generator, "Generator", s.financial,
        cap_cost_slope, segmented_techs, n_segs_by_tech, seg_min_size, seg_max_size, seg_yint
    )
    om_cost_per_kw["Generator"] = s.generator.om_cost_per_kw
    production_factor["Generator", :] = prodfactor(s.generator)
    fillin_techs_by_exportbin(techs_by_exportbin, s.generator, "Generator")
    if !s.generator.can_curtail
        push!(techs_no_curtail, "Generator")
    end
    return nothing
end

function setup_existing_boiler_inputs(s::AbstractScenario, max_sizes, min_sizes, existing_sizes, cap_cost_slope, boiler_efficiency)
    max_sizes["ExistingBoiler"] = s.existing_boiler.max_kw
    min_sizes["ExistingBoiler"] = 0.0
    existing_sizes["ExistingBoiler"] = 0.0
    cap_cost_slope["ExistingBoiler"] = 0.0
    boiler_efficiency["ExistingBoiler"] = s.existing_boiler.efficiency
    # om_cost_per_kw["ExistingBoiler"] = 0.0
    return nothing
end

function setup_present_worth_factors(s::AbstractScenario, techs::Techs)

    lvl_factor = Dict(t => 1.0 for t in techs.all)  # default levelization_factor of 1.0
    for (i, tech) in enumerate(techs.pv)  # replace 1.0 with actual PV levelization_factor (only tech with degradation)
        lvl_factor[tech] = levelization_factor(
            s.financial.analysis_years,
            s.financial.elec_cost_escalation_pct,
            s.financial.offtaker_discount_pct,
            s.pvs[i].degradation_pct  # TODO generalize for any tech (not just pvs)
        )
    end

    pwf_e = annuity(
        s.financial.analysis_years,
        s.financial.elec_cost_escalation_pct,
        s.financial.offtaker_discount_pct
    )

    pwf_om = annuity(
        s.financial.analysis_years,
        s.financial.om_cost_escalation_pct,
        s.financial.owner_discount_pct
    )
    pwf_fuel = Dict{String, Float64}()
    for t in techs.fuel_burning
        if t == "ExistingBoiler"
            pwf_fuel["ExistingBoiler"] = annuity(
                s.financial.analysis_years,
                s.financial.boiler_fuel_cost_escalation_pct,
                s.financial.offtaker_discount_pct
            )
        end
    end

    if s.financial.third_party_ownership
        pwf_offtaker = annuity(s.financial.analysis_years, 0.0, s.financial.offtaker_discount_pct)
        pwf_owner = annuity(s.financial.analysis_years, 0.0, s.financial.owner_discount_pct)
        third_party_factor = (pwf_offtaker * (1 - s.financial.offtaker_tax_pct)) /
                           (pwf_owner * (1 - s.financial.owner_tax_pct))
    else
        third_party_factor = 1.0
    end

    return lvl_factor, pwf_e, pwf_om, pwf_fuel, third_party_factor
end

"""
    setup_electric_utility_inputs(s::AbstractScenario)

Define the `time_steps_with_grid` and `time_steps_without_grid` (detministic outage).

NOTE: v1 of the API spliced the critical_loads_kw into the loads_kw during outages but this splicing is no longer needed
now that the constraints are properly applied over `time_steps_with_grid` and `time_steps_without_grid` using loads_kw
and critical_loads_kw respectively.
"""
function setup_electric_utility_inputs(s::AbstractScenario)
    if s.electric_utility.outage_end_time_step > 0 &&
            s.electric_utility.outage_end_time_step > s.electric_utility.outage_start_time_step
        time_steps_without_grid = Int[i for i in range(s.electric_utility.outage_start_time_step,
                                                    stop=s.electric_utility.outage_end_time_step)]
        if s.electric_utility.outage_start_time_step > 1
            time_steps_with_grid = append!(
                Int[i for i in range(1, stop=s.electric_utility.outage_start_time_step - 1)],
                Int[i for i in range(s.electric_utility.outage_end_time_step + 1,
                                     stop=length(s.electric_load.loads_kw))]
            )
        else
            time_steps_with_grid = Int[i for i in range(s.electric_utility.outage_end_time_step + 1,
                                       stop=length(s.electric_load.loads_kw))]
        end
    else
        time_steps_without_grid = Int[]
        time_steps_with_grid = Int[i for i in range(1, stop=length(s.electric_load.loads_kw))]
    end
    return time_steps_with_grid, time_steps_without_grid
end

"""
    adjust_load_profile(s::AbstractScenario, production_factor::DenseAxisArray)

Adjust the (critical_)loads_kw based off of (critical_)loads_kw_is_net
"""
function adjust_load_profile(s::AbstractScenario, production_factor::DenseAxisArray)
    if s.electric_load.loads_kw_is_net
        for pv in s.pvs if pv.existing_kw > 0
            s.electric_load.loads_kw .+= pv.existing_kw * production_factor[pv.name, :].data
        end end
    end

    if s.electric_load.critical_loads_kw_is_net
        for pv in s.pvs if pv.existing_kw > 0
            s.electric_load.critical_loads_kw .+= pv.existing_kw * production_factor[pv.name, :].data
        end end
    end
end

"""
    production_incentives(tech::AbstractTech, financial::Financial)

Intermediate function for building the PBI arrays in REoptInputs
"""
function production_incentives(tech::AbstractTech, financial::Financial)
    pwf_prod_incent = 0.0
    max_prod_incent = 0.0
    max_size_for_prod_incent = 0.0
    production_incentive_rate = 0.0
    T = typeof(tech)
    # TODO should Generator be excluded? (v1 has the PBI inputs for Generator)
    if !(nameof(T) in [:Generator, :Boiler, :Elecchl, :Absorpchl])
        if :degradation_pct in fieldnames(T)  # PV has degradation
            pwf_prod_incent = annuity_escalation(tech.production_incentive_years, -1*tech.degradation_pct,
                                                 financial.owner_discount_pct)
        else
            # prod incentives have zero escalation rate
            pwf_prod_incent = annuity(tech.production_incentive_years, 0, financial.owner_discount_pct)
        end
        max_prod_incent = tech.production_incentive_max_benefit
        max_size_for_prod_incent = tech.production_incentive_max_kw
        production_incentive_rate = tech.production_incentive_per_kwh
    end

    return pwf_prod_incent, max_prod_incent, max_size_for_prod_incent, production_incentive_rate
end

function fillin_techs_by_exportbin(techs_by_exportbin::Dict, tech::AbstractTech, tech_name::String)
    if tech.can_net_meter && :NEM in keys(techs_by_exportbin)
        push!(techs_by_exportbin[:NEM], tech_name)
        if tech.can_export_beyond_nem_limit && :EXC in keys(techs_by_exportbin)
            push!(techs_by_exportbin[:EXC], tech_name)
        end
    end

    if tech.can_wholesale && :WHL in keys(techs_by_exportbin)
        push!(techs_by_exportbin[:WHL], tech_name)
    end
    return nothing
end


"""
Abstract coarse quantizer type. The coarse quantizer is ment to
assign a point to a given number of Voronoi cells in which its
neighbors will be searched.
"""
abstract type AbstractCoarseQuantizer{D,T} end

# Naive coarse quantizer
"""
    NaiveQuantizer{D<:Distances.PreMetric, T<:AbstractFloat}

Coarse quantization structure based on brute force search.
The `vector` fields contains the coarse vectors while `distance`
contains the distance that is used to calculate the distance
from a point to the coarse vectors.
"""
struct NaiveQuantizer{D<:Distances.PreMetric, T<:AbstractFloat} <: AbstractCoarseQuantizer{D,T}
    vectors::Matrix{T}
end

Base.show(io::IO, cq::NaiveQuantizer{D,T}) where {D,T} = begin
    nrows, nclusters = size(cq.vectors)
    print(io, "NaiveQuantizer{$D,$T}, $nrows×$nclusters cluster centres")
end

Base.size(cq::NaiveQuantizer{D,T}) where {D,T} = size(cq.vectors)
Base.size(cq::NaiveQuantizer{D,T}, i::Int) where {D,T} = size(cq.vectors, i)

function coarse_search(cq::NaiveQuantizer{D,T}, point::Vector{T}, w::Int) where {D,T}
    coarse_distances = colwise(D(), cq.vectors, point)
    closest_clusters = sortperm(coarse_distances)[1:w]
    return closest_clusters, coarse_distances[closest_clusters]
end

_closest_cluster_residuals(cq::NaiveQuantizer{D,T},
                           point::Vector{T},
                           closest_clusters::Vector{Int}
                          ) where {D,T} = begin
    return point .- cq.vectors[:, closest_clusters]
end

_get_quantizer_vector(cq::NaiveQuantizer, idx::Int) = cq.vectors[:, idx]

# HNSW coarse quantizer
"""
    HNSWQuantizer{U<:Unsigned, V<:Vector{Vector{T}}, D<:Distances.PreMetric, T<:AbstractFloat}

Coarse quantization structure based on HNSW search structure.
The `hnsw` field contains the coarse vectors.
"""
struct HNSWQuantizer{U, V, D<:Distances.PreMetric, T<:AbstractFloat} <: AbstractCoarseQuantizer{D,T}
    hnsw::HierarchicalNSW{U,T,V,D}
end

Base.show(io::IO, cq::HNSWQuantizer{U,V,D,T}) where {U,V,D,T} = begin
    nrows, nclusters = size(cq)
    print(io, "HNSWQuantizer{$U,$V,$D,$T}, $nrows×$nclusters cluster centres")
end

Base.size(cq::HNSWQuantizer{D,T}) where {D,T} = length(cq.hnsw.data[1]), length(cq.hnsw.data)
Base.size(cq::HNSWQuantizer{D,T}, i::Int) where {D,T} = size(cq)[i]

function coarse_search(cq::HNSWQuantizer{U,V,D,T}, point::Vector{T}, w::Int) where {U,V,D,T}
    closest_clusters, coarse_distances = HNSW.knn_search(cq.hnsw, point, w)
    return Int.(closest_clusters), coarse_distances
end

_closest_cluster_residuals(cq::HNSWQuantizer{U,V,D,T},
                           point::Vector{T},
                           closest_clusters::Vector{Int}
                          ) where {U,V,D,T} = begin
    nclusters = length(closest_clusters)
    residuals = Matrix{T}(undef, size(point,1), nclusters)
    @inbounds @simd for i in eachindex(closest_clusters)
        residuals[:, i] = point - cq.hnsw.data[closest_clusters[i]]
    end
    return residuals
end

_get_quantizer_vector(cq::HNSWQuantizer, idx::Int) = cq.hnsw.data[idx]


using MzXML, Unitful
using MzXML.IntervalSets
using MzCore
using Test

@testset "MzXML" begin
    scans, info = MzXML.load("test32.mzXML")
    @test info[:msModel] == "API 3000"
    @test eltype(scans) == MzXML.MSscan{Float32,Float32}
    @test length(scans) == 2
    scan = scans[1]
    @test scan.polarity == '-'
    @test scan.msLevel == 2
    @test scan.basePeakMz == 321.4
    @test scan.totIonCurrent == 1.637827475594e06
    @test scan.retentionTime == 0.004*u"s"
    @test scans[2].retentionTime == 0.809*u"s"

    # MzCore traits
    @test intensitytype(scans[1]) === Float32
    @test mztype(scans[1]) === Float32
    axmz, axt = limits(scans)
    @test minimum(axmz.val) === 110.9f0
    @test maximum(axmz.val) === 587.6f0
    @test minimum(axt.val) === 0.004*u"s"
    @test maximum(axt.val) === 0.809*u"s"

    scans, info = MzXML.load("test32.mzXML"; timeinterval=0.2u"s" .. Inf*u"s")
    @test length(scans) == 1 && scans[1].retentionTime == 0.809*u"s"

    io = IOBuffer()
    show(io, scan)
    @test occursin("└─basePeak 321.4", String(take!(io)))

    scans, info = MzXML.load("test32.mzXML"; timeinterval=10.2u"s" .. Inf*u"s", timeshift=10u"s")
    @test length(scans) == 1 && scans[1].retentionTime == 10.809*u"s"

    idx = MzXML.index("test32.mzXML")
    @test idx == [1231, 4242]

    scans, info = MzXML.load("test64.mzXML")
    @test eltype(scans) == MzXML.MSscan{Float64,Float64}
    @test intensitytype(scans[1]) === Float64
    @test mztype(scans[1]) === Float64
end


module DDCModelsExample

# Types and functions package exports
export DDCbasic
export gen_data, estimateΠ
export estimate_model, calc_stderr

# Load additional packages
using Parameters
using LinearAlgebra, Distributions, Random
using GalacticOptim, Optim, ForwardDiff

# Include  the source files
include("types.jl")
include("support.jl")
include("models/DDCbasic.jl")
end


module Libxc
using Libxc_jll
const libxc = Libxc_jll.libxc

include("gen/libxc_common.jl")
include("gen/libxc.jl")
include("Functional.jl")
include("evaluate.jl")

const libxc_version = begin
    varray = zeros(Cint, 3)
    xc_version(pointer(varray, 1), pointer(varray, 2), pointer(varray, 3))
    VersionNumber(varray[1], varray[2], varray[3])
end
const libxc_doi = unsafe_string(Libxc.xc_reference_doi())

"""Return the list of available libxc functionals as strings"""
function available_functionals()
    n_xc = xc_number_of_functionals()
    max_string_length = xc_maximum_name_length()

    funcnames = Vector{String}(undef, n_xc)
    for i in 1:n_xc
        funcnames[i] = ' '^(max_string_length + 2)
    end
    xc_available_functional_names(funcnames)

    [Symbol(first(split(funcnames[i], "\0"))) for i in 1:n_xc]
end

export available_functionals
export Functional, evaluate, evaluate!
export is_lda, is_gga, is_mgga, is_hybrid, is_vv10, is_range_separated, is_global_hybrid

end # module


using OffsetArrays

function src1(meqn::Int, mbc::Int, mx::Int, xlower::Float64, dx::Float64,
              q::OffsetArray{Float64,2},
              maux::Int,
              aux::OffsetArray{Float64,2},
              t::Float64, dt::Float64)
#
#    ! Called to update q by solving source term equation
#    ! $q_t = \psi(q)$ over time dt starting at time t.
#    !
#    ! This default version does nothing.
#
#    implicit none
#    integer, intent(in) :: mbc,mx,meqn,maux
#    real(kind=8), intent(in) :: xlower,dx,t,dt
#    real(kind=8), intent(in) ::  aux(maux,1-mbc:mx+mbc)
#    real(kind=8), intent(inout) ::  q(meqn,1-mbc:mx+mbc)

end # subroutine src1


module ApproxInference

const VecI  = AbstractVector # Input  Vector
const VecO  = AbstractVector # Output Vector
const VecB  = AbstractVector # Buffer Vector
const VecIO = AbstractVector # In/Out Vector
const MatI  = AbstractMatrix # Input  Matrix
const MatO  = AbstractMatrix # Output Matrix
const MatB  = AbstractMatrix # Buffer Matrix
const MatIO = AbstractMatrix # In/Out Matrix

import LinearAlgebra: BLAS
import LinearAlgebra: LAPACK

if haskey(ENV, "BLAS_THREAD_NUM")
    BLAS.set_num_threads(parse(Int, ENV["BLAS_THREAD_NUM"]))
else
    BLAS.set_num_threads(4)
end

include("./linalg.jl")
include("./macros.jl")
include("./statistics.jl")

abstract type AbstractObjective <: Function end

macro objective(e::Expr)
    e.head ≡ :struct || error("Objective funtion should be defined as `struct ... end`.")
    @inbounds a = e.args[2]
    if a isa Expr && a.head ≡ :<:
        @inbounds a.args[2] = :AbstractObjective
    else
        @inbounds e.args[2] = Expr(:<:, a, :AbstractObjective)
    end
    return e
end

function optimize!     end
function get_residual! end
function get_jacobian! end

include("./lmpack.jl")
include("./vipack.jl")

optimizer(; nd::Int=1, ny::Int=1, args...) = optimizer(nd, ny; args...)
function optimizer(nd::Int, ny::Int; method::String="lm")
    if method ≡ "lm" || method ≡ "LM" || method ≡ "LevenbergMarquardt" || method ≡ "Levenberg-Marquardt"
        return LevenbergMarquardtOptimizer(nd, ny)
    elseif method ≡ "vi" || method ≡ "VI" || method ≡ "VariationalInference" || method ≡ "Variational-Inference"
        return VariationalInferenceOptimizer(nd, ny)
    end
end

export @objective, optimize!, get_residual!, get_jacobian!, optimizer

end # module


"""
    OneOf(tfms)
    OneOf(tfms, ps)

Apply one of `tfms` selected randomly with probability `ps` each
or uniformly chosen if no `ps` is given.
"""
struct OneOf{T<:Transform, D<:Sampleable} <: Transform
    tfms::Vector{T}
    dist::D
end

function OneOf(tfms::Vector{<:Transform}, ps = ones(length(tfms)) ./ length(tfms))
    @assert length(tfms) == length(ps)
    return OneOf(tfms, Categorical(ps))
end

function getrandstate(oneof::OneOf)
    i = rand(oneof.dist)
    return i, getrandstate(oneof.tfms[i])
end

function apply(oneof::OneOf, item::Item; randstate = getrandstate(oneof))
    i, tfmrandstate = randstate
    return apply(oneof.tfms[i], item; randstate = tfmrandstate)
end

function makebuffer(oneof::OneOf, items)
    return Tuple([makebuffer(tfm, items) for tfm in oneof.tfms])
end

function apply!(bufs, oneof::OneOf, item::Item; randstate = getrandstate(oneof))
    i, tfmrandstate = randstate
    buf = bufs[i]
    return apply!(buf, oneof.tfms[i], item; randstate = tfmrandstate)
end

"""
    Maybe(tfm, p = 0.5) <: Transform

With probability `p`, apply transformation `tfm`.
"""
Maybe(tfm, p = 0.5) = OneOf([tfm, Identity()], [p, 1-p])

struct OneOfProjective{T<:Transform, D<:Sampleable} <: ProjectiveTransform
    tfms::Vector{T}
    dist::D
end

function OneOf(tfms::Vector{<:ProjectiveTransform}, ps = ones(length(tfms)) ./ length(tfms))
    @assert length(tfms) == length(ps)
    return OneOfProjective(tfms, Categorical(ps))
end

function getrandstate(oneof::OneOfProjective)
    i = rand(oneof.dist)
    return i, getrandstate(oneof.tfms[i])
end

function Maybe(tfm::ProjectiveTransform, p = 1/2)
    return OneOf([tfm, Project(IdentityTransformation())], [p, 1-p])
end

function getprojection(oneof::OneOfProjective, bounds; randstate = getrandstate(oneof))
    i, tfmrandstate = randstate
    return getprojection(oneof.tfms[i], bounds; randstate = tfmrandstate)
end

function makebuffer(oneof::OneOfProjective, items)
    return [makebuffer(tfm, items) for tfm in oneof.tfms]
end


# This file is a part of RRRMC.jl. License is MIT: http://github.com/carlobaldassi/RRRMC.jl/LICENCE.md

module TLE

using ExtractMacro
using ..Interface
using ..Common

using ...RRRMC # this is silly but it's required for correct cross-linking in docstrings, apparently

export GraphTLE, GraphTopologicalLocalEntropy, TLEenergies

import ..Interface: energy, delta_energy, neighbors, allΔE,
                    update_cache!, delta_energy_residual,
                    cenergy, distances

mutable struct GraphTLE{M,γT,λT} <: DiscrGraph{Float64}
    N::Int
    Nk::Int
    neighb::Vector{Vector{Int}}
    neighb_jcs::Vector{Vector{Int}}
    neighb_Us::Vector{Vector{UnitRange{Int}}}
    cache1::LocalFields{Int} # replica-center
    cache2::LocalFields{Int} # topological
    ncache::Vector{Int}

    function GraphTLE(N::Integer, neighb::Vector{Vector{Int}}) where {M,γT,λT}
        isa(M, Int) || throw(ArgumentError("invalid parameter M, expected Int, given: $(typeof(M))"))
        M > 2 || throw(ArgumentError("M must be greater than 2, given: $M"))
        isa(γT, Float64) || throw(ArgumentError("invalid parameter γT, expected Float64, given: $(typeof(γT))"))
        N % (M + 1) == 0 || throw(ArgumentError("N must be divisible by M+1, given: N=$N M=$M"))
        Nk = N ÷ (M+1)
        length(neighb) == Nk || throw(ArgumentError("invalid neighb argument, expected length $Nk, given: $(length(neighb))"))
        for (i,ni) in enumerate(neighb)
            all(j->(1 ≤ j ≤ Nk), ni) || throw(ArgumentError("invalid neighb[$i] argument, all elements must be between 1 and $Nk"))
            i ∈ ni && throw(ArgumentError("invalid neighb[$i], contains itself"))
            lni = length(ni)
            for i1 in 1:lni, i2 in (i1+1):lni
                ni[i1] ≠ ni[i2] || throw(ArgumentError("duplicated element $i1 in neighb[$i]"))
            end
        end
        for (i,ni) in enumerate(neighb)
            for i1 in ni
                i ∈ neighb[i1] || throw(ArgumentError("neighb is not symmetrical, $i1 ∈ ∂$i but $i ∉ ∂$i1"))
            end
        end

        neighb_jcs = [[1 + (i1-1) * (M+1) for i1 in ni] for ni in neighb]
        neighb_Us = [[jinterval(i1, M) for i1 in ni] for ni in neighb]

        cache1 = LocalFields{Int}(N)
        cache2 = LocalFields{Int}(N)
        ncache = empty!(Array{Int}(undef, (M+1) * maximum(length.(neighb))))
        return new{M,γT,λT}(N, Nk, neighb, neighb_jcs, neighb_Us, cache1, cache2, ncache)
    end
end

@doc """
    GraphTLE{M,γT,λT}(N::Integer) <: DiscrGraph

An auxiliary `DiscrGraph` used to implement the interactions in the
Local Entropy Ensemble.

It is only useful when implementing other graph types; see [`GraphTopologicalLocalEntropy`](@ref).
""" -> GraphTLE{M,γT,λT}(N::Integer)

GraphTLE(X::GraphTLE{M,oldγ,oldλ}, newγ::Float64, newλ::Float64) where {M,oldγ,oldλ} = GraphTLE{M,newγ,newλ}(X.N, X.neighb)

# @generated function ΔElist{M,γT}(::Type{GraphTLE{M,γT}})
#     Expr(:tuple, ntuple(d->fk(2*(d - 1 - (M-1) >>> 0x1) - iseven(M), γT), M)...)
# end
# lstind(μ̄::Int, M::Int) = (μ̄ + M-1) >>> 0x1 + 1
#
# function getk{M,γT}(X::GraphTLE{M,γT}, μ̄::Int)
#     @inbounds k = ΔElist(GraphTLE{M,γT})[lstind(μ̄, M)]
#     return k
# end

function energy(X::GraphTLE{M,γT,λT}, C::Config) where {M,γT,λT}
    # @assert X.N == C.N
    @extract X : Nk neighb cache1 cache2
    @extract cache1 : lfields1=lfields lfields_last1=lfields_last
    @extract cache2 : lfields2=lfields lfields_last2=lfields_last
    @extract C : s

    n = 0
    j = 0
    for i = 1:Nk
        j += 1
        # @assert j == 1 + (i-1) * (M+1)
        jc = j
        σc = 2s[jc] - 1

        μ = 0
        for k = 2:(M+1)
            j += 1
            # @assert j == k + (i-1) * (M+1)
            σj = 2s[j] - 1
            lfields1[j] = σc * σj
            μ += σj
        end
        f = σc * μ
        lfields1[jc] = f
        n -= f
    end
    cache1.move_last = 0
    fill!(lfields_last1, 0.0)

    r = 0
    fill!(lfields2, 0.0)
    for (i1, ni) in enumerate(neighb)
        jc1 = 1 + (i1-1) * (M+1)
        σc1 = 2s[jc1] - 1
        μ = 0
        for i2 in ni
            jc2 = 1 + (i2-1) * (M+1)
            σc2 = 2s[jc2] - 1
            σc = σc1 * σc2

            for k = 2:(M+1)
                j1 = k + (i1-1) * (M+1)
                j2 = k + (i2-1) * (M+1)
                σj = (2s[j1] - 1) * (2s[j2] - 1)
                lfields2[j1] += σc * σj
                lfields2[j2] += σc * σj
                μ += σc2 * σj
            end
        end
        f = σc1 * μ
        lfields2[jc1] = 2f
        r -= f
    end
    @assert r % 2 == 0
    r ÷= 2
    lfields2 .÷= 2
    # @show n,r
    # @show n * γT, r * λT
    cache2.move_last = 0
    fill!(lfields_last2, 0.0)

    return n * γT + r * λT
end

function kinterval(move::Integer, M::Integer)
    j0 = move - ((move-1) % (M+1))
    j1 = j0 + M
    return j0:j1
end

function jinterval(i::Integer, M::Integer)
    a = (M+1) * (i-1) + 1
    b = (M+1) * i
    return a:b
end

function update_cache!(X::GraphTLE{M,γT,λT}, C::Config, move::Int) where {M,γT,λT}
    # @assert X.N == C.N
    # @assert 1 ≤ move ≤ C.N
    @extract C : N s
    @extract X : Nk neighb neighb_jcs neighb_Us cache1 cache2

    k = mod1(move, M+1)         # replica index
    i = (move-1) ÷ (M+1) + 1    # variable index
    jc = 1 + (i-1) * (M+1)      # corresponding reference
    if k ≠ 1
        jcs = neighb_jcs[i]
        # jcs = [1 + (i1-1) * (M+1) for i1 in neighb[i]]  # other neighboring references
        # js = [k + (i1-1) * (M+1) for i1 in neighb[i]]   # other neighboring variables
    else
        Ux = kinterval(move, M)                         # the whole slice
        # Us = [jinterval(i1, M) for i1 in neighb[i]]     # other neighboring slices
        Us = neighb_Us[i]
    end

    @extract cache1 : lfields1=lfields lfields_last1=lfields_last move_last1=move_last
    @extract cache2 : lfields2=lfields lfields_last2=lfields_last move_last2=move_last
    @assert move_last1 == move_last2
    if move_last1 == move
        @inbounds begin
            #for y in neighbors(X, move)
            if k ≠ 1
                lfields1[jc], lfields_last1[jc] = lfields_last1[jc], lfields1[jc]
                lfields1[move], lfields_last1[move] = lfields_last1[move], lfields1[move]
            else
                for y in Ux
                    lfields1[y], lfields_last1[y] = lfields_last1[y], lfields1[y]
                end
            end
            #lfields1[move] = -lfields1[move]
            #lfields_last1[move] = -lfields_last1[move]
        end

        @inbounds begin
            if k ≠ 1
                lfields2[jc], lfields_last2[jc] = lfields_last2[jc], lfields2[jc]
                lfields2[move], lfields_last2[move] = lfields_last2[move], lfields2[move]
                for jc1 in jcs
                    lfields2[jc1], lfields_last2[jc1] = lfields_last2[jc1], lfields2[jc1]
                    j1 = jc1 - 1 + k
                    lfields2[j1], lfields_last2[j1] = lfields_last2[j1], lfields2[j1]
                end
                # for j in js
                #     lfields2[j], lfields_last2[j] = lfields_last2[j], lfields2[j]
                # end
            else
                for y in Ux
                    lfields2[y], lfields_last2[y] = lfields_last2[y], lfields2[y]
                end
                for u in Us, y in u
                    lfields2[y], lfields_last2[y] = lfields_last2[y], lfields2[y]
                end
            end
        end
        return
    end

    @inbounds begin
        if k ≠ 1
            #jc = 1 + (i-1) * (M+1)
            σx = 2s[move] - 1
            σc = 2s[jc] - 1
            lfc = lfields1[jc]
            lfm = lfields1[move]
            lfields_last1[jc] = lfc
            lfields_last1[move] = lfm
            lfields1[jc] = lfc + 2 * (σc * σx)
            lfields1[move] = -lfm
        else
            for y in Ux
                lfy = lfields1[y]
                lfields_last1[y] = lfy
                lfields1[y] = -lfy
            end
        end
        #lfm = lfields1[move]
        #lfields_last1[move] = lfm
        #lfields1[move] = -lfm
    end
    cache1.move_last = move

    # lfields_bk1 = copy(lfields1)
    # energy(X, C)
    # lfields_bk1 ≠ lfields1 && @show move hcat(lfields1,lfields_bk1) find(lfields1 .≠ lfields_bk1)
    # @assert lfields_bk1 == lfields1

    @inbounds begin
        if k ≠ 1
            σx = 2s[move] - 1
            lfm = lfields2[move]
            lfields_last2[move] = lfm
            lfields2[move] = -lfm

            σc = 2s[jc] - 1
            lfc = lfields2[jc]
            lfields_last2[jc] = lfc
            # for (jc1,j1) in zip(jcs, js)
            for jc1 in jcs
                j1 = jc1 - 1 + k
                lfj1 = lfields2[j1]
                lfc1 = lfields2[jc1]
                lfields_last2[j1] = lfj1
                lfields_last2[jc1] = lfc1
                σj1 = 2s[j1] - 1
                σc1 = 2s[jc1] - 1
                σ = σx * σc * σj1 * σc1
                lfields2[j1] = lfj1 + 2 * σ
                lfields2[jc1] = lfc1 + 2 * σ
                lfc += 2σ
            end
            lfields2[jc] = lfc
        else
            for y in Ux
                lfy = lfields2[y]
                lfields_last2[y] = lfy
                lfields2[y] = -lfy
            end
            σc = 2s[jc] - 1
            for Ux1 in Us
                jc1 = first(Ux1)
                lfc1 = lfields2[jc1]
                lfields_last2[jc1] = lfc1
                σc1 = 2s[jc1] - 1
                for j1 = (jc1+1):last(Ux1)
                    lfj1 = lfields2[j1]
                    lfields_last2[j1] = lfj1
                    σj1 = 2s[j1] - 1
                    σx = 2s[jc+(j1-jc1)] - 1
                    σ = σx * σc * σj1 * σc1
                    lfields2[j1] = lfj1 + 2 * σ
                    lfc1 += 2 * σ
                end
                lfields2[jc1] = lfc1
            end
        end
    end
    cache2.move_last = move

    return
end

@inline function delta_energy(X::GraphTLE{M,γT,λT}, C::Config, move::Int) where {M,γT,λT}
    # @assert X.N == C.N
    # @assert 1 ≤ move ≤ C.N
    @extract X : cache1 cache2
    @extract cache1 : lfields1=lfields
    @extract cache2 : lfields2=lfields

    # @show 2γT * lfields1[move], 2λT * lfields2[move]
    @inbounds Δ = 2γT * lfields1[move] + 2λT * lfields2[move]
    return Δ
end

@inline function neighbors(X::GraphTLE{M}, j::Int) where {M}
    @extract X : neighb ncache
    k = mod1(j, M+1)         # replica index
    i = (j-1) ÷ (M+1) + 1    # variable index
    if k == 1
        resize!(ncache, M + (M+1) * length(neighb[i]))
        ncache[1:M] = j + (1:M)
        for (n,i1) in enumerate(neighb[i])
            ncache[M + jinterval(n, M)] = jinterval(i1, M)
        end
        # return (j+1):(j+M)
    else
        resize!(ncache, 1 + 2 * length(neighb[i]))
        ncache[1] = j - k + 1
        for (n,i1) in enumerate(neighb[i])
            ncache[2n] = 1 + (i1-1) * (M+1)
            ncache[2n+1] = k + (i1-1) * (M+1)
        end
    end
    return ncache
end

function allΔE(X::GraphTLE{M,γT,λT}) where {M,γT,λT}
    @extract X : neighb
    ΔE1 = iseven(M) ? insert!([4*d*γT for d = 0:(M÷2)], 2, 2γT) :
                      [2*(2d-1)*γT for d = 1:((M+1)÷2)]

    mn = M * maximum(length.(neighb))
    ΔE2 = [2*d*λT for d = -mn:mn]
    # @show ΔE1
    # @show ΔE2

    ΔE = sort!(unique(abs.(ΔE1 .+ ΔE2')))
    return tuple(ΔE...)
end

# Replicate an existsing graph

mutable struct GraphTopologicalLocalEntropy{M,γT,λT,G<:AbstractGraph} <: DoubleGraph{DiscrGraph{Float64},Float64}
    N::Int
    Nk::Int
    X0::GraphTLE{M,γT,λT}
    Xc::G
    X1::Vector{G}
    Cc::Config
    C1::Vector{Config}
    function GraphTopologicalLocalEntropy(N::Integer, neighb::Vector{Vector{Int}}, g0::G, Gconstr, args...) where {M,γT,λT,G}
        X0 = GraphTLE{M,γT,λT}(N, neighb)
        Nk = X0.Nk
        X1 = Array{G}(undef, M)
        Xc = g0
        for k = 1:M
            X1[k] = Gconstr(args...)
        end
        Cc = Config(Nk, init=false)
        C1 = [Config(Nk, init=false) for k = 1:M]
        return new{M,γT,λT,G}(N, Nk, X0, Xc, X1, Cc, C1)
    end
end

"""
    GraphTopologicalLocalEntropy(N::Integer, M::Integer, γ::Float64, λ::Float64, β::Float64, Gconstr, args...) <: DoubleGraph

A `DoubleGraph` which implements a "Local Entropy" model, given any other Ising model previously defined.
This simulates a replicated system in which each replica interacts with an extra "reference"
configuration.
This kind of graph can be simulated efficiently with [`rrrMC`](@ref).

`N` is the number of spins, `M` the number of replicas, `γ` the interaction strength of the replicas with the reference,
`λ` is the topological interaction strenght,
`β` the inverse temperature. `GConstr` is the (original) graph constructor, and `args` the
arguments to the contructor.

This is similar to [`GraphLocalEntropy`](@ref RRRMC.GraphLocalEntropy), but with the additional topological interaction.

See also [`TLEenergies`](@ref) and [`cenergy`](@ref).
"""
function GraphTopologicalLocalEntropy(Nk::Integer, M::Integer, γ::Float64, λ::Float64, β::Float64, Gconstr, args...)
    g0 = Gconstr(args...)
    getN(g0) == Nk || throw(ArgumentError("expected $Nk spins, got $(getN(g0))"))
    G = typeof(g0)
    neighb = [Int[neighbors(g0, i)...] for i = 1:Nk]
    return GraphTopologicalLocalEntropy{M,γ/β,λ/β,G}(Nk * (M+1), neighb, g0, Gconstr, args...)
end

function update_cache!(X::GraphTopologicalLocalEntropy{M}, C::Config, move::Int) where {M}
    @extract X : X0 Xc X1 Cc C1
    k = mod1(move, M+1)
    i = (move - 1) ÷ (M+1) + 1

    if k == 1
        #spinflip!(Xc, Cc, i)
        spinflip!(Cc, i) # Xc cache not updated!
    else
        spinflip!(X1[k-1], C1[k-1], i)
    end

    update_cache!(X0, C, move)
end

function energy(X::GraphTopologicalLocalEntropy{M}, C::Config) where {M}
    # @assert X.N == C.N
    @extract X : Nk X0 Xc X1 Cc C1
    @extract C : s

    E = energy(X0, C)

    s1 = Cc.s
    for (i,j) = enumerate(1:(M+1):(1 + (M+1) * (Nk-1)))
        s1[i] = s[j]
    end
    energy(Xc, Cc)

    for k = 1:M
        s1 = C1[k].s
        for (i,j) = enumerate((k+1):(M+1):(k+1 + (M+1) * (Nk-1)))
            s1[i] = s[j]
        end
        E += energy(X1[k], C1[k])
    end

    return E
end

"""
    TLEenergies(X::GraphTopologicalLocalEntropy)

Returns a Vector with the individual energy (as defined by the original model)
of each replica in a [`GraphTopologicalLocalEntropy`](@ref) graph.
"""
function TLEenergies(X::GraphTopologicalLocalEntropy{M}) where {M}
    @extract X : X1 C1

    Es = zeros(M)

    for k = 1:M
        Es[k] = energy(X1[k], C1[k])
    end

    return Es
end

function cenergy(X::GraphTopologicalLocalEntropy{M}) where {M}
    @extract X : Xc Cc
    return energy(Xc, Cc)
end

function delta_energy_residual(X::GraphTopologicalLocalEntropy{M}, C::Config, move::Int) where {M}
    @extract X : X1 C1

    k = mod1(move, M+1)
    k == 1 && return 0.0

    i = (move - 1) ÷ (M+1) + 1

    return delta_energy(X1[k-1], C1[k-1], i)
end

function delta_energy(X::GraphTopologicalLocalEntropy, C::Config, move::Int)
    return delta_energy(X.X0, C, move) +
           delta_energy_residual(X, C, move)
end

# This is type-unstable and inefficient. On the other hand, it is
# basically only written for testing purposes...
function neighbors(X::GraphTopologicalLocalEntropy{M}, i::Int) where {M}
    @extract X : X0 X1
    jts = neighbors(X0, i)

    k = mod1(i, M+1)

    k == 1 && return jts

    i1 = (i - 1) ÷ (M+1) + 1

    is = neighbors(X1[k-1], i1)

    return tuple(jts..., map(i->((i-1)*(M+1)+k), is)...)
end

function distances(X::GraphTopologicalLocalEntropy{M}) where {M}
    @extract X : C1
    # @extract C : s
    # @assert Nk == C.N ÷ (M+1)

    # rs = BitVector[s[(k-1)*(M+1) + (2:(M+1):end)] for k = 1:M]

    # return [sum(s1 .⊻ s2) for s1 in rs, s2 in rs]
    return [sum(C1[k1].s .⊻ C1[k2].s) for k1 = 1:M, k2 = 1:M]
end

end # module


module FFTW

using LinearAlgebra, Reexport, Preferences
@reexport using AbstractFFTs
using Base.Threads

import AbstractFFTs: Plan, ScaledPlan,
                     fft, ifft, bfft, fft!, ifft!, bfft!,
                     plan_fft, plan_ifft, plan_bfft, plan_fft!, plan_ifft!, plan_bfft!,
                     rfft, irfft, brfft, plan_rfft, plan_irfft, plan_brfft,
                     fftshift, ifftshift,
                     rfft_output_size, brfft_output_size,
                     plan_inv, normalization

export dct, idct, dct!, idct!, plan_dct, plan_idct, plan_dct!, plan_idct!

include("providers.jl")

function __init__()
    # If someone is trying to set the provider via the old environment variable, warn them that they
    # should instead use `set_provider!()` instead.
    if haskey(ENV, "JULIA_FFTW_PROVIDER")
        Base.depwarn("JULIA_FFTW_PROVIDER is deprecated; use FFTW.set_provider!() instead", :JULIA_FFTW_PROVIDER)
    end

    # Hook FFTW threads up to our partr runtime
    @static if fftw_provider == "fftw"
        fftw_init_threads()
    end
end

# most FFTW calls other than fftw_execute should be protected by a lock to be thread-safe
const fftwlock = ReentrantLock()

# protect thread-safety of expressions or whole functions by fftwlock:
macro exclusive(ex)
    if Meta.isexpr(ex, :function) || (Meta.isexpr(ex, :(=)) && Meta.isexpr(ex.args[1], :call))
        newbody = quote
            lock(fftwlock)
            try
                $(esc(ex.args[2]))
            finally
                unlock(fftwlock)
                destroy_deferred() # deallocate plans queued to be destroyed while we held the lock
            end
        end
        Expr(:function, esc(ex.args[1]), newbody)
    else
        return quote
            lock(fftwlock)
            try
                $(esc(ex))
            finally
                unlock(fftwlock)
                destroy_deferred() # deallocate plans queued to be destroyed while we held the lock
            end
        end
    end
end

include("fft.jl")
include("dct.jl")

include("precompile.jl")
_precompile_()

end # module


abstract type MatpowerROPFSimpleImaxInput <: AbstractInput end

"""
    read_input(input_type::T, instance_path::String) where T<:Type{MatpowerInput}

Read instance in `instance_path` depending on `input_type`.\n
Return a structure OPFProblems.
"""
function read_input(input_type::T, instance_path::String) where T<:Type{MatpowerROPFSimpleImaxInput}
  data = load_matpower(instance_path)

  # DataStructure and Gridstructure data:
  bus = SortedDict{String, SortedDict{String, Any}}()
  link = SortedDict{Link, SortedDict{String, Any}}()

  bus_id_line=SortedDict{Int, Int}()
  bus_id_name=SortedDict{Int, String}()

  checkfor(data, 2, "mpc.baseMVA")
  baseMVA = data[2,3]

  ## Building bus load and shunt information
  i_debut, i_fin = find_numarray(1, data)
  bustype = data[i_debut:i_fin, 2]                # weather bus is 1:"PQ" (generators are not accounted for) or 2:"PV"
  checkfor(data, i_debut-1, "mpc.bus")
  for i=i_debut:i_fin
    id = i-i_debut+1
    busname = bus_name(id)
    ## Adding MatpowerVolt structure (for each bus)
    if !haskey(bus, busname)
        bus[busname] = SortedDict{String, Any}()
    end
    bus[busname][volt_name()] = MatpowerVolt(busname, id, data[i,13], data[i,12])

    ## Matpower Load
    load = data[i,3] + im*data[i,4]
    if load != 0
      bus[busname]["Load"] = MatpowerLoad(load)
    end

    ## Matpower Shunt
    shunt = data[i,5] + im*data[i,6]
    if shunt != 0
      bus[busname]["Shunt"] = MatpowerROPFShunt(shunt)
    end

    bus_id_line[data[i,1]] = id
    bus_id_name[data[i,1]] = busname
  end

  ## Adding bus generator information
  gen2bus = SortedDict{Int, Int}()
  line2busgen = SortedDict{Int, Tuple{Int, Int}}()

  i_debut, i_fin = find_numarray(i_fin+1, data)
  checkfor(data, i_debut-1, "mpc.gen")

  genind, prevgen = 0, 0
  for i=i_debut:i_fin
    gen2bus[i-i_debut+1] = bus_id_line[data[i,1]]
    busname = bus_id_name[data[i,1]]

    S_min = S_max = 0
    if data[i, 1] == prevgen
      genind += 1
      S_min = bus[busname]["Gen"].power_min
      S_max = bus[busname]["Gen"].power_max
    else
      prevgen = data[i, 1]
      genind = 1
    end
    line2busgen[i-i_debut+1] = (data[i, 1], genind)

    if data[i, 8] > 0 #generator is on
      S_min += data[i,10] + im*data[i,5] #Smin = Pmin + i Qmin
      S_max += data[i,9] + im*data[i,4] #Smax = Pmax + i Qmax
    end
    bus[busname]["Gen"] = MatpowerGenerator(S_min, S_max, -1, [-1],true)
  end

  ## building link information
  i_debut, i_fin = find_numarray(i_fin+1, data)
  checkfor(data, i_debut-1, "mpc.branch")
  for i=i_debut:i_fin
    linkname = Link(bus_id_name[data[i,1]], bus_id_name[data[i,2]])
    if data[i, 11] == 0
      #@warn("$link $(linkname.orig)⟶$(linkname.dest) breaker out of service !")
    else
      rs, xs, bc = data[i,3:5]
      Smax = data[i,6]
      τ, θ = data[i, 9:10]
      if !haskey(link, linkname)
        link[linkname] = SortedDict{String, Any}()
      end

      nb_elem = length(link[linkname])
      link[linkname]["LinkMP_$(nb_elem+1)"] = MatpowerLine_π_Imax(baseMVA, rs, xs, bc, τ, θ, Smax)
    end
  end

  i_debut, i_fin = find_numarray(i_fin+1, data)
  if data[i_debut-1, 1] == "mpc.areas"
    @warn("Not loading mpc.areas data.")
    i_debut = i_fin+1
    i_debut, i_fin = find_numarray(i_fin+1, data)
  end

  bus_line_id = SortedDict([val=>key for (key,val) in bus_id_line])

  ## Adding generator cost information
  checkfor(data, i_debut-1, "mpc.gencost")
  genind, cur_genind = 0, data[i_debut, 1]
  for i=i_debut:i_fin
    buslineid, _ = line2busgen[i-i_debut+1]
    busid = bus_id_line[buslineid]

    busname = "BUS_$busid"

    cost_degree = data[i, 4]
    cost_coeffs = data[i, 5:(5+cost_degree-1)]
    if haskey(bus[busname], "Gen")
      bus[busname]["Gen"].cost_degree = cost_degree
      bus[busname]["Gen"].cost_coeffs = cost_coeffs
    else
      @warn("Gen not found at $busname")
    end
  end

  ## Removing generators from PQ buses
  for (busname, bus_elems) in bus
    i = bus_elems["Volt"].busid
    if bustype[i] == 1
      for (elemid, elem) in bus_elems
        if typeof(elem) == MatpowerGenerator
          delete!(bus_elems, elemid)
        end
      end
    end
  end

  ## Adding null generator to reference bus
  refind = findall(bustype .== 3)
  length(refind) == 1 || warn("/! refind = $refind")
  bus["BUS_$(refind[1])"]["Gen_reference"] = MatpowerGenerator(0,0,3,[0 0 0],true)

  ds = DataSource(bus, link)

  ## Building GridStructure
  node_linksin = node_linksout = SortedDict{String, SortedSet{Link}}()
  node_vars = SortedDict{String, SortedDict{String, Variable}}()
  link_vars = SortedDict{Link, SortedDict{String, Variable}}()
  gs = GridStructure(basecase_scenario_name(), node_linksin, node_linksout)

  node_formulations = SortedDict{String, SortedDict{String, Symbol}}()
  link_formulations = SortedDict{Link, SortedDict{String, Symbol}}()
  mp = MathematicalProgramming(node_formulations, link_formulations, node_vars, link_vars)
  return OPFProblems(basecase_scenario_name()=>Scenario(ds, gs, mp))
end

## Utils functions
function find_numarray(i_start, data)
  i_debut = i_start
  while !isa(data[i_debut, 1], Int)
    i_debut+=1
  end
  i_fin=i_debut
  while !isa(data[i_fin,1], SubString)
    i_fin += 1
  end
  i_debut, i_fin-1
end

checkfor(data, line_ind, name) = (data[line_ind, 1] == name) || error("Expected ", name, " at line ", line_ind, ", got ", data[line_ind,1], " instead.")

function load_matpower(filename)
  instance_name = split(filename, '.')[1]

  touch(instance_name*".temp")
  f = open(filename)
  out = open(instance_name*".temp", "w")

  # removing all ';' at end of lines
  while !eof(f)
    line = readline(f)
    if length(line) > 0 && line[1] != '%' && line[1] != 'f'
      s = split(line, ";")
      println(out, s[1])
    end
  end
  close(f)
  close(out)

  data = DelimitedFiles.readdlm(instance_name*".temp")
  rm(instance_name*".temp")
  return data
end


## Cootes images (from ICAAM project)

immutable CootesDataset <: FaceDataset
end

const COOTES_DATA_DIR = joinpath(Pkg.dir(), "FaceDatasets", "data", "cootes", "data")
const COOTES_IMG_HEIGHT = 480

function load_shape_from_mat(path::AbstractString)
    return matread(path)["annotations"]
end

function load_shapes(::Type{CootesDataset}; count=-1)
    files = sort(filter(x -> endswith(x, ".mat"), readdir(COOTES_DATA_DIR)))
    paths = map(x->joinpath(COOTES_DATA_DIR, x), files)
    n_use = count > 0 ? count : length(paths)
    shapes = Array(Matrix{Float64}, n_use)
    for k=1:n_use
        shape_xy = load_shape_from_mat(paths[k])
        shape_ij = [COOTES_IMG_HEIGHT .- shape_xy[:, 2] shape_xy[:, 1]]
        shapes[k] = shape_ij
    end
    return shapes
end

function load_images(::Type{CootesDataset}; count=-1)
    files = sort(filter(x->endswith(x, ".bmp"), readdir(COOTES_DATA_DIR)))
    paths = map(x->joinpath(COOTES_DATA_DIR, x), files)
    n_use = count > 0 ? count : length(paths)
    imgs = Array(Array{Float64, 3}, n_use)
    for i=1:n_use
        img_rgb = load(paths[i])
        img_rgb = convert(Image{RGB{Float64}}, img_rgb)  # ensure 3-channel RGB
        imgs[i] = convert(Array, separate(img_rgb))
    end
    return imgs
end


roomTiles(layer::Layer, room::Maple.Room) = layerName(layer) == "fgTiles" ? room.fgTiles : room.bgTiles

function validTiles(layer::String, removeTemplate::Bool=true)
    meta = layer == "fgTiles" ? fgTilerMeta : bgTilerMeta

    res = collect(keys(meta.paths))
    push!(res, '0')

    if removeTemplate
        index = findfirst(isequal('z'), res)
        if index !== nothing
            deleteat!(res, index)
        end
    end

    sort!(res)

    return res
end

validTiles(layer::Layer, removeTemplate::Bool=true) = validTiles(layerName(layer), removeTemplate)

function tileNames(name::String)
    meta = name == "fgTiles" ? fgTilerMeta : bgTilerMeta
    res = Dict{Any, Any}(
        '0' => "Air",
        "Air" => '0'
    )

    for (id, path) in meta.paths
        name = humanizeVariableName(splitdir(path)[2])

        if startswith(name, "Bg ")
            name = name[4:end]
        end

        res[name] = id
        res[id] = name
    end

    return res
end

tileNames(layer::Layer) = tileNames(layerName(layer))

const nodeType = Tuple{Number, Number}
const edgeType = Tuple{nodeType, nodeType}

function shrinkMatrix(m::Array{Bool, 2})
    h, w = size(m)

    lx, ly = h, w
    hx, hy = 1, 1

    for i in 1:h, j in 1:w
        if m[i, j]
            lx = min(lx, j)
            ly = min(ly, i)

            hx = max(hx, j)
            hy = max(hy, i)
        end
    end

    return m[ly:hy, lx:hx], lx, ly
end

function nodesToBrushPixels(nodes::Array{nodeType, 1})
    # Take one pass through to make the output as small as possible
    # Lowest and highest
    lx = ly = typemax(Int)
    hx = hy = typemin(Int)

    for node in nodes
        x, y = node

        lx = min(lx, x)
        ly = min(ly, y)

        hx = max(hx, x)
        hy = max(hy, y)
    end

    w = hx - lx + 1
    h = hy - ly + 1

    # Generate the output
    res = fill(false, (h, w))

    for node in nodes
        x, y = node

        res[y - ly + 1, x - lx + 1] = true
    end

    return res, lx, ly
end

function brushEdges(a::AbstractArray{Bool})
    res = edgeType[]
    h, w = size(a)

    for y in 1:h, x in 1:w
        # Add edges where there are no adjacent value
        if a[y, x]
            if !get(a, (y - 1, x), false)
                push!(res, ((x + 1, y), (x, y)))
            end

            if !get(a, (y + 1, x), false)
                push!(res, ((x, y + 1), (x + 1, y + 1)))
            end

            if !get(a, (y, x - 1), false)
                push!(res, ((x, y + 1), (x, y)))
            end

            if !get(a, (y, x + 1), false)
                push!(res, ((x + 1, y), (x + 1, y + 1)))
            end
        end
    end

    return res
end

function connectEdges!(edges::Array{edgeType, 1})
    res = []

    while length(edges) > 0
        changed = true

        tail, head = pop!(edges)
        nodes = [tail, head]

        while changed
            changed = false

            for (i, edge) in enumerate(edges)
                tail, head = edge

                tailc = tail == nodes[end]
                headc = head == nodes[end]

                if tailc || headc
                    push!(nodes, tailc ? head : tail)
                    deleteat!(edges, i)

                    changed = true

                    break
                end
            end
        end

        push!(res, nodes)
    end

    return res
end

mutable struct Brush
    name::String
    pixels::AbstractArray{Bool}
    offset::Tuple{Number, Number}
    nodes::Array{Array{nodeType, 1}, 1}
    rotation::Number

    Brush(name::String, pixels::AbstractArray{Bool}, offset::Tuple{Number, Number}=(1, 1)) = new(name, pixels, offset, connectEdges!(brushEdges(pixels)), 0)
    Brush(name::String, pixels::Array{T, 2}, offset::Tuple{Number, Number}=(1, 1)) where T <: Integer = Brush(name, pixels .!= 0, offset)
end

function rotationOffset(ctx::CairoContext, brush::Brush)
    bh, bw = size(brush.pixels)
    rotation = brush.rotation

    if rotation == 0
        translate(ctx, 0, 0)

    elseif rotation == 1
        translate(ctx, 0, -bh + 1)

    elseif rotation == 2
        translate(ctx, -bw + 1, -bh + 1)

    elseif rotation == 3
        translate(ctx, -bw + 1, 0)
    end
end

function drawBrush(brush::Brush, layer::Layer, x::Number, y::Number, thickness::Int=2)
    ctx = getSurfaceContext(layer.surface)

    if ctx.ptr != C_NULL
        ox, oy = brush.offset

        Cairo.save(ctx)

        scale(ctx, 8, 8)
        translate(ctx, x + ox - 1.5, y + oy - 1.5)
        rotate(ctx, brush.rotation * pi / 2)
        translate(ctx, -1.5, -1.5)
        rotationOffset(ctx, brush)

        for nodes in deepcopy.(brush.nodes)
            # Fixes weird off by one error in Cairo paths
            # 1 / 8 is one "pixel", taking into account the (8, 8) scale
            # This should be good enough, but might look bad at non default stroke thickness

            extend = ceil(thickness / 2)
            nodes[end] = nodes[end] .+ (extend / 8, extend / 8) .* sign.(nodes[end] .- nodes[end - 1])
            drawLines(ctx, nodes, colors.brush_bc, filled=true, fc=colors.brush_fc, thickness=thickness)
        end

        restore(ctx)
    end
end

function applyBrush!(brush::Brush, tiles::Maple.Tiles, material::Char, x::Number, y::Number)
    brushPixels = rotr90(brush.pixels, brush.rotation)
    tilesData = tiles.data

    bh, bw = size(brushPixels)
    th, tw = size(tilesData)
    ox, oy = brush.offset

    for j in 1:bh, i in 1:bw
        ty, tx = j + y + oy - 2, i + x + ox - 2

        # Manual slice assignment
        # Brushes can be applied "off screen"
        if 1 <= ty <= th && 1 <= tx <= tw
            if brushPixels[j, i]
                tilesData[ty, tx] = material
            end
        end
    end
end

function applyBrush!(brush::Brush, tiles::Maple.Tiles, materials::Array{Char, 2}, x::Number, y::Number)
    brushPixels = rotr90(brush.pixels, brush.rotation)
    #materials = rotr90(materials, brush.rotation)
    tilesData = tiles.data

    bh, bw = size(brushPixels)
    th, tw = size(tilesData)
    ox, oy = brush.offset

    for j in 1:bh, i in 1:bw
        ty, tx = j + y - oy, i + x - ox

        # Manual slice assignment
        # Brushes can be applied "off screen"
        if 1 <= ty <= th && 1 <= tx <= tw
            if brushPixels[j, i]
                tilesData[ty, tx] = materials[j, i]
            end
        end
    end
end

function findFill(tiles::Array{Char, 2}, x::Number, y::Number)
    target = tiles[y, x]
    stack = Tuple{Number, Number}[(x, y)]

    h, w = size(tiles)

    res = fill(false, (h, w))

    while length(stack) > 0
        tx, ty = pop!(stack)

        if 1 <=tx <= w && 1 <= ty <= h && !res[ty, tx]
            if target == tiles[ty, tx]
                res[ty, tx] = true

                push!(stack, (tx - 1, ty))
                push!(stack, (tx, ty - 1))
                push!(stack, (tx + 1, ty))
                push!(stack, (tx, ty + 1))
            end
        end
    end

    return res
end