#=
 TODOs:

    - get quadratic objective functions

=#

__precompile__()
module LinQuadOptInterface

using MathOptInterface
using MathOptInterface.Utilities

const MOI = MathOptInterface
const MOIU = MathOptInterface.Utilities

# functions
const Linear = MOI.ScalarAffineFunction{Float64}
const Quad   = MOI.ScalarQuadraticFunction{Float64}
const SinVar = MOI.SingleVariable
const VecVar = MOI.VectorOfVariables
const VecLin = MOI.VectorAffineFunction{Float64}
# sets
const LE = MOI.LessThan{Float64}
const GE = MOI.GreaterThan{Float64}
const EQ = MOI.EqualTo{Float64}
const IV = MOI.Interval{Float64}
const SOS1 = MOI.SOS1{Float64}
const SOS2 = MOI.SOS2{Float64}
# constraint references
const CI{F,S} = MOI.ConstraintIndex{F,S}
const LCI{S} = CI{Linear,S}
const VLCI{S} = CI{VecLin,S}
const QCI{S} = CI{Quad,S}
const SVCI{S} = CI{SinVar,S}
const VVCI{S} = CI{VecVar,S}
# variable reference
const VarInd = MOI.VariableIndex

const LinSets = Union{LE, GE, EQ, IV}
const VecLinSets = Union{MOI.Nonnegatives, MOI.Nonpositives, MOI.Zeros}

struct ConstraintMapping
    # rows in constraint matrix
    less_than::Dict{LCI{LE}, Int}
    greater_than::Dict{LCI{GE}, Int}
    equal_to::Dict{LCI{EQ}, Int}
    interval::Dict{LCI{IV}, Int}

    # vectors of rows in constraint matrix
    nonnegatives::Dict{VLCI{MOI.Nonnegatives}, Vector{Int}}
    nonpositives::Dict{VLCI{MOI.Nonpositives}, Vector{Int}}
    zeros::Dict{VLCI{MOI.Zeros}, Vector{Int}}

    # rows in quadratic constraint matrix
    q_less_than::Dict{QCI{LE}, Int}
    q_greater_than::Dict{QCI{GE}, Int}
    q_equal_to::Dict{QCI{EQ}, Int}

    # references to variable
    upper_bound::Dict{SVCI{LE}, VarInd}
    lower_bound::Dict{SVCI{GE}, VarInd}
    fixed_bound::Dict{SVCI{EQ}, VarInd}
    interval_bound::Dict{SVCI{IV}, VarInd}

    # vectors of rows in constraint matrix
    vv_nonnegatives::Dict{VVCI{MOI.Nonnegatives}, Vector{Int}}
    vv_nonpositives::Dict{VVCI{MOI.Nonpositives}, Vector{Int}}
    vv_zeros::Dict{VVCI{MOI.Zeros}, Vector{Int}}

    integer::Dict{SVCI{MOI.Integer}, VarInd}
    #=
     for some reason CPLEX doesn't respect bounds on a binary variable, so we
     should store the previous bounds so that if we delete the binary constraint
     we can revert to the old bounds
    =#
    binary::Dict{SVCI{MOI.ZeroOne}, Tuple{VarInd, Float64, Float64}}
    sos1::Dict{VVCI{SOS1}, Int}
    sos2::Dict{VVCI{SOS2}, Int}
    semicontinuous::Dict{SVCI{MOI.Semicontinuous{Float64}}, VarInd}
    semiinteger::Dict{SVCI{MOI.Semiinteger{Float64}}, VarInd}

end
ConstraintMapping() = ConstraintMapping(
    Dict{LCI{LE}, Int}(),
    Dict{LCI{GE}, Int}(),
    Dict{LCI{EQ}, Int}(),
    Dict{LCI{IV}, Int}(),
    Dict{VLCI{MOI.Nonnegatives}, Vector{Int}}(),
    Dict{VLCI{MOI.Nonpositives}, Vector{Int}}(),
    Dict{VLCI{MOI.Zeros}, Vector{Int}}(),
    Dict{QCI{LE}, Int}(),
    Dict{QCI{GE}, Int}(),
    Dict{QCI{EQ}, Int}(),
    Dict{SVCI{LE}, VarInd}(),
    Dict{SVCI{GE}, VarInd}(),
    Dict{SVCI{EQ}, VarInd}(),
    Dict{SVCI{IV}, VarInd}(),
    Dict{VVCI{MOI.Nonnegatives}, Vector{Int}}(),
    Dict{VVCI{MOI.Nonpositives}, Vector{Int}}(),
    Dict{VVCI{MOI.Zeros}, Vector{Int}}(),
    Dict{SVCI{MOI.Integer}, VarInd}(),
    Dict{SVCI{MOI.ZeroOne}, Tuple{VarInd, Float64, Float64}}(),
    Dict{VVCI{SOS1}, Int}(),
    Dict{VVCI{SOS2}, Int}(),
    Dict{SVCI{MOI.Semicontinuous{Float64}}, VarInd}(),
    Dict{SVCI{MOI.Semiinteger{Float64}}, VarInd}()
)

"""
    shift_references_after_delete_affine!(m, row)

This function updates all of the references in `m`
after we have deleted row `row` in the affine constraint matrix.
"""
function shift_references_after_delete_affine!(m, row)
    for scalar_affine in [
            cmap(m).less_than,
            cmap(m).greater_than,
            cmap(m).equal_to,
            cmap(m).interval
        ]
        for (key, val) in scalar_affine
            if val > row
                scalar_affine[key] -= 1
            end
        end
    end

    for vector_affine in [
            cmap(m).nonnegatives,
            cmap(m).nonpositives,
            cmap(m).zeros,
            cmap(m).vv_nonnegatives,
            cmap(m).vv_nonpositives,
            cmap(m).vv_zeros
        ]
        for (key, vals) in vector_affine
            for (i, val) in enumerate(vals)
                if val > row
                    vector_affine[key][i] -= 1
                end
            end
        end
    end
end

"""
    shift_references_after_delete_quadratic!(m, row)

This function updates all of the references in `m`
after we have deleted row `row` in the quadratic constraint matrix.
"""
function shift_references_after_delete_quadratic!(m, row)
    for scalar_quadratic in [
            cmap(m).q_less_than,
            cmap(m).q_greater_than,
            cmap(m).q_equal_to
        ]
        for (key, val) in scalar_quadratic
            if val > row
                scalar_quadratic[key] -= 1
            end
        end
    end
end

function Base.isempty(map::ConstraintMapping)

    ret = true
    ret = ret && isempty(map.less_than)
    ret = ret && isempty(map.greater_than)
    ret = ret && isempty(map.equal_to)
    ret = ret && isempty(map.interval)
    ret = ret && isempty(map.nonnegatives)
    ret = ret && isempty(map.nonpositives)
    ret = ret && isempty(map.zeros)
    ret = ret && isempty(map.q_greater_than)
    ret = ret && isempty(map.q_greater_than)
    ret = ret && isempty(map.q_equal_to)
    ret = ret && isempty(map.upper_bound)
    ret = ret && isempty(map.lower_bound)
    ret = ret && isempty(map.fixed_bound)
    ret = ret && isempty(map.interval_bound)
    ret = ret && isempty(map.vv_nonnegatives)
    ret = ret && isempty(map.vv_nonpositives)
    ret = ret && isempty(map.vv_zeros)
    ret = ret && isempty(map.integer)
    ret = ret && isempty(map.binary)
    ret = ret && isempty(map.sos1)
    ret = ret && isempty(map.sos2)
    ret = ret && isempty(map.semiinteger)
    ret = ret && isempty(map.semicontinuous)

    return ret
end

macro def(name, definition)
    return quote
        macro $(esc(name))()
            esc($(Expr(:quote, definition)))
        end
    end
end

@enum(ObjectiveType,
      SingleVariableObjective,
      AffineObjective,
      QuadraticObjective)

# Abstract + macro
abstract type LinQuadOptimizer <: MOI.AbstractOptimizer end
@def LinQuadOptimizerBase begin

    inner#::LinQuadOptInterface.LinQuadOptimizer

    name::String

    obj_type::LinQuadOptInterface.ObjectiveType
    single_obj_var::Nullable{MOI.VariableIndex}
    obj_sense::MOI.OptimizationSense

    last_variable_reference::UInt64
    variable_mapping::Dict{MOI.VariableIndex, Int}
    variable_names::Dict{MOI.VariableIndex, String}
    variable_names_rev::Dict{String, MOI.VariableIndex}
    variable_references::Vector{MOI.VariableIndex}

    variable_primal_solution::Vector{Float64}
    variable_dual_solution::Vector{Float64}

    last_constraint_reference::UInt64
    constraint_mapping::LinQuadOptInterface.ConstraintMapping

    constraint_constant::Vector{Float64}
    constraint_primal_solution::Vector{Float64}
    constraint_dual_solution::Vector{Float64}

    qconstraint_primal_solution::Vector{Float64}
    qconstraint_dual_solution::Vector{Float64}

    # TODO(odow): temp hack for constraint names
    constraint_names::Dict{Any, String}
    constraint_names_rev::Dict{String, Any}

    objective_constant::Float64

    termination_status::MOI.TerminationStatusCode
    primal_status::MOI.ResultStatusCode
    dual_status::MOI.ResultStatusCode
    primal_result_count::Int
    dual_result_count::Int

    solvetime::Float64
end

function MOI.isempty(m::LinQuadOptimizer)

    ret = true
    ret = ret && m.name == ""
    ret = ret && m.obj_type == AffineObjective
    ret = ret && isnull(m.single_obj_var)
    ret = ret && m.obj_sense == MOI.MinSense
    ret = ret && m.last_variable_reference == 0
    ret = ret && isempty(m.variable_mapping)
    ret = ret && isempty(m.variable_names)
    ret = ret && isempty(m.variable_names_rev)
    ret = ret && isempty(m.variable_references)
    ret = ret && isempty(m.variable_primal_solution)
    ret = ret && isempty(m.variable_dual_solution)
    ret = ret && m.last_constraint_reference == 0
    ret = ret && isempty(m.constraint_mapping)
    ret = ret && isempty(m.constraint_constant)
    ret = ret && isempty(m.constraint_primal_solution)
    ret = ret && isempty(m.constraint_dual_solution)
    ret = ret && isempty(m.qconstraint_primal_solution)
    ret = ret && isempty(m.qconstraint_dual_solution)
    ret = ret && isempty(m.constraint_names)
    ret = ret && isempty(m.constraint_names_rev)
    ret = ret && m.objective_constant == 0.0
    ret = ret && m.termination_status == MOI.OtherError
    ret = ret && m.primal_status == MOI.UnknownResultStatus
    ret = ret && m.dual_status == MOI.UnknownResultStatus
    ret = ret && m.primal_result_count == 0
    ret = ret && m.dual_result_count == 0
    ret = ret && m.solvetime == 0.0

    return ret
end
function MOI.empty!(m::M, env = nothing) where M<:LinQuadOptimizer
    m.name = ""
    m.inner = LinearQuadraticModel(M,env)

    m.obj_type = AffineObjective
    m.single_obj_var = nothing
    # we assume the default is minimization
    m.obj_sense = MOI.MinSense

    m.last_variable_reference = 0
    m.variable_mapping = Dict{MathOptInterface.VariableIndex, Int}()
    m.variable_names = Dict{MathOptInterface.VariableIndex, String}()
    m.variable_names_rev = Dict{String, MathOptInterface.VariableIndex}()
    m.variable_references = MathOptInterface.VariableIndex[]

    m.variable_primal_solution = Float64[]
    m.variable_dual_solution = Float64[]

    m.last_constraint_reference = 0
    m.constraint_mapping = LinQuadOptInterface.ConstraintMapping()

    m.constraint_constant = Float64[]
    m.constraint_primal_solution = Float64[]
    m.constraint_dual_solution = Float64[]

    m.qconstraint_primal_solution = Float64[]
    m.qconstraint_dual_solution = Float64[]

    m.constraint_names = Dict{Any, String}()
    m.constraint_names_rev = Dict{String, Any}()

    m.objective_constant = 0.0

    m.termination_status = MathOptInterface.OtherError
    m.primal_status = MathOptInterface.UnknownResultStatus
    m.dual_status = MathOptInterface.UnknownResultStatus
    m.primal_result_count = 0
    m.dual_result_count = 0

    m.solvetime = 0.0

    nothing
end

function MOI.get(m::LinQuadOptimizer, ::MOI.Name)
    m.name
end
MOI.canget(m::LinQuadOptimizer, ::MOI.Name) = true
function MOI.set!(m::LinQuadOptimizer, ::MOI.Name, name::String)
    m.name = name
end
MOI.canset(m::LinQuadOptimizer, ::MOI.Name) = true

function MOI.supportsconstraint(m::LinQuadOptimizer, ft::Type{F}, st::Type{S}) where F <: MOI.AbstractFunction where S <: MOI.AbstractSet
    (ft,st) in supported_constraints(m)
end
function MOI.supports(m::LinQuadOptimizer, ::MOI.ObjectiveFunction{F}) where F <: MOI.AbstractFunction
    F in supported_objectives(m)
end

# a useful helper function
function deleteref!(dict::Dict, i::Int, ref)
    for (key, val) in dict
        if val > i
            dict[key] -= 1
        end
    end
    delete!(dict, ref)
end

include("variables.jl")
include("constraints.jl")
include("objective.jl")
include("solve.jl")
include("copy.jl")

include("solver_interface.jl")

include("mockoptimizer.jl")

end


module TreeArrays

using Base: @_inline_meta, @_propagate_inbounds_meta, @pure

using StaticArrays, StructArrays, OffsetArrays
using Coordinates

export
    TreeSize,
    TreeIndex,
    TreeLinearIndex,
    TreeCartesianIndex,
    LeafNode,
    StructLeafNode,
    @StructLeafNode,
    Node,
    DynamicNode,
    DynamicHashNode,
    HashNode,
    TreeView,
    TreeArray,
    StructTreeArray,
    continuousview,
    spotview,
    eachleaf!,
    nleaves

const THREADS_THRESHOLD = 1 << 13

include("utils.jl")
include("MaskedArray.jl")
include("PropertyArray.jl")
include("AbstractNode.jl")
include("LeafNode.jl")
include("Node.jl")
include("HashNode.jl")
include("TreeIndex.jl")
include("TreeView.jl")
include("TreeArray.jl")
include("ContinuousView.jl")
include("iterators.jl")
include("show.jl")

end # module


@testset "MIDI pitches" begin
    @testset "types ok and compact" begin
        @test isbitstype(MidiInterval)
        @test isbitstype(MidiIC)
        @test isbitstype(Pitch{MidiInterval})
        @test isbitstype(Pitch{MidiIC})
    end

    @testset "constructors" begin
        @test midi(3) == midi(3)
        @test midip(60) == midip(60)
        @test midic(15) == midic(3)
        @test midipc(63) == midipc(3)
    end

    @testset "macro constructors" begin
        @test (@midi [1,2,3]) == midi.([1,2,3])
        @test (@midip [1,2,3]) == midip.([1,2,3])
        @test (@midic [1,2,3]) == midic.([1,2,3])
        @test (@midipc [1,2,3]) == midipc.([1,2,3])
    end

    @testset "String conversion" begin
        @test string(midi(3)) == "i3"
        @test string(midip(63)) == "p63"
        @test string(midic(15)) == "ic3"
        @test string(midipc(63)) == "pc3"
    end

    @testset "Int conversion" begin
        @test Int(midi(3)) == 3
        @test Int(midip(63)) == 63
        @test Int(midic(15)) == 3
        @test Int(midipc(63)) == 3
    end

    @testset "hashing" begin
        @test hash(midi(3)) == hash(midi(3))
        @test hash(midip(3)) == hash(midip(3))
        @test hash(midic(3)) == hash(midic(3))
        @test hash(midipc(3)) == hash(midipc(3))
    end

    @testset "interval interface" begin
        @test midi(3) + midi(10) == midi(13)
        @test midi(-3) + midi(4) == midi(1)
        @test midi(3) + midi(-4) == midi(-1)
        @test midi(3) - midi(4) == midi(-1)
        @test -midi(5) == midi(-5)
        @test -midi(7) == midi(0) - midi(7)

        @test zero(MidiInterval) == midi(0)
        @test zero(midi(3)) == midi(0)

        @test midi(2) * 7 == midi(14)
        @test midi(-3) * 4 == midi(-12)
        @test midi(4) * -3 == midi(-12)
        @test 6 * midi(2) == midi(12)
        @test 4 * midi(-4) == midi(-16)
        @test 5 * midi(4) == midi(20)

        @test tomidi(midi(42)) == midi(42)
        @test octave(MidiInterval) == midi(12)
        @test sign(midi(3)) == 1
        @test sign(midi(0)) == 0
        @test sign(midi(-3)) == -1
        @test abs(midi(-3)) == midi(3)

        @test ic(midi(15)) == midic(3)
        @test ic(midi(-3)) == midic(9)
        @test embed(midi(42)) == midi(42)
        @test intervaltype(MidiInterval) == MidiInterval
        @test intervalclasstype(MidiInterval) == MidiIC

        for i in -2:2
            @test isstep(midi(i)) == true
        end
        for i in -13:-3
            @test isstep(midi(i)) == false
        end
        for i in 3:13
            @test isstep(midi(i)) == false
        end

        @test chromsemi(MidiInterval) == midi(1)
    end

    @testset "interval class interface" begin
        @test midic(3) + midic(10) == midic(1)
        @test midic(-3) + midic(4) == midic(1)
        @test midic(3) + midic(-4) == midic(11)
        @test midic(3) - midic(4) == midic(11)
        @test -midic(5) == midic(-5)
        @test -midic(7) == midic(0) - midic(7)

        @test zero(MidiIC) == midic(0)
        @test zero(midic(3)) == midic(0)

        @test midic(2) * 7 == midic(14)
        @test midic(-3) * 4 == midic(-12)
        @test midic(4) * -3 == midic(-12)
        @test 6 * midic(2) == midic(12)
        @test 4 * midic(-4) == midic(-16)
        @test 5 * midic(4) == midic(20)

        @test tomidi(midic(42)) == midic(42)
        @test octave(MidiIC) == midic(0)
        @test sign(midic(3)) == 1
        @test sign(midic(0)) == 0
        @test sign(midic(-3)) == -1
        @test abs(midic(-3)) == midic(3)

        @test ic(midic(15)) == midic(3)
        @test ic(midic(-3)) == midic(9)
        @test embed(midic(3)) == midi(3)
        @test embed(midic(-3)) == midi(9)
        @test intervaltype(MidiIC) == MidiInterval
        @test intervalclasstype(MidiIC) == MidiIC

        for i in -2:2
            @test isstep(midic(i)) == true
        end
        for i in -9:-3
            @test isstep(midic(i)) == false
        end
        for i in 3:9
            @test isstep(midic(i)) == false
        end

        @test chromsemi(MidiIC) == midic(1)
    end

    @testset "pitch interface" begin
        @test topitch(midi(3)) == midip(3)
        @test tointerval(midip(42)) == midi(42)

        @test midip(63) + midi(7) == midip(70)
        @test midip(63) + midi(-3) == midip(60)
        @test midip(63) - midi(7) == midip(56)
        @test midip(67) - midip(61) == midi(6)

        @test pc(midip(63)) == midipc(3)
        @test embed(midip(8)) == midip(8)
    end

    @testset "pitch class interface" begin
        @test topitch(midic(3)) == midipc(3)
        @test tointerval(midipc(42)) == midic(42)

        @test midipc(63) + midic(7) == midipc(70)
        @test midipc(63) + midic(-3) == midipc(60)
        @test midipc(63) - midic(7) == midipc(56)
        @test midipc(67) - midipc(61) == midic(6)

        @test pc(midipc(63)) == midipc(3)
        @test embed(midipc(8)) == midip(8)
    end
end


"""
    Post(opinion, weight, source_agent, published_at)

Provide data structure for posts of agents within agent-based simulation.

# Examples
```julia-repl
julia>Post(0, 0, 1, 1)
Post(0.0, 0.0 , 1, 1, Int[], 0, 0, 0)
```

# Arguments
- `opinion`: opinion value between -1 and 1
- `weight`: weight value for importance of post
- `source_agent`: ID of origin agent of post
- `published_at`: publication time of post
- `seen_by`: List of agents that saw it in their feed
- `like_count`: Number of post likes
- `dislike_count`: Number of post dislikes
- `share_count`: Number of post shares

See also: [Agent](@ref), [update_feed!](@ref)
"""
mutable struct Post
    opinion::Float64
    weight::Float64
    source_agent::Int64
    published_at::Int64
    seen_by::Array{Int64, 1}
    like_count::Int64
    dislike_count::Int64
    share_count::Int64

    function Post(
        opinion,
        weight,
        source_agent,
        published_at
    )

        if opinion < -1 || opinion > 1
            error("invalid opinion value")
        end

        if weight < 0
            error("invalid weight value")
        end

        new(
            opinion,
            weight,
            source_agent,
            published_at,
            Int64[],
            0,
            0,
            0
        )
    end
end

Base.:isless(x::Post, y::Post) = x.weight < y.weight

# suppress output of include()
;


mutable struct MyRecur{T,S,I,P}
  cell::T
  state::S
  input::I
  p::P
end
function MyRecur(cell)
  p = initial_params(cell)
  MyRecur(cell, cell.state0, zero(cell.state0), p)
end
function (m::MyRecur)(x, p=m.p)
  batchsize = size(x,2)
  m.input = x
  h′, y = m.cell(m.state, m.input, x, p)
  Inf ∈ h′ && return fill(Inf32, size(y,1), batchsize)
  m.state = h′
  return y
end
# function (m::MyRecur)(x::AbstractArray{T, 3}, p=m.p) where T
#   h = [m(view(x, :, :, i), p) for i in 1:size(x, 3)]
#   sze = size(h[1])
#   reshape(reduce(hcat, h), sze[1], sze[2], length(h))
# end
Base.show(io::IO, m::MyRecur) = print(io, "MyRecur(", m.cell, ")")
initial_params(m::MyRecur) = initial_params(m.cell)
paramlength(m::MyRecur) = length(m.p)
Flux.functor(::Type{<:MyRecur}, m) = (m.p), re -> MyRecur(m.cell, m.state, m.input, re...)
Flux.trainable(m::MyRecur) = (m.p,)
get_bounds(m::MyRecur, T::DataType=eltype(m.state)) = get_bounds(m.cell, T)
Flux.reset!(m::MyRecur) = reset_state!(m, m.p)
@views function reset_state!(m::MyRecur, p=m.p)
  #m.p = p
  pl = length(p)
  state = p[reshape(pl-size(m.cell.state0,1)+1:pl, :, 1)]
  m.state = state
  m.input = reshape(zeros(eltype(m.input), size(state,1)), :, 1)
  nothing
end
# TODO: reset_state! for cell with train_u0=false

function Base.getproperty(m::MyRecur{T,S,I,P}, s::Symbol) where {T,S,I,P}
  if s === :cell
    return getfield(m, :cell)
  elseif s === :state
    return getfield(m, :state)
  elseif s === :input
    return getfield(m, :input)
  elseif s === :p
    return getfield(m, :p)
  else
    return getfield(m, s)
  end
end

ode_solve_kwargs(; abstol=1e-4, reltol=1e-3, save_everystep=false, save_start=false, save_end=true, kwargs...) =
  (abstol=abstol, reltol=reltol, save_everystep=save_everystep, save_start=save_start, save_end=save_end, kwargs...)

struct BCTRNNCell{W,SOLVER,SENSE,PROB,LB,UB,S,P,KW}
  wiring::W

  solver::SOLVER
  sensealg::SENSE
  prob::PROB
  lb::LB
  ub::UB
  state0::S
  p::P
  kwargs::KW

  function BCTRNNCell(wiring, solver, sensealg, prob, lb, ub, state0, p; kwargs...)
    new{typeof(wiring),typeof(solver),typeof(sensealg),typeof(prob),typeof(lb),typeof(ub),typeof(state0),typeof(p), typeof(kwargs)}(
                      wiring, solver, sensealg, prob, lb, ub, state0, p, kwargs)
  end
end

function Base.getproperty(m::BCTRNNCell{W,SOLVER,SENSE,PROB,LB,UB,S,P,KW}, s::Symbol) where {W,SOLVER,SENSE,PROB,LB,UB,S,P,KW}
  if s === :s_in
    return getproperty(getfield(m, :wiring), s)
  elseif s === :n_total
    return getproperty(getfield(m, :wiring), s)
  # elseif s === :n_in
  #   return getproperty(getfield(m, :wiring), s)
  elseif s === :n_out
    return getproperty(getfield(m, :wiring), s)
  elseif s === :wiring
    return getfield(m, :wiring)
  elseif s === :solver
    return getfield(m, :solver)
  elseif s === :sensealg
    return getfield(m, :sensealg)
  elseif s === :prob
    return getfield(m, :prob)
  elseif s === :lb
    return getfield(m, :lb)
  elseif s === :ub
    return getfield(m, :ub)
  elseif s === :state0
    return getfield(m, :state0)
  elseif s === :p
    return getfield(m, :p)
  elseif s === :kwargs
    return getfield(m, :kwargs)
  else
    return getfield(m, s)
  end
end

function BCTRNNCell(wiring, solver, sensealg, prob, lb, ub; kwargs...)
  u0 = prob.u0
  p = prob.p
  p_ode = p[wiring.s_in+1:end]
  state0 = reshape(u0, :, 1)
  θ = vcat(p_ode, u0)
  BCTRNNCell(wiring, solver, sensealg, prob, lb, ub, state0, θ; kwargs...)
end

function BCTRNNCell(wiring, solver, sensealg, odef, u0, tspan, p, lb, ub; mtkize=false, gen_jac=false, kwargs...)
  _prob = ODEProblem{true}(odef, u0, tspan, p)
  p_ode = @view p[wiring.s_in+1:end]
  prob = mtkize_prob(_prob, odef, u0, tspan, p, mtkize, gen_jac)

  # eqs = ModelingToolkit.equations(sys)
  # sts = ModelingToolkit.states(sys)
  # noiseeqs = 0.1f0 .* sts
  # ps = ModelingToolkit.parameters(sys)
  # @named sde = SDESystem(eqs,noiseeqs,ModelingToolkit.get_iv(sys),sts,ps)
  # prob = SDEProblem(sde,u0,tspan,p)

  state0 = reshape(u0, :, 1)
  θ = vcat(p_ode, u0)

  BCTRNNCell(wiring, solver, sensealg, prob, lb, ub, state0, θ; kwargs...)
end

function (m::BCTRNNCell)(h, _last_input, x::AbstractArray, p=m.p)
  # size(h) == (N,1) at the first MTKNODECell invocation. Need to duplicate batchsize times
  nstates = size(h,1)
  batchsize = size(x,2)
  num_reps = batchsize-size(h,2)+1
  u0s = repeat(h, 1, num_reps)
  last_input = repeat(_last_input, 1, num_reps)

  tspan_h = m.prob.tspan
  tspan_o = (tspan_h[2], tspan_h[2]+1)

  p_ode = @view p[1:end-nstates]

  out_idxs = _output_idxs(m.wiring.output_mapping, m.wiring.n_total)

  #sol = solve_ode(m, u0s, tspan_h, p_ode, last_input)
  sol = solve_ode(m, u0s, tspan_h, p_ode, x)
  h′ = sol[:, end, :]

  Inf ∈ h′ && return (fill(Inf32, nstates, batchsize), fill(Inf32, length(out_idxs), batchsize))
  NaN ∈ h′ && return (fill(Inf32, nstates, batchsize), fill(Inf32, length(out_idxs), batchsize))

  return (h′, (@view h′[out_idxs, :]))

  # sol = solve_ode(m, h′, tspan_o, p_ode, x)
  # h′ = sol[:, end, :]

  # return (h′, h′[out_idxs, :])
end

function solve_ode(m, u0s, tspan, p_ode, x)
  batchsize = size(u0s, 2)
  prob = m.prob
  function prob_func(prob,i,repeat)
    u0 = @view u0s[:,i]
    stim = @view x[:,i]
    p = vcat(stim, p_ode)

    # condition(u,t,integrator) = any((u .> 10) .| (u .< -10))
    # function affect!(integrator)
    #   for ii in 1:length(integrator.u)
    #     integrator.u[ii] = clamp(integrator.u[ii], -one(integrator.u[ii]), one(integrator.u[ii]))
    #   end
    # end
    # cb = ContinuousCallback(condition,affect!)

    remake(prob; u0, tspan, p)
  end
  function output_func(sol,i)

    # Zygote.@ignore begin
    #   if i == 1 && length(sol.t) > 1
    #     @show length(sol.t)
    #     fig = plot(xlabel="t", ylabel="step size",title="Adaptive step size")
    #     steps1 = (@view sol.t[2:end]) - (@view sol.t[1:end-1])
    #     plot!(fig, (@view sol.t[2:end]),steps1)
    #     display(fig)
    #   end
    # end
    (sol.retcode != :Success || NaN ∈ sol) && return fill(Inf32, size(u0s,1)), false
    (@view sol[:, end]), false
  end

  kwargs = ode_solve_kwargs(; m.kwargs...)
  ensemble_prob = EnsembleProblem(prob; prob_func, output_func, safetycopy=false) # TODO: safetycopy ???
  sa = Array(solve(ensemble_prob, m.solver, EnsembleThreads(), trajectories=batchsize;
              sensealg=m.sensealg,
              kwargs...
  ))

  sa
end

Base.show(io::IO, m::BCTRNNCell) = print(io, "BCTRNNCell(", m.s_in, ",", m.n_out, ")")
initial_params(m::BCTRNNCell) = m.p
paramlength(m::BCTRNNCell) = length(m.p)
Flux.functor(::Type{<:BCTRNNCell}, m) = (m.p,), re -> BCTRNNCell(m.wiring, m.solver, m.sensealg, m.prob, m.lb, m.ub, m.state0, re...)
Flux.trainable(m::BCTRNNCell) = (m.p,)

function get_bounds(m::BCTRNNCell, T::DataType=eltype(m.p))
  m.lb, m.ub
end

function mtkize_prob(prob, f, u0, tspan, p, mtkize, gen_jac)
  mtkize == false && return prob

  sys = ModelingToolkit.modelingtoolkitize(prob)
  #sys = ModelingToolkit.structural_simplify(sys)

  gen_jac == false && return ODEProblem{true}(sys,u0,tspan,p, tgrad=true)

  jac = eval(ModelingToolkit.generate_jacobian(sys)[2])
  odef = ODEFunction{true}(f, jac=jac)
  ODEProblem{true}(odef,u0,tspan,p, tgrad=true)
end


#Contains all the evaluation stuff
using JLD2
using PyPlot
using FileIO
using Statistics

a= load("Results/10_26_evaluate_pend/1_all.jld2")["RESULT"]
b= load("Results/10_26_evaluate_pend/2_all.jld2")["RESULT"]
c= load("Results/10_26_evaluate_pend/3-4_all.jld2")["RESULT"]

a_all = 4
iters = 500

RE  = zeros(a_all,iters)
A2C = zeros(a_all,iters)
TNPG= zeros(a_all,iters)
TRPO= zeros(a_all,iters)
PPO = zeros(a_all,iters)
TD3 = zeros(a_all,iters)

RE[1,:]   = a[1:6:end]
A2C[1,:]  = a[2:6:end]
TNPG[1,:] = a[3:6:end]
TRPO[1,:] = a[4:6:end]
PPO[1,:]  = a[5:6:end]
TD3[1,:]  = a[6:6:end]

RE[2,:]   = b[1:6:end]
A2C[2,:]  = b[2:6:end]
TNPG[2,:] = b[3:6:end]
TRPO[2,:] = b[4:6:end]
PPO[2,:]  = b[5:6:end]
TD3[2,:]  = b[6:6:end]

RE[3,:]   = c[1:6:3000]
A2C[3,:]  = c[2:6:3000]
TNPG[3,:] = c[3:6:3000]
TRPO[3,:] = c[4:6:3000]
PPO[3,:]  = c[5:6:3000]
TD3[3,:]  = c[6:6:3000]

RE[4,:]   = c[3001:6:end]
A2C[4,:]  = c[3002:6:end]
TNPG[4,:] = c[3003:6:end]
TRPO[4,:] = c[3004:6:end]
PPO[4,:]  = c[3005:6:end]
TD3[4,:]  = c[3006:6:end]

scale = 1
vis   = 0.5
offset = 0

function f_filt(x)
    l = length(x)
    tau = 0.1
    y = zeros(l)
    y[1] = x[1]

    for i = 2:l
        y[i] = (1-tau)*y[i-1] + tau *x[i]
    end

    return y
end

re_mean = f_filt(vec(mean(RE,dims = 1)) .+ offset)
re_std  = f_filt(scale*vec(std(RE,dims=1)))

a2c_mean = f_filt(vec(mean(A2C,dims = 1)) .+ offset)
a2c_std  = f_filt(scale*vec(std(A2C,dims=1)))

tnpg_mean = f_filt(vec(mean(TNPG,dims = 1)) .+ offset)
tnpg_std  = f_filt(scale*vec(std(TNPG,dims=1)))

trpo_mean = f_filt(vec(mean(TRPO,dims = 1)) .+ offset)
trpo_std  = f_filt(scale*vec(std(TRPO,dims=1)))

ppo_mean = f_filt(vec(mean(PPO,dims = 1)) .+ offset)
ppo_std  = f_filt(scale*vec(std(PPO,dims=1)))

td3_mean = f_filt(vec(mean(TD3,dims = 1)) .+ offset)
td3_std  = f_filt(scale*vec(std(TD3,dims=1)))

figure("pend_v0")
clf()
grid(alpha=0.25)
fill_between(1:iters,re_mean .+ re_std, re_mean .- re_std, alpha=vis)
plot(re_mean, label="REINFORCE")
fill_between(1:iters,a2c_mean .+ a2c_std, a2c_mean .- a2c_std, alpha=vis)
plot(a2c_mean, label="A2C")
fill_between(1:iters,tnpg_mean .+ tnpg_std, tnpg_mean .- tnpg_std, alpha=vis)
plot(tnpg_mean, label="TNPG")
fill_between(1:iters,trpo_mean .+ trpo_std, trpo_mean .- trpo_std, alpha=vis)
plot(trpo_mean, label="TRPO")
fill_between(1:iters,ppo_mean .+ ppo_std, ppo_mean .- ppo_std, alpha=vis)
plot(ppo_mean, label="PPO")
fill_between(1:iters,td3_mean .+ td3_std, td3_mean .- td3_std, alpha=vis)
plot(td3_mean, label="TD3")
xlabel("Iterations", fontsize = 16)
ylabel("Cost", fontsize = 16)
#legend()


struct SRTM <: RasterDataSource end

# SRTM Mirror with 5x5 degree tiles
const SRTM_URI = URI(scheme = "https", host = "srtm.csi.cgiar.org", path = "/wp-content/uploads/files/srtm_5x5/TIFF")

function _raster_tile_stem(tile_index::CartesianIndex)
    y, x = tile_index.I
    "srtm_$(lpad(x, 2, '0'))_$(lpad(y, 2, '0'))"
end

_rastername(::Type{SRTM}, tile_index::CartesianIndex{2}) = _raster_tile_stem(tile_index) * ".tif"
_rasterpath(T::Type{SRTM}, tile_index::CartesianIndex{2}) = joinpath(rasterpath(), "SRTM", _rastername(T, tile_index))

_zipname(::Type{SRTM}, tile_index::CartesianIndex{2}) = _raster_tile_stem(tile_index) * ".zip"
_zipurl(T::Type{SRTM}, tile_index::CartesianIndex{2}) = joinpath(SRTM_URI, _zipname(T, tile_index))
_zippath(T::Type{SRTM}, tile_index::CartesianIndex{2}) = joinpath(rasterpath(), "SRTM", "zips", _zipname(T, tile_index))

function _getraster(T::Type{SRTM}, tile_index::CartesianIndex{2})
    raster_path = _rasterpath(T, tile_index)
    if !isfile(raster_path)
        zip_path = _zippath(T, tile_index)
        _maybe_download(_zipurl(T, tile_index), zip_path)
        mkpath(dirname(raster_path))
        raster_name = _rastername(T, tile_index)
        zf = ZipFile.Reader(zip_path)
        write(raster_path, read(_zipfile_to_read(raster_name, zf)))
        close(zf)
    end
    return raster_path
end

# Adapted from https://github.com/centreborelli/srtm4/blob/master/src/srtm4.c#L87-L117
function _wgs84_to_tile_index(x, y)
    y = clamp(y, -60, 60)
    # tiles longitude indexes go from 1 to 72,
    # covering the range from -180 to +180
    tile_x = (1 + floor(Int, (x + 180) / 5)) % 72
    tile_x = tile_x == 0 ? 72 : tile_x

    tile_y = 1 + floor(Int, (60 - y) / 5)
    tile_y = tile_y == 25 ? 24 : tile_y
    CartesianIndex(tile_y, tile_x)
end

function bounds_to_tile_indices(::Type{SRTM}, bounds::NTuple{4,Real})
    minx, miny, maxx, maxy = bounds
    _min = _wgs84_to_tile_index(minx, miny)
    _max = _wgs84_to_tile_index(maxx, maxy)
    _min:_max
end

for op in (:getraster, :rastername, :rasterpath, :zipname, :zipurl, :zippath)
    _op = Symbol('_', op) # Name of internal function
    @eval begin
        # Broadcasting function dispatch
        $_op(T::Type{SRTM}, tile_index::CartesianIndices) = $(_op).(T, tile_index)
        # Bounds to tile indices dispatch
        $_op(T::Type{SRTM}, bounds::NTuple{4,Real}) = $_op(T, bounds_to_tile_indices(T, bounds))

        # Public function definition with key-word arguments
        function $op(T::Type{SRTM}; bounds=nothing, tile_index=nothing)
            if isnothing(bounds) & isnothing(tile_index)
                :op === :getraster || return joinpath(rasterpath(), "SRTM")
                throw(ArgumentError("One of `bounds` or `tile_index` kwarg must be specified"))
            elseif !isnothing(bounds) & !isnothing(tile_index)
                throw(ArgumentError("Only one of `bounds` or `tile_index` should be specified. " *
                                    "found `bounds`=$bounds and `tile_index`=$tile_index"))
            else
                # Call the internal function without key-word arguments
                return $_op(T, isnothing(tile_index) ? bounds : tile_index)
            end
        end
    end
end


abstract type Branch <: Device end

include("./branches/lines.jl")
include("./branches/transformers.jl")
include("./branches/dc_lines.jl")
include("../utils/ybus_calculations.jl")
include("../utils/ptdf_calculations.jl")


struct Object
    surface::Surface
    frontMaterial::Union{Nothing, Material}
    backMaterial::Union{Nothing, Material}

    function Object(surface::Surface, material::Material)
        new(surface, material, nothing)
    end
end

function hit(obj::Object, ray::Ray, range::NamedTuple{(:min, :max), Tuple{T, T}} where T <: AbstractFloat)::Tuple{Bool, AbstractFloat, Union{Nothing, HitPoint}, Union{Nothing, Material}}
    succeeded::Bool, t::AbstractFloat, hp::Union{Nothing, HitPoint} = hit(obj.surface, ray, range)
    if !succeeded
        return false, NaN, nothing, nothing
    end
    local m::Union{Nothing, Material}
    if hp.front
        m = obj.frontMaterial
    else
        m = obj.backMaterial
    end
    m != nothing, t, hp, m
end


# Parser
# ======

# A set of parser parameters.
struct ParserParameters
    delim::UInt8
    quot::UInt8
    trim::Bool
    skip::Int
    colnames::Union{Vector{Symbol},Nothing}
    chunksize::Int

    function ParserParameters(delim::Char, quot::Char, trim::Bool, skip::Integer, header::Any, chunksize::Integer)
        if delim ∉ ALLOWED_DELIMITERS
            throw(ArgumentError("delimiter $(repr(delim)) is not allowed"))
        elseif quot ∉ ALLOWED_QUOTECHARS
            throw(ArgumentError("quotation character $(repr(quot)) is not allowed"))
        elseif delim == quot
            throw(ArgumentError("delimiter and quotation character cannot be the same character"))
        elseif delim == ' ' && trim
            throw(ArgumentError("delimiting with space and space trimming are exclusive"))
        elseif quot == ' ' && trim
            throw(ArgumentError("quoting with space and space trimming are exclusive"))
        elseif skip < 0
            throw(ArgumentError("skip cannot be negative"))
        elseif chunksize < 0
            throw(ArgumentError("chunk size cannot be negative"))
        elseif chunksize ≥ MAX_TOKEN_START
            throw(ArgumentError("chunk size must be less than $(MAX_TOKEN_START)"))
        end
        if header != nothing
            colnames = Symbol.(collect(header))
        else
            colnames = nothing
        end
        return new(
            UInt8(delim),
            UInt8(quot),
            trim,
            skip,
            colnames,
            chunksize,
        )
    end
end

# Integer
# -------

function fillcolumn!(col::Vector{Int}, nvals::Int, mem::Memory, tokens::Matrix{Token}, c::Int, quot::UInt8)
    @inbounds for i in 1:nvals
        start, length = location(tokens[c,i])
        col[end-nvals+i] = parse_integer(mem, start, length)
    end
    return col
end

function fillcolumn!(col::Vector{Union{Int,Missing}}, nvals::Int, mem::Memory, tokens::Matrix{Token}, c::Int, quot::UInt8)
    @inbounds for i in 1:nvals
        t = tokens[c,i]
        if ismissing(t)
            col[end-nvals+i] = missing
        else
            start, length = location(t)
            col[end-nvals+i] = parse_integer(mem, start, length)
        end
    end
    return col
end

const SAFE_INT_LENGTH = sizeof(string(typemax(Int))) - 1

@inline function parse_integer(mem::Memory, start::Int, length::Int)
    stop = start + length - 1
    if length > SAFE_INT_LENGTH
        # safe but slow fallback
        buf = IOBuffer()
        for i in start:stop
            write(buf, mem[i])
        end
        return parse(Int, String(take!(buf)))
    end
    i = start
    b = mem[i]
    if b == UInt8('-')
        sign = -1
        i += 1
    elseif b == UInt8('+')
        sign = +1
        i += 1
    else
        sign = +1
    end
    n::Int = 0
    while i ≤ stop
        @inbounds b = mem[i]
        n = 10n + (b - UInt8('0'))
        i += 1
    end
    return sign * n
end

# Float
# -----

function fillcolumn!(col::Vector{Float64}, nvals::Int, mem::Memory, tokens::Matrix{Token}, c::Int, quot::UInt8)
    @inbounds for i in 1:nvals
        start, length = location(tokens[c,i])
        col[end-nvals+i] = parse_float(mem, start, length)
    end
    return col
end

function fillcolumn!(col::Vector{Union{Float64,Missing}}, nvals::Int, mem::Memory, tokens::Matrix{Token}, c::Int, quot::UInt8)
    @inbounds for i in 1:nvals
        t = tokens[c,i]
        if ismissing(t)
            col[end-nvals+i] = missing
        else
            start, length = location(tokens[c,i])
            col[end-nvals+i] = parse_float(mem, start, length)
        end
    end
    return col
end

@inline function parse_float(mem::Memory, start::Int, length::Int)
    return ccall(:jl_strtod_c, Cdouble, (Ptr{UInt8}, Ptr{Cvoid}), mem.ptr + start - 1, C_NULL)
end

function fillcolumn!(col::Vector{Bool}, nvals::Int, mem::Memory, tokens::Matrix{Token}, c::Int, quot::UInt8)
    @inbounds for i in 1:nvals
        start, length = location(tokens[c,i])
        col[end-nvals+i] = parse_bool(mem, start, length)
    end
    return col
end

@inline function parse_bool(mem::Memory, start::Int, length::Int)
    c = mem[start]
    return (c == UInt8('f') || c == UInt8('F')) ? false : true
end

# String
# ------

const N_CACHED_STRINGS = 8

function fillcolumn!(col::Vector{String}, nvals::Int, mem::Memory, tokens::Matrix{Token}, c::Int, quot::UInt8)
    cache = StringCache(N_CACHED_STRINGS)
    usecache = true
    @inbounds for i in 1:nvals
        t = tokens[c,i]
        start, length = location(t)
        if kind(t) & QSTRING != 0
            s = qstring(mem, start, length, quot)
        elseif usecache
            s = allocate!(cache, mem.ptr + start - 1, length % UInt64)
        else
            s = unsafe_string(mem.ptr + start - 1, length)
        end
        col[end-nvals+i] = s
        if usecache && i % 4096 == 0 && 10 * cache.stats.hit < cache.stats.hit + cache.stats.miss
            # stop using cache because the cache hit rate is too low
            usecache = false
        end
    end
    if !usecache
        @debug "Cache is turned off"
    end
    @debug cache
    return col
end

function fillcolumn!(col::Vector{Union{String,Missing}}, nvals::Int, mem::Memory, tokens::Matrix{Token}, c::Int, quot::UInt8)
    cache = StringCache(N_CACHED_STRINGS)
    usecache = true
    @inbounds for i in 1:nvals
        t = tokens[c,i]
        if ismissing(t)
            col[end-nvals+i] = missing
        else
            start, length = location(t)
            if kind(t) & QSTRING != 0
                s = qstring(mem, start, length, quot)
            elseif usecache
                s = allocate!(cache, mem.ptr + start - 1, length % UInt64)
            else
                s = unsafe_string(mem.ptr + start - 1, length)
            end
            col[end-nvals+i] = s
            if usecache && i % 4096 == 0 && 10 * cache.stats.hit < cache.stats.hit + cache.stats.miss
                # stop using cache because the cache hit rate is too low
                usecache = false
            end
        end
    end
    if !usecache
        @debug "Cache is turned off"
    end
    @debug cache
    return col
end

function qstring(mem::Memory, start::Int, length::Int, quot::UInt8)
    i = start
    skip = false
    buf = IOBuffer()
    while i < start + length
        if skip
            skip = false
        else
            x = mem[i]
            write(buf, x)
            skip = x == quot
        end
        i += 1
    end
    return String(take!(buf))
end

# Date
# ----

const DATE_REGEX = r"^\d{4}-\d{2}-\d{2}$"

function parse_date(s::String)
    return Date(s, dateformat"y-m-d")
end

function parse_date(col::Vector{String})
    out = Vector{Date}(undef, length(col))
    for i in 1:length(col)
        out[i] = parse_date(col[i])
    end
    return out
end

function parse_date(col::Vector{Union{String,Missing}})
    out = Vector{Union{String,Missing}}(undef, length(col))
    for i in 1:length(col)
        x = col[i]
        if x === missing
            out[i] = missing
        else
            out[i] = parse_date(x)
        end
    end
    return out
end

function is_date_like(col::Vector{<:Union{String,Missing}})
    i = 1
    n = 0
    while i ≤ length(col) && n < 3
        x = col[i]
        if x isa String
            if !occursin(DATE_REGEX, x)
                return false
            end
            n += 1
        end
        i += 1
    end
    return n > 0
end

# DateTime
# --------

const DATETIME_REGEX = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(:?\.\d+)?$"

function parse_datetime(s::String)
    return DateTime(s, dateformat"y-m-dTH:M:S.s")
end

function is_datetime_like(col::Vector{<:Union{String,Missing}})
    # Check if the first three strings (if any) are datetime-like.
    i = 1
    n = 0
    while i ≤ length(col) && n < 3
        x = col[i]
        if x isa String
            if !occursin(DATETIME_REGEX, x)
                return false
            end
            n += 1
        end
        i += 1
    end
    return n > 0
end

function parse_datetime(col::Vector{String})
    out = Vector{DateTime}(undef, length(col))
    for i in 1:length(col)
        out[i] = parse_datetime(col[i])
    end
    return out
end

function parse_datetime(col::Vector{Union{String,Missing}})
    out = Vector{Union{DateTime,Missing}}(undef, length(col))
    for i in 1:length(col)
        x = col[i]
        if x === missing
            out[i] = missing
        else
            out[i] = parse_datetime(x)
        end
    end
    return out
end


include("..\\Vehicle\\Spacecraft.jl")
include("..\\solver\\GlobalDictionaries.jl")
include("..\\Environment\\LoadEnvironment.jl")
include("SimulationOptions.jl")
include("Propagator.jl")
import DifferentialEquations: solve, ODEProblem

#A function that executes the a single vehicle simulation. Defines the interface to the users
function sim(vehicle::Type{<:AbstractSpacecraft}, env::Environment, opt::SimulationOptions)

    #Get solver and simulation time  from Simulation options
    solver = ODEALG[opt.ODE_SolverType]
    SimulationTime = opt.SimulationTime
    ####################################################
    #Get EOM (ODEProblem) from the environment structure
    eom = EOM(vehicle, env)
    prob = ODEProblem(eom, vehicle.InitialState, (0, SimulationTime))

    ####################################################
    #Get callbacks from the vehicle systems  (vehiclesystems)

    ####################################################
    #Solve the resulting differential equations
    error("not yet implemented")
    return solve(prob, solver, callback = vehiclesystems, reltol=1e-8, abstol=1e-8)
    ####################################################
end


module Libxc
using Libdl

# Load in `deps.jl`, complaining if it does not exist
const depsjl_path = joinpath(@__DIR__, "..", "deps", "deps.jl")
if !isfile(depsjl_path)
    error("Libxc not installed properly, run Pkg.build(\"Libxc\"), restart Julia and try again")
end
include(depsjl_path)

# Module initialization function
function __init__()
    # Always check your dependencies from `deps.jl`
    check_deps()
end

include("version.jl")
include("xc.jl")
include("functionals_info.jl")

export Functional
include("Functional.jl")

export evaluate_lda!
export evaluate_gga!
include("evaluate.jl")

end # module


# Modelo de separador de corrente de 4 saidas
type Splitter4
	Splitter4()=begin
		new(
			fill(Fracao(Dict{Symbol,Any}(
				:Brief=>"Fracao de massa"
			)),(3)),
			Corrente(Dict{Symbol,Any}(
				:Symbol=>"_{in}",
				:PosX=>1,
				:PosY=>0.5
			)),
			Corrente(Dict{Symbol,Any}(
				:Symbol=>"_{out1}",
				:PosX=>0,
				:PosY=>0.19
			)),
			Corrente(Dict{Symbol,Any}(
				:Symbol=>"_{out2}",
				:PosX=>0,
				:PosY=>0.4
			)),
			Corrente(Dict{Symbol,Any}(
				:Symbol=>"_{out3}",
				:PosX=>0,
				:PosY=>0.62
			)),
			Corrente(Dict{Symbol,Any}(
				:Symbol=>"_{out4}",
				:PosX=>0,
				:PosY=>0.85
			)),
			[
				:(Fout1.P = Fin.P),
				:(Fout1.T = Fin.T),
				:(Fout1.S = Fin.S),
				:(Fout1.H = Fin.H),
				:(Fout2.P = Fin.P),
				:(Fout2.T = Fin.T),
				:(Fout2.S = Fin.S),
				:(Fout2.H = Fin.H),
				:(Fout3.P = Fin.P),
				:(Fout3.T = Fin.T),
				:(Fout3.S = Fin.S),
				:(Fout3.H = Fin.H),
				:(Fout4.P = Fin.P),
				:(Fout4.T = Fin.T),
				:(Fout4.S = Fin.S),
				:(Fout4.H = Fin.H),
				:(Fout1.F = Fin.F * y(1)),
				:(Fout2.F = Fin.F * y(2)),
				:(Fout3.F = Fin.F * y(3)),
				:(Fout4.F = Fin.F - Fout1.F - Fout2.F - Fout3.F),
			],
			[
				"","","","","","","","","","","","","","","","","","","","",
			],
			[:y,:Fin,:Fout1,:Fout2,:Fout3,:Fout4,]
		)
	end
	y::Array{Fracao}
	Fin::Corrente
	Fout1::Corrente
	Fout2::Corrente
	Fout3::Corrente
	Fout4::Corrente
	equations::Array{Expr,1}
	equationNames::Array{String,1}
	variables::Array{Symbol,1}
	attributes::Dict{Symbol,Any}
end
export Splitter4
function setEquationFlow(in::Splitter4)
	addEquation(1)
	addEquation(2)
	addEquation(3)
	addEquation(4)
	addEquation(5)
	addEquation(6)
	addEquation(7)
	addEquation(8)
	addEquation(9)
	addEquation(10)
	addEquation(11)
	addEquation(12)
	addEquation(13)
	addEquation(14)
	addEquation(15)
	addEquation(16)
	addEquation(17)
	addEquation(18)
	addEquation(19)
	#Fout4.F = Fin.F * y(4);
	addEquation(20)
end
function atributes(in::Splitter4,_::Dict{Symbol,Any})
	fields::Dict{Symbol,Any}=Dict{Symbol,Any}()
	fields[:Pallete]=true
	fields[:Icon]="icon/splitter4"
	drive!(fields,_)
	return fields
end
Splitter4(_::Dict{Symbol,Any})=begin
	newModel=Splitter4()
	newModel.attributes=atributes(newModel,_)
	newModel
end


function event_occurred(obj::FieldAbstractDetector, arg0::FieldSpacecraftState, arg1::jboolean)
    return jcall(obj, "eventOccurred", Action, (FieldSpacecraftState, jboolean), arg0, arg1)
end

function g(obj::FieldAbstractDetector, arg0::FieldSpacecraftState)
    return jcall(obj, "g", RealFieldElement, (FieldSpacecraftState,), arg0)
end

function get_handler(obj::FieldAbstractDetector)
    return jcall(obj, "getHandler", FieldEventHandler, ())
end

function get_max_check_interval(obj::FieldAbstractDetector)
    return jcall(obj, "getMaxCheckInterval", RealFieldElement, ())
end

function get_max_iteration_count(obj::FieldAbstractDetector)
    return jcall(obj, "getMaxIterationCount", jint, ())
end

function get_threshold(obj::FieldAbstractDetector)
    return jcall(obj, "getThreshold", RealFieldElement, ())
end

function init(obj::FieldAbstractDetector, arg0::FieldSpacecraftState, arg1::FieldAbsoluteDate)
    return jcall(obj, "init", void, (FieldSpacecraftState, FieldAbsoluteDate), arg0, arg1)
end

function is_forward(obj::FieldAbstractDetector)
    return jcall(obj, "isForward", jboolean, ())
end

function reset_state(obj::FieldAbstractDetector, arg0::FieldSpacecraftState)
    return jcall(obj, "resetState", FieldSpacecraftState, (FieldSpacecraftState,), arg0)
end

function with_handler(obj::FieldAbstractDetector, arg0::FieldEventHandler)
    return jcall(obj, "withHandler", FieldEventDetector, (FieldEventHandler,), arg0)
end

function with_max_check(obj::FieldAbstractDetector, arg0::RealFieldElement)
    return jcall(obj, "withMaxCheck", FieldEventDetector, (RealFieldElement,), arg0)
end

function with_max_iter(obj::FieldAbstractDetector, arg0::jint)
    return jcall(obj, "withMaxIter", FieldEventDetector, (jint,), arg0)
end

function with_threshold(obj::FieldAbstractDetector, arg0::RealFieldElement)
    return jcall(obj, "withThreshold", FieldEventDetector, (RealFieldElement,), arg0)
end


"""
Un object de classe Solution encapsule les éléments d'une solution

Args:
    inst (Instance): instance du problème
    planes (Vector{Plane}): Avions
    x (Vector{Int}): Dates d'atterrissage des avions
    costs (Vector{Float64}): Coût de chaque avion
    cost (Float64): Coût total de la solution
    timing_algo_solver
"""
mutable struct Solution
    inst::Instance
    planes::Vector{Plane}
    x::Vector{Int}
    costs::Vector{Float64}
    cost::Float64
    timing_algo_solver     # :earliest, :lp, ...
    # lpTimingSolver       # solver lp si c'est :lp qui est choisi
    # earliestTimingSolver # solver dp si c'est :dp qui est choisi

    # l'attribut contiendra l'objet solver dont le type dépend du symbole
    # contenu dans l'attribut timing_algo_solver précédent
    solver # e.g. objet de type LpTimingSolver ou EarliestimingSolver

    # Le constructeur peut prendre
    # - soit une instance qui sert à initialiser la solution
    # - soit un autre objet solution pour en faire une copie
    #
    function Solution(inst::Instance; update::Bool=true, algo=:ARGS)
        this = new()
        this.inst = inst
        this.planes = copy(inst.planes)
        # @show inst.nb_planes
        this.x = zeros(Int, inst.nb_planes)
        this.costs = zeros(inst.nb_planes)
        this.cost = typemax(Float64) # valeur Inf pour un Float64)

        # @show algo
        # @show Args.t_algos()
        if algo == :ARGS
            this.timing_algo_solver = Args.get("timing_algo_solver")
        else
            if String(algo) in Args.t_algos()
                this.timing_algo_solver = algo
            else
                error("timing_algo_solver \"$(algo)\" inconnu !")
            end
        end
        # this.solver = :nothing # sera initialisé lors du premier solve!()
        init_solver(this, this.timing_algo_solver)

        ln3("Solution: Construction d'une solution avec $(this.timing_algo_solver)")
        if update
            solve!(this)
        end
        return this
    end

    # ATTENTION la référence au timingSolver est copiées superficiellement !
    function Solution(sol::Solution)
        this = new()
        # On recopie une solution existante
        this.inst = sol.inst
        this.planes = copy(sol.planes)
        this.x = copy(sol.x)
        this.costs = copy(sol.costs)
        this.cost = sol.cost
        this.timing_algo_solver = sol.timing_algo_solver
        this.solver = sol.solver
        # solve!(this)  # ceci est en principe inutile
        return this
    end

    function Solution(instance::Instance, planes::Vector{Plane}, x)
        solution = new(instance, planes, x, zeros(Int, instance.nb_planes), 0,
                       "none", "none")
        update_costs!(solution)
        return solution
    end
end

"""Retourne la solution """
function Solution(inst::Instance, filename::String; update::Bool=true, algo=:ARGS)
    this = Solution(inst, update=false, algo=:ARGS)
    readsol(this, filename)
end

"""Recopie dans la solution *sol* le contenu de la solution *other*."""
function copy!(sol::Solution, other::Solution)
    sol.inst = other.inst
    copyto!(sol.planes, other.planes)
    copyto!(sol.x,      other.x)
    copyto!(sol.costs,  other.costs)
    sol.cost = other.cost  # ceci maintient la cohérence du coût avec la solution
    sol.timing_algo_solver = other.timing_algo_solver
    sol.solver = other.solver # ATTENTION : COPIE SUPERFICIELLE
    return sol   # pratique pour enchainer les appels de méthode (en POO !)
end

"""
Crée et affecte l'attribut solver pour la résolution des dates d'atterrissage.

Le type du solver créée est mémorisé dans l'attribut symbole
``sol.timing_algo_solver``
(par exemple :ealiest, :lp, ...)
"""
function init_solver(sol::Solution, algo::Symbol)
    if algo == :lp
        sol.solver = LpTimingSolver(sol.inst)
    elseif algo == :earliest
        # sol.solver = nothing # pas de TimingSolver (inutile) pour :earliest
        sol.solver = EarliestTimingSolver(sol.inst)
    else
        error("init_solver:  algo invalide : $(algo)")
    end
    sol.timing_algo_solver = algo
end

"""
    current_algo_symbol(sol::Solution)

Return le symbol de l'algo à partir du TimingSolver mémorisé dans la
variable sol.solver
Si l'objet n'est pas un solver, le symbol :earliest est retourné
"""
function current_algo_symbol(sol::Solution)
    if sol.solver !== nothing
        return symbol(sol.solver)
    else
        return :earliest
    end
end

"""
    hash(sol::Solution)

Calcul du hachage d'une solution.
Deux solutions identiques doivent donner le même hachage.
Deux solutions sont considérées comme identique ssi elles ont les avions
dans le même ordre, et avec les mêmes dates d'atterrissage.
HYP: les deux solutions sont à jour (par solve!)
"""
function hash(sol::Solution)
    # on construit une chaine de caractères représentative de la solution
    # (ce qui revient plus ou moins à sérialiser l'objet sol)
    io = IOBuffer()
    for p in sol.planes
        print(io, p.name, ":", round(Int, sol.x[p.id]), "-")
    end
    return Base.hash(String(take!(io)))
end

"""Retourne le tableaux des noms d'avions dans l'ordre de la solution."""
function get_names(sol::Solution)
    # RUBY: @planes.map{| plane |  plane.name}
    map(p::Plane->p.name, sol.planes)
end

"""
Affecte la solution en fonction de l'ordre des noms d'avion définit
en paramètre

Example:
   ``set_from_names!(mysol, [3, 4, 5, 6, 8, 9, 7, 1, 10, 2])``

Attention:
    Couteux O(n^2), mais pratique pour affichage et tests.
"""
function set_from_names!(sol::Solution, names)
    if length(names) != length(sol.planes)
        error("\nERREUR : taille de names $(length(names)) incorrecte " *
              "attendue : $(length(sol.planes))")
    end
    # sol.planes .= get_plane_from_name.(sol.inst, names)
    for i in 1:length(names)
        sol.planes[i] = get_plane_from_name(sol.inst, names[i])
    end
    return sol
end

"""
Trie les avions de l'objet sol selon leur date d'atterrissage croissante.

La solution n'est pas recalculée, mais les vecteurs *x* et *planes* sont
triés de manière cohérente.
"""
function sort!(sol::Solution)
    # Pour cela, on récupère l'ordre correspondant au tri des dates d'atterrissage
    # (le résultat est la permutation correspondant au tri à effectuer)
    permu = sortperm(sol.x)

    # On applique cette permutation aux deux tableaux de notre solution pour les
    # trier dans le même ordre
    sol.x = sol.x[permu]
    sol.planes = sol.planes[permu]
    sol.costs = sol.costs[permu]
end

function Base.show(io::IO, sol::Solution)
    Base.write(io, to_s(sol))
end

"""
Convertit une Solution en chaine

Example:
    ``"cost=123.456  :[p2,p5,p1,...,p10]"``

"""
function to_s(sol::Solution)
    # prc = Args.args[:cost_precision] + 2 # cmt le 30/10/2018 car 7 par défaut
    prc = Args.args[:cost_precision]
    cost = round(sol.cost, digits=prc)

    txt = "cost=" * rpad(cost, 8) * " :" *
    # "[" * sol.planes.map {|plane| plane.name }.join(",") << "]"
    "[" * join(map( p->p.name, sol.planes), ",") * "]"
    return txt
end

function to_s_long(sol::Solution)
    io = IOBuffer()
    prc = Args.args[:cost_precision]
    cost = round(sol.cost, digits=prc)
    viol_cost = get_viol_penality(sol)

    println(io, "# ALP solution version 1.0")
    println(io, "name $(sol.inst.name)  # nb_planes=$(length(sol.planes))")
    if viol_cost != 0.0
        println(io, "# Solution **invalide** ")
        println(io, "viols_cost=$(viol_cost)")
    else
        println(io, "# Solution valide")
    end
    println(io, "timestamp $(Dates.now())")
    println(io, "cost $(cost)")
    println(io, "order [", join([p.name for p in sol.planes], ","), "]")
    println(io)
    println(io, "#       name     t   dt  cost        # comments")
    for i in 1:length(sol.planes)
        p = sol.planes[i]
        print(io, @sprintf("landing %4s ", p.name) )
        print(io, @sprintf(" %4s ", sol.x[i]) )
        print(io, @sprintf("%4s  ", sol.x[i] - p.target) )

        # Ceci est horrible, mais la chaine de formatage de @sprintf ne peut
        # pas contenir de variable !
        #   print(io, @sprintf("% 7.$(prc)f ", sol.costs[i]) ) # CECI PLANTE
        if prc <= 5
            print(io, @sprintf("%9.5f ", sol.costs[i]) )
        else
            print(io, @sprintf("%11.7f ", sol.costs[i]) )
        end

        print(io, "  # ")
        print(io, @sprintf("E=%-4d ", p.lb) )
        print(io, @sprintf("T=%-4d ", p.target) )
        print(io, @sprintf("L=%-4d ", p.ub))
        print(io, "  ")
        print(io, @sprintf(" ep=%3.2f  ", p.ep) )
        print(io, @sprintf(" tp=%3.2f  ", p.tp) )

        # On affiche les temps d'écart et temps attendu entre les trois derniers
        # prédessesseurs pour vérifier le respect des temps d'écart.
        print(io, " sep ")
        isbad = false
        # sep_string = ""
        for j in i-1:-1:max(i-3,1)
            p_prev = sol.planes[j]
            sep_real = sol.x[i]-sol.x[j]
            sep_th = get_sep(sol.inst, p_prev, p)
            # sep_string *= " $sep_real($sep_th)"
            if sep_real < sep_th
                isbad = true
            end
            print(io, " $sep_real($sep_th)")
        end
        if isbad
            # print(io, to_sc(" VIOL "), sep_string)
            println(io, to_sc(" VIOL "))
        else
            # print(io, " -ok- ", sep_string)
            println(io, " -ok- ")
        end
    end
    println(io)
    String(take!(io))
end

"""
Vérifie la faisabilité de la solution par rapport aux dates d'atterrissage
réelle de la solution.

Retourne un coût de pénalité (nul si solution valide) si des contraintes sont
violées.
Ce coût prend en compte :

- l'amplitude des viols de la contrainte de séparation,
- l'amplitude des viols des bornes lb et ub de chaque avion.

Cette méthode suppose que l'information sol.x de la date d'atterrissage
réelle de chaque avion est à jour.

Les viols sont détectés même si l'inégalité triangulaire n'est pas respectée
dans l'instance (instance incorrecte par hypothèse)

Si level est suffisant : affiche message si solution non faisable
"""
function get_viol_penality(sol::Solution)
    total_penality = 0.0
    unit_penality = 1000.0
    SEP_MAX = sep_max(sol.inst)

    ### ASSERT : p1=sol.planes[1] est bien positionné sinon il

    # On commence par prendre en compte la pénalité de placement du premier avion
    # On calcule l'écart (viol sur les bornes) de la date d'atterissage x[1]
    # cet écart est positif sauf en cas de viol.
    p1 = sol.planes[1]
    dt = min(sol.x[1]-p1.lb, p1.ub-sol.x[1])
    if dt < 0
        total_penality += -dt*unit_penality
    end

    for i2 in 2:length(sol.planes)
        p2 = sol.planes[i2]

        # Étape 1 : calcul du viol des temps de séparation pour p2

        # parcours les prédecesseurs en partant de de p2
        for i1 in i2-1:-1:1
            if sol.x[i2] > sol.x[i1] + SEP_MAX
                # tous les temps d'écarts des prédécesseurs sont respectés
                break
            end
            p1 = sol.planes[i1]
            # s_val = sol.inst.sep_mat[p1.kind, p2.kind]
            s_val = get_sep(sol.inst, p1, p2)
            if sol.x[i2] < sol.x[i1] + s_val
                total_penality += unit_penality * (sol.x[i1] + s_val  - sol.x[i2])
                # il faudrait pousser j
                feasible = false
                if lg4()>=4
                    println("ERREUR écart entre type des avions $(p1.name)->$(p2.name)")
                    println("   x1=$(sol.x[i1]) pour $(p1)")
                    println("   x2=$(sol.x[i2]) pour $(p2)")
                    println("   écart réel=$(sol.x[i2]-sol.x[i1]) au lieu de $(s_val)")
                end
            end
        end

        # Étape 2 : calcul du viol de la Upper Bound
        if sol.x[i2] > p2.ub
            total_penality += unit_penality * (sol.x[i2] - p2.ub)
            lg4() && println("UB infaisable pour l'avion $(p2.name)")
        end

        # Étape 3 : calcul du viol de la Lower Bound
        if sol.x[i2] < p2.lb
            total_penality += unit_penality *  (p2.lb - sol.x[i2])
            lg4() && println("LB infaisable pour l'avion $(p2.name)")
        end
    end
    if lg4()>=4 && total_penality==0.0
        println("solution faisable.")
    end
    feasible = (total_penality==0.0)
    if !feasible && lg4()
        println("get_viol_penality: Infaisable : total_penality=", total_penality)
    end
    return total_penality
end

"""
Vérifie la faisabilité dates d'atterrissage x de la solution par rapport
aux contraintes de l'instance.

- Si pas de viol, retourne une chaine vide
- Si viol, retourne un texte avec une ligne par contrainte violée.

Les contraintes testées sont les suivante :

- la date d'atterissage doit entre dans l'intervale permi pour l'avion
- tous les temps de séparation doivent être respertés

Cette méthode suppose que l'information *sol.x* de la date d'atterrissage
réelle de chaque avion est à jour.

Les viols sont détectés même si l'inégalité triangulaire n'est pas respectée
dans l'instance (instance incorrecte par hypothèse).
"""
function get_viol_description(sol::Solution)
    io = IOBuffer()

    nbviols = 0
    for i in 1:length(sol.planes)
        p = sol.planes[i]
        x = sol.x[i] ; lb = p.lb ; ub = p.ub
        if sol.x[i] - p.lb < 0
            println(io, p.name, " atterrit trop tôt  x=$x < lb=$lb")
            nbviols += 1
        end
        if p.ub - sol.x[i] < 0
            println(io, p.name, " atterrit trop tard ub=$ub < x=$x")
            nbviols += 1
        end
    end

    SEP_MAX = sep_max(sol.inst)
    for i2 in 2:length(sol.planes)
        p2 = sol.planes[i2]
        x2 = sol.x[i2]
        # parcours les prédecesseurs en partant de de p2
        for i1 in i2-1:-1:1
            p1 = sol.planes[i1]
            x1 = sol.x[i1]
            # On pourrait ne pas explorer tout si l'on se contentait de détecter
            # une invalidité. Mais on veut afficher **tous** les viols
            #   if x2 > x1 + SEP_MAX
            #       # tous les temps d'écarts des prédécesseurs sont respectés
            #       # (s'il n'y a aucun e viol avant x1)
            #       break
            #   end

            sep = get_sep(sol.inst, p1, p2)
            if x2 < x1 + sep
                print(io,  "$(p1.name)->$(p2.name) : écart insuffisant ")
                print(io,  "x[$(p1.name)]=$x1->x[$(p2.name)]=$x2 => ")
                println(io, "sep($(p1.name),$(p2.name))=$(sol.x[i2]-sol.x[i1]) ",
                "au lieu de ", get_sep(sol.inst, p1, p2) )
                nbviols += 1
            end
        end
    end
    violtxt = String(take!(io))
    return (nbviols, violtxt)
end

"""
Calcul la faisabilité intrinsèque de la solution en fonction de l'ordre
imposé pour les avions.

Ne modifie pas la solution.

N'utilise pas l'attribut `x[i]` de la solution associée au placement
de chaque avion.

Note:
    Si on peut placer les avions au plus tôt en respectant les temps de
    séparation sans violer la borne sup du placement des avions suivants,
    alors la solution est faisable.
"""
function is_feasable(sol::Solution; param_bidon=nothing)
    # ASSUME_TRINEQ = Args.get(:assume_trineq)
    ASSUME_TRINEQ = false # L'inégalité triangulaire n'est pas présupposée

    n = length(sol.planes)
    # x : vecteur des placements au plus tôt des avions
    x = Vector{Int}(undef,n)

    i1 = 1
    p1 = sol.planes[i1]
    x[i1] = p1.lb
    # @assert (x[i1] <= p1.ub), "Instance infaisable pour avion $(p1) !"
    @assert x[i1] <= p1.ub

    for i2 in 2:length(sol.planes)
        p2 = sol.planes[i2]
        x[i2] = max(p2.lb, x[i1] + get_sep(sol.inst, p1, p2))
        if !ASSUME_TRINEQ
            # inégalité triangulaire non vérifiée : on doit vérifier
            # les prédécesseurs antérieurs à p1
            for i_prev in i1-1 : -1 : 1
                p_prev - sol.planes[i_prev]
                this_sep = x[i2] - x[i_prev]
                if this_sep >= MAX_SEP
                    # c'est bon : p2 est forcément suffisamment loin devant
                    # tous ces prédécesseurs
                    break
                end
                # on retarde p2 pour satisfaire cette séparation
                x[i2] = max(x[i2], x[i_prev] + get_sep(sol.inst, p_prev, p2))
            end
        end
        # maintenant que p2 est placé au plus tôt en respectant les contraintes
        # de séparation, on vérifie qu'il n'est pas hors borne supérieure
        if x[i2] > p2.ub
            # if lg1() println("viol x[$(i2)]=$(x[i2]) mais $(p2).ub=$(p2.ub)") end
            return false
        end
    end
    # Tous les avions ont pu être placés au plus tôt.
    # La solution est donc timing-faisable
    return true
end

"""

TODO:
    prévoir paramètre from_index=1 pour permettre un recalcul partiel.

do_update_cost (false par défaut) : si vrai, met à jours les coûts.

Notes:
    - l'option do_update_cost n'est utile que si ce n'est pas déjà fait
      par le TimingSolver.
    - Elle ne semble plus utilisée (sauf par solve_to_earliest!(sol)).
    - Il est conseillé d'appeler explicitement la méthode update_costs!(sol)
      plutot que d'utiliser cette option.
    - Si aucun TimingSolver n'a été spécifié lors de la création de la solution
      alors on utilise la résolution au plus tot.
"""
function solve!(sol::Solution; do_update_cost::Bool=false)
    if sol.solver === nothing
        init_solver(sol, sol.timing_algo_solver)
    end
    solve!(sol.solver, sol)
    if do_update_cost
        update_costs!(sol)
    end
    return sol
end

"""
Mélange la solution et mets à jour (par défaut) les timings

Args:
    do_update (Bool): si vrai, résoud le pb de timing

Note:
    L'option do_update ne semble plus utilisée.
"""
function shuffle!(sol::Solution; do_update::Bool=true)
    Random.shuffle!(sol.planes)
    if do_update
        solve!(sol) # NEW 03/04/2019
    end
    return sol
end

"""Remue légèrement la solution et mets à jour (par défaut) les timings"""
function disturb!(sol::Solution;
                  idx_first=1, idx_last=-1, shift_max=1,
                  nb_shift=10, do_update::Bool=true,
                  )
    # VÉRIFICATION DU DOMAINE DES PARAMÈTRES
    if idx_last == -1   idx_last = length(sol.planes)   end
    @assert 1 <  idx_last  <= length(sol.planes)
    @assert 1 <= idx_first <  idx_last

    # shift_max == -1 => on prend le maximum possible
    if shift_max == -1  shift_max = idx_last-idx_first  end
    @assert shift_max >= 1 "shift_max doit être stricement positif (ou -1 pour max)"

    # APPLICATION DES MOUVEMENTS (de type shift)
    for i in 1:nb_shift
        dist = rand(1:shift_max)
        idx1 = rand(idx_first:idx_last)
        idx2_min = max(idx1-dist, idx_first)
        idx2_max = min(idx1+dist, idx_last)
        idx2 = rand(idx2_min:idx2_max)
        shift!(sol, idx1, idx2, do_update=false)
    end

    # RECALCUL ÉVENTUELLE DES COÛTS DE LA SOLUTION
    if do_update
        # do_update_cost est utile ici par la solution obtenue peut-être infaisable
        # et il faut pouvoir évaluer le coût de cette solution
        solve!(sol, do_update_cost=true) # mise à jour après le dernier shift
    end
end

"""
Met à jour les coûts de chaque avion (sol.costs) et le coût global (sol.cost)
à partir des dates d'atterrissage supposées connues (sol.x).

Caution:
    Précondition: le Vector s.costs est dans le même ordre que s.planes
"""
function update_costs!(sol::Solution; add_viol_penality = true )

    # if !isnothing(x)
    #     copy!(sol.x, x)
    # end

    # Mise à jour des coûts individuels de chaque avion
    for i in 1:length(sol.planes)
        p = sol.planes[i]
        sol.costs[i] = get_cost(p, sol.x[i])
    end

    # Mise à jour du coût global de la solution
    #
    sol.cost = sum(sol.costs)
    # @show Args.args[:cost_precision]
    # @show sol.cost
    sol.cost = round(sol.cost, digits=Args.args[:cost_precision]) # 30/10/2018
    # @show sol.cost

    if lg4()>=4 println("get_viol_penality=$(get_viol_penality(sol))") end
    if add_viol_penality
        sol.cost += get_viol_penality(sol)
    end
    return sol
end

"""
Met à jour la date d'atterrissage de chaque avion au plus tôt, de façon à
respecter les contraintes de précédence au mieux.

Important:
  Les contraintes de séparation sont **toujours respectées**.
  Au besoin on viole la borne  d'atterrissage des avions successeurs (p.ub)
"""
function solve_to_earliest!(sol::Solution; do_update_cost=true)
    # println("="^70)
    # println("=== BEGIN solve_to_earliest! pour $(to_s(s))")
    # println("")
    SEP_MAX = sep_max(sol.inst)
    # Méthode : on parcourt tous les successeurs (plane2) et on s'assure qu'ils
    # sont suffisamment éloignés de **tous** leurs prédécesseurs (les plane1).
    for i2 in 1:length(sol.planes)
        plane2 = sol.planes[i2]
        # @show plane2
        sol.x[i2] = plane2.lb # date au plus tot
        # parcours les prédecesseurs de plane2 du plus proche au plus éloigné
        for i1 in i2-1:-1:1
            # @show sol.planes[i1]
            if (sol.x[i2] - sol.x[i1]) >= SEP_MAX
                # i2 est suffisamment en avance quels que soit le type de
                # l'avions i1 qui le précède
                break
            end
            plane1 = sol.planes[i1]
            if sol.x[i2] < sol.x[i1] + sol.inst.sep_mat[plane1.kind, plane2.kind]
                # il faut pousser plane2
                sol.x[i2] = sol.x[i1] + sol.inst.sep_mat[plane1.kind, plane2.kind]
            end
        end
        if sol.x[i2] > plane2.ub
            if lg4()>=4 println("pb infaisable pour l'avion $(plane2.name)" ) end
            # TODO: On pourrait ajouter un coût de pénalité pour les
            # contraintes de bornes non respectées
        end
    end
    if do_update_cost
        update_costs!(sol)
    end
    return sol
end

"""
Permute deux avions d'indices `idx1` et  `idx2` dans la solution.

Si un indice n'est pas défini, il est choisi aléatoirement.

Returns:
    indices des avions permutés
"""
function swap!(sol::Solution, idx1=-1, idx2=-1; do_update=true)
    # RUBY: idx1 = rand(@planes.size)  if not idx1
    if  idx1 == -1
        idx1 = rand(1:length(sol.planes))
    end
    if  idx2 == -1
        idx2 = rand(1:length(sol.planes))
    end
    # Il se peut que idx1 == idx2 mais tant pis
    sol.planes[idx1], sol.planes[idx2] = sol.planes[idx2], sol.planes[idx1]
    if idx1 != idx2 && do_update
        solve!(sol)
    end
    return (idx1, idx2)
end

"""
Déplace avions dans la solution

Args:
    idx1: indice de l'avion d'origine
    idx2: indice de l'avion cible

Si un indice n'est pas défini, il est choisi aléatoirement.

Returns:
    indices des avions permutés
"""
function shift!(sol::Solution, idx1=-1, idx2=-1; do_update=true)
    if  idx1 == -1
        idx1 = rand(1:length(sol.planes))
    end
    if  idx2 == -1
        idx2 = rand(1:length(sol.planes))
    end
    # Il se peut que idx1 == idx2 mais tant pis
    if idx1==idx2
        do_update && solve!(sol)
        return (idx1, idx1)
    elseif abs(idx1-idx2)==1
        return swap!(sol, idx1, idx2, do_update=do_update)
    end
    # println("AVANT SHIFT $idx1->$idx2 : ", to_s(sol))
    # 03/05/2019 : shift! est définie dans le fichier utils/array.jl
    shift!(sol.planes, idx1, idx2)

    if do_update
        solve!(sol)
    end
    # println("APRES SHIFT $idx1->$idx2 : ", to_s(sol))
    return (idx1, idx2)
end

"""
Permute des avions dans la solution.

Args:
    indices1: tableau des indices d'avions à permuter
    indices2: tableau des nouveaux indices des avions permutés
    do_update : recalcule le coût (true par défaut)

HYPOTHESE : indices2 est une permutation valide de indices1

Les avions de la solution ont pour indice 0:nb_planes-1
Par exemple, si indices1 vaut [0, 2, 7, 12]
et si indices2 vaut [12, 7, 0, 2],
L'opération consiste à permuter les avions positionnés en :
   ``0, 2, 7, 12``
de façon à ce que leur nouvelle position soit en :
  ``12, 7, 0, 2``

Exemple:
    On veut transformer la liste d'avion :
    ``
    @plane = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
                0   1*  2   3   4   5   6*  7*  8    9
    ``
    en :
    ``
    @plane = [10, 70, 30, 40, 50, 60, 80, 20, 90, 100]
                0   1*  2   3   4   5   6*  7*  8    9
    ``
    Pour cela l'appel doit être de la forme :
    ``mysol.permu! [1, 6, 7], [7, 1, 6]``

TODO: effectuer une mise à jour différencielle du coût
"""
function permu!(sol::Solution, indices1, indices2; do_update=true)
    sol.planes[indices1] = sol.planes[indices2]
    # sol.planes[[indices1...]] = sol.planes[[indices2...]] # avec des tuples iiii
    if do_update
        solve!(sol)
    end
    return indices2
end

"""
Effectue un tri de la solution courante selon le critère passer en paramètre.

Si aucun critère presort n'est imposé en argumant, celui spécifié par
l'option --presort est utilisé

Le timing de la solution est mis à jour.
"""
function initial_sort!(sol::Solution; presort=:ARGS, solve=true)
    if presort == :ARGS
        presort = Args.get("presort")
    end

    if presort == :target
        Base.sort!(sol.planes, by=p->p.target)
    elseif presort == :rtarget
        Base.sort!(sol.planes, by=p->p.target, rev=true)
    elseif presort == :lb
        Base.sort!(sol.planes, by=p->p.lb)
    elseif presort == :rlb
        Base.sort!(sol.planes, by=p->p.lb, rev=true)
    elseif presort == :ub
        Base.sort!(sol.planes, by=p->p.ub)
    elseif presort == :rub
        Base.sort!(sol.planes, by=p->p.ub, rev=true)
    elseif presort == :mean
        Base.sort!(sol.planes, by=p->(p.lb + p.ub))
    elseif presort == :bary
        Base.sort!(sol.planes, by=p->(p.lb + p.target + p.ub))
    elseif presort == :shuffle
        # Base.shuffle!(sol.planes)
        Random.shuffle!(sol.planes)
    elseif presort==:none
        # rien à faire
    else
        error("Valeur incorrecte pour presort : ", presort)
    end

    # Autre solution pour trier de type d'avions (A GARDER COMME EXEMPLES) :
    # On passe la fonction qui compare deux éléments :
    # Base.sort!(sol.planes, lt=(p1,p2)->p1.lb < p2.lb) # Marche aussi (comparator)
    # Base.sort!(sol.planes, rev=true,lt=(p1,p2)->p1.lb < p2.lb) # ok aussi THE WORST CASE
    solve!(sol, do_update_cost=true)
    return nothing
end

"""
Enregistre la solution dans un fichier.

Par défaut, le nom est construit automatiquement à partir des caractéristiques
de l'instance et du coût de la solution (arrondi à deux décimales)
"""
function write(sol::Solution, filename="")
    dir = Args.args[:outdir] # e.g "_tmp"
    if !isdir(dir)
        dir = "."
    end
    cost = round(sol.cost, digits=2) # cost intervient dans le nom du fichier
    if filename == ""
        # filename = "seqata_p$(sol.inst.nb_planes)=$(cost).sol"
        filename = guess_solname(sol)
    end
    # solpath = dir*"/"*filename
    solpath = "$dir/$filename"
    open(solpath, "w") do fh
        # Base.write(fh, to_s(sol))
        # Base.write(fh, "\n")
        Base.write(fh, to_s_long(sol))
    end
end

"""
Infère le nom de la solution.

Le nom est de la forme:
``alp_01_p10=700.0.sol``
"""
function  guess_solname(sol::Solution)
    prc = Args.args[:cost_precision]
    cost = round(sol.cost, digits=prc)
    return "$(sol.inst.name)=$(cost).sol"
end

"""Fonction d'impression de haut niveau."""
function print_sol(sol::Solution, msg::AbstractString= "")
    if msg!=""  println(msg) end
    println("\nmeilleure solution trouvée :")
    println("="^70)
    println(to_s(sol))
    println("="^70)
    println(to_s_long(sol))
    println("="^70)
end

# END TYPE Solution


#==============================================================================#
# SNS/test/runtests.jl
#
# Copyright Sam O'Connor 2014 - All rights reserved
#==============================================================================#

using AWSCore
using AWSLambda
using AWSSNS
using Retry
using SymDict
using JSON
using Base.Test

AWSCore.set_debug_level(1)

#-------------------------------------------------------------------------------
# Load credentials...
#-------------------------------------------------------------------------------

aws = AWSCore.aws_config(
                         region = "ap-northeast-1",
                         lambda_bucket = "ocaws.jl.lambdatest.tokyo",
                         #region = "us-east-1",
                         #lambda_bucket = "ocaws.jl.lambdatest",
                         lambda_packages = ["Requests",
                                            "Nettle",
                                            "LightXML",
                                            "JSON",
                                            "DataStructures",
                                            "StatsBase",
                                            "DataFrames",
                                            "DSP",
                                            "GZip",
                                            "ZipFile",
                                            "IniFile",
                                            "SymDict",
                                            "XMLDict",
                                            "Retry"
                                           ])

#create_jl_lambda_base(aws)

#using AWSS3
#s3_copy(aws, "ocaws.jl.lambdatest", "jl_lambda_base.zip",
#             to_bucket="ocaws.jl.lambdatest.tokyo", to_path= "jl_lambda_base.zip")

#-------------------------------------------------------------------------------
# Lambda tests
#-------------------------------------------------------------------------------

# Count primes in the cloud...

λ = @λ aws function count_primes(low::Int, high::Int)
    count = length(primes(low, high))
    println("$count primes between $low and $high.")
    return count
end

@test invoke_lambda(aws, "count_primes", low = 10, high = 100)[:jl_data] == "21"

# Count primes in parallel...

function count_primes(low::Int, high::Int)
    w = 500000000
    counts = amap(λ, [(i, min(high,i + w)) for i = low:w:high])
    count = sum(counts)
    println("$count primes between $low and $high.")
    return count
end

@test count_primes(10, 10000000000) == 455052507

mktempdir() do tmp
    cd(tmp) do

        # Create a test module under "tmp"...

        mkpath("TestModule")
        open(io->write(io, """
            module TestModule

            export test_function

            __precompile__()

            test_function(x) = x * x

            end
        """), "TestModule/TestModule.jl", "w")

        push!(LOAD_PATH, "TestModule")

        # Create a lambda that uses the TestModule...
        λ = @λ aws function lambda_test(x)

            # Check that precompile cache is being used...
            @assert !Base.stale_cachefile("/var/task/TestModule/TestModule.jl",
                                          "/var/task/TestModule.ji")
            using TestModule
            return test_function(x)
        end

        @test λ(4) == 16
    end
end

#-------------------------------------------------------------------------------
# API Gateway tests
#-------------------------------------------------------------------------------

if false

for api in apigateway_restapis(aws)
    apigateway(aws, "DELETE", "/restapis/$(api["id"])")
end

apigateway_create(aws, "count_primes", (:low, :high))

end

#==============================================================================#
# End of file.
#==============================================================================#


# helper functions to use adm along with POMDPs.jl

"""
    state_to_scene(pomdp::SingleOCPOMDP, s::SingleOCState)
Convert an SingleOCState object to a scene
"""
function state_to_scene(pomdp::SingleOCPOMDP, s::SingleOCState)
    scene = Scene()
    ego = Vehicle(s.ego, pomdp.ego_type, EGO_ID)
    ped = Vehicle(s.ped, pomdp.ped_type, 2)
    push!(scene, ego)
    push!(scene, ped)
    return scene
end

"""
    rand_scene(rng::AbstractRNG, b::Dict{Int64, SparseCat}, env::CrosswalkEnv)
Sample a scene from a belief representation
"""
function rand_scene(rng::AbstractRNG, b::Dict{Int64, SparseCat}, env::CrosswalkEnv)
    scene = Scene()
    for id in keys(b)
        s = rand(rng, b[id])
        if scene.n == 0
            ego = Vehicle(s.ego, pomdp.ego_type, EGO_ID)
            push!(scene, ego)
        end
        ped = Vehicle(s.ped, PEDESTRIAN_DEF, id)
        push!(scene, ped)
    end
    return scene
end


@testset "Test Joins" begin
    y = rand(10)
    z = rand(10)

    t  = table((x=1:10,   y=y), pkey=:x)
    t2 = table((x=1:2:20, z=z), pkey=:x)

    @testset "how = :inner" begin
        t_inner = table((x = 1:2:9, y = y[1:2:9], z = z[1:5]), pkey = :x)
        @test isequal(join(t, t2; how=:inner), t_inner)
    end
    @testset "how = :left" begin
        # Missing
        z_left = Union{Float64,Missing}[missing for i in 1:10]
        z_left[1:2:9] = z[1:5]
        t_left = table((x = 1:10, y = y, z = z_left))
        @test isequal(join(t, t2; how=:left), t_left)

        # DataValue
        z_left2 = [DataValue{Float64}() for i in 1:10]
        z_left2[1:2:9] = z[1:5]
        t_left2 = table((x=1:10, y = y, z = z_left2))
        @test isequal(join(t, t2, how=:left, missingtype=DataValue), t_left2)
    end
    @testset "how = :outer" begin
        # Missing
        x_outer = union(1:10, 1:2:20)
        y_outer = vcat(y, fill(missing, 5))
        z_left = Union{Float64,Missing}[missing for i in 1:10]
        z_left[1:2:9] = z[1:5]
        z_outer = vcat(z_left, z[6:10])
        t_outer = table((x=x_outer, y=y_outer, z=z_outer); pkey=:x)
        @test isequal(join(t, t2; how=:outer), t_outer)

        # DataValue
        y_outer2 = vcat(y, fill(DataValue{Float64}(), 5))
        z_left2 = [DataValue{Float64}() for i in 1:10]
        z_left2[1:2:9] = z[1:5]
        z_outer2 = vcat(z_left2, z[6:10])
        t_outer2 = table((x=x_outer, y=y_outer2, z=z_outer2); pkey=:x)
        @test isequal(join(t, t2; how=:outer, missingtype=DataValue), t_outer2)
    end
    @testset "how = :anti" begin
        t_anti = table((x=2:2:10, y=y[2:2:10]), pkey=:x)
        @test isequal(join(t, t2; how=:anti), t_anti)
    end
    @testset "stringarray" begin
        a = table((sa=StringArray(["a","b","c"]), id=1:3))
        b = table((x=1:3, id=1:2:5))
        t_outer = table((id=[1,2,3,5], sa=["a","b","c",missing], x=[1,missing,2,3]), pkey=:id)
        t_outer2 = table((id=[1,2,3,5], sa=["a","b","c",NA], x=[1,NA,2,3]), pkey=:id)
        @test isequal(join(a, b, how=:outer, lkey=:id, rkey=:id), t_outer)
        @test isequal(join(a, b, how=:outer, lkey=:id, rkey=:id, missingtype=DataValue), t_outer2)
    end
    @testset "empty" begin
        t1 = table((key = ["a","b","c"], ))
        t2 = table((key = ["a","b"], value = [1,2]))
        res = join(t1, t2, how = :left, lkey = :key, rkey = :key)
        @test isequal(res, table((key = ["a","b","c"], value = [1,2,missing])))
    end
end


#-------------------------------------------------------------------
#* EMSO Model Library (EML) Copyright (C) 2004 - 2007 ALSOC.
#*
#* This LIBRARY is free software; you can distribute it and/or modify
#* it under the therms of the ALSOC FREE LICENSE as available at
#* http://www.enq.ufrgs.br/alsoc.
#*
#* EMSO Copyright (C) 2004 - 2007 ALSOC, original code
#* from http://www.rps.eng.br Copyright (C) 2002-2004.
#* All rights reserved.
#*
#* EMSO is distributed under the therms of the ALSOC LICENSE as
#* available at http://www.enq.ufrgs.br/alsoc.
#*--------------------------------------------------------------------
#* Author: Gerson Balbueno Bicca
#* $Id$
#*--------------------------------------------------------------------
type Clearances_Main
	Clearances_Main()=begin
		new(
			DanaInteger(Dict{Symbol,Any}(
				:Brief=>"Number of Sealing Strips pairs",
				:Lower=>1
			)),
			length(Dict{Symbol,Any}(
				:Brief=>"Height Under Shell Inlet Nozzle",
				:Lower=>1E-6
			)),
			length(Dict{Symbol,Any}(
				:Brief=>"Height Under Shell Outlet Nozzle",
				:Lower=>1E-6
			)),
			length(Dict{Symbol,Any}(
				:Brief=>"Bundle-to-Shell Clearance",
				:Symbol=>"L_{cf}",
				:Lower=>1E-8
			)),
			length(Dict{Symbol,Any}(
				:Brief=>"Baffle-to-Shell Clearance",
				:Symbol=>"L_{cd}",
				:Lower=>1E-8
			)),
			length(Dict{Symbol,Any}(
				:Brief=>"Tube-to-Baffle Clearance",
				:Symbol=>"L_{td}",
				:Lower=>1E-8
			)),
			[:SealStrip,:Hinozzle_Shell,:Honozzle_Shell,:BundleToShell,:BaffleToShell,:TubeToBaffle,],
		)
	end
	SealStrip::DanaInteger
	Hinozzle_Shell::length
	Honozzle_Shell::length
	BundleToShell::length
	BaffleToShell::length
	TubeToBaffle::length
	parameters::Array{Symbol,1}
	attributes::Dict{Symbol,Any}
end
export Clearances_Main
function atributes(in::Clearances_Main,_::Dict{Symbol,Any})
	fields::Dict{Symbol,Any}=Dict{Symbol,Any}()
	fields[:Pallete]=false
	fields[:Brief]="Main parameters for diametral clearances in a shell and tube heat exchanger."
	drive!(fields,_)
	return fields
end
Clearances_Main(_::Dict{Symbol,Any})=begin
	newModel=Clearances_Main()
	newModel.attributes=atributes(newModel,_)
	newModel
end


using JEMSS

println("Loading sim.")
sim = initSim(joinpath(@__DIR__, "input/sim_config.xml"))

println("Simulating.")
simulate!(sim)

println("Writing stats files.")
writeStatsFiles(sim)

# # Animate, if you like:
# animate!(sim)


function add_two_digit(d1::Float64, d2::Float64)
    return d1 + d2
end


##
# Base hotfixes
##
module BaseHacks
    @info "activating BaseHacks..."

    import Base.IteratorsMD: inc
    import Base: tail

    function unsafe_inc(i::Int64)
        Base.llvmcall("""
        %i = add nsw i64 %0, 1
        ret i64 %i
        """, Int64, Tuple{Int64}, i)
    end

    @inline inc(state::Tuple{Int}, start::Tuple{Int}, stop::Tuple{Int}) = (unsafe_inc(state[1]),)
    @inline function inc(state, start, stop)
         if state[1] < stop[1]
           return (unsafe_inc(state[1]),tail(state)...)
         end
         newtail = inc(tail(state), tail(start), tail(stop))
         (start[1], newtail...)
    end

    @inline function Base.getindex(r::AbstractUnitRange, s::AbstractUnitRange{<:Integer})
        @boundscheck checkbounds(r, s)
        f = first(r)
        st = oftype(f, f + first(s)-1)
        Base._range(st, nothing, nothing, length(s)) #gotta use this version of range instead of const-propping through the kwarg version.
    end

    #FIXME This is a separate eachindex for scalar cases so that swizzles can use
    #a different thunk for the scalar case and avoid recursion depth limiting.
    Base.eachindex(args::AbstractArray{<:Any, 0}...) = CartesianIndices(())

    #=
    #Avoid checking shapes in eachindex when inbounds scopes
    Base.@propagate_inbounds function Base.eachindex(A::AbstractArray, B::AbstractArray)
        eachindex(IndexStyle(A,B), A, B)
    end
    Base.@propagate_inbounds function Base.eachindex(A::AbstractArray, B::AbstractArray...)
        eachindex(IndexStyle(A,B...), A, B...)
    end
    Base.@propagate_inbounds function Base.eachindex(::IndexLinear, A::AbstractArray, B::AbstractArray...)
        indsA = eachindex(IndexLinear(), A)
        @boundscheck Base._all_match_first(X->eachindex(IndexLinear(), X), indsA, B...) || Base.throw_eachindex_mismatch(IndexLinear(), A, B...)
        indsA
    end
    Base.@propagate_inbounds function Base.eachindex(::IndexCartesian, A::AbstractArray, B::AbstractArray...)
        axsA = axes(A)
        @boundscheck Base._all_match_first(axes, axsA, B...) || Base.throw_eachindex_mismatch(IndexCartesian(), A, B...)
        CartesianIndices(axsA)
    end
    =#

    #=
    Base.@propagate_inbounds function Base.getindex(iter::CartesianIndices{N,<:NTuple{N,Base.OneTo}}, I::Vararg{Int, N}) where {N}
        @boundscheck checkbounds(iter, I...)
        CartesianIndex(I)
    end

    Base.@propagate_inbounds function Base.getindex(iter::CartesianIndices{N,R}, I::Vararg{Int, N}) where {N,R}
        @boundscheck checkbounds(iter, I...)
        CartesianIndex(I .- first.(Base.axes1.(iter.indices)) .+ first.(iter.indices))
    end
    =#

    @info "BaseHacks is online. all your Base are belong to us."
end


# Extended Sparse matrix interface

"""
    sparse_from_coo(::Type{T} where T,I,J,V,m,n)
"""
function sparse_from_coo(::Type{T} where T,I,J,V,m,n)
  @abstractmethod
end

"""
    fill_entries!(A::AbstractSparseMatrix,v)

Fills the non-zero entries in the sparse matrix A with a given value v.
"""
function fill_entries!(A::AbstractSparseMatrix,v)
  @abstractmethod
end

"""
    push_coo!(::Type{T} where T, I,J,V,ik,jk,vk)

Inserts entries in COO vectors for further building a sparse matrix of type T.
"""
function push_coo!(::Type{T} where T,I,J,V,ik,jk,vk)
  @abstractmethod
end

"""
    is_entry_stored(::Type{T} where T,i,j) -> Bool

Tells if the entry with coordinates `[i,j]` will be stored when calling function push_coo!
"""
function is_entry_stored(::Type{T} where T,i,j)
  @abstractmethod
end

"""
    finalize_coo!(::Type{T} where T,I,J,V,m,n)

Check and insert diagonal entries in COO vectors if needed.
"""
function finalize_coo!(::Type{T} where T,I,J,V,m,n)
  @abstractmethod
end

"""
"""
function allocate_coo_vectors(::Type{M},n::Integer) where M
  return (zeros(Int,n), zeros(Int,n), zeros(Float64,n))
end

function allocate_coo_vectors(::Type{M},n::Integer) where {Tv,Ti,M<:AbstractSparseMatrix{Tv,Ti}}
  return (zeros(Ti,n), zeros(Ti,n), zeros(Tv,n))
end


abstract type AbstractReservesFormulation <: AbstractServiceFormulation end
struct RangeReserve <: AbstractReservesFormulation end
struct StepwiseCostReserve <: AbstractReservesFormulation end
############################### Reserve Variables` #########################################
"""
This function add the variables for reserves to the model
"""
function AddVariableSpec(
    ::Type{ActiveServiceVariable},
    ::PSIContainer,
    service::T,
) where {T <: PSY.Reserve}
    return AddVariableSpec(;
        variable_name = make_variable_name(PSY.get_name(service), T),
        binary = false,
        lb_value_func = x -> 0,
        devices_filter_func = x -> PSY.get_available(x),
    )
end

function AddVariableSpec(
    ::Type{T},
    ::Type{U},
    ::PSIContainer,
) where {T <: ServiceRequirementVariable, U <: PSY.ReserveDemandCurve}
    return AddVariableSpec(;
        variable_name = make_variable_name(T, U),
        binary = false,
        lb_value_func = x -> 0.0,
    )
end

################################## Reserve Requirement Constraint ##########################
function service_requirement_constraint!(
    psi_container::PSIContainer,
    service::SR,
    ::ServiceModel{SR, T},
) where {SR <: PSY.Reserve, T <: AbstractReservesFormulation}
    parameters = model_has_parameters(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)
    initial_time = model_initial_time(psi_container)
    @debug initial_time
    time_steps = model_time_steps(psi_container)
    name = PSY.get_name(service)
    constraint = get_constraint(psi_container, make_constraint_name(REQUIREMENT, SR))
    reserve_variable = get_variable(psi_container, name, SR)
    use_slacks = get_services_slack_variables(psi_container.settings)

    ts_vector = get_time_series(psi_container, service, "requirement")

    use_slacks && (slack_vars = reserve_slacks(psi_container, name))

    requirement = PSY.get_requirement(service)
    if parameters
        param = get_parameter_array(
            psi_container,
            UpdateRef{SR}(SERVICE_REQUIREMENT, "requirement"),
        )
        for t in time_steps
            param[name, t] = PJ.add_parameter(psi_container.JuMPmodel, ts_vector[t])
            if use_slacks
                resource_expression = sum(reserve_variable[:, t]) + slack_vars[t]
            else
                resource_expression = sum(reserve_variable[:, t])
            end
            constraint[name, t] = JuMP.@constraint(
                psi_container.JuMPmodel,
                resource_expression >= param[name, t] * requirement
            )
        end
    else
        for t in time_steps
            constraint[name, t] = JuMP.@constraint(
                psi_container.JuMPmodel,
                sum(reserve_variable[:, t]) >= ts_vector[t] * requirement
            )
        end
    end
    return
end

function service_requirement_constraint!(
    psi_container::PSIContainer,
    service::SR,
    ::ServiceModel{SR, T},
) where {SR <: PSY.StaticReserve, T <: AbstractReservesFormulation}
    parameters = model_has_parameters(psi_container)
    initial_time = model_initial_time(psi_container)
    @debug initial_time
    time_steps = model_time_steps(psi_container)
    name = PSY.get_name(service)
    constraint = get_constraint(psi_container, make_constraint_name(REQUIREMENT, SR))
    reserve_variable = get_variable(psi_container, name, SR)
    use_slacks = get_services_slack_variables(psi_container.settings)

    use_slacks && (slack_vars = reserve_slacks(psi_container, name))

    requirement = PSY.get_requirement(service)
    for t in time_steps
        resource_expression = JuMP.GenericAffExpr{Float64, JuMP.VariableRef}()
        JuMP.add_to_expression!(resource_expression, sum(reserve_variable[:, t]))
        if use_slacks
            resource_expression += slack_vars[t]
        end
        constraint[name, t] =
            JuMP.@constraint(psi_container.JuMPmodel, resource_expression >= requirement)
    end

    return
end

function cost_function!(
    psi_container::PSIContainer,
    service::SR,
    ::ServiceModel{SR, T},
) where {SR <: PSY.Reserve, T <: AbstractReservesFormulation}
    reserve = get_variable(psi_container, PSY.get_name(service), SR)
    for r in reserve
        JuMP.add_to_expression!(psi_container.cost_function, r, DEFAULT_RESERVE_COST)
    end
    return
end

function service_requirement_constraint!(
    psi_container::PSIContainer,
    service::SR,
    ::ServiceModel{SR, StepwiseCostReserve},
) where {SR <: PSY.Reserve}
    initial_time = model_initial_time(psi_container)
    @debug initial_time
    time_steps = model_time_steps(psi_container)
    name = PSY.get_name(service)
    constraint = get_constraint(psi_container, make_constraint_name(REQUIREMENT, SR))
    reserve_variable = get_variable(psi_container, name, SR)
    requirement_variable = get_variable(psi_container, SERVICE_REQUIREMENT, SR)

    for t in time_steps
        constraint[name, t] = JuMP.@constraint(
            psi_container.JuMPmodel,
            sum(reserve_variable[:, t]) >= requirement_variable[name, t]
        )
    end

    return
end

function AddCostSpec(
    ::Type{T},
    ::Type{StepwiseCostReserve},
    psi_container::PSIContainer,
) where {T <: PSY.Reserve}
    return AddCostSpec(;
        variable_type = ServiceRequirementVariable,
        component_type = T,
        has_status_variable = false,
        has_status_parameter = false,
        variable_cost = PSY.get_variable,
        start_up_cost = nothing,
        shut_down_cost = nothing,
        fixed_cost = nothing,
        sos_status = NO_VARIABLE,
    )
end

function add_to_cost!(
    psi_container::PSIContainer,
    spec::AddCostSpec,
    service::SR,
    component_name::String,
) where {SR <: PSY.Reserve}
    time_steps = model_time_steps(psi_container)
    use_forecast_data = model_uses_forecasts(psi_container)
    if !use_forecast_data
        error("StepwiseCostReserve is only supported with forecast")
    end
    variable_cost_forecast = get_time_series(psi_container, service, "variable_cost")
    variable_cost_forecast = map(PSY.VariableCost, variable_cost_forecast)
    for t in time_steps
        variable_cost!(psi_container, spec, component_name, variable_cost_forecast[t], t)
    end
    return
end

function cost_function!(
    psi_container::PSIContainer,
    service::SR,
    model::ServiceModel{SR, StepwiseCostReserve},
) where {SR <: PSY.ReserveDemandCurve}
    spec = AddCostSpec(SR, model.formulation, psi_container)
    @debug SR, spec
    add_to_cost!(psi_container, spec, service, PSY.get_name(service))
    return
end

function modify_device_model!(
    devices_template::Dict{Symbol, DeviceModel},
    service_model::ServiceModel{<:PSY.Reserve, <:AbstractReservesFormulation},
    contributing_devices::Vector{<:PSY.Device},
)
    device_types = unique(typeof.(contributing_devices))
    for dt in device_types
        for (device_model_name, device_model) in devices_template
            # add message here when it exists
            device_model.device_type != dt && continue
            service_model in device_model.services && continue
            push!(device_model.services, service_model)
        end
    end

    return
end

function include_service!(
    constraint_info::T,
    services,
    ::ServiceModel{SR, <:AbstractReservesFormulation},
) where {
    T <: Union{AbstractRangeConstraintInfo, AbstractRampConstraintInfo},
    SR <: PSY.Reserve{PSY.ReserveUp},
}
    for (ix, service) in enumerate(services)
        push!(
            constraint_info.additional_terms_ub,
            make_constraint_name(PSY.get_name(service), SR),
        )
    end
    return
end

function include_service!(
    constraint_info::T,
    services,
    ::ServiceModel{SR, <:AbstractReservesFormulation},
) where {
    T <: Union{AbstractRangeConstraintInfo, AbstractRampConstraintInfo},
    SR <: PSY.Reserve{PSY.ReserveDown},
}
    for (ix, service) in enumerate(services)
        push!(
            constraint_info.additional_terms_lb,
            make_constraint_name(PSY.get_name(service), SR),
        )
    end
    return
end

function add_device_services!(
    constraint_info::T,
    device::D,
    model::DeviceModel,
) where {
    T <: Union{AbstractRangeConstraintInfo, AbstractRampConstraintInfo},
    D <: PSY.Device,
}
    for service_model in get_services(model)
        if PSY.has_service(device, service_model.service_type)
            services =
                (s for s in PSY.get_services(device) if isa(s, service_model.service_type))
            @assert !isempty(services)
            include_service!(constraint_info, services, service_model)
        end
    end
    return
end

function add_device_services!(
    constraint_data_in::AbstractRangeConstraintInfo,
    constraint_data_out::AbstractRangeConstraintInfo,
    device::D,
    model::DeviceModel{D, <:AbstractStorageFormulation},
) where {D <: PSY.Storage}
    for service_model in get_services(model)
        if PSY.has_service(device, service_model.service_type)
            services =
                (s for s in PSY.get_services(device) if isa(s, service_model.service_type))
            @assert !isempty(services)
            if service_model.service_type <: PSY.Reserve{PSY.ReserveDown}
                for service in services
                    push!(
                        constraint_data_in.additional_terms_ub,
                        make_constraint_name(
                            PSY.get_name(service),
                            service_model.service_type,
                        ),
                    )
                end
            elseif service_model.service_type <: PSY.Reserve{PSY.ReserveUp}
                for service in services
                    push!(
                        constraint_data_out.additional_terms_ub,
                        make_constraint_name(
                            PSY.get_name(service),
                            service_model.service_type,
                        ),
                    )
                end
            end
        end
    end
    return
end