################################################################################
## MOI wrapper
#################################################################################
## This file is largely inspired from MOI wrappers existing in other
## JuliaOpt packages:
## https://www.juliaopt.org/
## The authors are indebted to the developers of JuliaOpt for
## the current MOI wrapper.
##
## In particular, KNITRO.jl and IpOPT.jl where used as an inspiration

RESHOP_OPTIONS = ["solver"]

# TODO(Xhub) is this useful?
const MOIU = MathOptInterface.Utilities

# Scalar Functions
const SF = Union{MOI.ScalarAffineFunction{Float64},
				 MOI.ScalarQuadraticFunction{Float64}}

# Vector Types
const VAF = MOI.VectorAffineFunction{Float64}
const VOV = MOI.VectorOfVariables

const ALLV = Union{MOI.SingleVariable,MOI.VectorOfVariables}

# Scalar Sets
const SS = Union{MOI.EqualTo{Float64},
					  MOI.GreaterThan{Float64},
					  MOI.LessThan{Float64},
					  MOI.Interval{Float64}}
# LinSets
const LS = Union{MOI.EqualTo{Float64},
					  MOI.GreaterThan{Float64},
					  MOI.LessThan{Float64}}
# VecLinSets
const VLS = Union{MOI.Nonnegatives,
				  MOI.Nonpositives,
				  MOI.Zeros}

# NonConvex Sets
const NCS = Union{MOI.ZeroOne,
									MOI.Integer,
									MOI.Semicontinuous,
									MOI.Semiinteger,
									MOI.SOS1,
									MOI.SOS2}

# NonPolyhedral Cones
const NPC = Union{MOI.SecondOrderCone,
MOI.RotatedSecondOrderCone,
MOI.ExponentialCone,
MOI.DualExponentialCone,
MOI.PowerCone,
MOI.DualPowerCone}

mutable struct Optimizer <: MOI.AbstractOptimizer
    # Get number of solve for restart.
    number_solved::Int
    sense::MOI.OptimizationSense
    status::Cint

    vov_mapping::Dict{MOI.ConstraintIndex{VOV, <: Union{VLS, MOI.SOS1{Float64}, MOI.SOS2{Float64}}}, Vector{Cuint}}
    vaf_mapping::Dict{MOI.ConstraintIndex{VAF, <: VLS}, Vector{Cuint}}
    quadfn_mapping::Dict{MOI.ConstraintIndex{MOI.ScalarQuadraticFunction{Float64}, <: SS}, Cuint}
    sos_sets::Dict{MOI.ConstraintIndex{VOV, <: Union{MOI.SOS1{Float64}, MOI.SOS2{Float64}}}, Union{MOI.SOS1{Float64}, MOI.SOS2{Float64}}}
    fake_cons_name::Dict{MOI.ConstraintIndex, String}
    delvars::Set{MOI.VariableIndex}

    ctx::Union{Ptr{context}, Nothing}
    ctx_dest::Ptr{context}
    mdl::Ptr{reshop_model}
    mdl_solver::Ptr{reshop_model}
    options::Ptr{reshop_options}
    rhp_options::Dict{String, Union{Cdouble, Cint, Bool, Cstring, String}}
    gams_dir::String
    avar_cache::Union{Ptr{abstract_var}, Nothing}
    solver_name::String
    start_nl_cons::Int
    len_nl_cons::Cuint
end

function helper_options(ctx, options, reshop_opts::Ptr{reshop_options})
    solver_name = ""
    rhp_options = Dict{String, Union{Cdouble, Cint, Bool, Cstring, String}}()
    for (name, value) in options
        if string(name) == "solver"
            solver_name = value
        else
            res = reshop_option_set(reshop_opts, string(name), value)
            if (res != 0)
                 rhp_set_option(ctx, string(name), value)
            end
            rhp_options[string(name)] = value
        end
    end
    return (solver_name, rhp_options)
end

function Optimizer(;options...)
    reshop_set_printops(stdout)
    # Create ReSHOP context.
    ctx = ctx_alloc()

    # TODO this is quite a hack just for the "output" option.
    # Refactoring option in ReSHOP will enable us to move on
    reshop_opts = reshop_options_alloc()
    solver_name, rhp_options = helper_options(ctx, options, reshop_opts)

    model = Optimizer(0, MOI.FEASIBILITY_SENSE, 0,
                      Dict{MOI.ConstraintIndex{VOV, <: VLS}, Cuint}(), Dict{MOI.ConstraintIndex{VAF, <: VLS}, Cuint}(),
                      Dict{MOI.ConstraintIndex{MOI.ScalarQuadraticFunction{Float64}, <: SS}, Cuint}(),
                      Dict{MOI.ConstraintIndex{VOV, <: Union{MOI.SOS1{Float64}, MOI.SOS2{Float64}}}, Union{MOI.SOS1{Float64}, MOI.SOS2{Float64}}}(),
                      Dict{MOI.ConstraintIndex,String}(), Set{MOI.VariableIndex}(),
                      ctx, Ptr{context}(C_NULL), Ptr{reshop_model}(C_NULL), Ptr{reshop_model}(C_NULL),
                      reshop_opts, rhp_options, "", nothing, solver_name, -1, 0)

    finalizer(MOI.empty!, model)
    return model
end

# Print Optimizer.
function Base.show(io::IO, model::Optimizer)
    println(io, "A MathOptInterface model with backend:")
    println(io, model.ctx)
    return
end

# copy
MOIU.supports_default_copy_to(model::Optimizer, copy_names::Bool) = true
function MOI.copy_to(model::Optimizer, src::MOI.ModelLike; kws...)
    return MOI.Utilities.automatic_copy_to(model, src; kws...)
end

# Some utilities.
number_variables(model::Optimizer) = ctx_numvar(model.ctx) - length(model.delvars)
number_constraints(model::Optimizer) = ctx_m(model.ctx)

# Getter for solver's name.
function MOI.get(model::Optimizer, ::MOI.SolverName)
    if model.solver_name == ""
        return "ReSHOP"
    else
        return "ReSHOP ($(model.solver_name))"
    end
end

# MOI.Silent.
MOI.supports(model::Optimizer, ::MOI.Silent) = true
function MOI.get(model::Optimizer, ::MOI.Silent)
    return reshop_option_get(model.options, "output") == 0xF
end

function MOI.set(model::Optimizer, ::MOI.Silent, value)
    val = value ? 0xF : 0x8
    reshop_option_set(model.options, "output", val)
    return
end

# MOI.TimeLimitSec.
MOI.supports(model::Optimizer, ::MOI.TimeLimitSec) = false

# MOI.RawParameters
function MOI.supports(model::Optimizer, param::MOI.RawParameter)
    name = param.name
    if name in RESHOP_OPTIONS
        return true
    end
    return false
end

function MOI.set(model::Optimizer, p::MOI.RawParameter, value)
    if !MOI.supports(model, p)
        throw(MOI.UnsupportedAttribute)
    end
    if p.name == "solver"
         model.solver_name = value
    end
    #reshop_option_set(model.options, p.name, value)
    return
end

function MOI.get(model::Optimizer, p::MOI.RawParameter)
    if haskey(model.options, p.name)
        return model.options[p.name]
    end
    error("RawParameter with name $(p.name) is not set.")
end

##################################################
# Optimize
##################################################
function MOI.optimize!(model::Optimizer)
    model.number_solved += 1
    # If gams_dir already existed (from a past solve), get rid of it
    # TODO(xhub) model.gams_dir could be a list of directory to remove
    if (!isempty(model.gams_dir))
        try
            rm(model.gams_dir, recursive=true, force=true)
        catch
            iswin && run(`cmd /C RMDIR /s /q $(model.gams_dir)`)
        end
    end
    # TODO check if gams_dir and ctx_dest already exists, do not reallocate then.
    model.ctx_dest, model.gams_dir = reshop_setup_gams()

    # Calling from emp, we already have a mdl object
    if model.mdl == C_NULL
        model.mdl = reshop_alloc(model.ctx)
    end
    model.mdl_solver = reshop_alloc(model.ctx_dest)
    model.status = reshop_solve(model.mdl, model.mdl_solver, model.ctx_dest, model.solver_name)
    reshop_postprocess(model.mdl_solver)

    return
end

function MOI.empty!(model::Optimizer)
    ctx_dealloc(model.ctx_dest)
    model.ctx_dest = C_NULL
    reshop_free(model.mdl)
    model.mdl = C_NULL
    reshop_free(model.mdl_solver)
    model.mdl_solver = C_NULL
    if model.ctx != nothing
        ctx_dealloc(model.ctx)
        model.ctx = ctx_alloc()
        helper_options(model.ctx, model.rhp_options, model.options)
    end

    model.number_solved = 0
    model.vov_mapping = Dict()
    model.vaf_mapping = Dict()
    model.quadfn_mapping = Dict()
    model.sos_sets       = Dict()
    model.fake_cons_name = Dict()
    model.delvars    = Set()
    # TODO(xhub) cleanup avar?
    # TODO(xhub) check that it is not necessary to do anything here
    #set_options(model, model.options)
    if (!isempty(model.gams_dir))
        try
            rm(model.gams_dir, recursive=true, force=true)
        catch
            iswin && run(`cmd /C RMDIR /s /q $(model.gams_dir)`)
        end
    end
    model.start_nl_cons = -1
    model.len_nl_cons = 0
    return
end

function MOI.is_empty(model::Optimizer)
    return model.number_solved == 0 &&
           length(model.vov_mapping) == 0 &&
           length(model.vaf_mapping) == 0 &&
           length(model.quadfn_mapping) == 0 &&
           length(model.sos_sets) == 0 &&
           ctx_m(model.ctx) == 0 &&
           ctx_numvar(model.ctx) == 0
end

include(joinpath("MOI_wrapper", "getters.jl"))
include(joinpath("MOI_wrapper", "supports.jl"))
include(joinpath("MOI_wrapper", "variables.jl"))
include(joinpath("MOI_wrapper", "constraints.jl"))
include(joinpath("MOI_wrapper", "objective.jl"))
include(joinpath("MOI_wrapper", "results.jl"))
include(joinpath("MOI_wrapper", "utils.jl"))
include(joinpath("MOI_wrapper", "changes.jl"))
include(joinpath("MOI_wrapper", "nlp.jl"))


using Random: rand

mutable struct RandomModel <: ALifeModel
    num_organisms
    num_genotypes
    current_organisms::Vector{Organism}
    current_time

    RandomModel(num_organisms, num_genotypes) = new(num_organisms, num_genotypes, [], 0)
end

struct IntGenotype <: Genotype
    genotype::Int
end

function init!(model::RandomModel)
    model.current_organisms = _get_random_organism(model)
end

function iterate!(model::RandomModel)
    current_organisms = model.current_organisms
    model.current_organisms = _get_random_organism(model)
    model.current_time += 1
    return current_organisms
end

function _get_random_organism(model::RandomModel)
    new_genotypes = rand(1:model.num_genotypes, model.num_organisms)

    if model.current_organisms == []
        return [Organism(IntGenotype(new), Ancestor(), model.current_time, model.current_time+1)
                    for new in new_genotypes]
    else
        return [Organism(IntGenotype(new), old, model.current_time, model.current_time+1)
                    for (new, old) in zip(new_genotypes, model.current_organisms)]
    end
end


sprites = [
  [
    0b11110000,  # ****
    0b10010000,  # *  *
    0b10010000,  # *  *
    0b10010000,  # *  *
    0b11110000   # ****
  ]
  [
    0b00100000,  #  *
    0b01100000,  # **
    0b00100000,  #  *
    0b00100000,  #  *
    0b01110000   # ***
  ]
  [
    0b11110000,  # ****
    0b00010000,  #    *
    0b11110000,  # ****
    0b10000000,  # *
    0b11110000   # ****
  ]
  [
    0b11110000,  # ****
    0b00010000,  #    *
    0b11110000,  # ****
    0b00010000,  #    *
    0b11110000   # ****
  ]
  [
    0b10010000,  # *  *
    0b10010000,  # *  *
    0b11110000,  # ****
    0b00010000,  #    *
    0b00010000   #    *
  ]
  [
    0b11110000,  # ****
    0b10000000,  # *
    0b11110000,  # ****
    0b00010000,  #    *
    0b11110000   # ****
  ]
  [
    0b11110000,  # ****
    0b10000000,  # *
    0b11110000,  # ****
    0b10010000,  # *  *
    0b11110000   # ****
  ]
  [
    0b11110000,  # ****
    0b00010000,  #    *
    0b00100000,  #   *
    0b01000000,  #  *
    0b01000000   #  *
  ]
  [
    0b11110000,  # ****
    0b10010000,  # *  *
    0b11110000,  # ****
    0b10010000,  # *  *
    0b11110000   # ****
  ]
  [
    0b11110000,  # ****
    0b10010000,  # *  *
    0b11110000,  # ****
    0b00010000,  #    *
    0b11110000   # ****
  ]
  [
    0b11110000,  # ****
    0b10010000,  # *  *
    0b11110000,  # ****
    0b10010000,  # *  *
    0b10010000   # *  *
  ]
  [
    0b11100000,  # ***
    0b10010000,  # *  *
    0b11100000,  # ***
    0b10010000,  # *  *
    0b11100000   # ***
  ]
  [
    0b11110000,  # ****
    0b10000000,  # *
    0b10000000,  # *
    0b10000000,  # *
    0b11110000   # ****
  ]
  [
    0b11100000,  # ***
    0b10010000,  # *  *
    0b10010000,  # *  *
    0b10010000,  # *  *
    0b11100000   # ***
  ]
  [
    0b11110000,  # ****
    0b10000000,  # *
    0b11110000,  # ****
    0b10000000,  # *
    0b11110000   # ****
  ]
  [
    0b11110000,  # ****
    0b10000000,  # *
    0b11110000,  # ****
    0b10000000,  # *
    0b10000000   # *
  ]
]


# This function includes the whole simulation over the incident data set
function part2_simulation!(districts::DataFrame,
                            incidents::DataFrame,
                            sim_data::DataFrame,
                            ressource_flow::Array{Int64,3},
                            drivingtime::Array{Float64,2},
                            traffic::Array{Float64,2},
                            nearby_radius::Float64,
                            drop_incident::Int64,
                            exchange_reserve::Vector{Int64})
# locations: state the current district ceters
    locations  = Vector{Int64}(vec(sort!(unique(districts[:,:location]))))

# location_dictionary: dictionary to match the locations to columns in the simulation
    location_district = Dict(locations[i] => i for i = 1:length(locations))

# District BAs grouped by department location
    districts_grouped = groupby(districts, [:location])

# card_priorities: number of priorities in the incident data set
# card_districts:  number of districts in the district district layout
# card_incidents:  number of incidents in our simulation
# list of districts:
    card_priorities  = maximum(incidents[:,:priority])
    card_districts   = length(locations)
    card_incidents   = nrow(incidents)

#  incident_queue: array that saves all incidents not fulfilled happening
#  exchange_queue: array that saves the incidents for exchanges
#  queue_used:     states the number of cases in a queue
#  queue_change:   states whether a queue has changed to save unnecessary sorting
#  candidates:     lists all possible candidates and their fit for an exchange
#  cin:            counts the current incident row in our DataFrames
    incident_queue   = Array{Union{Missing,Int},3}(missing,max_queue,card_districts,card_priorities)
    exchange_queue   = Array{Union{Missing,Int64},1}(missing,max_queue*card_districts)
    queue_used       = Array{Int64,2}(undef,card_districts,card_priorities) .= 1
    queue_change     = Array{Bool,2}(undef,card_districts,card_priorities) .= false
    candidates       = Vector{Union{Missing,Float64}}(missing, card_districts)
    patrol_location  = Matrix{Int64}(undef,max_queue,card_districts) .= 0
    patrol_area_time = Matrix{Int64}(undef, size(patrol_location,1), card_districts) .= 0
    patrol_drive     = Vector{Float64}(undef, size(patrol_location,1)) .= 0
    cin              = 1

# sim_start:  epoch (in minutes) of the first incident
# sim_length: epoch (in minutes) ends 5 minutes after the incident_minute
#             of the last incidents in our incidents DataFrame
    sim_start   = minimum(sim_data[:,:incident_minute])
    sim_length  = maximum(sim_data[:,:incident_minute]) + 5

# adjust the start of the incidents to write the results and all steps
# directly to the corresponding row of the ressource_flow matrix
    sim_data[:,:incident_minute] .-= sim_start - 1

#  Starting simulation
    for mnt = 1:(sim_length - sim_start)
        # fill the queues with all new incidents of the corresponding minute
        cin = fill_queues!(sim_data::DataFrame,
                     incidents::DataFrame,
                     location_district::Dict,
                     incident_queue::Array{Union{Missing,Int},3},
                     queue_used::Array{Int64,2},
                     queue_change::Array{Bool,2},
                     cin::Int64,
                     mnt::Int64,
                     card_incidents::Int64,
                     card_priorities::Int64,
                     card_districts::Int64,
                     max_queue::Int64)
        # determine the location of all cars currently on patrol
        determine_patrol_location!(patrol_location,patrol_area_time,drivingtime,ressource_flow,locations,districts_grouped,mnt)

        # start the allocation of ressources to incidents in the own district
        for p = 1:card_priorities
            dispatch = "own_department"
            for k = 1:2
                for i = 1:card_districts
                    for j = 1:max_queue
                        if ismissing(incident_queue[j,i,p]) == false
                            current_incident = incident_queue[j,i,p]
                            # if district has capacity at district center
                            # or if cars are working at the backlog at the district center
                            # and the average car has a backlog below the threshold "backlog_max"
                            # and the incident has at least one missing car
                            if ressource_flow[mnt,i,1] > 0 || (ressource_flow[mnt,i,5] > 0 &&
                               ressource_flow[mnt,i,7] < backlog_max * sum(ressource_flow[mnt,i,1:6])) &&
                               sim_data[current_incident,:cars_missing] > 0
                               fastest_time = 0
                               incident_dispatch!(sim_data::DataFrame,
                                                   incidents::DataFrame,
                                                   ressource_flow::Array{Int64,3},
                                                   drivingtime::Array{Float64,2},
                                                   traffic::Array{Float64,2},
                                                   locations::Vector{Int64},
                                                   current_incident::Int64,
                                                   mnt::Int64,
                                                   i::Int64,
                                                   k::Int64,
                                                   dispatch::String,
                                                   fastest_time::Int64)
                            end
                        else
                            break
                        end
                    end
                end
            end
            # check whether cars on patrol within the own district can serve as backup
            #######
            dispatch = "own_patrol"
            k = 1
            for i = 1:card_districts
                for j = 1:max_queue
                    if ismissing(incident_queue[j,i,p]) == false && ressource_flow[mnt,i,6] > 0
                        current_incident = incident_queue[j,i,p]
                        # if the district has cars currently patroling the own district
                        # and there hasn't been a car dispatched to the incident and the
                        # priority is high enough to warrant a dispatch
                        if ressource_flow[mnt,i,6] > 0 &&
                            sim_data[current_incident,:cars_dispatched] == 0 &&
                            incidents[current_incident,:priority] <= patrol_prio
                            fastest_time = determine_patrol_driving_time(i::Int64,
                                                                            incidents::DataFrame,
                                                                            traffic::Array{Float64,2},
                                                                            patrol_location::Matrix{Int64},
                                                                            patrol_drive::Vector{Float64},
                                                                            drivingtime::Array{Float64,2},
                                                                            current_incident::Int64,
                                                                            max_drive::Float64,
                                                                            mnt::Int64)
                            if fastest_time < patrol_time
                                incident_dispatch!(sim_data::DataFrame,
                                                    incidents::DataFrame,
                                                    ressource_flow::Array{Int64,3},
                                                    drivingtime::Array{Float64,2},
                                                    traffic::Array{Float64,2},
                                                    locations::Vector{Int64},
                                                    current_incident::Int64,
                                                    mnt::Int64,
                                                    i::Int64,
                                                    k::Int64,
                                                    dispatch::String,
                                                    fastest_time::Int64)
                            end
                        end
                    else
                        break
                    end
                end
            end
            # start the exchange of cars to foreign districts to fulfill all incidents
            # faster if an incident happens is within a predefined perimeter. The
            # exchange queue is fulfilled stepwise starting with the hightes priority
            dispatch = "exchange_department"
            if p <= exchange_prio
                # fill the exchange queue with all unfullfilled incidents
                fill_exchange_queue!(sim_data::DataFrame,
                                    incident_queue::Array{Union{Missing,Int64},3},
                                    exchange_queue::Array{Union{Missing,Int64},1},
                                    max_queue::Int64,
                                    card_districts::Int64,
                                    p::Int64)
                # fullfill all incidents possible within the exchange queue
                fulfill_exchange_queue!(incidents::DataFrame,
                                        sim_data::DataFrame,
                                        ressource_flow::Array{Int64,3},
                                        drivingtime::Array{Float64,2},
                                        exchange_queue::Array{Union{Missing,Int64},1},
                                        candidates::Vector{Union{Missing,Float64}},
                                        locations::Vector{Int64},
                                        exchange_reserve::Vector{Int64},
                                        card_districts::Int64,
                                        mnt::Int64,
                                        dispatch::String,
                                        nearby_radius::Float64,
                                        p::Int64)
            end
        end
        # prepare the queues and the ressource_flow for the nest minute
        prepare_next_minute!(sim_data::DataFrame,
                             incidents::DataFrame,
                             ressource_flow::Array{Int64,3},
                             incident_queue::Array{Union{Missing,Int64},3},
                             queue_used::Array{Int64,2},
                             queue_change::Array{Bool,2},
                             mnt::Int64,
                             drop_incident::Int64)
    end
    return sim_data, ressource_flow
end


module Shapefile

import GeoInterface, DBFTables, Tables

struct Rect
    left::Float64
    bottom::Float64
    right::Float64
    top::Float64
end

struct Interval
    left::Float64
    right::Float64
end

struct Point <: GeoInterface.AbstractPoint
    x::Float64
    y::Float64
end

struct PointM <: GeoInterface.AbstractPoint
    x::Float64
    y::Float64
    m::Float64  # measure
end

struct PointZ <: GeoInterface.AbstractPoint
    x::Float64
    y::Float64
    z::Float64
    m::Float64  # measure
end

struct Polyline <: GeoInterface.AbstractMultiLineString
    MBR::Rect
    parts::Vector{Int32}
    points::Vector{Point}
end

struct PolylineM <: GeoInterface.AbstractMultiLineString
    MBR::Rect
    parts::Vector{Int32}
    points::Vector{Point}
    measures::Vector{Float64}
end

struct PolylineZ <: GeoInterface.AbstractMultiLineString
    MBR::Rect
    parts::Vector{Int32}
    points::Vector{Point}
    zvalues::Vector{Float64}
    measures::Vector{Float64}
end

struct Polygon <: GeoInterface.AbstractMultiPolygon
    MBR::Rect
    parts::Vector{Int32}
    points::Vector{Point}
end

Base.show(io::IO, p::Polygon) =
    print(io, "Polygon(", length(p.points), " Points)")

struct PolygonM <: GeoInterface.AbstractMultiPolygon
    MBR::Rect
    parts::Vector{Int32}
    points::Vector{Point}
    measures::Vector{Float64}
end

struct PolygonZ <: GeoInterface.AbstractMultiPolygon
    MBR::Rect
    parts::Vector{Int32}
    points::Vector{Point}
    zvalues::Vector{Float64}
    measures::Vector{Float64}
end

struct MultiPoint <: GeoInterface.AbstractMultiPoint
    MBR::Rect
    points::Vector{Point}
end

struct MultiPointM <: GeoInterface.AbstractMultiPoint
    MBR::Rect
    points::Vector{Point}
    measures::Vector{Float64}
end

struct MultiPointZ <: GeoInterface.AbstractMultiPoint
    MBR::Rect
    points::Vector{Point}
    zvalues::Vector{Float64}
    measures::Vector{Float64}
end

struct MultiPatch <: GeoInterface.AbstractGeometry
    MBR::Rect
    parts::Vector{Int32}
    parttypes::Vector{Int32}
    points::Vector{Point}
    zvalues::Vector{Float64}
    # measures::Vector{Float64}  # (optional)
end

const SHAPETYPE = Dict{Int32,DataType}(
    0 => Missing,
    1 => Point,
    3 => Polyline,
    5 => Polygon,
    8 => MultiPoint,
    11 => PointZ,
    13 => PolylineZ,
    15 => PolygonZ,
    18 => MultiPointZ,
    21 => PointM,
    23 => PolylineM,
    25 => PolygonM,
    28 => MultiPointM,
    31 => MultiPatch,
)

mutable struct Handle{T<:Union{<:GeoInterface.AbstractGeometry,Missing}}
    code::Int32
    length::Int32
    version::Int32
    shapeType::Int32
    MBR::Rect
    zrange::Interval
    mrange::Interval
    shapes::Vector{T}
end

Base.length(shp::Handle) = length(shp.shapes)

function Base.read(io::IO, ::Type{Rect})
    minx = read(io, Float64)
    miny = read(io, Float64)
    maxx = read(io, Float64)
    maxy = read(io, Float64)
    Rect(minx, miny, maxx, maxy)
end

function Base.read(io::IO, ::Type{Point})
    x = read(io, Float64)
    y = read(io, Float64)
    Point(x, y)
end

function Base.read(io::IO, ::Type{PointM})
    x = read(io, Float64)
    y = read(io, Float64)
    m = read(io, Float64)
    PointM(x, y, m)
end

function Base.read(io::IO, ::Type{PointZ})
    x = read(io, Float64)
    y = read(io, Float64)
    z = read(io, Float64)
    m = read(io, Float64)
    PointZ(x, y, z, m)
end

function Base.read(io::IO, ::Type{Polyline})
    box = read(io, Rect)
    numparts = read(io, Int32)
    numpoints = read(io, Int32)
    parts = Vector{Int32}(undef, numparts)
    read!(io, parts)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    Polyline(box, parts, points)
end

function Base.read(io::IO, ::Type{PolylineM})
    box = read(io, Rect)
    numparts = read(io, Int32)
    numpoints = read(io, Int32)
    parts = Vector{Int32}(undef, numparts)
    read!(io, parts)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    mrange = Vector{Float64}(undef, 2)
    read!(io, mrange)
    measures = Vector{Float64}(undef, numpoints)
    read!(io, measures)
    PolylineM(box, parts, points, measures)
end

function Base.read(io::IO, ::Type{PolylineZ})
    box = read(io, Rect)
    numparts = read(io, Int32)
    numpoints = read(io, Int32)
    parts = Vector{Int32}(undef, numparts)
    read!(io, parts)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    zrange = Vector{Float64}(undef, 2)
    read!(io, zrange)
    zvalues = Vector{Float64}(undef, numpoints)
    read!(io, zvalues)
    mrange = Vector{Float64}(undef, 2)
    read!(io, mrange)
    measures = Vector{Float64}(undef, numpoints)
    read!(io, measures)
    PolylineZ(box, parts, points, zvalues, measures)
end

function Base.read(io::IO, ::Type{Polygon})
    box = read(io, Rect)
    numparts = read(io, Int32)
    numpoints = read(io, Int32)
    parts = Vector{Int32}(undef, numparts)
    read!(io, parts)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    Polygon(box, parts, points)
end

function Base.read(io::IO, ::Type{PolygonM})
    box = read(io, Rect)
    numparts = read(io, Int32)
    numpoints = read(io, Int32)
    parts = Vector{Int32}(undef, numparts)
    read!(io, parts)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    mrange = Vector{Float64}(undef, 2)
    read!(io, mrange)
    measures = Vector{Float64}(undef, numpoints)
    read!(io, measures)
    PolygonM(box, parts, points, measures)
end

function Base.read(io::IO, ::Type{PolygonZ})
    box = read(io, Rect)
    numparts = read(io, Int32)
    numpoints = read(io, Int32)
    parts = Vector{Int32}(undef, numparts)
    read!(io, parts)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    zrange = Vector{Float64}(undef, 2)
    read!(io, zrange)
    zvalues = Vector{Float64}(undef, numpoints)
    read!(io, zvalues)
    mrange = Vector{Float64}(undef, 2)
    read!(io, mrange)
    measures = Vector{Float64}(undef, numpoints)
    read!(io, measures)
    PolygonZ(box, parts, points, zvalues, measures)
end

function Base.read(io::IO, ::Type{MultiPoint})
    box = read(io, Rect)
    numpoints = read(io, Int32)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    MultiPoint(box, points)
end

function Base.read(io::IO, ::Type{MultiPointM})
    box = read(io, Rect)
    numpoints = read(io, Int32)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    mrange = Vector{Float64}(undef, 2)
    read!(io, mrange)
    measures = Vector{Float64}(undef, numpoints)
    read!(io, measures)
    MultiPointM(box, points, measures)
end

function Base.read(io::IO, ::Type{MultiPointZ})
    box = read(io, Rect)
    numpoints = read(io, Int32)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    zrange = Vector{Float64}(undef, 2)
    read!(io, zrange)
    zvalues = Vector{Float64}(undef, numpoints)
    read!(io, zvalues)
    mrange = Vector{Float64}(undef, 2)
    read!(io, mrange)
    measures = Vector{Float64}(undef, numpoints)
    read!(io, measures)
    MultiPointZ(box, points, zvalues, measures)
end

function Base.read(io::IO, ::Type{MultiPatch})
    box = read(io, Rect)
    numparts = read(io, Int32)
    numpoints = read(io, Int32)
    parts = Vector{Int32}(undef, numparts)
    read!(io, parts)
    parttypes = Vector{Int32}(undef, numparts)
    read!(io, parttypes)
    points = Vector{Point}(undef, numpoints)
    read!(io, points)
    zrange = Vector{Float64}(undef, 2)
    read!(io, zrange)
    zvalues = Vector{Float64}(undef, numpoints)
    read!(io, zvalues)
    # mrange = Vector{Float64}(2)
    # read!(io, mrange)
    # measures = Vector{Float64}(numpoints)
    # read!(io, measures)
    MultiPatch(box, parts, parttypes, points, zvalues) #,measures)
end

function Base.read(io::IO, ::Type{Handle})
    code = bswap(read(io, Int32))
    read!(io, Vector{Int32}(undef, 5))
    fileSize = bswap(read(io, Int32))
    version = read(io, Int32)
    shapeType = read(io, Int32)
    MBR = read(io, Rect)
    zmin = read(io, Float64)
    zmax = read(io, Float64)
    mmin = read(io, Float64)
    mmax = read(io, Float64)
    jltype = SHAPETYPE[shapeType]
    shapes = Vector{Union{jltype,Missing}}(undef, 0)
    file = Handle(
        code,
        fileSize,
        version,
        shapeType,
        MBR,
        Interval(zmin, zmax),
        Interval(mmin, mmax),
        shapes,
    )
    while (!eof(io))
        num = bswap(read(io, Int32))
        rlength = bswap(read(io, Int32))
        shapeType = read(io, Int32)
        if shapeType === Int32(0)
            push!(shapes, missing)
        else
            push!(shapes, read(io, jltype))
        end
    end
    file
end

function Base.:(==)(a::Rect, b::Rect)
    a.left == b.left &&
    a.bottom == b.bottom && a.right == b.right && a.top == b.top
end

include("table.jl")
include("geo_interface.jl")
include("shx.jl")

end # module


# =========================================================================================
"""
    readmcsf{T <: AbstractFloat}(
        stream::IOStream,
              ::Type{T}=Float64;
        # kwargs
        domain::Symbol=:none
    )

Reads a volume model from the given GAMer-generated mcsf file.

!!! note
    This file type does not support charge models! Hence, the charge list of the returning
    `Model` object is empty and has to be set separately.

# Arguments
 * `domain` Element domain (solute `:Ω`, solvent `:Σ`, or `:none`)

# Return type
`Model{T, Tetrahedron{T}}`

# Aliases

    readmcsf{T}(
        fname ::String,
              ::Type{T}=Float64;
        # kwargs
        domain::Symbol=:none
    )

Reads the model using a file name rather than a `IOStream` object.

    readmcsf{T}(
        fnameΩ::String,
        fnameΣ::String,
              ::Type{T}=Float64
    )

Reads the contents of two separate files (given by name), sets the element domains to `:Ω`
or `:Σ`, respectively, and returns a single (merged) `Model` object.
"""
function readmcsf(
        stream::IOStream,
              ::Type{T}=Float64;
        domain::Symbol=:none
    ) where T <: AbstractFloat
    nodes = readmcsf_nodes(stream, T)
    Model(nodes, readmcsf_elements(stream, nodes, domain=domain))
end

function readmcsf(
        fname ::String,
              ::Type{T}=Float64;
        domain::Symbol=:none
    ) where T
    open(fh -> readmcsf(fh, T, domain=domain), fname)
end

function readmcsf(
        fnameΩ::String,
        fnameΣ::String,
              ::Type{T}=Float64
    ) where T
    meshunion(readmcsf(fnameΩ, T, domain=:Ω), readmcsf(fnameΣ, T, domain=:Σ))
end

# =========================================================================================
"""
    readmcsf_nodes{T <: AbstractFloat}(
        stream::IOStream,
              ::Type{T}=Float64
    )

Reads all nodes from the given GAMer-generated mcsf file.

# Return type
`Vector{Vector{T}}`
"""
function readmcsf_nodes(
        stream::IOStream,
              ::Type{T}=Float64
    ) where T <: AbstractFloat
    nodes = Vector{T}[]
    seek(stream, "vert=[")
    for line in eachline(stream)
        startswith(line, "%") && continue        # skip comments
        startswith(line, "];") && break          # all nodes read
        push!(nodes, [parse(T, e) for e in split(line)[3:5]])
    end
    nodes
end

# =========================================================================================
"""
    readmcsf_elements{T <: AbstractFloat}(
        stream::IOStream,
        nodes ::Vector{Vector{T}};
        # kwargs
        domain::Symbol=:none
    )

Reads all elements from the given GAMer-generated mcsf file.

# Return type
`Vector{Tetrahedron{T}}`
"""
function readmcsf_elements(
        stream::IOStream,
        nodes ::Vector{Vector{T}};
        domain::Symbol=:none
    ) where T <: AbstractFloat
    elements = Tetrahedron{T}[]
    seek(stream, "simp=[")
    for line in eachline(stream)
        startswith(line, "%") && continue        # skip comments
        startswith(line, "];") && break          # all simplices read
        push!(
            elements,
            Tetrahedron([nodes[parse(Int, e)+1] for e in split(line)[8:end]]..., domain)
        )
    end
    elements
end


export single_point_crossover,
    random_node_crossover,
    aligned_node_crossover,
    proportional_crossover,
    output_graph_crossover,
    subgraph_crossover,
    crossover

"single point crossover, genes from p1 up to a random point, then genes from p2"
function single_point_crossover(cfg::NamedTuple, c1::CGPInd, c2::CGPInd)
    cpoint = rand(2:(min(length(c1.chromosome), length(c2.chromosome))-2))
    if rand(Bool)
        ngenes = deepcopy([c1.chromosome[1:cpoint]; c2.chromosome[(cpoint+1):end]])
        return CGPInd(cfg, ngenes)
    else
        ngenes = deepcopy([c2.chromosome[1:cpoint]; c1.chromosome[(cpoint+1):end]])
        return CGPInd(cfg, ngenes)
    end
end

"returns random input genes from c1 and c2 equally, not a full crossover operator"
function random_inputs(c1::CGPInd, c2::CGPInd)
    child_inputs = zeros(c1.n_in)
    parent = rand(Bool, c1.n_in)
    child_inputs[parent] = c1.chromosome[1:c1.n_in][parent]
    child_inputs[.~parent] = c2.chromosome[1:c1.n_in][.~parent]
    child_inputs
end

"returns random output genes from c1 and c2 equally, not a full crossover operator"
function random_outputs(c1::CGPInd, c2::CGPInd)
    child_outputs = zeros(c1.n_out)
    parent = rand(Bool, c1.n_out)
    s = length(c1.chromosome) - c1.n_out + 1
    child_outputs[parent] = c1.chromosome[s:end][parent]
    child_outputs[.~parent] = c2.chromosome[s:end][.~parent]
    child_outputs
end

"take random nodes from each parent equally, up to the size of the smaller parent"
function random_node_crossover(cfg::NamedTuple, c1::CGPInd, c2::CGPInd)
    l = min(length(c1.chromosome), length(c2.chromosome)) - max(c1.n_out, c2.n_out)
    p = rand(Bool, Int64(round(l / 3)))
    p_nodes = repeat(p, inner=3)
    append!(p_nodes, rand(Bool, c1.n_out))
    genes = rand(length(c1.chromosome))
    genes[p_nodes] = c1.chromosome[p_nodes]
    genes[.~p_nodes] = c2.chromosome[.~p_nodes]
    CGPInd(cfg, genes)
end

"align nodes based on position, then take from each parent equally, only works on PCGP"
function aligned_node_crossover(cfg::NamedTuple, c1::CGPInd, c2::CGPInd)
    p1_pos = get_positions(c1)
    p2_pos = get_positions(c2)
    min_nodes = min(length(c1.nodes)-c1.n_in, length(c2.nodes)-c2.n_in)
    p1_inds = c1.n_in + collect(1:min_nodes)
    p1_nodes = Array{Int64}(undef, 0)
    p2_nodes = Array{Int64}(undef, 0)
    for node in c1.n_in+(1:min_nodes)
        if rand() < 0.5
            i = indmin(abs.(p1_pos[p1_inds] - p2_pos[node]))
            append!(p1_nodes, [p1_inds[i]])
            p1_inds = deleteat!(p1_inds, i)
        else
            append!(p2_nodes, [node])
        end
    end
    genes = [random_inputs(c1, c2); random_outputs(c1, c2); get_genes(
        c1, p1_nodes); get_genes(c2, p2_nodes)]
    CGPInd(cfg, genes)
end

"choose a point between the genes from the two parents for a child individual"
function proportional_crossover(cfg::NamedTuple, c1::CGPInd, c2::CGPInd)
    genes = Array{Float64}(undef, 0)
    if length(c1.chromosome) == length(c2.chromosome)
        r = rand(length(c1.chromosome))
        genes = ((1 .- r) .* c1.chromosome) .+ (r .* c2.chromosome)
    elseif length(c1.chromosome) > length(c2.chromosome)
        r = rand(length(c2.chromosome))
        genes = ((1 .- r) .* c1.chromosome[1:length(c2.chromosome)]) .+ (r .* c2.chromosome)
        genes = [genes; c1.chromosome[(length(c2.chromosome)+1):end]]
    else
        r = rand(length(c1.chromosome))
        genes = ((1 .- r) .* c1.chromosome) .+ (r .* c2.chromosome[1:length(c1.chromosome)])
        genes = [genes; c2.chromosome[(length(c1.chromosome)+1):end]]
    end
    CGPInd(cfg, genes)
end

"crossover with a list of nodes"
function node_crossover(cfg::NamedTuple, c1::CGPInd, c2::CGPInd,
                        c1_nodes::Array{Int16}, c2_nodes::Array{Int16},
                        output_genes::AbstractArray = Array{Float64}(undef, 0))
    s = length(c1.chromosome) - c1.n_out
    if length(output_genes) == 0
        for o in 1:c1.n_out
            g = rand() < 0.5 ? c1.chromosome[s+o] : c2.chromosome[s+o]
            push!(output_genes, g)
        end
    end
    genes = Array{Float64}(undef, 0)
    for i in (c1.n_in+1):length(c1.nodes)
        if i in c1_nodes
            if i in c2_nodes
                g = rand() < 0.5 ? get_genes(c1, i) : get_genes(c2, i)
                append!(genes, g)
            else
                append!(genes, get_genes(c1, i))
            end
        else
            if i in c2_nodes
                append!(genes, get_genes(c2, i))
            else
                append!(genes, rand(3))
            end
        end
    end
    append!(genes, output_genes)
    CGPInd(cfg, genes)
end

"split outputs equally between parents, then construct a child from their corresponding input graphs"
function output_graph_crossover(cfg::NamedTuple, c1::CGPInd, c2::CGPInd)
    p1_nodes = Array{Int16}(undef, 0)
    p2_nodes = Array{Int16}(undef, 0)
    output_genes = Array{Float64}(undef, 0)
    s = length(c1.chromosome) - c1.n_out
    for output in 1:c1.n_out
        if rand() < 0.5
            append!(p1_nodes, get_output_trace(c1, output))
            push!(output_genes, c1.chromosome[s+output])
        else
            append!(p2_nodes, get_output_trace(c2, output))
            push!(output_genes, c2.chromosome[s+output])
        end
    end
    c1_nodes = sort(unique(p1_nodes))
    c2_nodes = sort(unique(p2_nodes))
    node_crossover(cfg, c1, c2, c1_nodes, c2_nodes, output_genes)
end

"Take subgraphs from both parents equally, adding all nodes of the chosen subgraphs to the child"
function subgraph_crossover(cfg::NamedTuple, c1::CGPInd, c2::CGPInd)
    fc1 = forward_connections(c1)
    fc2 = forward_connections(c2)
    c1_nodes = []; c2_nodes = []
    for i in (c1.n_in+1):min(length(c1.nodes), length(c2.nodes))
        if rand(Bool)
            append!(c1_nodes, fc1[i])
        else
            append!(c2_nodes, fc2[i])
        end
    end
    if length(c1.nodes) < length(c2.nodes)
        for i in length(c1.nodes):length(c2.nodes)
            if rand(Bool)
                append!(c2_nodes, fc2[i])
            end
        end
    elseif length(c2.nodes) < length(c1.nodes)
        for i in length(c2.nodes):length(c1.nodes)
            if rand(Bool)
                append!(c1_nodes, fc1[i])
            end
        end
    end
    c1_nodes = Array{Int16}(unique(intersect(collect((c1.n_in+1):length(c1.nodes)), c1_nodes)))
    c2_nodes = Array{Int16}(unique(intersect(collect((c2.n_in+1):length(c2.nodes)), c2_nodes)))
    node_crossover(cfg, c1, c2, c1_nodes, c2_nodes)
end


using JLD
include("Transformation.jl")
include("AbstractSystem.jl")
include("Tree.jl")
include("Evaluation.jl")
include("BackPropogation.jl")
include("Facility.jl")
import Base.push!

type Axiom
    tree1 :: Tree
    tree2 :: Tree
    index :: Dict
end

function Base.show(io :: IO, m :: Axiom)
    print(io, "Axiom[")
    print(io, m.tree1)
    print(io, ", ")
    print(io, m.tree2)
    print(io, "]")
end

function beautify(m :: Axiom)
    string("Axiom:\n", beautify(m.tree1, 1), "\n", beautify(m.tree2, 1), "\n")
end

Axiom(tree1 :: Tree, tree2 :: Tree) = Axiom(tree1, tree2, add!(index(tree1), index(tree2)))

Axiom(skeleton1, skeleton2) = Axiom(toTree(skeleton1), toTree(skeleton2))

function push!(index :: Dict, ops :: Array)
    n = length(ops)
    for i in 1:n
        for t in index[i]
            t.op = ops[i]
        end
    end
end

function push!(index :: Dict, ops :: Dict)
    for key in keys(ops)
        if haskey(index, key)
            ts = index[key]
            for t in ts
                t.op = ops[key]
            end
        end
    end
end

function push!(axiom :: Axiom, ops)
    push!(axiom.index, variables)
end

function init_axiom!(axiom :: Axiom, variables)
    push!(axiom.index, variables)
    init_tree!(axiom.tree1)
    init_tree!(axiom.tree2)
end

init_axioms! = distribute(init_axiom!)

## add some loss function to deal with degenerating problem?

function loss(a, b)
    b * (1. - a * b)
end

function train_axiom!(axiom :: Axiom, variables, n = 1, randomize = identity)
    push!(axiom.index, variables)
    d1 :: Array{Float64, 1} = axiom.tree1.value[:d]
    d2 :: Array{Float64, 1} = axiom.tree2.value[:d]
    v1 :: Array{Float64, 1} = axiom.tree1.value[:value]
    v2 :: Array{Float64, 1} = axiom.tree2.value[:value]
    for i in 1:n
        randomize(variables)
        push!(axiom.index, variables)
        eval_tree!(axiom.tree2)
        eval_tree!(axiom.tree1)
        for j in 1:length(d1)
            d1[j] = loss(v1[j], v2[j])
            d2[j] = loss(v2[j], v1[j])
        end
        bp_tree!(axiom.tree1)
        bp_tree!(axiom.tree2)
    end
end

train_axioms! = distribute(train_axiom!)

## to prevent degeneration problem, we use anti-traing to deal with the problem.

function anti_train_axiom!(axiom :: Axiom, n = 1, randomize = identity)
    d1 :: Array{Float64, 1} = axiom.tree1.value[:d]
    d2 :: Array{Float64, 1} = axiom.tree2.value[:d]
    v1 :: Array{Float64, 1} = axiom.tree1.value[:value]
    v2 :: Array{Float64, 1} = axiom.tree2.value[:value]
    for i in 1:n
        randomize(axiom.tree1)
        randomize(axiom.tree2)
        eval_tree!(axiom.tree2)
        eval_tree!(axiom.tree1)
        for j in 1:length(d1)
            d1[j] = - loss(v1[j], v2[j])
            d2[j] = - loss(v2[j], v1[j])
        end
        bp_tree!(axiom.tree1)
        bp_tree!(axiom.tree2)
    end
end

anti_train_axioms! = distribute(anti_train_axiom!)

include("DataStructs.jl")

@load "infinity.jld"

## The Repl facility, note that for the program structure parsing, we rely on the julia parser;
## and we also need to read the variable name from string.
include("Repl.jl")
r = REPL(Env.empty, Seq.empty)

a1 = repl(r, "function(x) x end").value;
a2 = repl(r, "var y = function(x) x end").value;
a3 = repl(r, "y").value;
a4 = repl(r, "function(x) x(x) end").value;
a5 = repl(r, "begin var y = function(x) x end; y end").value;
as = [a1, a2, a3, a4, a5];

pair_computation(function(x, y) sum(abs(as[x] - as[y])) end, 1:5)


module Balloons

export deflate, inflate!, inflate

abstract type CompressionAlgorithm end

"""
    Packbits Compression

A simple byte-oriented run-length encoding as defined by the TIFF Spec 6.0
"""
struct Packbits <: CompressionAlgorithm end

inflate!(pb::Packbits, input, output) = inflate!(pb, reinterpret(UInt8, input), reinterpret(UInt8, output))

function inflate!(::Packbits, input::AbstractVector{UInt8}, output::AbstractVector{UInt8})
    ipos, opos = 1, 1

    while opos <= length(output)
        n = reinterpret(Int8, input[ipos])
        ipos += 1
        if 0 <= n <= 127
            output[opos:opos+n-1] = view(input, ipos:ipos+n-1)
            ipos += n
            opos += n
        elseif -127 <= n <= -1
            nxt = input[ipos]
            output[opos:(opos-n-1)] .= nxt
            opos += -n
            ipos += 1
        end
    end
end

"""
    Lempel-Ziv-Welch Compression

A robust universal lossless compression algorithm. This implementation supports
variable-width compression codes.
"""
struct LZW <: CompressionAlgorithm
    initial_codewidth::Int
    initial_dictsize::Int

    clearcode::UInt32
    endcode::UInt32
end

LZW() = LZW(9, 256, 256, 257)

mask(t::Type{T}, p::Integer, l::Integer) where {T} = (typemax(T) << (sizeof(T)*8-l)) >> p

extract(packed::T, p::Integer, l::Integer) where {T} = (packed & mask(T, p, l)) >> (sizeof(T)*8 - p - l)

"""
    stuff(v, a, loc)

Stuff bits of `a` into 32-bit unsigned integer `v` at a given byte location `loc`

```jldoctest
julia> Balloons.stuff(zero(UInt32), 0xff, 1)
0xff000000

julia> Balloons.stuff(zero(UInt32), 0xff, 4)
0x000000ff
```
"""
function stuff(v::UInt32, a::UInt8, loc::T) where {T <: Integer}
    @assert 0 < loc <= 4
    v += (zero(UInt32) + a) << ((4-loc)*8)
end

function initdict!(lzw::LZW, dict)
    empty!(dict)

    for i in 0:(lzw.initial_dictsize - 1)
        dict[i] = UInt8[i]
    end
    dict[lzw.clearcode] = [0x00]
    dict[lzw.endcode] = [0x00]
end

function nextchar(input, ρ, bitwidth)
    i = ρ÷8+1
    v = stuff(zero(UInt32), input[i], 1)

    for j in 1:min(3, length(input)-i)
        v = stuff(v, input[i+j], j+1)
    end

    extract(v, mod(ρ, 8), bitwidth)
end

function inflate(lzw::LZW, input::Vector{UInt8})
    bitwidth = lzw.initial_codewidth

    ρ = 0 # position in bits

    output = IOBuffer()

    dict = Dict{UInt32, Vector{UInt8}}()
    k = lzw.clearcode

    entry = UInt8[]
    prev_entry = UInt8[]

    while true
        # increase bitwidth if we're out of space
        if length(dict) >= 2^bitwidth - 1
            bitwidth += 1
        end

        k = nextchar(input, ρ, bitwidth)
        if k == lzw.endcode
            break
        end

        # reset dictionary and variables on detecting a clear code
        if k == lzw.clearcode
            ρ += bitwidth

            # reset everything back to baseline
            initdict!(lzw, dict)
            sz = length(dict)
            bitwidth = lzw.initial_codewidth

            # get next character
            k = nextchar(input, ρ, bitwidth)
            (k == lzw.endcode) && break
            entry = dict[k]
            write(output, entry)
            prev_entry = entry
        else
            if haskey(dict, k)
                entry = dict[k]
            else
                entry = vcat(prev_entry, first(prev_entry))
            end

            write(output, entry)
            if length(prev_entry) != 0
                dict[length(dict)] = vcat(prev_entry, first(entry))
            end
            prev_entry = entry
        end

        ρ += bitwidth
    end

    seekstart(output)
    readavailable(output)
end

end


"""
`module  JAC.PlasmaShift`
    ... a submodel of JAC that contains all methods for computing plasma shifts and related properties between for some level or
        between some initial and final-state multiplets.
"""
module PlasmaShift

    using Printf, ..Basics, ..Defaults, ..Nuclear, ..ManyElectron, ..Radial, ..TableStrings

    #============================
    """
    `@enum   PlasmaModel`  ... defines a enumeration for the (allowed) plasma models.

        + NoPlasmaModel      ... No plasma model defined.
        + DebyeHueckel       ... Debye-Hueckel plasma model.
        + IonSphere          ... Ion-sphere (not yet supported).
    """
    @enum   PlasmaModel    NoPlasmaModel    DebyeHueckel    DebeyBox    IonSphere

    """
    `PlasmaModel(sa::String)`  ... constructor for a given String.
    """
    function PlasmaModel(sa::String)
        if       sa == "No Model"              wa = NoPlasmaModel
        elseif   sa == "Debye-Hueckel"         wa = DebyeHueckel
        elseif   sa == "DebeyBox"              wa = DebeyBox
        elseif   sa == "IonSphere"             wa = IonSphere
        else     error("stop a")
        end

        return( UseGauge(wa) )
    end

    # `Base.show(io::IO, model::PlasmaModel)`  ... prepares a proper printout of the variable model::PlasmaModel.
    function Base.show(io::IO, model::PlasmaModel)
        print(io, string(model) )
    end

    # `Base.string(model::PlasmaModel)`  ... provides a proper printout of the variable model::PlasmaModel.
    function Base.string(model::PlasmaModel)
        if       model == NoPlasmaModel     return("No plasma model")
        elseif   model == DebyeHueckel      return("Debye-Hueckel model")
        elseif   model == DebeyBox          return("Debey-box model")
        elseif   model == IonSphere         return("Ion-sphere model")
        else     error("stop a")
        end
    end          ==========================================#

    """
    `abstract type PlasmaShift.AbstractPlasmaModel`
        ... defines an abstract and a number of singleton types for the the (allowed) plasma models.

        + NoPlasmaModel      ... No plasma model defined.
        + DebyeHueckel       ... Debye-Hueckel plasma model.
        + IonSphere          ... Ion-sphere (not yet supported).
    """
    abstract type  AbstractPlasmaModel                      end
    struct         NoPlasmaModel  <:  AbstractPlasmaModel   end
    struct         DebyeHueckel   <:  AbstractPlasmaModel   end
    struct         IonSphere      <:  AbstractPlasmaModel   end

    # `Base.show(io::IO, model::AbstractPlasmaModel)`  ... prepares a proper printout of the variable model::AbstractPlasmaModel.
    function Base.show(io::IO, model::AbstractPlasmaModel)
        print(io, string(model) )
    end

    # `Base.string(model::AbstractPlasmaModel)`  ... provides a proper printout of the variable model::AbstractPlasmaModel.
    function Base.string(model::AbstractPlasmaModel)
        if       model == NoPlasmaModel()     return("No plasma model")
        elseif   model == DebyeHueckel()      return("Debye-Hueckel model")
        elseif   model == DebeyBox()          return("Debey-box model")
        elseif   model == IonSphere()         return("Ion-sphere model")
        else     error("stop a")
        end
    end

    """
    `struct  PlasmaShift.Settings`  ... defines a type for the details and parameters of computing level energies with plasma interactions.

        + plasmaModel      ::AbstractPlasmaModel        ... Specify a particular plasma model, e.g. ion-sphere, debye.
        + lambdaDebye      ::Float64                    ... The lambda parameter of different plasma models.
        + ionSphereR0      ::Float64                    ... The effective radius of the ion-sphere model.
        + NoBoundElectrons ::Int64                      ... Effective number of bound electrons.
    """
    struct Settings
        plasmaModel        ::AbstractPlasmaModel
        lambdaDebye        ::Float64
        ionSphereR0        ::Float64
        NoBoundElectrons   ::Int64
    end

    """
    `PlasmaShift.Settings()`  ... constructor for a standard instance of PlasmaShift.Settings.
    """
    function Settings()
        Settings(DebyeHueckel(), 0.25, 0., 0)
    end

    # `Base.show(io::IO, settings::PlasmaShift.Settings)`  ... prepares a proper printout of the settings::PlasmaShift.Settings.
    function Base.show(io::IO, settings::PlasmaShift.Settings)
        println(io, "plasmaModel:            $(settings.plasmaModel)  ")
        println(io, "lambdaDebye:            $(settings.lambdaDebye)  ")
        println(io, "ionSphereR0:            $(settings.ionSphereR0)  ")
        println(io, "NoBoundElectrons:       $(settings.NoBoundElectrons)  ")
    end

    """
    `struct  PlasmaShift.AugerSettings`
        ... defines a type for the details and parameters of computing Auger rates with plasma interactions.

        + plasmaModel            ::AbstractPlasmaModel           ... Specify a particular plasma model, e.g. ion-sphere, debye.
        + lambdaDebye            ::Float64                       ... The lambda parameter of different plasma models.
        + ionSphereR0            ::Float64                       ... The effective radius of the ion-sphere model.
        + NoBoundElectrons       ::Int64                         ... Effective number of bound electrons.
        + printBeforeComputation ::Bool                          ... True, if all energies and lines are printed before their evaluation.
        + selectLines            ::Bool                          ... True, if lines are selected individually for the computations.
        + selectedLines          ::Array{Tuple{Int64,Int64},1}   ... List of lines, given by tupels (inital-level, final-level).
    """
    struct AugerSettings
        plasmaModel              ::AbstractPlasmaModel
        lambdaDebye              ::Float64
        ionSphereR0              ::Float64
        NoBoundElectrons         ::Int64
        printBeforeComputation   ::Bool
        selectLines              ::Bool
        selectedLines            ::Array{Tuple{Int64,Int64},1}
    end

    """
    `PlasmaShift.AugerSettings()`  ... constructor for a standard instance of PlasmaShift.AugerSettings.
    """
    function AugerSettings()
        AugerSettings(DebyeHueckel(), 0.25, 0., 0, true, false, Tuple{Int64,Int64}[])
    end

    # `Base.show(io::IO, settings::PlasmaShift.AugerSettings)`  ... prepares a proper printout of the settings::PlasmaShift.AugerSettings.
    function Base.show(io::IO, settings::PlasmaShift.AugerSettings)
        println(io, "plasmaModel:             $(settings.plasmaModel)  ")
        println(io, "lambdaDebye:             $(settings.lambdaDebye)  ")
        println(io, "ionSphereR0:             $(settings.ionSphereR0)  ")
        println(io, "NoBoundElectrons:        $(settings.NoBoundElectrons)  ")
        println(io, "printBeforeComputation:  $(settings.printBeforeComputation)  ")
        println(io, "selectLines:             $(settings.selectLines)  ")
        println(io, "selectedLines:           $(settings.selectedLines)  ")
    end

    """
    `struct  PlasmaShift.PhotoSettings`
        ... defines a type for the details and parameters of computing photoionization rates with plasma interactions.

        + plasmaModel            ::AbstractPlasmaModel           ... Specify a particular plasma model, e.g. ion-sphere, debye.
        + lambdaDebye            ::Float64                       ... The lambda parameter of different plasma models.
        + ionSphereR0            ::Float64                       ... The effective radius of the ion-sphere model.
        + NoBoundElectrons       ::Int64                         ... Effective number of bound electrons.
        + multipoles             ::Array{Basics.EmMultipole}     ... Specifies the multipoles of the radiation field that are to be included.
        + gauges                 ::Array{Basics.UseGauge}        ... Specifies the gauges to be included into the computations.
        + photonEnergies         ::Array{Float64,1}              ... List of photon energies.
        + printBeforeComputation ::Bool                          ... True, if all energies and lines are printed before their evaluation.
        + selectLines            ::Bool                          ... True, if lines are selected individually for the computations.
        + selectedLines          ::Array{Tuple{Int64,Int64},1}   ... List of lines, given by tupels (inital-level, final-level).
    """
    struct PhotoSettings
        plasmaModel              ::AbstractPlasmaModel
        lambdaDebye              ::Float64
        ionSphereR0              ::Float64
        NoBoundElectrons         ::Int64
        multipoles               ::Array{Basics.EmMultipole}
        gauges                   ::Array{Basics.UseGauge}
        photonEnergies           ::Array{Float64,1}
        printBeforeComputation   ::Bool
        selectLines              ::Bool
        selectedLines            ::Array{Tuple{Int64,Int64},1}
    end

    """
    `PlasmaShift.PhotoSettings()`  ... constructor for a standard instance of PlasmaShift.PhotoSettings.
    """
    function PhotoSettings()
        PhotoSettings(DebyeHueckel(), 0.25, 0., 0, [E1], [Basics.UseCoulomb], Float64[], true, false, Tuple{Int64,Int64}[])
    end

    # `Base.show(io::IO, settings::PlasmaShift.PhotoSettings)`  ... prepares a proper printout of the settings::PlasmaShift.PhotoSettings.
    function Base.show(io::IO, settings::PlasmaShift.PhotoSettings)
        println(io, "plasmaModel:             $(settings.plasmaModel)  ")
        println(io, "lambdaDebye:             $(settings.lambdaDebye)  ")
        println(io, "ionSphereR0:             $(settings.ionSphereR0)  ")
        println(io, "NoBoundElectrons:        $(settings.NoBoundElectrons)  ")
        println(io, "multipoles:              $(settings.multipoles)  ")
        println(io, "gauges:                  $(settings.gauges)  ")
        println(io, "photonEnergies:          $(settings.photonEnergies)  ")
        println(io, "printBeforeComputation:  $(settings.printBeforeComputation)  ")
        println(io, "selectLines:             $(settings.selectLines)  ")
        println(io, "selectedLines:           $(settings.selectedLines)  ")
    end

    """
    `PlasmaShift.computeOutcomes(multiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                                     asfSettings::AsfSettings, settings::PlasmaShift.Settings; output=true)`
        ... to compute a new multiplet from the plasma-modified Hamiltonian matrix as specified by the given
            settings. The diagonalization of the (plasma-modified) Hamiltonian matrix follows the asfSettings that
            were applied before for the computation of the multiplet; warnings are issued if features were selected
            that are not supported for plasma calculations, such as the Breit interaction, QED shifts and others.
            The energies and plasma shifts (with regard to the unperturbed multiplet) are printed to screen
            and into the summary file. The generated (plasma) multiplet is returned if output=true and nothing otherwise,
    """
    function computeOutcomes(multiplet::Multiplet, nm::Nuclear.Model, grid::Radial.Grid,
                             asfSettings::AsfSettings, settings::PlasmaShift.Settings; output=true)
        println("")
        printstyled("PlasmaShift.computeOutcomes(): The computation of plasma-shifts starts now ... \n", color=:light_green)
        printstyled("---------------------------------------------------------------------------------- \n", color=:light_green)
        #
        basis        = multiplet.levels[1].basis
        newMultiplet = Basics.perform("computation: CI for plasma", basis, nm, grid, asfSettings, settings)
        # Print all results to screen
        PlasmaShift.displayResults(stdout, multiplet, newMultiplet, settings)
        printSummary, iostream = Defaults.getDefaults("summary flag/stream")
        if  printSummary    PlasmaShift.displayResults(iostream, multiplet, newMultiplet, settings)   end
        #
        if    output    return( newMultiplet )
        else            return( nothing )
        end
    end

    """
    `PlasmaShift.displayResults(stream::IO, multiplet::Multiplet, newMultiplet::Multiplet, settings::PlasmaShift.Settings)`
        ... to display the energies, M_ms and F-parameters, etc. for the  selected levels. A neat table is printed but nothing
            is returned otherwise.
    """
    function  displayResults(stream::IO, multiplet::Multiplet, pMultiplet::Multiplet, settings::PlasmaShift.Settings)
        println(stream, " ")
        println(stream, " ")
        println(stream, "  Plasma shifts for $(settings.plasmaModel):")

        # Write out the relevant plasma parameters
        if settings.plasmaModel == PlasmaShift.DebyeHueckel()
            println(stream, "\n     + lambda = $(settings.lambdaDebye)")
            println(stream,   "     + Plasma screening included perturbatively in CI matrix but not in SCF field.")
        else  error("Unsupported plasma model = $(plasmaSettings.plasmaModel)")
        end

        println(stream, " ")
        println(stream, "  ", TableStrings.hLine(64))
        sa = "  ";   sb = "  "
        sa = sa * TableStrings.center(10, "Level"; na=2);                             sb = sb * TableStrings.hBlank(12)
        sa = sa * TableStrings.center(10, "J^P";   na=4);                             sb = sb * TableStrings.hBlank(14)
        sa = sa * TableStrings.center(18, "Energy w/o plasma"; na=4)
        sb = sb * TableStrings.center(18, TableStrings.inUnits("energy"); na=4)
        sa = sa * TableStrings.center(14, "Delta E";     na=4)
        sb = sb * TableStrings.center(14, TableStrings.inUnits("energy"); na=4)
        println(stream, sa);    println(stream, sb);    println(stream, "  ", TableStrings.hLine(64))
        #
        for  i  in  1:length(multiplet.levels)
            sa  = "  ";    sym = LevelSymmetry( multiplet.levels[i].J, multiplet.levels[i].parity);
                        newsym = LevelSymmetry( pMultiplet.levels[i].J, pMultiplet.levels[i].parity)
            sa = sa * TableStrings.center(10, TableStrings.level(multiplet.levels[i].index); na=2)
            sa = sa * TableStrings.center(10, string(sym); na=6)
            energy = multiplet.levels[i].energy
            sa = sa * @sprintf("%.8e", Defaults.convertUnits("energy: from atomic", energy))              * "     "
            if  sym == newsym   deltaE = pMultiplet.levels[i].energy - energy;
                                sa     = sa * @sprintf("%.8e", Defaults.convertUnits("energy: from atomic", deltaE))
            else                sa     = sa * "Level crossing."
            end
            println(stream, sa )
        end
        println(stream, "  ", TableStrings.hLine(64), "\n\n")
        #
        return( nothing )
    end

end # module


function OceanTidesWave(arg0::jint, arg1::jint, arg2::jint, arg3::Vector{Vector{jdouble}})
    return OceanTidesWave((jint, jint, jint, Vector{Vector{jdouble}}), arg0, arg1, arg2, arg3)
end

function add_contribution(obj::OceanTidesWave, arg0::BodiesElements, arg1::Vector{Vector{jdouble}}, arg2::Vector{Vector{jdouble}})
    return jcall(obj, "addContribution", void, (BodiesElements, Vector{Vector{jdouble}}, Vector{Vector{jdouble}}), arg0, arg1, arg2)
end

function get_doodson(obj::OceanTidesWave)
    return jcall(obj, "getDoodson", jint, ())
end

function get_max_degree(obj::OceanTidesWave)
    return jcall(obj, "getMaxDegree", jint, ())
end

function get_max_order(obj::OceanTidesWave)
    return jcall(obj, "getMaxOrder", jint, ())
end


# References:
# [1] 3-Axis Digital Accelerometer ADXL345 - D07925-0-5/09(0)

module PiADXL345

export adxl_open

using BBSPI
using PiGPIOMEM

"""
GPIOPin with inverted logic (Low = `true`, High = `false`).
"""
struct ActiveLowPin
    pin::GPIOPin
end
Base.setindex!(p::ActiveLowPin, v) = p.pin[] = iszero(v) ? 1 : 0
Base.getindex(p::ActiveLowPin) = iszero(p.pin[])

"""
Bit-delay for bit-bangged SPI.
"""
BBSPI.delay(s::BBSPI.SPISlave) = PiGPIOMEM.spin(50)

struct ADXL345{T} <: AbstractChannel{Tuple{Float64,Float64,Float64}}
    spi::T
    buf::Vector{UInt8}
    ADXL345(spi::T) where T = new{T}(spi, zeros(UInt8, 10))
end

"""
    adxl_open(;cs=chip select output pin,
              sdo=master input pin,
              sda=master output pin,
              scl=clock output pin)::ADXL345

Open ADXL345 conencted to GPIO pins `cs`, `sdo`, `sda` and `scl`.

Methods must be defined for `setindex!(::PinType, v)` and `getindex(::PinType)`.
See `help?> SPISlave`.
"""
function adxl_open(;cs=nothing, scl=nothing, sda=nothing, sdo=nothing)

    adxl_open(BBSPI.SPISlave(cs=ActiveLowPin(cs),
                             clk=ActiveLowPin(scl),
                             mosi=sda,
                             miso=sdo))
end

function adxl_open(spi)

    spi.chip_select[] = 0
    spi.clock[] = 0

    adxl = ADXL345(spi)

    @assert adxl_is_connected(adxl)

    adxl_enable(adxl)
    adxl_bw_rate_12hz5(adxl)

    adxl
end

adxl_is_connected(adxl) = adxl_read(adxl, 0) == 0xE5 # DEVID [1, p15]

adxl_enable(adxl) = adxl_write(adxl, 0x2D, 0b00001000) # POWER_CTL [1, p16]

adxl_offset(adxl) = [adxl_read(adxl, 0x1E), # OFSX [1, p15]
                     adxl_read(adxl, 0x1F), # OFSY [1, p15]
                     adxl_read(adxl, 0x20)] # OFSZ [1, p15]

function adxl_set_offset(adxl, x, y, z)
    adxl_write(adxl, 0x1E, unsigned(Int8(x))) # OFSX [1, p15]
    adxl_write(adxl, 0x1F, unsigned(Int8(y))) # OFSY [1, p15]
    adxl_write(adxl, 0x20, unsigned(Int8(z))) # OFSZ [1, p15]
end

adxl_bw_rate_12hz5(adxl) = adxl_write(adxl, 0x2C, 0b00000111) # BW_RATE [1, p16]
adxl_bw_rate_25hz(adxl)  = adxl_write(adxl, 0x2C, 0b00001000) # BW_RATE [1, p16]
adxl_bw_rate_50hz(adxl)  = adxl_write(adxl, 0x2C, 0b00001001) # BW_RATE [1, p16]
adxl_bw_rate_100hz(adxl) = adxl_write(adxl, 0x2C, 0b00001010) # BW_RATE [1, p16]

"""
    adxl_read(adxl, address, n=1)

Read `n` bytes from `address`.
"""
function adxl_read(adxl, address, n)
    n += 1
    cout = UInt8[0b11000000 | address] # SPI header [1, p9 Figure 5]
    BBSPI.transfer(adxl.spi, cout, view(adxl.buf, 1:n))
    return view(adxl.buf, 2:n)
end

adxl_read(adxl, address) = adxl_read(adxl, address, 1)[1]

"""
    adxl_write(adxl, address, v)

Write byte `v` to `address`.
"""
function adxl_write(adxl, address, v)
    cout = UInt8[address, v] # SPI header [1, p9 Figure 5]
    BBSPI.transfer(adxl.spi, cout)
    nothing
end

"""
    take!(::ADXL345)

Read [x,y,z] vector from ADXL345.
"""
function Base.take!(adxl::ADXL345)

    v = adxl_read(adxl, 0x32, 6) # DATAX0... Register [1, p18]
    x = signed(v[1] | UInt16(v[2]) << 8)/256 # 0.0039g/LSB [1, Table 1, p3]
    y = signed(v[3] | UInt16(v[4]) << 8)/256
    z = signed(v[5] | UInt16(v[6]) << 8)/256
    (x, y, z)
end

pitch_and_roll(v) = (pitch = atan(-v[1], hypot(v[2], v[3])) * 180 / π,
                     roll  = atan( v[2],             v[3])  * 180 / π)

function adxl_demo()
    adxl = adxl_open(cs=GPIOPin(26; output=true),
                     scl=GPIOPin(21; output=true),
                     sda=GPIOPin(20; output=true),
                     sdo=GPIOPin(16))

    while true
        t = time()
        v = take!(adxl)
        pr = pitch_and_roll(v)
        print(round(pr.pitch; digits=3), " ",
              round(pr.roll; digits=3), "      \r")
        sleep(max(0, 0.08 - (time() - t)))
    end
end

end # module


using DSGE, Test

# Initialize model object
m = AnSchorfheide(testing = true)
m <= Setting(:forecast_horizons, 12)

# Define dummy scenario
scen = Scenario(:testscen, "Test Scenario", [:obs_gdp, :obs_cpi], [:g_sh, :rm_sh], "REF")

# Check compute_scenario_system throws error before zeroing out measurement error
@testset "Check compute_scenario_system error throwing" begin
    @test_throws ErrorException compute_scenario_system(m, scen)
end

# Zero out measurement error (or else filtering shocks won't work exactly)
for para in [:e_y, :e_π, :e_R]
    m[para].value = 0
end

# Test compute_scenario_system
@testset "Test compute_scenario_system, filter_shocks!, and shock_scaling" begin
    global sys = compute_scenario_system(m, scen)
    @test all(x -> x == 0, sys[:CCC])
    @test all(x -> x == 0, sys[:DD])
    @test all(x -> x == 0, sys[:DD_pseudo])
    for shock in keys(m.exogenous_shocks)
        ind = m.exogenous_shocks[shock]
        if shock in scen.instrument_names
            @test sys[:QQ][ind, ind] != 0
        else
            @test sys[:QQ][ind, ind] == 0
        end
    end

    # Test filter_shocks!
    for var in scen.target_names
        scen.targets[var] = rand(forecast_horizons(m))
    end
    global forecastshocks = filter_shocks!(m, scen, sys)
    for var in scen.instrument_names
        ind = m.exogenous_shocks[var]
        @test scen.instruments[var] == forecastshocks[ind, :]
    end

    s_T = zeros(n_states_augmented(m))
    global _, forecastobs,_ ,_  = forecast(m, sys, s_T, shocks = forecastshocks)
    for var in scen.target_names
        ind = m.observables[var]
        @test scen.targets[var] ≈ forecastobs[ind, :]
    end

    # Test scenario with shock scaling
    global scale = Scenario(:scaledscen, "Test Shock Scaling Scenario",
                            [:obs_gdp, :obs_cpi], [:g_sh, :rm_sh], "REF",
                            shock_scaling = 2.0)
    scale.targets = scen.targets

    global forecastshocks = filter_shocks!(m, scale, sys)
    for var in scale.target_names
        ind = m.observables[var]
        @test scale.targets[var] == scen.targets[var]
    end
    for var in scale.instrument_names
        ind = m.exogenous_shocks[var]
        @test scale.instruments[var] == forecastshocks[ind, :]
    end
end


"""
    model_input(year::Int, d::Dict{Symbol,DataFrame}, set::Dict, idx::Dict)

# Arguments
- `year::Int`: year for which to perform calibration
- `d::Dict{Symbol,DataFrame}` of DataFrames containing the model data.
- `set::Dict` of Arrays describing region, sector, final demand, etc.
- `idx::Dict` of parameter indices

# Returns
- `d::Dict{Symbol,Dict}` of model data with all zeros filled
- `set::Dict` of model indices, with added arrays of tuples indicating which (region,sector)
    or (region,good) combinations are relevant to the zero-profit and market-clearing
    conditions.
- `idx::Dict` of parameter indices.
"""
function model_input(year::Any, d::Dict, set::Dict, idx::Dict=Dict())
    return model_input(ensurearray(year), d, set, idx)
end

function model_input(year::Array{Int,1}, d::Dict, set::Dict, idx::Dict=Dict())
    @info("Preparing model data for $year.")
    if length(year) > 1
        d2 = Dict(k => filter_with(df, (yr = year,); extrapolate=true) for (k, df) in d)

        isempty(idx) && (idx = Dict(k => findindex(df) for (k, df) in d2))
        (set, idx) = _model_set!(d2, set, idx)

        d1 = Dict(k => filter_with(df, (yr = year,); drop=true) for (k, df) in d)
        d1 = Dict(k => convert_type(Dict, fill_zero(df; with=set)) for (k, df) in d1)
        return (d1, set, idx)
    else
        d = Dict(k => filter_with(df, (yr = year,); drop=true) for (k, df) in d)

        isempty(idx) && (idx = Dict(k => findindex(df) for (k, df) in d))
        (set, idx) = _model_set!(d, set, idx)

        d = Dict(k => convert_type(Dict, fill_zero(df; with=set)) for (k, df) in d)
        return (d, set, idx)
    end
end

"""
    _model_set!(d::Dict{Symbol,DataFrame}, set::Dict, idx::Dict)
This function returns subsets intended to limit the size of the model by including only
non-zero values when mapping zero-profit and market-clearing conditions.

# Arguments
- `d::Dict{Symbol,DataFrame}` of DataFrames containing the model data.
- `set::Dict` of Arrays describing region, sector, final demand, etc.
- `idx::Dict` of parameter indices

# Returns
- `d::Dict{Symbol,Dict}` of model data with all zeros filled
- `set::Dict` of model indices, with added arrays of tuples indicating which (region,sector)
    or (region,good) combinations are relevant to the zero-profit and market-clearing
    conditions.
- `idx::Dict` of indices, updated to include those used to define the newly-added sets.
"""
function _model_set!(d::Dict{Symbol,DataFrame}, set::Dict, idx::Dict)
    (set[:A], idx[:A]) = nonzero_subset(d[:a0] + d[:rx0])
    (set[:Y], idx[:Y]) = nonzero_subset(combine_over(d[:ys0], :g))
    (set[:X], idx[:X]) = nonzero_subset(d[:s0])
    (set[:PA], idx[:PA]) = nonzero_subset(d[:a0])
    (set[:PD], idx[:PD]) = nonzero_subset(d[:xd0])
    (set[:PK], idx[:PK]) = nonzero_subset(d[:kd0])
    (set[:PY], idx[:PY]) = nonzero_subset(d[:s0])

    if :yr in idx[:kd0]
        set[:PKT] = convert_type(Array{Tuple}, unique(d[:kd0][:,setdiff(idx[:kd0], [:yr])]))
    end

    return (set, idx)
end

"""
    nonzero_subset(df::DataFrame)

# Returns
- `x::Array{Tuple,1}` of all parameter indices corresponding with non-zero values
- `idx::Array{Symbol,1}` of parameter indices in `df`
"""
function nonzero_subset(df::DataFrame)
    idx = findindex(df)
    val = convert_type(Array{Tuple}, dropzero(df)[:,idx])
    return (val, idx)
end


module AutoML

using XGBoost
using MLBase

include("preprocess/preprocess.jl")
include("feature_engineer/feature_engineer.jl")
include("model/model.jl")
include("evaluation/evaluation.jl")

end # module


# xaxis(args...; kw...) = Axis(:x, args...; kw...)
# yaxis(args...; kw...) = Axis(:y, args...; kw...)
# zaxis(args...; kw...) = Axis(:z, args...; kw...)

# -------------------------------------------------------------------------

function Axis(sp::Subplot, letter::Symbol, args...; kw...)
    # init with values from _plot_defaults
    d = KW(
        :letter => letter,
        # :extrema => (Inf, -Inf),
        :extrema => Extrema(),
        :discrete_map => Dict(),   # map discrete values to discrete indices
        :continuous_values => zeros(0),
        :use_minor => false,
        :show => true,  # show or hide the axis? (useful for linked subplots)
    )

    # get defaults from letter version, unless match
    for (k,v) in _axis_defaults
        lk = Symbol(letter, k)
        lv = _axis_defaults_byletter[lk]
        d[k] = (lv == :match ? v : lv)
    end

    # merge!(d, _axis_defaults)
    d[:discrete_values] = []

    # update the defaults
    update!(Axis([sp], d), args...; kw...)
end

function get_axis(sp::Subplot, letter::Symbol)
    axissym = Symbol(letter, :axis)
    if haskey(sp.attr, axissym)
        sp.attr[axissym]
    else
        sp.attr[axissym] = Axis(sp, letter)
    end::Axis
end

function process_axis_arg!(d::KW, arg, letter = "")
    T = typeof(arg)
    arg = get(_scaleAliases, arg, arg)

    if typeof(arg) <: Font
        d[Symbol(letter,:tickfont)] = arg
        d[Symbol(letter,:guidefont)] = arg

    elseif arg in _allScales
        d[Symbol(letter,:scale)] = arg

    elseif arg in (:flip, :invert, :inverted)
        d[Symbol(letter,:flip)] = true

    elseif T <: AbstractString
        d[Symbol(letter,:guide)] = arg

    # xlims/ylims
    elseif (T <: Tuple || T <: AVec) && length(arg) == 2
        sym = typeof(arg[1]) <: Number ? :lims : :ticks
        d[Symbol(letter,sym)] = arg

    # xticks/yticks
    elseif T <: AVec
        d[Symbol(letter,:ticks)] = arg

    elseif arg == nothing
        d[Symbol(letter,:ticks)] = []

    elseif typeof(arg) <: Number
        d[Symbol(letter,:rotation)] = arg

    elseif typeof(arg) <: Function
        d[Symbol(letter,:formatter)] = arg

    else
        warn("Skipped $(letter)axis arg $arg")

    end
end

# update an Axis object with magic args and keywords
function update!(axis::Axis, args...; kw...)
    # first process args
    d = axis.d
    for arg in args
        process_axis_arg!(d, arg)
    end

    # then override for any keywords... only those keywords that already exists in d
    for (k,v) in kw
        if haskey(d, k)
            if k == :discrete_values
                # add these discrete values to the axis
                for vi in v
                    discrete_value!(axis, vi)
                end
            else
                d[k] = v
            end
        end
    end

    # replace scale aliases
    if haskey(_scaleAliases, d[:scale])
        d[:scale] = _scaleAliases[d[:scale]]
    end

    axis
end

# -------------------------------------------------------------------------

Base.show(io::IO, axis::Axis) = dumpdict(axis.d, "Axis", true)
# Base.getindex(axis::Axis, k::Symbol) = getindex(axis.d, k)
Base.setindex!(axis::Axis, v, ks::Symbol...) = setindex!(axis.d, v, ks...)
Base.haskey(axis::Axis, k::Symbol) = haskey(axis.d, k)
Base.extrema(axis::Axis) = (ex = axis[:extrema]; (ex.emin, ex.emax))

const _scale_funcs = Dict{Symbol,Function}(
    :log10 => log10,
    :log2 => log2,
    :ln => log,
)
const _inv_scale_funcs = Dict{Symbol,Function}(
    :log10 => exp10,
    :log2 => exp2,
    :ln => exp,
)

# const _label_func = Dict{Symbol,Function}(
#     :log10 => x -> "10^$x",
#     :log2 => x -> "2^$x",
#     :ln => x -> "e^$x",
# )

const _label_func = Dict{Symbol,Function}(
    :log10 => x -> "10^$x",
    :log2 => x -> "2^$x",
    :ln => x -> "e^$x",
)

scalefunc(scale::Symbol) = x -> get(_scale_funcs, scale, identity)(Float64(x))
invscalefunc(scale::Symbol) = x -> get(_inv_scale_funcs, scale, identity)(Float64(x))
labelfunc(scale::Symbol, backend::AbstractBackend) = get(_label_func, scale, string)

function optimal_ticks_and_labels(axis::Axis, ticks = nothing)
    amin,amax = axis_limits(axis)

    # scale the limits
    scale = axis[:scale]
    sf = scalefunc(scale)

    # get a list of well-laid-out ticks
    scaled_ticks = if ticks == nothing
        optimize_ticks(
            sf(amin),
            sf(amax);
            k_min = 5, # minimum number of ticks
            k_max = 8, # maximum number of ticks
        )[1]
    else
        map(sf, filter(t -> amin <= t <= amax, ticks))
    end
    unscaled_ticks = map(invscalefunc(scale), scaled_ticks)

    labels = if any(isfinite, unscaled_ticks)
        formatter = axis[:formatter]
        if formatter == :auto
            # the default behavior is to make strings of the scaled values and then apply the labelfunc
            map(labelfunc(scale, backend()), Showoff.showoff(scaled_ticks, :plain))
        elseif formatter == :scientific
            Showoff.showoff(unscaled_ticks, :scientific)
        else
            # there was an override for the formatter... use that on the unscaled ticks
            map(formatter, unscaled_ticks)
        end
    else
        # no finite ticks to show...
        String[]
    end

    # @show unscaled_ticks labels
    # labels = Showoff.showoff(unscaled_ticks, scale == :log10 ? :scientific : :auto)
    unscaled_ticks, labels
end

# return (continuous_values, discrete_values) for the ticks on this axis
function get_ticks(axis::Axis)
    ticks = axis[:ticks]
    ticks in (nothing, false) && return nothing

    dvals = axis[:discrete_values]
    cv, dv = if !isempty(dvals) && ticks == :auto
        # discrete ticks...
        axis[:continuous_values], dvals
    elseif ticks == :auto
        # compute optimal ticks and labels
        optimal_ticks_and_labels(axis)
    elseif typeof(ticks) <: AVec
        # override ticks, but get the labels
        optimal_ticks_and_labels(axis, ticks)
    elseif typeof(ticks) <: NTuple{2}
        # assuming we're passed (ticks, labels)
        ticks
    else
        error("Unknown ticks type in get_ticks: $(typeof(ticks))")
    end
    # @show ticks dvals cv dv

    # TODO: better/smarter cutoff values for sampling ticks
    if length(cv) > 30
        rng = Int[round(Int,i) for i in linspace(1, length(cv), 15)]
        cv[rng], dv[rng]
    else
        cv, dv
    end
end

# -------------------------------------------------------------------------

function reset_extrema!(sp::Subplot)
    for asym in (:x,:y,:z)
        sp[Symbol(asym,:axis)][:extrema] = Extrema()
    end
    for series in sp.series_list
        expand_extrema!(sp, series.d)
    end
end

function expand_extrema!(ex::Extrema, v::Number)
    ex.emin = min(v, ex.emin)
    ex.emax = max(v, ex.emax)
    ex
end

function expand_extrema!(axis::Axis, v::Number)
    expand_extrema!(axis[:extrema], v)
end

# these shouldn't impact the extrema
expand_extrema!(axis::Axis, ::Void) = axis[:extrema]
expand_extrema!(axis::Axis, ::Bool) = axis[:extrema]

function expand_extrema!{MIN<:Number,MAX<:Number}(axis::Axis, v::Tuple{MIN,MAX})
    ex = axis[:extrema]
    ex.emin = min(v[1], ex.emin)
    ex.emax = max(v[2], ex.emax)
    ex
end
function expand_extrema!{N<:Number}(axis::Axis, v::AVec{N})
    ex = axis[:extrema]
    for vi in v
        expand_extrema!(ex, vi)
    end
    ex
end

function expand_extrema!(sp::Subplot, d::KW)
    vert = isvertical(d)

    # first expand for the data
    for letter in (:x, :y, :z)
        data = d[if vert
            letter
        else
            letter == :x ? :y : letter == :y ? :x : :z
        end]
        axis = sp[Symbol(letter, "axis")]

        if isa(data, Volume)
            expand_extrema!(sp[:xaxis], data.x_extents)
            expand_extrema!(sp[:yaxis], data.y_extents)
            expand_extrema!(sp[:zaxis], data.z_extents)
        elseif eltype(data) <: Number || (isa(data, Surface) && all(di -> isa(di, Number), data.surf))
            if !(eltype(data) <: Number)
                # huh... must have been a mis-typed surface? lets swap it out
                data = d[letter] = Surface(Matrix{Float64}(data.surf))
            end
            expand_extrema!(axis, data)
        elseif data != nothing
            # TODO: need more here... gotta track the discrete reference value
            #       as well as any coord offset (think of boxplot shape coords... they all
            #       correspond to the same x-value)
            d[letter], d[Symbol(letter,"_discrete_indices")] = discrete_value!(axis, data)
            expand_extrema!(axis, d[letter])
        end
    end

    # # expand for fillrange/bar_width
    # fillaxis, baraxis = sp.attr[:yaxis], sp.attr[:xaxis]
    # if isvertical(d)
    #     fillaxis, baraxis = baraxis, fillaxis
    # end

    # expand for fillrange
    fr = d[:fillrange]
    if fr == nothing && d[:seriestype] == :bar
        fr = 0.0
    end
    if fr != nothing
        axis = sp.attr[vert ? :yaxis : :xaxis]
        if typeof(fr) <: Tuple
            for fri in fr
                expand_extrema!(axis, fri)
            end
        else
            expand_extrema!(axis, fr)
        end
    end

    # expand for bar_width
    if d[:seriestype] == :bar
        dsym = vert ? :x : :y
        data = d[dsym]

        bw = d[:bar_width]
        if bw == nothing
            bw = d[:bar_width] = mean(diff(data))
        end
        axis = sp.attr[Symbol(dsym, :axis)]
        expand_extrema!(axis, maximum(data) + 0.5maximum(bw))
        expand_extrema!(axis, minimum(data) - 0.5minimum(bw))
    end

end

function expand_extrema!(sp::Subplot, xmin, xmax, ymin, ymax)
    expand_extrema!(sp[:xaxis], (xmin, xmax))
    expand_extrema!(sp[:yaxis], (ymin, ymax))
end

# -------------------------------------------------------------------------

# push the limits out slightly
function widen(lmin, lmax)
    span = lmax - lmin
    # eps = max(1e-16, min(1e-2span, 1e-10))
    eps = max(1e-16, 0.03span)
    lmin-eps, lmax+eps
end

# figure out if widening is a good idea.  if there's a scale set it's too tricky,
# so lazy out and don't widen
function default_should_widen(axis::Axis)
    should_widen = false
    if axis[:scale] == :identity && !is_2tuple(axis[:lims])
        for sp in axis.sps
            for series in series_list(sp)
                if series.d[:seriestype] in (:scatter,) || series.d[:markershape] != :none
                    should_widen = true
                end
            end
        end
    end
    should_widen
end

# using the axis extrema and limit overrides, return the min/max value for this axis
function axis_limits(axis::Axis, should_widen::Bool = default_should_widen(axis))
    ex = axis[:extrema]
    amin, amax = ex.emin, ex.emax
    lims = axis[:lims]
    if (isa(lims, Tuple) || isa(lims, AVec)) && length(lims) == 2
        if isfinite(lims[1])
            amin = lims[1]
        end
        if isfinite(lims[2])
            amax = lims[2]
        end
    end
    if amax <= amin && isfinite(amin)
        amax = amin + 1.0
    end
    if !isfinite(amin) && !isfinite(amax)
        amin, amax = 0.0, 1.0
    end
    if should_widen
        widen(amin, amax)
    else
        amin, amax
    end
end

# -------------------------------------------------------------------------

# these methods track the discrete (categorical) values which correspond to axis continuous values (cv)
# whenever we have discrete values, we automatically set the ticks to match.
# we return (continuous_value, discrete_index)
function discrete_value!(axis::Axis, dv)
    cv_idx = get(axis[:discrete_map], dv, -1)
    # @show axis[:discrete_map], axis[:discrete_values], dv
    if cv_idx == -1
        ex = axis[:extrema]
        cv = max(0.5, ex.emax + 1.0)
        expand_extrema!(axis, cv)
        push!(axis[:discrete_values], dv)
        push!(axis[:continuous_values], cv)
        cv_idx = length(axis[:discrete_values])
        axis[:discrete_map][dv] = cv_idx
        cv, cv_idx
    else
        cv = axis[:continuous_values][cv_idx]
        cv, cv_idx
    end
end

# continuous value... just pass back with axis negative index
function discrete_value!(axis::Axis, cv::Number)
    cv, -1
end

# add the discrete value for each item.  return the continuous values and the indices
function discrete_value!(axis::Axis, v::AVec)
    n = length(v)
    cvec = zeros(n)
    discrete_indices = zeros(Int, n)
    for i=1:n
        cvec[i], discrete_indices[i] = discrete_value!(axis, v[i])
    end
    cvec, discrete_indices
end

# add the discrete value for each item.  return the continuous values and the indices
function discrete_value!(axis::Axis, v::AMat)
    n,m = size(v)
    cmat = zeros(n,m)
    discrete_indices = zeros(Int, n, m)
    for i=1:n, j=1:m
        cmat[i,j], discrete_indices[i,j] = discrete_value!(axis, v[i,j])
    end
    cmat, discrete_indices
end

function discrete_value!(axis::Axis, v::Surface)
    map(Surface, discrete_value!(axis, v.surf))
end

# -------------------------------------------------------------------------

function pie_labels(sp::Subplot, series::Series)
    d = series.d
    if haskey(d,:x_discrete_indices)
        dvals = sp.attr[:xaxis].d[:discrete_values]
        [dvals[idx] for idx in d[:x_discrete_indices]]
    else
        d[:x]
    end
end

# -------------------------------------------------------------------------

# compute the line segments which should be drawn for this axis
function axis_drawing_info(sp::Subplot)
    xaxis, yaxis = sp[:xaxis], sp[:yaxis]
    xmin, xmax = axis_limits(xaxis)
    ymin, ymax = axis_limits(yaxis)
    xticks = get_ticks(xaxis)
    yticks = get_ticks(yaxis)
    spine_segs = Segments(2)
    grid_segs = Segments(2)

    if !(xaxis[:ticks] in (nothing, false))
        f = scalefunc(yaxis[:scale])
        invf = invscalefunc(yaxis[:scale])
        t1 = invf(f(ymin) + 0.015*(f(ymax)-f(ymin)))
        t2 = invf(f(ymax) - 0.015*(f(ymax)-f(ymin)))

        push!(spine_segs, (xmin,ymin), (xmax,ymin)) # bottom spine
        # push!(spine_segs, (xmin,ymax), (xmax,ymax)) # top spine
        for xtick in xticks[1]
            push!(spine_segs, (xtick, ymin), (xtick, t1)) # bottom tick
            push!(grid_segs,  (xtick, t1),   (xtick, t2)) # vertical grid
            # push!(spine_segs, (xtick, ymax), (xtick, t2)) # top tick
        end
    end

    if !(yaxis[:ticks] in (nothing, false))
        f = scalefunc(xaxis[:scale])
        invf = invscalefunc(xaxis[:scale])
        t1 = invf(f(xmin) + 0.015*(f(xmax)-f(xmin)))
        t2 = invf(f(xmax) - 0.015*(f(xmax)-f(xmin)))

        push!(spine_segs, (xmin,ymin), (xmin,ymax)) # left spine
        # push!(spine_segs, (xmax,ymin), (xmax,ymax)) # right spine
        for ytick in yticks[1]
            push!(spine_segs, (xmin, ytick), (t1, ytick)) # left tick
            push!(grid_segs,  (t1, ytick),   (t2, ytick)) # horizontal grid
            # push!(spine_segs, (xmax, ytick), (t2, ytick)) # right tick
        end
    end

    xticks, yticks, spine_segs, grid_segs
end


module Fracciones

export Fraccion, numerador, denominador, fraccion, reciproco, @fraccion

include("fraccion.jl")

end # module


"""
  Profiling for all matpower instances
    To be adapted...
"""
include(joinpath(ROOT,"..","src_PowSysMod", "PowSysMod_body.jl"))

struct Measurement
  nb_buses
  time
  bytes
  gctime
  nbpoolallocs
  nbmalloc
  summarysize
end

## Input
files_m = [split(name, ".")[1] for name in readdir(joinpath("instances", "matpower"))]
files_dat = [split(name, ".")[1] for name in readdir(joinpath("instances", "matpower_QCQP"))]

instances = intersect(files_m, files_dat)

nb_repeat = 3

instancename = sort(collect(instances))[1]

data_OPFpbs = Dict{String, Measurement}()
data_Problems = Dict{String, Measurement}()
data_errors = Dict{String, Float64}()

instnb = 1
for instancename in sort(collect(instances))
  println("\n-- Working on $instancename ($instnb/$(length(instances)))")
  nb_buses = parse(matchall(r"\d+", instancename)[1])

  ## Building OPF_problems
  t = bytes = gctime = nbpoolallocs = nbmalloc = sumsize = 0
  OPFpbs = build_OPFproblems(MatpowerSimpleInput, [joinpath("instances", "matpower", "$instancename.m")])
  for i=1:nb_repeat
    print("$i ")
    OPFpbs, t_, bytes_, gctime_, gc_ = @timed build_OPFproblems(MatpowerSimpleInput, [joinpath("instances", "matpower", "$instancename.m")])
    t += t_/nb_repeat
    bytes += bytes_/nb_repeat
    gctime += gctime_/nb_repeat
    nbpoolallocs += gc_.poolalloc/nb_repeat
    nbmalloc += gc_.malloc/nb_repeat
    sumsize += Base.summarysize(OPFpbs)/nb_repeat
  end
  data_OPFpbs[instancename] = Measurement(nb_buses, t, bytes, gctime, nbpoolallocs, nbmalloc, sumsize)

  for (busname, busdata) in OPFpbs["BaseCase"].ds.bus
    if haskey(busdata, "Gen_reference")
      OPFpbs["BaseCase"].mc.node_formulations[busname]["Gen_reference"] = :None
    end
  end

  ## Building OPF problem
  t = bytes = gctime = nbpoolallocs = nbmalloc = sumsize = 0
  pb = build_Problem!(OPFpbs, "BaseCase")
  for i=1:nb_repeat
    print("$i ")
    pb, t_, bytes_, gctime_, gc_ = @timed build_Problem!(OPFpbs, "BaseCase")
    t += t_/nb_repeat
    bytes += bytes_/nb_repeat
    gctime += gctime_/nb_repeat
    nbpoolallocs += gc_.poolalloc/nb_repeat
    nbmalloc += gc_.malloc/nb_repeat
    sumsize += Base.summarysize(pb)/nb_repeat
  end
  data_Problems[instancename] = Measurement(nb_buses, t, bytes, gctime, nbpoolallocs, nbmalloc, sumsize)

  export_to_dat(pb, "my$instancename.dat")
  filename_dat = joinpath("instances", "matpower_QCQP", "$instancename.dat")
  error, _ = compare_dat("my$instancename.dat", filename_dat)
  rm("my$instancename.dat")
  data_errors[instancename] = error

  instnb+=1
end

# output
filename = "buildOPFpbs_$(Dates.format(now(), "yy_u_dd_HH_MM")).csv"
touch(filename)

open(filename, "w") do outfile
  print(outfile, "instancename;nb_buses;time;bytes;gctime;nbpoolallocs;nbmalloc;summarysize;dat_error\n")

  insancenames_ord = sort(collect(keys(data_OPFpbs)), by=x->data_OPFpbs[x].nb_buses)

  for instancename in insancenames_ord
    measure = data_OPFpbs[instancename]
    print(outfile, "$instancename;$(measure.nb_buses);$(measure.time);$(measure.bytes);$(measure.gctime);$(measure.nbpoolallocs);$(measure.nbmalloc);$(measure.summarysize);$(data_errors[instancename])\n")
  end
end

filename = "build_Problem_$(Dates.format(now(), "yy_u_dd_HH_MM")).csv"
touch(filename)

open(filename, "w") do outfile
  print(outfile, "instancename;nb_buses;time;bytes;gctime;nbpoolallocs;nbmalloc;summarysize;dat_error\n")

  insancenames_ord = sort(collect(keys(data_Problems)), by=x->data_Problems[x].nb_buses)

  for instancename in insancenames_ord
    measure = data_Problems[instancename]
    print(outfile, "$instancename;$(measure.nb_buses);$(measure.time);$(measure.bytes);$(measure.gctime);$(measure.nbpoolallocs);$(measure.nbmalloc);$(measure.summarysize);$(data_errors[instancename])\n")
  end
end


module FlujoFluidos

# Write your package code here.

export Reynolds

include("accesorios.jl")
include("fluidos.jl")
include("fricción.jl")
include("tuberias.jl")

end


# define atom and module

struct Atom
    name::String
    Z::Int
    Ne::Int
    orbitals::Vector{STONG}
    R::Vec3{Float64}
    Atom(name::AbstractString, Z::Integer, Ne::Integer, orbitals::Vector{STONG{N}}, R::Vec3) where N = new(
        name, Z%Int, Ne%Int, orbitals, float(R)
    )
end

Atom(name::AbstractString, R::Vec3) = begin
    name = titlecase(name)
    number = atomic_number[name]
    Rf = float(R)
    number <= 2 && return Atom(name, number, number, [
        STONG(coefficient_data[name]["1s"], Rf)
    ], Rf)
    number <= 10 && return Atom(name, number, number, [
        STONG(coefficient_data[name]["1s"], Rf),
        STONG(coefficient_data[name]["2s"], Rf),
        STONG(coefficient_data[name]["2p"], Rf; type=:px),
        STONG(coefficient_data[name]["2p"], Rf; type=:py),
        STONG(coefficient_data[name]["2p"], Rf; type=:pz)
    ], Rf)
    error("atomic_number > 10 atoms are not support yet")
end

const atomic_number = Dict{String, Int}(
    "H"  => 1,
    "He" => 2,
    "Li" => 3,
    "Be" => 4,
    "B"  => 5,
    "C"  => 6,
    "N"  => 7,
    "O"  => 8,
    "F"  => 9,
    "Ne" => 10,
    "Na" => 11,
    "Mg" => 12,
    "Al" => 13,
    "Si" => 14,
    "P"  => 15,
    "S"  => 16,
    "Cl" => 17,
    "Ar" => 18,
    "K"  => 19,
    "Ca" => 20
)

hydrogen_atom(R::Vec3) = Atom("H", R)
helium_atom(R::Vec3)   = Atom("He", R)
carbon_atom(R::Vec3)   = Atom("C", R)
oxygen_atom(R::Vec3)   = Atom("O", R)
fluorine_atom(R::Vec3) = Atom("F", R)

struct Molecule
    name::String
    charge::Int
    atoms::Vector{Atom}
    Molecule(name::AbstractString, charge::Integer, atoms::Vector{Atom}) = new(name, charge%Int, atoms)
end

Molecule(name::AbstractString, atoms::Vector{Atom}) = Molecule(name, 0, atoms)
Molecule(atoms::Vector{Atom}) = Molecule("", 0, atoms)


using Test
using Basics.AbstractDataStructure: DataStructure, ins!, del!, at, search, len, ordered_data_structure_p
using Basics.LinkedLists #: LinkedList, ins!, del!, at, search, len, ordered_data_structure_p
using Basics.SkipLists #: SkipList, ins!, del!, at, search, len, ordered_data_structure_p

function test_order_via_interface(ds::DataStructure{Int, Int})
    @test issorted([i for i in ds])
    inserted_zero = ins!(ds, 0)
    inserted_three = ins!(ds, 3)
    inserted_minus_two = ins!(ds, -2)
    inserted_zero = ins!(ds, 1)
    inserted_zero = ins!(ds, 0)
    @test issorted([i for i in ds])
    del!(ds, search(ds, 1)[1])
    @test issorted([i for i in ds])
end

function test_correctness_via_interface(ds::DataStructure{Int, Int})
    # Results when empty
    @test len(ds) == 0
    @test_throws KeyError at(ds, 1)
    @test search(ds, 1) == Int[]
    @test_throws KeyError del!(ds, 1)
    # Insert one 0
    first_inserted = ins!(ds, 0)
    @test first_inserted isa Int
    @test at(ds, first_inserted) == 0
    @test len(ds) == 1
    @test search(ds, 0) == [first_inserted]
    @test search(ds, 1) == Int[]
    @test_throws KeyError del!(ds, first_inserted + 1)
    # Delete it and run some tests
    @test del!(ds, first_inserted) == 0
    @test_throws KeyError at(ds, first_inserted)
    @test search(ds, 0) == Int[]
    @test len(ds) == 0
    # Insert 0, 3, -2
    inserted_zero = ins!(ds, 0)
    @test at(ds, inserted_zero) == 0
    inserted_three = ins!(ds, 3)
    @test at(ds, inserted_three) == 3
    inserted_minus_two = ins!(ds, -2)
    @test at(ds, inserted_minus_two) == -2
    @test search(ds, -2) == [inserted_minus_two]
    @test (inserted_zero, inserted_three, inserted_minus_two) isa Tuple{Int,Int,Int}
    @test len(ds) == 3
    @test search(ds, 1) == Int[]
    @test_throws KeyError del!(ds, search(ds, -2)[1] * search(ds, 0)[1] * search(ds, 3)[1])
    # Delete 3, check that 0 and -2 are left
    three = del!(ds, search(ds, 3)[1])
    @test three == 3
    @test len(ds) == 2
    @test at(ds, search(ds, -2)[1]) == -2
    @test at(ds, search(ds, 0)[1]) == 0
    @test search(ds, 3) == Int[]
    @test at(ds, search(ds, -2)[1]) == -2
    # Insert another -2, check that there are two of them
    inserted_second_minus_two = ins!(ds, -2)
    @test length(search(ds, -2)) == 2
    @test length(search(ds, 0)) == 1
    @test len(ds) == 3
    # Empty and run order tests
    if ordered_data_structure_p(ds)
        del!(ds, search(ds, -2)[1])
        del!(ds, search(ds, -2)[1])
        del!(ds, search(ds, 0)[1])
        @test len(ds) == 0
        test_order_via_interface(ds)
    end
end

@testset "Interface contracts" begin
    test_correctness_via_interface(LinkedList{Int}())
    test_correctness_via_interface(SkipList{Int}())
end