# Set the IPP library path
@static if Sys.isapple()
    ipplib = ENV["IPPROOT"] * "/lib"
end

@static if Sys.islinux()
    ipplib = ENV["IPPROOT"] * "/lib/intel64"
end

@static if Sys.iswindows()
    error("Unsupported OS: Windows")
end

const ippC1 = 1
const ippBorderRepl = 1

const ippNearest = 1
const ippCubic = 6
const ippSuper = 8
const ippLanczos = 16

mutable struct IppiSize
    width::Cint
    height::Cint
end

mutable struct IppiPoint
    x::Cint
    y::Cint
end

mutable struct IppiBorderSize
    borderLeft::Cint
    borderTop::Cint
    borderRight::Cint
    borderBottom::Cint
end

function resizeCubic32fC1R(src::Matrix{Float32}, width::Integer, height::Integer)
    # a destination buffer
    dst = Matrix{Float32}(undef, width, height)

    dims = size(src)
    srcSize = IppiSize(dims[1], dims[2])
    dstSize = IppiSize(width, height)

    specSize = Ref{Cint}(0)
    initSize = Ref{Cint}(0)

    # Spec and init buffer sizes
    status = ccall(
        (:ippiResizeGetSize_32f, ipplib * "/libippi.so"),
        Cint,
        (IppiSize, IppiSize, Cint, Cint, Ref{Cint}, Ref{Cint}),
        srcSize,
        dstSize,
        ippCubic,
        0,
        specSize,
        initSize,
    )

    if status != 0
        error("ippiResizeGetSize_32f::$status")
    end

    # Memory allocation
    pInitBuf =
        ccall((:ippsMalloc_8u, ipplib * "/libipps.so"), Ptr{Cvoid}, (Cint,), initSize[])

    pSpec = ccall((:ippsMalloc_8u, ipplib * "/libipps.so"), Ptr{Cvoid}, (Cint,), specSize[])

    if pInitBuf == C_NULL || pSpec == C_NULL
        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pInitBuf)
        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)
        error("NULL pInitBuf or pSpec")
    end

    valueB = 0.0f0
    valueC = 0.0f0

    # Filter initialization
    status = ccall(
        (:ippiResizeCubicInit_32f, ipplib * "/libippi.so"),
        Cint,
        (IppiSize, IppiSize, Cfloat, Cfloat, Ptr{Cvoid}, Ptr{Cvoid}),
        srcSize,
        dstSize,
        valueB,
        valueC,
        pSpec,
        pInitBuf,
    )
    ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pInitBuf)

    if status != 0
        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)
        error("ippiResizeCubicInit_32f::$status")
    end

    borderSize = Ref{IppiBorderSize}(IppiBorderSize(0, 0, 0, 0))

    status = ccall(
        (:ippiResizeGetBorderSize_32f, ipplib * "/libippi.so"),
        Cint,
        (Ptr{Cvoid}, Ref{IppiBorderSize}),
        pSpec,
        borderSize,
    )

    if status != 0
        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)
        error("ippiResizeGetBorderSize_32f::$status")
    end

    bufSize = Ref{Cint}(0)

    # General transform function
    status = ccall(
        (:ippiResizeGetBufferSize_32f, ipplib * "/libippi.so"),
        Cint,
        (Ptr{Cvoid}, IppiSize, Cint, Ref{Cint}),
        pSpec,
        dstSize,
        ippC1,
        bufSize,
    )

    if status != 0
        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)
        error("ippiResizeGetBufferSize_32f::$status")
    end

    pBuffer =
        ccall((:ippsMalloc_8u, ipplib * "/libipps.so"), Ptr{Cvoid}, (Cint,), bufSize[])

    if pBuffer != C_NULL
        srcOffset = IppiPoint(0, 0)
        dstOffset = IppiPoint(0, 0)

        status = ccall(
            (:ippiResizeGetSrcRoi_32f, ipplib * "/libippi.so"),
            Cint,
            (Ptr{Cvoid}, IppiPoint, IppiSize, Ref{IppiPoint}, Ref{IppiSize}),
            pSpec,
            dstOffset,
            dstSize,
            srcOffset,
            srcSize,
        )

        if status == 0
            # finally resize the image
            srcStep = strides(src)[2] * sizeof(Float32)
            dstStep = strides(dst)[2] * sizeof(Float32)

            status = ccall(
                (:ippiResizeCubic_32f_C1R, ipplib * "/libippi.so"),
                Cint,
                (
                    Ptr{Cfloat},
                    Cint,
                    Ptr{Cfloat},
                    Cint,
                    IppiPoint,
                    IppiSize,
                    Cint,
                    Ptr{Cfloat},
                    Ptr{Cvoid},
                    Ptr{Cvoid},
                ),
                src,
                srcStep,
                dst,
                dstStep,
                dstOffset,
                dstSize,
                ippBorderRepl,
                C_NULL,
                pSpec,
                pBuffer,
            )

        end

        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pBuffer)
    end

    # Release memory
    ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)

    return dst
end

function resizeNearest8uC1R(src::Matrix{UInt8}, width::Integer, height::Integer)
    # a destination buffer
    dst = Matrix{UInt8}(undef, width, height)

    dims = size(src)
    srcSize = IppiSize(dims[1], dims[2])
    dstSize = IppiSize(width, height)

    specSize = Ref{Cint}(0)
    initSize = Ref{Cint}(0)

    # Spec and init buffer sizes
    status = ccall(
        (:ippiResizeGetSize_8u, ipplib * "/libippi.so"),
        Cint,
        (IppiSize, IppiSize, Cint, Cint, Ref{Cint}, Ref{Cint}),
        srcSize,
        dstSize,
        ippNearest,
        0,
        specSize,
        initSize,
    )

    if status != 0
        error("ippiResizeGetSize_8u::$status")
    end

    # Memory allocation
    pSpec = ccall((:ippsMalloc_8u, ipplib * "/libipps.so"), Ptr{Cvoid}, (Cint,), specSize[])

    if pSpec == C_NULL
        error("NULL pSpec")
    end

    # Filter initialization
    status = ccall(
        (:ippiResizeNearestInit_8u, ipplib * "/libippi.so"),
        Cint,
        (IppiSize, IppiSize, Ptr{Cvoid}),
        srcSize,
        dstSize,
        pSpec,
    )

    if status != 0
        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)
        error("ippiResizeNearestInit_8u::$status")
    end

    bufSize = Ref{Cint}(0)

    # General transform function
    status = ccall(
        (:ippiResizeGetBufferSize_8u, ipplib * "/libippi.so"),
        Cint,
        (Ptr{Cvoid}, IppiSize, Cint, Ref{Cint}),
        pSpec,
        dstSize,
        ippC1,
        bufSize,
    )

    if status != 0
        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)
        error("ippiResizeGetBufferSize_8u::$status")
    end

    pBuffer =
        ccall((:ippsMalloc_8u, ipplib * "/libipps.so"), Ptr{Cvoid}, (Cint,), bufSize[])

    if pBuffer != C_NULL
        srcOffset = IppiPoint(0, 0)
        dstOffset = IppiPoint(0, 0)

        status = ccall(
            (:ippiResizeGetSrcRoi_8u, ipplib * "/libippi.so"),
            Cint,
            (Ptr{Cvoid}, IppiPoint, IppiSize, Ref{IppiPoint}, Ref{IppiSize}),
            pSpec,
            dstOffset,
            dstSize,
            srcOffset,
            srcSize,
        )

        if status == 0
            # finally resize the image
            srcStep = strides(src)[2] * sizeof(UInt8)
            dstStep = strides(dst)[2] * sizeof(UInt8)

            status = ccall(
                (:ippiResizeNearest_8u_C1R, ipplib * "/libippi.so"),
                Cint,
                (Ptr{UInt8}, Cint, Ptr{UInt8}, Cint, IppiPoint, IppiSize, Cint, Ptr{Cvoid}),
                src,
                srcStep,
                dst,
                dstStep,
                dstOffset,
                dstSize,
                pSpec,
                pBuffer,
            )

        end

        ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pBuffer)
    end

    # Release memory
    ccall((:ippsFree, ipplib * "/libipps.so"), Cvoid, (Ptr{Cvoid},), pSpec)

    return dst
end


function add_constrained_variables(
    dual_problem,
    primal_model,
    variable_parameters,
)
    single_or_vector_variables_types =
        MOIU.sorted_variable_sets_by_cost(dual_problem.dual_model, primal_model)
    params = Set(variable_parameters)
    for S in single_or_vector_variables_types
        if S <: MOI.AbstractVectorSet
            _add_constrained_variables(
                dual_problem.primal_dual_map,
                primal_model,
                S,
                params,
            )
        elseif S <: MOI.AbstractScalarSet
            _add_constrained_variable(
                dual_problem.primal_dual_map,
                primal_model,
                S,
                params,
            )
        end
    end
    return
end
const NO_CONSTRAINT = CI{Nothing,Nothing}(0)
# Function barrier for the type unstability of `F` and `S`.
function _add_constrained_variables(
    m::PrimalDualMap,
    primal_model,
    ::Type{S},
    params,
) where {S<:MOI.AbstractVectorSet}
    cis = MOI.get(
        primal_model,
        MOI.ListOfConstraintIndices{MOI.VectorOfVariables,S}(),
    )
    for ci in cis
        f = MOI.get(primal_model, MOI.ConstraintFunction(), ci)
        if all(
            vi -> !haskey(m.constrained_var_idx, vi) && !(vi in params),
            f.variables,
        )
            for (i, vi) in enumerate(f.variables)
                m.constrained_var_idx[vi] = (ci, i)
            end
            # Placeholder to indicate this constraint is part of constrained variables,
            # it will be replaced later with a dual constraints
            m.constrained_var_dual[ci] = NO_CONSTRAINT
        end
    end
    return
end
function _add_constrained_variable(
    m::PrimalDualMap,
    primal_model,
    ::Type{S},
    params,
) where {S<:MOI.AbstractScalarSet}
    cis = MOI.get(
        primal_model,
        MOI.ListOfConstraintIndices{MOI.VariableIndex,S}(),
    )
    for ci in cis
        f = MOI.get(primal_model, MOI.ConstraintFunction(), ci)
        if !haskey(m.constrained_var_idx, f) && !(f in params)
            set = MOI.get(primal_model, MOI.ConstraintSet(), ci)
            if !iszero(MOI.constant(set))
                continue
            end
            m.constrained_var_idx[f] = (ci, 1)
            # Placeholder to indicate this constraint is part of constrained variables,
            # it will be replaced later with a dual constraints
            m.constrained_var_dual[ci] = NO_CONSTRAINT
        end
    end
    return
end


__precompile__(true)

module DynClust
using StatsBase
using Distributions
using ComputationalResources
using Distributed
using ProgressMeter
using Images
using OffsetArrays

function __init__()
    # Enable `using` to load additional modules in this folder
    push!(LOAD_PATH, dirname(@__FILE__))
    # Now check for any resources that your package supports
    if haveresource(ArrayFireLibs)
        # User has indicated support for the ArrayFire libraries, so load your relevant code
        @eval using DynClustAF
    end
    # Put additional resource checks here
    # Don't forget to clean up!
    pop!(LOAD_PATH)
end

include("utilityFunctions.jl")
include("multiTest.jl")
include("denoising.jl")
include("clustering.jl")

export  runDenoising,
        getDenoisingResults,
        runClustering,
        getClusteringResults

end # module


module AssociativeArrays

export AssociativeArray

import Base: getindex, setindex!, size, similar
using Base.Test

immutable AssociativeArray{T,N,C<:Associative} <: AbstractArray{T,N}
    data::C
    dims::NTuple{N,Int}
end

AssociativeArray{K,V,N}(data::Associative{K,V}, dims::NTuple{N,Int}) = AssociativeArray{V, N, typeof(data)}(data, dims)
AssociativeArray(data::Associative, dims::Int) = AssociativeArray(data, (dims,))

# outer constructor infers the types from the data
# triangular dispatch constraints are always satisfied if you use this constructor
# these constructors use run time type functions so are unstable.
# function AssociativeArray(data, dims)
#     nt, t = eltype(data).types
#     a=length(nt.types)
#     n = max(1, a)
#     C = typeof(data)
#     return AssociativeArray{t, n, C}(data, dims)
# end
# #special case for the 1D case so you can index by plain integers rather than (Int,)
# function AssociativeArray(data, dims::Integer)
#     nt, t = eltype(data).types
#     return AssociativeArray{t, 1, typeof(data)}(data, (dims,))
# end

# this doesn't work because you can't pass a Dict you need to instantiate
# is with something in order to instantiate it.
# type SparseVec{T,N,C<:Associative}
#   data::C{T,N}
#   dims::N
# end

# this doesn't work because you can't nest parametric types.
# type SparseVec{T,N,C<:Associative{N,T}}
#     data::C
#     dims::N
# end

# satisfying the interface
size(sa::AssociativeArray) = sa.dims
@inline getindex(sa::AssociativeArray, indices...) = sa.data[indices]
@inline setindex!(sa::AssociativeArray, x, indices...) = setindex!(sa.data, x, indices)
similar{T}(sa::AssociativeArray, ::Type{T}, dims) = AssociativeArray(Dict{typeof(dims), T}(), dims)
#special 1d case
@inline getindex{T,C<:Associative}(sa::AssociativeArray{T,1,C}, index::Integer) = sa.data[index]
@inline setindex!{T,C<:Associative}(sa::AssociativeArray{T,1,C}, x, index::Integer) = setindex!(sa.data, x, index)

# TODO enable abbreviated printing.
# println(AssociativeArray(Dict{NTuple{2, Int}, Float64}(), (150,150)))

type SparseVector{T,C}
    data::C
    length::NTuple{1, Int}
end
end


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

@everywhere module TestDecisionOperators

using StochasticPrograms
using Test
using LinearAlgebra
using MathOptInterface
const MOI = MathOptInterface
const MOIU = MOI.Utilities
const MA = MOI.MutableArithmetics

function DecisionModel()
    m = Model()
    m.ext[:decisions] = Decisions(Val{1}())
    return m
end

macro decision_variable(m, x, args...)
    return esc(quote
        @variable($m, $x, $((args)...), set = StochasticPrograms.DecisionSet(1))
    end)
end

macro test_expression(expr)
    return esc(
        quote
            @test JuMP.isequal_canonical(@expression(model, $expr), $expr)
        end,
    )
end

macro test_expression_with_string(expr, str)
    return esc(
        quote
            realized_expr = @inferred $expr
            @test string(realized_expr) == $str
            @test JuMP.isequal_canonical(
                @expression(model, $expr),
                realized_expr,
            )
        end,
    )
end

function test_promotion()
    I = Int
    V = VariableRef
    D = DecisionRef
    VA = GenericAffExpr{Float64,V}
    DA = GenericAffExpr{Float64,D}
    VQ = GenericQuadExpr{Float64,V}
    DQ = GenericQuadExpr{Float64,D}
    A = DecisionAffExpr{Float64}
    Q = DecisionQuadExpr{Float64}
    # DecisionRef--Number
    @test promote_type(D, I) == A
    @test promote_type(I, D) == A
    # DecisionAffExpr--Number
    @test promote_type(A, I) == A
    @test promote_type(I, A) == A
    # DecisionAffExpr--VariableRef
    @test promote_type(A, V) == A
    @test promote_type(V, A) == A
    # DecisionAffExpr--DecisionRef
    @test promote_type(A, D) == A
    @test promote_type(D, A) == A
    # DecisionAffExpr--_VariableAffExpr
    @test promote_type(A, VA) == A
    @test promote_type(VA, A) == A
    # DecisionAffExpr--_DecisionAffExpr
    @test promote_type(A, DA) == A
    @test promote_type(DA, A) == A
    # DecisionQuadExpr--Number
    @test promote_type(Q, I) == Q
    @test promote_type(I, Q) == Q
    # DecisionQuadExpr--VariableRef
    @test promote_type(Q, V) == Q
    @test promote_type(V, Q) == Q
    # DecisionQuadExpr--DecisionRef
    @test promote_type(Q, D) == Q
    @test promote_type(D, Q) == Q
    # DecisionQuadExpr--_VariableAffExpr
    @test promote_type(Q, VA) == Q
    @test promote_type(VA, Q) == Q
    # DecisionQuadExpr--_DecisionAffExpr
    @test promote_type(Q, DA) == Q
    @test promote_type(DA, Q) == Q
    # DecisionQuadExpr--DecisionAffExpr
    @test promote_type(Q, A) == Q
    @test promote_type(A, Q) == Q
    # DecisionQuadExpr--_VariableQuadExpr
    @test promote_type(Q, VQ) == Q
    @test promote_type(VQ, Q) == Q
    # DecisionQuadExpr--_DecisionQuadExpr
    @test promote_type(Q, DQ) == Q
    @test promote_type(DQ, Q) == Q
end

function test_uniform_scaling()
    model = DecisionModel()
    x = @decision_variable(model, x)
    y = @variable(model, y)
    @test_expression_with_string x + 2I "x + 2"
    @test_expression_with_string (x + y + 1) + I "x + y + 2"
    @test_expression_with_string x - 2I "x - 2"
    @test_expression_with_string (x + y - 1) - I "x + y - 2"
    @test_expression_with_string 2I + x "x + 2"
    @test_expression_with_string I + (x + 1) "x + 2"
    @test_expression_with_string 2I - x "-x + 2"
    @test_expression_with_string I - (x - 1) "-x + 2"
    @test_expression_with_string I * x "x"
    @test_expression_with_string I * (x + y + 1) "x + y + 1"
    @test_expression_with_string (x + 1) * I "x + 1"
end

function test_basic_operators()
    model = DecisionModel()
    x = @decision_variable(model, x)
    y = @variable(model, y)
    z = @decision_variable(model, z)

    aff = @inferred 7.1 * x + 2.5
    @test_expression_with_string 7.1 * x + 2.5 "7.1 x + 2.5"
    aff2 = @inferred 1.2 * y + 1.2 * x
    @test_expression_with_string 1.2 * y + 1.2 * x "1.2 x + 1.2 y"
    q = @inferred 2.5 * y * z + aff
    @test_expression_with_string 2.5 * y * z + aff "7.1 x + 2.5 z*y + 2.5"
    q2 = @inferred 8 * x * z + aff2
    @test_expression_with_string 8 * x * z + aff2 "8 z*x + 1.2 x + 1.2 y"
    @test_expression_with_string 2 * x * x + 1 * y * y + z + 3 "2 x² + z + y² + 3"
end

function test_basic_operators_number()
    model = DecisionModel()
    x = @decision_variable(model, x)
    y = @variable(model, y)
    z = @decision_variable(model, z)
    w = @variable(model, w)

    aff = @inferred 7.1 * x + 2.5
    quad = @inferred 2.5 * y * z + aff
    # Number--DecisionRef
    @test_expression_with_string 4.13 + x "x + 4.13"
    @test_expression_with_string 3.16 - x "-x + 3.16"
    @test_expression_with_string 5.23 * x "5.23 x"
    @test_throws ErrorException 2.94 / x
    # Number--DecisionAffExpr
    @test_expression_with_string 1.5 + aff "7.1 x + 4"
    @test_expression_with_string 1.5 - aff "-7.1 x - 1"
    @test_expression_with_string 2 * aff "14.2 x + 5"
    @test_throws ErrorException 2 / aff
    # Number--DecisionQuadExpr
    @test_expression_with_string 1.5 + quad "7.1 x + 2.5 z*y + 4"
    @test_expression_with_string 1.5 - quad "-7.1 x - 2.5 z*y - 1"
    @test_expression_with_string 2 * quad "14.2 x + 5 z*y + 5"
    @test_throws ErrorException 2 / quad
end

function test_basic_operators_decision()
    model = DecisionModel()
    x = @decision_variable(model, x)
    y = @variable(model, y)
    z = @decision_variable(model, z)
    w = @variable(model, w)

    aff = @inferred 7.1 * x + 2.5
    vaff = @inferred 7.1 * y + 2.5
    quad = @inferred 2.5 * y * z + aff
    vquad = @inferred 2.5 * y * w + vaff

    # DecisionRef unary
    @test (+x) === x
    @test_expression_with_string -x "-x"
    # DecisionRef--Number
    @test_expression_with_string x + 4.13 "x + 4.13"
    @test_expression_with_string x - 4.13 "x - 4.13"
    @test_expression_with_string x * 4.13 "4.13 x"
    @test_expression_with_string x / 2.00 "0.5 x"
    @test x == x
    @test transpose(x) === x
    @test conj(x) === x
    @test_expression_with_string x - x "0"
    @test_expression_with_string x^2 "x²"
    @test_expression_with_string x^1 "x"
    @test_expression_with_string x^0 "1"
    @test_throws ErrorException x^3
    @test_throws ErrorException x^1.5
    # DecisionRef--VariableRef
    @test_expression_with_string w + x "x + w"
    @test_expression_with_string w - x "-x + w"
    @test_expression_with_string w * x "x*w"
    @test_throws ErrorException w / x
    @test_expression_with_string y * z - x "-x + z*y"
    # DecisionRef--DecisionRef
    @test_expression_with_string z + x "z + x"
    @test_expression_with_string z - x "z - x"
    @test_expression_with_string z * x "z*x"
    @test_throws ErrorException z / x
    # VariableRef--DecisionAffExpr
    @test_expression_with_string w + aff "7.1 x + w + 2.5"
    @test_expression_with_string w - aff "-7.1 x + w - 2.5"
    @test_expression_with_string w * aff "7.1 x*w + 2.5 w"
    @test_throws ErrorException w / aff
    @test_throws MethodError w ≤ aff
    @test_expression_with_string 7.1 * w - aff "-7.1 x + 7.1 w - 2.5"
    # DecisionRef--_VariableAffExpr
    @test_expression_with_string z + vaff "z + 7.1 y + 2.5"
    @test_expression_with_string z - vaff "z - 7.1 y - 2.5"
    @test_expression_with_string z * vaff "2.5 z + 7.1 z*y"
    @test_throws ErrorException z / vaff
    @test_throws MethodError z ≤ vaff
    @test_expression_with_string 7.1 * x - vaff "7.1 x - 7.1 y - 2.5"
    # DecisionRef--DecisionAffExpr
    @test_expression_with_string z + aff "z + 7.1 x + 2.5"
    @test_expression_with_string z - aff "z - 7.1 x - 2.5"
    @test_expression_with_string z * aff "7.1 z*x + 2.5 z"
    @test_throws ErrorException z / aff
    @test_throws MethodError z ≤ aff
    @test_expression_with_string 7.1 * x - aff "0 x - 2.5"
    # VariableRef--_DecisionQuadExpr
    @test_expression_with_string w + quad "7.1 x + 2.5 z*y + w + 2.5"
    @test_expression_with_string w - quad "-7.1 x - 2.5 z*y + w - 2.5"
    @test_throws ErrorException w * quad
    @test_throws ErrorException w / quad
    # DecisionRef--_VariableQuadExpr
    @test_expression_with_string x + vquad "x + 2.5 y*w + 7.1 y + 2.5"
    @test_expression_with_string x - vquad "x - 2.5 y*w - 7.1 y - 2.5"
    @test_throws ErrorException x * vquad
    @test_throws ErrorException x / vquad
    # DecisionRef--QuadExpr
    @test_expression_with_string z + quad "7.1 x + z + 2.5 z*y + 2.5"
    @test_expression_with_string z - quad "-7.1 x + z - 2.5 z*y - 2.5"
    @test_throws ErrorException z * quad
    @test_throws ErrorException z / quad
end

function test_basic_operators_affexpr()
    model = DecisionModel()
    x = @decision_variable(model, x)
    y = @variable(model, y)
    z = @decision_variable(model, z)
    w = @variable(model, w)

    aff = @inferred 7.1 * x + y + 2.5
    vaff = @inferred 2.4 * y + 1.2
    quad = @inferred 2.5 * y * z + aff
    vquad = @inferred w * w + vaff
    # DecisionAffExpr unary
    @test_expression_with_string +aff "7.1 x + y + 2.5"
    @test_expression_with_string -aff "-7.1 x - y - 2.5"
    # AffExpr--Number
    @test_expression_with_string aff + 1.5 "7.1 x + y + 4"
    @test_expression_with_string aff - 1.5 "7.1 x + y + 1"
    @test_expression_with_string aff * 2 "14.2 x + 2 y + 5"
    @test_expression_with_string aff / 2 "3.55 x + 0.5 y + 1.25"
    @test_throws MethodError aff ≤ 1
    @test aff == aff
    @test_throws MethodError aff ≥ 1
    @test_expression_with_string aff - 1 "7.1 x + y + 1.5"
    @test_expression_with_string aff^2 "50.41 x² + 35.5 x + 14.2 x*y + y² + 5 y + 6.25"
    @test_expression_with_string (7.1 * x + y + 2.5)^2 "50.41 x² + 35.5 x + 14.2 x*y + y² + 5 y + 6.25"
    @test_expression_with_string aff^1 "7.1 x + y + 2.5"
    @test_expression_with_string (7.1 * x + y + 2.5)^1 "7.1 x + y + 2.5"
    @test_expression_with_string aff^0 "1"
    @test_expression_with_string (7.1 * x + y + 2.5)^0 "1"
    @test_throws ErrorException aff^3
    @test_throws ErrorException (7.1 * x + y + 2.5)^3
    @test_throws ErrorException aff^1.5
    @test_throws ErrorException (7.1 * x + y + 2.5)^1.5
    # DecisionAffExpr--VariableRef
    @test_expression_with_string aff + w "7.1 x + w + y + 2.5"
    @test_expression_with_string aff - w "7.1 x - w + y + 2.5"
    @test_expression_with_string aff * w "7.1 x*w + w*y + 2.5 w"
    @test_throws ErrorException aff / w
    @test_expression_with_string aff - y "7.1 x + 0 y + 2.5"
    # DecisionAffExpr--DecisionRef
    @test_expression_with_string aff + z "z + 7.1 x + y + 2.5"
    @test_expression_with_string aff - z "-z + 7.1 x + y + 2.5"
    @test_expression_with_string aff * z "7.1 z*x + 2.5 z + z*y"
    @test_throws ErrorException aff / z
    @test_expression_with_string aff - 7.1 * x "0 x + y + 2.5"
    # DecisionAffExpr--_VariableAffExpr
    @test_expression_with_string aff + vaff "7.1 x + 3.4 y + 3.7"
    @test_expression_with_string aff - vaff "7.1 x - 1.4 y + 1.3"
    @test_expression_with_string aff * vaff "8.52 x + 17.04 x*y + 2.4 y² + 7.2 y + 3"
    @test_throws ErrorException aff / vaff
    # DecisionAffExpr--DecisionAffExpr
    @test_expression_with_string aff + aff "14.2 x + 2 y + 5"
    @test_expression_with_string aff - (x + y) "6.1 x + 0 y + 2.5"
    @test_expression_with_string aff * aff "50.41 x² + 35.5 x + 14.2 x*y + y² + 5 y + 6.25"
    @test_throws ErrorException aff / aff
    @test string((x + x) * (x + 3)) == string((x + 3) * (x + x))
    @test_expression_with_string aff - aff "0 x + 0 y"
    # _VariableAffExpr--DecisionQuadExpr
    @test_expression_with_string vaff + quad "7.1 x + 2.5 z*y + 3.4 y + 3.7"
    @test_expression_with_string vaff - quad "-7.1 x - 2.5 z*y + 1.4 y - 1.3"
    @test_throws ErrorException vaff * quad
    @test_throws ErrorException vaff / quad
    # DecisionAffExpr--_VariableQuadExpr
    @test_expression_with_string aff + vquad "7.1 x + w² + 3.4 y + 3.7"
    @test_expression_with_string aff - vquad "7.1 x - w² - 1.4 y + 1.3"
    @test_throws ErrorException aff * vquad
    @test_throws ErrorException aff / vquad
    # DecisionAffExpr--DecisionQuadExpr
    @test_expression_with_string aff + quad "14.2 x + 2.5 z*y + 2 y + 5"
    @test_expression_with_string aff - quad "0 x - 2.5 z*y + 0 y"
    @test_throws ErrorException aff * quad
    @test_throws ErrorException aff / quad
end

function test_basic_operators_quadexpr()
    model = DecisionModel()
    x = @decision_variable(model, x)
    y = @variable(model, y)
    z = @decision_variable(model, z)
    w = @variable(model, w)

    aff = @inferred 7.1 * x + y + 2.5
    vaff = @inferred 2.4 * y + 1.2
    quad = @inferred x^2 + 2.5 * y * z + aff
    vquad = @inferred w * w + vaff

    # DecisionQuadExpr unary
    @test_expression_with_string +quad "x² + 7.1 x + 2.5 z*y + y + 2.5"
    @test_expression_with_string -quad "-x² - 7.1 x - 2.5 z*y - y - 2.5"
    # DecisionQuadExpr--Number
    @test_expression_with_string quad + 1.5 "x² + 7.1 x + 2.5 z*y + y + 4"
    @test_expression_with_string quad - 1.5 "x² + 7.1 x + 2.5 z*y + y + 1"
    @test_expression_with_string quad * 2 "2 x² + 14.2 x + 5 z*y + 2 y + 5"
    @test_expression_with_string quad / 2 "0.5 x² + 3.55 x + 1.25 z*y + 0.5 y + 1.25"
    @test quad == quad
    # DecisionQuadExpr--VariableRef
    @test_expression_with_string quad + w "x² + 7.1 x + 2.5 z*y + y + w + 2.5"
    @test_expression_with_string quad - w "x² + 7.1 x + 2.5 z*y + y - w + 2.5"
    @test_throws ErrorException quad * w
    @test_throws ErrorException quad / w
    # DecisionQuadExpr--DecisionRef
    @test_expression_with_string quad + z "x² + 7.1 x + z + 2.5 z*y + y + 2.5"
    @test_expression_with_string quad - z "x² + 7.1 x - z + 2.5 z*y + y + 2.5"
    @test_throws ErrorException quad * z
    @test_throws ErrorException quad / z
    # DecisionQuadExpr--_VariableAffExpr
    @test_expression_with_string quad + vaff "x² + 7.1 x + 2.5 z*y + 3.4 y + 3.7"
    @test_expression_with_string quad - vaff "x² + 7.1 x + 2.5 z*y - 1.4 y + 1.3"
    @test_throws ErrorException quad * vaff
    @test_throws ErrorException quad / vaff
    # DecisionQuadExpr--DecisionAffExpr
    @test_expression_with_string quad + aff "x² + 14.2 x + 2.5 z*y + 2 y + 5"
    @test_expression_with_string quad - aff "x² + 0 x + 2.5 z*y + 0 y"
    @test_throws ErrorException quad * aff
    @test_throws ErrorException quad / aff
    # DecisionQuadExpr--_VariableQuadExpr
    @test_expression_with_string quad + vquad "x² + 7.1 x + 2.5 z*y + w² + 3.4 y + 3.7"
    @test_expression_with_string quad - vquad "x² + 7.1 x + 2.5 z*y - w² - 1.4 y + 1.3"
    @test_throws ErrorException quad * vquad
    @test_throws ErrorException quad / vquad
    # DecisionQuadExpr--DecisionQuadExpr
    @test_expression_with_string quad + (x^2 + x*y + y^2) "2 x² + 7.1 x + 2.5 z*y + x*y + y² + y + 2.5"
    @test_expression_with_string quad - (x^2 + x*y + y^2) "0 x² + 7.1 x - x*y + 2.5 z*y - y² + y + 2.5"
    @test_throws ErrorException quad * quad
    @test_throws ErrorException quad / quad
end

function test_dot()
    model = DecisionModel()
    @decision_variable(model, 0 ≤ x[1:3] ≤ 1)

    @test_expression_with_string dot(x[1], x[1]) "x[1]²"
    @test_expression_with_string dot(2, x[1]) "2 x[1]"
    @test_expression_with_string dot(x[1], 2) "2 x[1]"

    c = vcat(1:3)
    @test_expression_with_string dot(c, x) "x[1] + 2 x[2] + 3 x[3]"
    @test_expression_with_string dot(x, c) "x[1] + 2 x[2] + 3 x[3]"

    A = [1 3; 2 4]
    @decision_variable(model, 1 ≤ y[1:2, 1:2] ≤ 1)
    @test_expression_with_string dot(A, y) "y[1,1] + 2 y[2,1] + 3 y[1,2] + 4 y[2,2]"
    @test_expression_with_string dot(y, A) "y[1,1] + 2 y[2,1] + 3 y[1,2] + 4 y[2,2]"

    B = ones(2, 2, 2)
    @decision_variable(model, 0 ≤ z[1:2, 1:2, 1:2] ≤ 1)
    @test_expression_with_string dot(B, z) "z[1,1,1] + z[2,1,1] + z[1,2,1] + z[2,2,1] + z[1,1,2] + z[2,1,2] + z[1,2,2] + z[2,2,2]"
    @test_expression_with_string dot(z, B) "z[1,1,1] + z[2,1,1] + z[1,2,1] + z[2,2,1] + z[1,1,2] + z[2,1,2] + z[1,2,2] + z[2,2,2]"

    @objective(model, Max, dot(x, ones(3)) - dot(y, ones(2, 2)))
    for i in 1:3
        JuMP.set_start_value(x[i], 1)
    end
    for i in 1:2, j in 1:2
        JuMP.set_start_value(y[i, j], 1)
    end
    for i in 1:2, j in 1:2, k in 1:2
        JuMP.set_start_value(z[i, j, k], 1)
    end
    @test dot(c, JuMP.start_value.(x)) ≈ 6
    @test dot(A, JuMP.start_value.(y)) ≈ 10
    @test dot(B, JuMP.start_value.(z)) ≈ 8
end

function test_higher_level()
    model = DecisionModel()
    @decision_variable(model, 0 ≤ matrix[1:3, 1:3] ≤ 1, start = 1)
    @testset "sum(::Matrix{DecisionRef})" begin
        @test_expression_with_string sum(matrix) "matrix[1,1] + matrix[2,1] + matrix[3,1] + matrix[1,2] + matrix[2,2] + matrix[3,2] + matrix[1,3] + matrix[2,3] + matrix[3,3]"
    end

    @testset "sum(::Matrix{T}) where T<:Real" begin
        @test sum(JuMP.start_value.(matrix)) ≈ 9
    end
    @testset "sum(::Array{VariableRef})" begin
        @test string(sum(matrix[1:3, 1:3])) == string(sum(matrix))
    end
    @testset "sum(affs::Array{DecisionAffExpr})" begin
        @test_expression_with_string sum([
            2 * matrix[i, j] for i in 1:3, j in 1:3
        ]) "2 matrix[1,1] + 2 matrix[2,1] + 2 matrix[3,1] + 2 matrix[1,2] + 2 matrix[2,2] + 2 matrix[3,2] + 2 matrix[1,3] + 2 matrix[2,3] + 2 matrix[3,3]"
    end
    @testset "sum(quads::Array{DecisionQuadExpr})" begin
        @test_expression_with_string sum([
            2 * matrix[i, j]^2 for i in 1:3, j in 1:3
        ]) "2 matrix[1,1]² + 2 matrix[2,1]² + 2 matrix[3,1]² + 2 matrix[1,2]² + 2 matrix[2,2]² + 2 matrix[3,2]² + 2 matrix[1,3]² + 2 matrix[2,3]² + 2 matrix[3,3]²"
    end
    S = [1, 3]
    @decision_variable(model, x[S], start = 1)
    @testset "sum(::DenseAxisArray{DecisionRef})" begin
        @test_expression sum(x)
        @test length(string(sum(x))) == 11
        @test occursin("x[1]", string(sum(x)))
        @test occursin("x[3]", string(sum(x)))
    end
    @testset "sum(::DenseAxisArray{T}) where T<:Real" begin
        @test sum(JuMP.start_value.(x)) == 2
    end

    @decision_variable(model, 0 ≤ dense_matrix[i in 1:3, a in [:a,:b]] ≤ 1, start = 1)
    @testset "sum(::DenseAxisArray{DecisionRef})" begin
        @test_expression_with_string sum(dense_matrix) "dense_matrix[1,a] + dense_matrix[2,a] + dense_matrix[3,a] + dense_matrix[1,b] + dense_matrix[2,b] + dense_matrix[3,b]"
    end
end

function runtests()
    @testset "DecisionOperators" begin
        for name in names(@__MODULE__; all = true)
            if !startswith("$(name)", "test_")
                continue
            end
            f = getfield(@__MODULE__, name)
            @testset "$(name)" begin
                f()
            end
        end
    end
end

end


#=
	This is a file for simulation based conflict resolution time computation.
=#

using Distributions, PyPlot
push!(LOAD_PATH, "../../VICAS/src/")
push!(LOAD_PATH, "../../DeepCorrection")
push!(LOAD_PATH, "../../DeepCorrection/src")
push!(LOAD_PATH, "../TrajectoryViz")

include("../TrajectoryViz/trajViz.jl")

const BEARING_DIST =   [[0.1047 0.0489];
   						[0.3142 0.0474];
   						[0.5236 0.0474];
   						[0.7330 0.0459];
   						[0.9425 0.0425];
   						[1.1519 0.0403];
   						[1.3614 0.0365];
   						[1.5708 0.0333];
   						[1.7802 0.0304];
   						[1.9897 0.0269];
   						[2.1991 0.0237];
   						[2.4086 0.0210];
   						[2.6180 0.0194];
   						[2.8274 0.0183];
   						[3.0369 0.0181];
   						[3.2463 0.0181];
   						[3.4558 0.0183];
   						[3.6652 0.0194];
   						[3.8746 0.0210];
   						[4.0841 0.0237];
   						[4.2935 0.0269];
   						[4.5029 0.0304];
   						[4.7124 0.0333];
   						[4.9218 0.0365];
   						[5.1313 0.0403];
   						[5.3407 0.0425];
   						[5.5501 0.0459];
   						[5.7596 0.0474];
   						[5.9690 0.0474];
   						[6.1785 0.0489];]

function distance(own::Ownship, int::Ownship)
	return sqrt((own.x - int.x)^2 + (own.y - int.y)^2)
end

function checkWithinSensing(own::Ownship, int::Ownship)
	if distance(own, int) < SENSING_RANGE * 1.02
		return true
	else
		return false
	end
end

function checkDest(ac::Ownship)
	if sqrt((ac.x - ac.x_dest)^2 + (ac.y - ac.y_dest)^2) < 50
		return true
	else
		return false
	end
end

function getIntruderStartDestCoords(ownship::Ownship)
	# Use encounter model:
	intruder_dist = Categorical(vec(BEARING_DIST[:, 2]))
	intruder_angle = BEARING_DIST[:, 1][rand(intruder_dist)]
	if intruder_angle > π
		intruder_angle -= 2 * π
	end # intruder_angle ∈ [-pi, pi], centered at 0
	init_angle = norm_angle(ownship.heading + intruder_angle)
	dest_angle = norm_angle(init_angle + deg2rad(rand() * 270 + 45)) # 45 - 315 uniform?
	x_init = ownship.x + cos(init_angle) * SENSING_RANGE
	y_init = ownship.y + sin(init_angle) * SENSING_RANGE
	x_dest = ownship.x + cos(dest_angle) * SENSING_RANGE * 1.2
	y_dest = ownship.y + sin(dest_angle) * SENSING_RANGE * 1.2
	return x_init, y_init, x_dest, y_dest
end

function addIntruders!(sr::Float64, ownship::Ownship, intArr::Vector{Ownship}, CASType::Symbol, policy)
	numNewIntruders = rand(Poisson(sr))
	for i = 1:numNewIntruders
		x0, y0, x_dest, y_dest = getIntruderStartDestCoords(ownship)
		ac = initOwnship(x0, y0, x_dest, y_dest, CASType, policy)
		push!(intArr, ac)
	end
	return numNewIntruders
end

function reset(sr::Float64, CASType::Symbol, policy)
	x0 = 0.0
	y0 = 0.0
	x_dest = 10000.0
	y_dest = 0.0
	ownship = initOwnship(x0, y0, x_dest, y_dest, CASType, policy)
	intArr = Ownship[]
	addIntruders!(sr, ownship, intArr, CASType, policy)
	return ownship, intArr
end

function step!(sr::Float64, ownship::Ownship, intArr::Vector{Ownship}, CASType::Symbol, policy)
	stepResTime = length(intArr) * dt

	acArr = vcat(ownship, intArr)

	for i in 1 : length(acArr)
		intCoords = Vector{Float64}()
		for j in 1 : length(acArr)
			if j != i
				intCoords = vcat(intCoords, [acArr[j].x, acArr[j].y, acArr[j].heading])
			end
		end

		numInt = length(intArr)
		if numInt > 0 && CASType != :NOCAS
			# Get state
			numObservedInt = getState!(acArr[i], numInt, intCoords)
			if numObservedInt > 0
				# Get action and advisory
				getAdvisory!(acArr[i])
			else
				acArr[i].advisory = :COC
			end
		else
			acArr[i].advisory = :COC
		end
	end

	# Update dynamics
	for ac in acArr
		updateOwnshipCoords!(ac)
	end

	# Check within sensing and check dest for intruders
	i = 1
	while i <= length(intArr)
		if checkDest(intArr[i]) || !checkWithinSensing(ownship, intArr[i])
		   deleteat!(intArr, i)
		end
		i += 1
	end

	numNewIntruders = addIntruders!(sr, ownship, intArr, CASType, policy)

	return stepResTime, numNewIntruders
end

function render_env(sr::Float64, t::Int64, ownship::Ownship, intruders::Vector{Ownship})
	All_AC = vcat(ownship, intruders)
	AIRSPACE_DIM = 1500
	num_sections = ownship.policy.env.num_sections
	for ac in All_AC
		if ac.advisory == :COC
			scatter(ac.x, ac.y, marker="o", color="blue", s=12)
		else
			scatter(ac.x, ac.y, marker="o", color="red", s=12)
		end

		arrow_len = AIRSPACE_DIM / 6
		arrow(
			ac.x, ac.y,
			cos(ac.heading) * arrow_len * 0.8,
			sin(ac.heading) * arrow_len * 0.8,
			head_width=AIRSPACE_DIM / 20,
			width=1,
			head_length=AIRSPACE_DIM / 20,
			overhang=0.5,
			head_starts_at_zero="true",
			facecolor="black",
			length_includes_head="true")

		scatter(ac.x_dest, ac.y_dest, marker=",", color="magenta", s=12)

		PyPlot.plot([ac.x; ac.x_dest], [ac.y; ac.y_dest],
			linestyle="--", color="black", linewidth=0.5)
	end

	# alert circle
	th = linspace(-pi, pi, 30)
	if ownship.advisory == :COC
		PyPlot.plot(ownship.x + SENSING_RANGE * cos.(th),
			ownship.y + SENSING_RANGE * sin.(th),
			linestyle="--", color="green", linewidth=0.4)
	else
		PyPlot.plot(ownship.x + SENSING_RANGE * cos.(th),
			ownship.y + SENSING_RANGE * sin.(th),
			linestyle="--", color="red", linewidth=0.4)
	end

	# NMAC circle
	PyPlot.plot(ownship.x + NMAC_RANGE * cos.(th),
			ownship.y + NMAC_RANGE * sin.(th),
			linestyle="--", color="red", linewidth=0.4)

	# plot sections and highlight the closest intruder in each section
	section_angle = 2 * pi / num_sections
	for i in 0 : num_sections / 2 - 1
		th_1 = i * section_angle + ownship.heading
		th_2 = th_1 + pi
		PyPlot.plot(ownship.x + SENSING_RANGE * [cos(th_1), cos(th_2)], ownship.y + SENSING_RANGE * [sin(th_1), sin(th_2)],
			linestyle="--", color="red", linewidth=0.4)
	end

	# for i in 1 : env.num_sections
	# 	rho = env.full_state[(i - 1) * STATE_DIM + IND_rho]
	# 	if rho != TERM_VAR
	# 		theta = env.full_state[(i - 1) * STATE_DIM + IND_theta]
	# 		scatter(rho * cos(theta), rho * sin(theta),
	# 			marker="o", color="green", s=40, linewidths=0.6)
	# 	end
	# end

	xlabel("X (m)")
	ylabel("Y (m)")
	title(string(sr) * ", " * string(t) * ": " * string(length(intruders)))
	axis("equal")
	# ax = gca()
	# ax[:set_xlim]([-AIRSPACE_DIM * 1.5, AIRSPACE_DIM * 1.5])
	# ax[:set_ylim]([-AIRSPACE_DIM * 1.5, AIRSPACE_DIM * 1.5])
	pause(0.1)
	clf()
end

function resTime(;
				spawnRateSet::Union{Vector{Int64},Int64}=[25, 35, 45],
				CASType::Symbol=:correctedSector,
				policyFile::AbstractString="",
				numEvalEposide::Int64=50,
				maxEpisodeLen::Int64=2000,
				render::Bool=false)

	println("CASType = ", string(CASType))
	avgResTimeArr = Dict{Float64, Float64}()
	stdResTimeArr = Dict{Float64, Float64}()
	problem_file = "../../DeepCorrection/logs/" * policyFile * "/final_problem.jld"
	weights_file = "../../DeepCorrection/logs/" * policyFile * "/final_weights.jld"
	policy = restore(problem_file=problem_file, weights_file=weights_file)

	for sr in spawnRateSet
		sr /= 100
		epResTimeArr = Float64[]

		for ep in 1 : numEvalEposide
			# initiate ownship and intruders (reset)
			ownship, intArr = reset(sr, CASType, policy)
			totalResTime = 0.0
			totalNumIntruders = 0
			numNewIntruders = length(intArr)
			totalNumIntruders += numNewIntruders

			for t = 1 : maxEpisodeLen
				stepResTime, numNewIntruders = step!(sr, ownship, intArr, CASType, policy)
				totalResTime += stepResTime
				totalNumIntruders += numNewIntruders
				if render
					render_env(sr, t, ownship, intArr)
				end
				if checkDest(ownship) # ownship reaches destination
					break
				end
			end # for t
			push!(epResTimeArr, totalResTime / totalNumIntruders)
		end # for ep

		avgResTimeArr[sr] = mean(epResTimeArr)
		stdResTimeArr[sr] = std(epResTimeArr)
		println("sr = ", sr, ", avgResTime = ", avgResTimeArr[sr])
	end # for sr
	return avgResTimeArr, stdResTimeArr
end


using FermiCG
using Test

@testset "strings" begin
    display(StringCI.string_to_index("110010"))
    @test StringCI.string_to_index("110010") == 19
end


module Eval

using Ekztazy

c = Client()
g = ENV["TESTGUILD"]

command!(c, g, "eval", "evaluates a string as julia code", legacy=false, options=Options(
    [String, "str", "the string to evaluate"]
)) do ctx, str::String
    @info "$str"
    reply(c, ctx, content="```julia\n$(eval(Meta.parse(str)))\n```")
end

start(c)

end


"""
    State1D

A data type to represent one dimensional states

# Fields
    - `s::Float64` position
    - `v::Float64` speed [m/s]
"""
struct State1D
    s::Float64 # position
    v::Float64 # speed [m/s]
end
Base.write(io::IO, ::MIME"text/plain", s::State1D) = @printf(io, "%.16e %.16e", s.s, s.v)
function Base.read(io::IO, ::MIME"text/plain", ::Type{State1D})
    i = 0
    tokens = split(strip(readline(io)), ' ')
    s = parse(Float64, tokens[i+=1])
    v = parse(Float64, tokens[i+=1])
    return State1D(s,v)
end

"""
    Vehicle1D
A specific instance of the Entity type defined in Records.jl to represent vehicles in 1d environments.
"""
const Vehicle1D = Entity{State1D, VehicleDef, Int64}

"""
    Scene1D

A specific instance of the Frame type defined in Records.jl to represent a list of vehicles in 1d environments.

# constructors
    Scene1D(n::Int=100)
    Scene1D(arr::Vector{Vehicle1D})
"""
const Scene1D = Frame{Vehicle1D}
Scene1D(n::Int=100) = Frame(Vehicle1D, n)
Scene1D(arr::Vector{Vehicle1D}) = Frame{Vehicle1D}(arr, length(arr))

get_center(veh::Vehicle1D) = veh.state.s
get_footpoint(veh::Vehicle1D) = veh.state.s
get_front(veh::Vehicle1D) = veh.state.s + veh.def.length/2
get_rear(veh::Vehicle1D) = veh.state.s - veh.def.length/2


module TestDict

using ConcurrentCollections
using ConcurrentCollections.Implementations: clusters, migrate!
using SyncBarriers: Barrier, cycle!
using Test
using ..Utils: ⊏

function test_expand_and_shrink(n = 17)
    d = ConcurrentDict{Int,Int}()
    @testset "expand" begin
        @testset for i in 1:n
            d[i] = -i
            @testset for k in 1:i
                @test d[k] == -k
            end
        end
    end
    nfull = length(d.slots)
    @testset "shrink" begin
        @testset for i in n:-1:1
            @test pop!(d, i) == -i
            @testset for k in 1:i-1
                @test d[k] == -k
            end
        end
    end
    @test length(d.slots) < nfull
    return d
end

function test_parallel_expand(n = 2^10, basesize = 8)
    d = ConcurrentDict{Int,Int}(pairs(1:n))
    nslots = length(d.slots)
    migrate!(d, true; basesize)
    @test nslots < length(d.slots)
    diffs = Pair{Int,Int}[]
    for k in 1:n
        v = d[k]
        if v != k
            push!(diffs, k => v)
        end
    end
    @test diffs == []
    return d
end

function test_dict()
    @testset for npairs in [2, 100]
        test_dict(npairs)
    end
end

function test_dict(npairs)
    @testset for Key in [Int8, Int32, Int64], Value in [Int]
        d = ConcurrentDict{Key,Value}()
        @testset for trial in 1:2
            for i in 1:npairs
                d[i] = -i
            end
            @testset "getindex" begin
                vals = [d[i] for i in 1:2]
                @test vals == [-1, -2]
            end
            @testset "iterate" begin
                kvs = sort!(collect(d))
                @test kvs == [i => -i for i in 1:npairs]
                ks = sort!(collect(keys(d)))
                @test ks == 1:npairs
                vs = sort!(collect(values(d)))
                @test vs == reverse((1:npairs) .* -1)
            end
        end
        @testset "maybepop!" begin
            @test maybeget(d, 1) === Some(-1)
            @test maybepop!(d, 1) === Some(-1)
            @test maybeget(d, 1) === nothing
            @test maybepop!(d, 1) === nothing
            @test 1 ∉ sort!(collect(keys(d)))
        end
        @testset "length_upper_bound" begin
            @test length_upper_bound(d) == mapreduce(_ -> 1, +, d)
        end

        @testset "typed constructor" begin
            ks = Key.(1:npairs)
            vs = Value.(1:npairs)
            @testset "$label" for (label, dict) in [
                "ConcurrentDict{Key,Value}(zip(ks, vs))" =>
                    ConcurrentDict{Key,Value}(zip(ks, vs)),
                "ConcurrentDict{Key,Value}(map(=>, ks, vs))" =>
                    ConcurrentDict{Key,Value}(map(=>, ks, vs)),
            ]
                @test sort!(collect(keys(dict))) == ks
                @test sort!(collect(values(dict))) == ks
            end
        end

        @testset "clusters" begin
            @test length(clusters(d)::Vector{UnitRange{Int}}) > 0
        end
    end

    @testset "ConcurrentDict{String,_}()" begin
        d = ConcurrentDict{String,Int}()
        str(i) = string(i; pad = 3)
        @testset for trial in 1:2
            for i in 1:npairs
                d[str(i)] = -i
            end
            @testset "getindex" begin
                vals = [d[str(i)] for i in 1:2]
                @test vals == [-1, -2]
            end
            @testset "iterate" begin
                kvs = sort!(collect(d))
                @test kvs == [str(i) => -i for i in 1:npairs]
            end
        end
        @testset "maybepop!" begin
            @test maybeget(d, "001") === Some(-1)
            @test maybepop!(d, "001") === Some(-1)
            @test maybeget(d, "001") === nothing
            @test maybepop!(d, "001") === nothing
            @test "001" ∉ sort!(collect(keys(d)))
        end
        @testset "clusters" begin
            @test length(clusters(d)::Vector{UnitRange{Int}}) > 0
        end
    end
end

function test_pairs_constructor()
    @testset for Key in [Int8, Int32, Int64], Value in [Int]
        @testset "$label" for (label, dict) in [
            "ConcurrentDict{_,_}(_ => _, ...)" =>
                ConcurrentDict{Key,Value}(Key(0) => Value(1), Key(2) => Value(3)),
            "ConcurrentDict(_ => _, ...)" =>
                ConcurrentDict(Key(0) => Value(1), Key(2) => Value(3)),
            "ConcurrentDict{_,_}(false => true, ...)" =>
                ConcurrentDict{Key,Value}(false => true, 2 => 3),
            "ConcurrentDict(false => true, ...)" =>
                ConcurrentDict(false => true, Key(2) => Value(3)),
        ]
            @test dict[0] == 1
            @test dict[2] == 3
        end
    end
end

function test_shrink()
    @testset for Key in [Int8, Int32, Int64], Value in [Int]
        ks = vs = 1:100
        d = ConcurrentDict{Key,Value}(zip(ks, vs))
        nslots = length(d.slots)
        for k in ks
            pop!(d, k)
        end
        @test collect(d) == []
        @test length(d.slots) < nslots
    end
end

function random_mutation!(dict; nkeys = 8, repeat = 2^20, ntasks = Threads.nthreads())
    ks = 1:nkeys
    locals = [
        (
            popped = zeros(valtype(dict), nkeys),  # sum of popped values
            added = zeros(valtype(dict), nkeys),   # sum of all inserted values
        ) for _ in 1:ntasks
    ]
    @sync for (; popped, added) in locals
        Threads.@spawn begin
            for _ in 1:repeat
                k = rand(ks)
                if rand(Bool)
                    y = maybepop!(dict, k)
                    if y !== nothing
                        popped[k] += something(y)
                    end
                else
                    added[k] += 1
                    modify!(dict, k) do ref
                        Base.@_inline_meta
                        Some(ref === nothing ? 1 : ref[] + 1)
                    end
                end
            end
        end
    end
    return locals
end

function test_random_mutation(; kwargs...)
    dict = ConcurrentDict{Int,Int}()
    nkeys = 16
    locals = random_mutation!(dict; kwargs..., nkeys)
    actual = zeros(valtype(dict), nkeys)
    desired = zeros(valtype(dict), nkeys)
    for (k, v) in dict
        actual[k] = v
    end
    for (; popped, added) in locals
        actual .+= popped
        desired .+= added
    end
    @test actual == desired
end

function phased_push_pop!(
    dict;
    nkeys = 16,
    repeat = 2^10,
    phases = 2^10,
    ntasks = Threads.nthreads(),
)
    locals = [
        (
            popped = zeros(valtype(dict), nkeys * phases),  # sum of popped values
            added = zeros(valtype(dict), nkeys * phases),   # sum of all inserted values
        ) for _ in 1:ntasks
    ]
    barrier = Barrier(ntasks)
    @sync for (itask, (; popped, added)) in enumerate(locals)
        Threads.@spawn begin
            for p in 1:phases
                k0 = (p - 1) * nkeys + 1
                ks = k0:k0+nkeys-1
                for _ in 1:repeat
                    k = rand(ks)
                    added[k] += 1
                    modify!(dict, k) do ref
                        Base.@_inline_meta
                        Some(ref === nothing ? 1 : ref[] + 1)
                    end
                end
                spin = 10_000  # spin for a few μs
                cycle!(barrier[itask], spin)
                for k in ks
                    popped[k] += something(maybepop!(dict, k), 0)
                end
            end
        end
    end
    return locals
end

function test_phased_push_pop(; nkeys = 16, phases = 2^10, kwargs...)
    dict = ConcurrentDict{Int,Int}()
    locals = phased_push_pop!(dict; kwargs..., nkeys, phases)
    actual = zeros(valtype(dict), nkeys * phases)
    desired = zeros(valtype(dict), nkeys * phases)
    for k in eachindex(actual)
        actual[k] = get(dict, k, 0)
    end
    for (; popped, added) in locals
        actual .+= popped
        desired .+= added
    end
    @test actual == desired
end

function concurrent_intern!(ys, xs)
    dict = ConcurrentDict{eltype(xs),eltype(xs)}()
    Threads.@threads for i in eachindex(ys, xs)
        x = xs[i]
        ys[i] = get!(dict, x, x)
    end
    return ys
end

function test_get_default()
    d = ConcurrentDict(:a => 111)
    @test get!(d, :a, 222) == 111
    @test get!(d, :b, 222) == 222
    @test Dict(d) == Dict(:a => 111, :b => 222)
end

function test_get_function()
    d = ConcurrentDict(:a => 111)
    @test get!(() -> 222, d, :a) == 111
    @test get!(() -> 222, d, :b) == 222
    @test Dict(d) == Dict(:a => 111, :b => 222)
end

concurrent_intern(xs) = concurrent_intern!(similar(xs), xs)

function check_concurrent_intern(nitems)
    strs = [string(i)^50 for i in 1:10]
    xs = map(s -> sprint(print, s), rand(strs, nitems))
    ys = concurrent_intern(xs)
    puv = map(pointer_from_objref, unique(ys))
    upv = unique(map(pointer_from_objref, ys))
    @test length(puv) == length(upv)
    @test puv == upv
end

function test_concurrent_intern()
    @testset for trial in 1:5, nitems in [2^n for n in 7:10]
        check_concurrent_intern(nitems)
    end
end

function test_print()
    d = ConcurrentDict(111 => 222, 333 => 444)
    str = sprint(show, "text/plain", d)
    @test "ConcurrentDict" ⊏ str
    @test "111 => 222" ⊏ str
    @test "333 => 444" ⊏ str
end

end  # module


##############################################################################
##
## Matrix constructors
##
##############################################################################

for (f, basef) in ((:dataeye, :eye), )
    @eval begin
        ($f)(n::Int) = DataArray(($basef)(n), falses(n, n))
        ($f)(n::Int, p::Int) = DataArray(($basef)(n, p), falses(n, p))
    end
end
for (f, basef) in ((:datadiagm, :diagm), )
    @eval begin
        ($f)(vals::Vector) = DataArray(($basef)(vals), falses(length(vals), length(vals)))
    end
end

##############################################################################
##
## Extract the matrix diagonal
##
##############################################################################

function diag{T}(dm::DataMatrix{T})
    return DataArray(diag(dm.data), diag(dm.na))
end

##############################################################################
##
## Size information
##
##############################################################################

nrow(dm::DataMatrix) = size(dm, 1)
ncol(dm::DataMatrix) = size(dm, 2)


include("ranking_criteria.jl")

abstract type AbstractRanks end

"""
    struct MeasurementRanks

Type for the results of the measurement ranking.

Fields:
* `names::Vector{Symbol}`: Names of the measurements.
* `values::Vector{Float64}`: Values of the ranking, calculated according to the criterion.
* `criterion<:AbstractRankingCriterion`: Criterion that was used for ranking.

Constructors:

```julia
function MeasurementRanks(
    names::Vector{Symbol},
    values::Vector{Float64},
    criterion::AbstractRankingCriterion;
    order::Symbol = :values,
    rev::Bool = false
)
```
Keyword arguments:
* `order::Symbol = :values`: Specifies how the `names` and `values` are sorted. Further options are
`:names` for alphabetical sorting based on the measurement names or `:none` for keeping the initial order of measurements.
* `rev::Bool=false`: Switch to invert the order.
"""
struct MeasurementRanks <: AbstractRanks
    names::Vector{Symbol}
    values::Vector{Float64}
    criterion::AbstractRankingCriterion

    function MeasurementRanks(
        names::Vector{Symbol},
        values::Vector{<:Real},
        criterion::AbstractRankingCriterion;
        order::Symbol = :values,
        rev::Bool = false
    )
        sorted_names, sorted_values = sort_ranking_results(order, names, values, rev=rev)
        return new(sorted_names, sorted_values, criterion)
    end
end

"""
    struct UncertaintyRanks

Type for the results of the unceertainty ranking.

Fields:
* `names::Vector{Symbol}`: Names of the uncertainty categories.
* `values::Vector{Float64}`: Values of the ranking, calculated according to the criterion.
* `criterion<:AbstractRankingCriterion`: Criterion that was used for ranking.

Constructors:

```julia
function UncertaintyRanks(
    names::Vector{Symbol},
    values::Vector{Float64},
    criterion::AbstractRankingCriterion;
    order::Symbol = :values,
    rev::Bool = false
)
```
Keyword arguments:
* `order::Symbol = :values`: Specifies how the `names` and `values` are sorted. Further options are
`:names` for alphabetical sorting based on the uncertainty names or `:none` for keeping the initial order of uncertainty categories.
* `rev::Bool=false`: Switch to invert the order.
"""
struct UncertaintyRanks <: AbstractRanks
    names::Vector{Symbol}
    values::Vector{Float64}
    criterion::AbstractRankingCriterion

    function UncertaintyRanks(
        names::Vector{Symbol},
        values::Vector{<:Real},
        criterion::AbstractRankingCriterion;
        order::Symbol = :values,
        rev::Bool = false
    )
        sorted_names, sorted_values = sort_ranking_results(order, names, values, rev=rev)
        return new(sorted_names, sorted_values, criterion)
    end
end

"""
    rank_measurements(
        model::EFTfitterModel;
        sampling_algorithm::BAT.AbstractSamplingAlgorithm = MCMCSampling(),
        criterion = SmallestIntervalsSum(p=0.9),
        order = :values,
        rev = true
    )

Computes a ranking of the individual measurement in the `EFTfitterModel` based
on a certain ranking criterion by performing the sampling according to the specified `sampling_algorithm`.
Returns a `MeasurementRank` object.

By default, the summed width of all one-dimensional marginalized smallest intervals containing 90% of the posterior
probability is used as the ranking criterion.
The measurements are ranked by the relative increase of this value.

Available ranking criteria:

* `SmallestIntervalsSum(p=0.9)`: summed width of all one-dimensional marginalized smallest intervals containing p=90% of the posterior probability
* `SmallestInterval(key=:C1, p=0.9)`: width of the one-dimensional marginalized smallest interval of parameter `:C1` containing p=90% of the posterior probability.
"""
function rank_measurements(
    model::EFTfitterModel;
    sampling_algorithm::BAT.AbstractSamplingAlgorithm = MCMCSampling(),
    criterion = SumOfSmallestIntervals(),
    order = :values,
    rev = true
)
    @info "Begin ranking of measurements"
    original = criterion_value(criterion, model, sampling_algorithm=sampling_algorithm)

    deactivated_models, measurement_keys = measurement_models(model)
    values = [criterion_value(criterion, m, sampling_algorithm=sampling_algorithm) for m in deactivated_models]

    relative_increase = calculate_ranks(original, values)

    return MeasurementRanks(collect(measurement_keys), relative_increase, criterion, order=order, rev=rev)
end

"""
    rank_uncertainties(
        model::EFTfitterModel;
        sampling_algorithm::BAT.AbstractSamplingAlgorithm = MCMCSampling(),
        criterion = SmallestIntervalsSum(p=0.9),
        order = :values,
        rev = true
    )

Computes a ranking of the types of uncertainty in the `EFTfitterModel` based
on a certain ranking criterion by performing the sampling according to the specified `sampling_algorithm`.
Returns a `MeasurementRank` object.

By default, the summed width of all one-dimensional marginalized smallest intervals containing 90% of the posterior
probability is used as the ranking criterion.
The uncertainty types are ranked by the relative decrease of this value.

Available ranking criteria:

* `SmallestIntervalsSum(p=0.9)`: summed width of all one-dimensional marginalized smallest intervals containing p=90% of the posterior probability
* `SmallestInterval(key=:C1, p=0.9)`: width of the one-dimensional marginalized smallest interval of parameter `key=:C1` containing p=90% of the posterior probability
"""
function rank_uncertainties(
    model::EFTfitterModel;
    sampling_algorithm::BAT.AbstractSamplingAlgorithm = MCMCSampling(),
    criterion = SumOfSmallestIntervals(),
    order = :values,
    rev = true
)
    @info "Begin ranking of uncertainties"
    original = criterion_value(criterion, model, sampling_algorithm=sampling_algorithm)

    deactivated_models, uncertainty_keys = uncertainty_models(model)
    values = [criterion_value(criterion, m, sampling_algorithm=sampling_algorithm) for m in deactivated_models]
    relative_decrease = -1*calculate_ranks(original, values)

    return UncertaintyRanks(collect(uncertainty_keys), relative_decrease, criterion, order=order, rev=rev)
end

"""
    function measurement_models(model::EFTfitterModel)

Creates a `Vector` of `EFTfitterModel` where always one of the initially active measurements is deactivated at a time.
Returns a `Vector` of `EFTfitterModel` and a `Vector` of `Symbol` with the names of the currently deactivated measurement.
"""
function measurement_models(model::EFTfitterModel)
    m = model.measurements
    measurement_keys = collect(keys(m))

    if length(measurement_keys) < 2
         throw(ArgumentError("Only one measurement active. Ranking of measurements not possible."))
    end

    models = Vector{EFTfitterModel}(undef, length(m))
    for i in 1:length(m)
        meas = collect(values(model.measurements))
        meas[i] = deactivate_measurement(meas[i])
        meas_nt = namedtuple(measurement_keys, meas)
        models[i] = EFTfitterModel(model.parameters, meas_nt, model.correlations)
    end

    return models, measurement_keys
end

"""
    function uncertainty_models(model::EFTfitterModel)

Creates a `Vector` of `EFTfitterModel` where always one of the initially active uncertainty types is deactivated at a time.
Returns a `Vector` of `EFTfitterModel` and a `Vector` of `Symbol` with the names of the currently deactivated uncertainty types.
"""
function uncertainty_models(model::EFTfitterModel)
    cors = model.correlations
    uncertainty_keys = collect(keys(cors))

    if length(uncertainty_keys) < 2
         throw(ArgumentError("Only one type of uncertainties active. Ranking of uncertainties not possible."))
    end

    models = Vector{EFTfitterModel}(undef, length(cors))
    for i in 1:length(cors)
        cors = collect(values(model.correlations))
        cors[i] = deactivate_uncertainty(cors[i])
        cors_nt = namedtuple(uncertainty_keys, cors)
        models[i] = EFTfitterModel(model.parameters, model.measurements, cors_nt)
    end

    return models, uncertainty_keys
end

function deactivate_measurement(meas::Measurement)
    return Measurement(meas.observable, meas.value, meas.uncertainties, false)
end

function deactivate_uncertainty(cor::Correlation)
    return Correlation(cor.matrix, false)
end

"""
    function criterion_value(
        criterion::AbstractRankingCriterion,
        model::EFTfitterModel;
        sampling_algorithm::BAT.AbstractSamplingAlgorithm=MCMCSampling()
    )

Computes the value of the `criterion` for the `model` using the `sampling_algorithm`.

Available ranking criteria:

* `SmallestIntervalsSum(p=0.9)`: summed width of all one-dimensional marginalized smallest intervals containing p=90% of the posterior probability
* `SmallestInterval(key=:C1, p=0.9)`: width of the one-dimensional marginalized smallest interval of parameter `key=:C1` containing p=90% of the posterior probability
"""
function criterion_value(
    criterion::AbstractRankingCriterion,
    model::EFTfitterModel;
    sampling_algorithm::BAT.AbstractSamplingAlgorithm=MCMCSampling()
)
    posterior = PosteriorDensity(EFTfitterDensity(model), model.parameters)
    samples = BAT.bat_sample(posterior, sampling_algorithm).result
    criterion_value = apply_criterion(criterion, samples)
end

function calculate_ranks(original_value::Real, values::AbstractArray{<:Real})
    return relative_increase = (values.-original_value)./original_value
end

function calculate_ranks(original_values::AbstractArray{<:Real}, values::AbstractArray{<:AbstractArray{<:Real, 1}, 1})
    relative_increase = [sum((val.-original_values)./original_values) for val in values]
end

function sort_ranking_results(
    order::Symbol,
    names::Vector{Symbol},
    values::Vector{<:Real};
    rev=false
)
    if order == :names
        sorted = sortperm(names, rev=rev)
        return names[sorted], values[sorted]

    elseif order == :values
        sorted = sortperm(values, rev=rev)
        return names[sorted], values[sorted]

    elseif order == :none
        return names, values

    else
        return names, values
    end
end


#=
Metadata for offline datasets included in this respository...
=#

data_manifest = OrderedDict(
"IDRC_2002_Pharma_Shootout" =>
    OrderedDict(
        "usage" => ["Calibration transfer", "Regression"],
        "keywords" => [ "API", "Pharmaceuticals", "IDRC", "shootout"],
        "property values" => ["weight","hardness","assay"],
        "measurements" => ["NIR"],
        "spectral range" => "600-1398nm",
        "spectral resolution" => "2nm",
        "observations" => "654",
        "format" => "csv",
        "description" => "\"Spectra of 654 pharmaceutical tablets from two Multitab spectrometers (FossNIRsystems, Silverspring, MD) will be the basis of the ShootOut at the next Chambersburg meeting (August 10-16, 2002). All tablets were measured in the transmittance mode.  Each individual tablet was subsequently analyzed for ASSAY value, tablet weight, and tablet hardness. All log (1/T) spectra cover the region from 600 to 1898 nm in 2 nm increments.\"... \"Data from each instrument has been split into a calibration file (155 spectra), and a validation file (459 spectra). In addition, a small test file (40 spectra) from each instrument is avaliable for testing calibrations. The challenge is to develop the best possible calibration for the ASSAY value on one instrument, and to provide the best means of transferring the calibration to the other instrument. The reference method has an error of +/- 1.3 mg on these tablets with a nominal value of 200 mg per tablet. The data set includes tablets with a wide ASSAY range, 152 to 239 mg, for developing calibration(s). No deliberate outliers are included in the data set, but there may be outliers due to experimental errors. The name of the active ingredient in these tablets can not be disclosed for proprietary reasons.\"",
        "URL" => "https://web.archive.org/web/20031204005143/http://www.idrc-chambersburg.org/shootout_2002.htm",
        "Date Accessed" => "7/3/2020",
        "references" => ["Gary E. Ritchie, Chairman NIRVWoG and Sr. Scientist, Pharmaceutical Analysis/New Technology, Purdue Pharma L.P., 444 Saw Mill River Road, Ardsley, NY 10502",
                        "http://www.foss.dk/c/p/solutions/products/showprodfamily.asp?prodfamilypkid=98"],
        "permission" => "Public domain. Permission to redistribute given to Dr. Casey Kneale via e-mail in 2018."
    ),

"Tecator_Meat_Data" =>
    OrderedDict(
        "usage" => ["Regression"],
        "keywords" => ["food", "fat", "meat"],
        "property values" => ["fat","water","protein"],
        "format" => "csv",
        "measurements" => ["NIR"],
        "spectral range" => "850-1050nm",
        "spectral resolution" => "",
        "observations" => "240",
        "description" => "Statement of permission from Tecator (the original data source).These data are recorded on a Tecator Infratec Food and Feed Analyzer working in the wavelength range 850 - 1050 nm by the Near Infrared Transmission (NIT) principle. Each sample contains finely chopped pure meat with different moisture, fat and protein contents.If results from these data are used in a publication we want you to mention the instrument and company name (Tecator) in the publication. In addition, please send a preprint of your article to Karin Thente, Tecator AB, Box 70, S-263 21 Hoganas, Sweden. The data are available in the public domain with no responsability from the original data source. The data can be redistributed as long as this permission note is attached. For more information about the instrument - call Perstorp Analytical's representative in your area.",
        "URL" => "https://www.openml.org/d/505",
        "Date Accessed" => "7/3/2020",
        "references" => [""],
        "permission" => "Public domain with statement of permission from Tecator."
    ),

"Cargill_Corn" =>
    OrderedDict(
        "usage" => ["calibration transfer", "regression"],
        "keywords" => ["food", "corn"],
        "property values" => ["moisture", "oil", "protein", "starch"],
        "format" => "csv",
        "measurements" => ["NIR"],
        "spectral range" => "1100-2498nm",
        "spectral resolution" => "2nm",
        "observations" => "80",
        "description" => "This dataset contains 80 NIR measures of corn taken from 3 different instruments( M5, MP5,and MP6 ).
        Measures of NBS glass standards (with replicates) on each instrument are also contained in seperate files.",
        "URL" => "https://eigenvector.com/resources/data-sets/",
        "Date Accessed" => "7/3/2020",
        "references" => [""],
        "permission" => "Public domain. Permission to redistribute given to Dr. Casey Kneale granted by Mike Blackburn 6/3/2020"
    ),

# "Southwest_Research_Institute_Diesel_Fuels" =>
#     OrderedDict(
#         "usage" => ["regression", "impution", "semisupervised learning"],
#         "keywords" => ["diesel", "fuel", "oil", "gas", "SWRI"],
#         "property values" => ["bp50", "CN", "d4052", "freeze", "total"],
#         "format" => "csv",
#         "measurements" => ["NIR"],
#         "spectral range" => "750-1550nm",
#         "spectral resolution" => "2nm",
#         "observations" => "785",
#         "description" => "Various diesel fuels were measured with a single NIR instrument
#         bp50 - boilingpoint (*C) at 50% recovery(ASTM D 86)
#         CN - Cetane Number (ASTM D 613)
#         d4052 - density, g/mL (15*C ASTM D 4052)
#         freeze - freezing temperature of the fuel, deg C
#         total - total aromatics (mass% ASTM D 5186)
#         visc - viscosity, cSt(40*C)",
#         "URL" => "https://eigenvector.com/wp-content/uploads/2019/06/SWRI_Diesel_NIR_CSV.zip",
#         "Date Accessed" => "7/3/2020",
#         "references" => [""],
#         "permission" => "Public Domain. Collection funded by the US Army."
#     ),

"Fruit_Puree_Adulteration" =>
    OrderedDict(
        "usage" => ["Classification", "One class classification"],
        "keywords" => ["food", "fruit", "adulteration", "interferent"],
        "format" => "csv",
        "measurements" => ["MIR"],
        "spectral range" => "899.327-1802.564 cm-1",
        "spectral resolution" => "3.86cm-1",
        "observations" => "983",
        "description" => "...\"a collection of 983 Mid-infrared spectra collected from
        different authenticated fruit purees in one of two classes:
        \"Strawberry\" (purees prepared from fresh whole fruits by the researchers)
        \"NON-Strawberry\" (diverse collection of other purees, including: strawberry adulterated with other fruits and sugar solutions; raspberry; apple; blackcurrant; blackberry; plum; cherry; apricot; grape juice and mixtures of these.\"
         ",
        "URL" => "https://csr.quadram.ac.uk/example-datasets-for-download/",
        "Date Accessed" => "7/3/2020",
        "references" => ["\"Use of Fourier transform infrared spectroscopy and partial least squares regression for the detection of adulteration of strawberry purees\" Holland JK, Kemsley EK, Wilson RH. (1998). Journal of the Science of Food and Agriculture, 76, 263-269"],
        "permission" => "BBSRC policy on data sharing. Free to analyse and redistribute the data; if the original source is cited."
    ),

"Fresh_Meats" =>
    OrderedDict(
        "usage" => ["Classification", "Authentication"],
        "keywords" => ["food", "chicken", "pork", "turkey"],
        "format" => "csv",
        "measurements" => ["MIR"],
        "spectral range" => "1005.3495-1867.864 cm-1",
        "spectral resolution" => "1.93cm-1",
        "observations" => "120",
        "description" => "...\"120 Mid-infrared (MIR) spectra of fresh minced meats – chicken, pork and turkey. Duplicate acquisitions from 60 independent samples. Raw data matrix size [448 x 120]. Obtained using Fourier transform infrared (FTIR) spectroscopy with attenuated total reflectance (ATR) sampling\"",
        "URL" => "https://csr.quadram.ac.uk/example-datasets-for-download/",
        "Date Accessed" => "7/3/2020",
        "references" => ["\"Mid-infrared spectroscopy and authenticity problems in selected meats: a feasibility study\" Al-Jowder O, Kemsley E K, Wilson R. H.(1997) Food Chemistry 59 195-20."],
        "permission" => "BBSRC policy on data sharing. Free to analyse and redistribute the data; if the original source is cited."
    ),

"Olive_Oils_Quadrum" =>
    OrderedDict(
        "usage" => ["Classification", "Provenance"],
        "keywords" => ["food", "olive", "oil"],
        "format" => "csv",
        "measurements" => ["FTIR"],
        "spectral range" => "798.892-1896.8085 cm-1",
        "spectral resolution" => "1.93 cm-1",
        "observations" => "120",
        "description" => "...\" contains a collection of 120 Mid-infrared spectra collected from 60 different authenticated extra virgin olive oils, supplied to the Institute of Food Research, UK, by the International Olive Oil Council.
        Spectra were acquired from each oil using attenuated total reflectance (ATR) sampling.  The acquisition order was randomized with respect to the country of origin code.
        Once all the samples had been examined once, a second acquisition session commenced, to produce a second spectrum from each sample.  Again the acquisition order was randomized with respect to country of origin.  thus, duplicate spectra were collected from all samples.
        \"",
        "URL" => "https://csr.quadram.ac.uk/example-datasets-for-download/",
        "Date Accessed" => "7/3/2020",
        "references" => ["\"FTIR spectroscopy and multivariate analysis can distinguish the geographic origin of extra virgin olive oils\" Tapp H.S. et al, J. Agric. Food Chem. 51 (21) 6110-5 (2003) "],
        "permission" => "BBSRC policy on data sharing. Free to analyse and redistribute the data; if the original source is cited."
    ),

"Instant_Coffee_Quadrum" =>
    OrderedDict(
        "usage" => ["Classification", "Provenance", "Fraud"],
        "keywords" => ["food", "instant", "coffee", "arabica", "robusta"],
        "format" => "csv",
        "measurements" => ["MIR", "DRIFT", "MIR-DRIFT"],
        "spectral range" => "810.548-1910.644 cm-1",
        "spectral resolution" => "3.86 cm-1",
        "observations" => "56",
        "description" => "...\"Mid-infrared (MIR) spectra of authenticated freeze-dried coffee samples (arabica and robusta species, respectively 29 and 27 of each). Raw data matrix size is [286 x 56]. Obtained by Fourier transform infrared (FTIR) spectroscopy with diffuse reflectance (DRIFT) sampling.\"",
        "URL" => "https://csr.quadram.ac.uk/example-datasets-for-download/",
        "Date Accessed" => "7/3/2020",
        "references" => [   "\"Discrimination of Arabica and Robusta in Instant Coffee by Fourier Transform Infrared Spectroscopy and Chemometrics\" Briandet R et al, J. Agric. Food Chem., 44 (1),170–174 (1996)",
                            "\"Near- and Mid-Infrared Spectroscopies in Food Authentication: Coffee Varietal Identification\" Downey G. et al, J. Agric. Food Chem. 45 (11) 4357-4361 (1997)."],
        "permission" => "BBSRC policy on data sharing. Free to analyse and redistribute the data; if the original source is cited."
    ),

"Raman_PUFA" =>
    OrderedDict(
        "usage" => ["Regression"],
        "keywords" => ["food", "polyunsaturated", "PUFA", "fat"],
        "property values" => ["PUFA%emul", "%PUFA"],
        "format" => "csv",
        "measurements" => ["Raman"],
        "spectral range" => "675-1697 cm-1",
        "spectral resolution" => "1.0 cm-1",
        "observations" => "69",
        "description" => " see citation for more details. ",
        "URL" => "https://data.mendeley.com/datasets/3ympjxywdm/1",
        "Date Accessed" => "7/4/2020",
        "references" => [ "Rimal, Raju; Almøy, Trygve; Sæbø, Solve (2019), \"Data for: Comparison of Multi-response Prediction Methods\", Mendeley Data, v1 http://dx.doi.org/10.17632/3ympjxywdm.1 "],
        "permission" => "CC BY NC 3.0."
    ),

"Dough_NIR" =>
    OrderedDict(
        "usage" => ["Regression"],
        "keywords" => ["food", "dough", "biscuit"],
        "property values" => ["fat", "sucrose", "flour", "water"],
        "format" => "csv",
        "measurements" => ["Raman"],
        "spectral range" => "1000-2498 nm",
        "spectral resolution" => "2.0 nm",
        "observations" => "73",
        "description" => "see citation for more details. ",
        "URL" => "https://data.mendeley.com/datasets/3ympjxywdm/1",
        "Date Accessed" => "7/4/2020",
        "references" => [ "Brown, P.J., Fearn, T. and Vannucci, M. (2001). \"Bayesian wavelet regression on curves with application to a spectroscopic calibration problem.\" Journal of the American Statistical Association 96, 398-408.",
                            "Rimal, Raju; Almøy, Trygve; Sæbø, Solve (2019), \"Data for: Comparison of Multi-response Prediction Methods\", Mendeley Data, v1 http://dx.doi.org/10.17632/3ympjxywdm.1 "],
        "permission" => "CC BY NC 3.0. Permission to redistribute given to Dr. Casey Kneale by Marina Vannucci.",
        "Usage Statement" => "Please acknowledge the following paper if utilizing \"Brown, P.J., Fearn, T. and Vannucci, M. (2001). \"Bayesian wavelet regression on curves with application to a spectroscopic calibration problem.\" Journal of the American Statistical Association 96, 398-408."
    ),

"Penicillin_Monitoring" =>
    OrderedDict(
        "usage" => ["Regression", "control", "monitoring", "ode"],
        "keywords" => ["fermentation", "process", "semisynthetic"],
        "property values" => ["penicillin"],
        "format" => "csv",
        "measurements" => ["Raman"],
        "spectral range" => "?-? cm-1",
        "spectral resolution" => "? cm-1",
        "observations" => "?",
        "description" => "100 batches of penicillin femernations monitored via Raman spectroscopy.
        See references for more details. ",
        "URL" => "https://data.mendeley.com/datasets/pdnjz7zz5x/2",
        "Date Accessed" => "7/4/2020",
        "references" => [ "Goldrick, Stephen (2019), \"Data for: Modern day monitoring and control challenges outlined on an industrial-scale benchmark fermentation process\", Mendeley Data, v2 http://dx.doi.org/10.17632/pdnjz7zz5x.2",
                        "Goldrick S., Duran-Villalobos C., K. Jankauskas, Lovett D., Farid S. S, Lennox B., (2019) Modern day control challenges for industrial-scale fermentation processes. Computers and Chemical Engineering."],
        "permission" => "CC BY NC 4.0. Permission given to Dr. Casey Kneale to redistribute via LinkedIn provided a statement is displayed on GitHub.",
        "Usage Statement" => "Please acknowledge the following paper if utilizing the spectral data which can be freely downloaded at www.industrialpenicillinsimulation.com. Goldrick S., Duran-Villalobos C., K. Jankauskas, Lovett D., Farid S. S, Lennox B., (2019) Modern day control challenges for industrial-scale fermentation processes. Computers and Chemical Engineering."
    )
)


function simplify(c::Chain; rules = naive, mod = @__MODULE__)
	eg = EGraph(c)

	settermtype!(eg, :Chain, 0, Chain)
	settermtype!(eg, :Chain, 1, Chain)
	settermtype!(eg, :Chain, 2, Chain)
	settermtype!(eg, :Gate, 2, Gate)
	settermtype!(eg, :Ctrl, 2, Ctrl)

	saturate!(eg, rules; mod = @__MODULE__)
	c_op = extract!(eg, astsize)
	return postprocess(c_op)
end


using EconModels

# Test promote rules and converts

test1 = Setting(:test1, 22.0) # short constructor
vint = Setting(:data_vintage, "REF", true, "vint", "Data vintage") # full constructor

@test promote_rule(Setting{Float64}, Float16) == Float64
@test promote_rule(Setting{Bool}, Bool) == Bool
@test promote_rule(Setting{ASCIIString}, AbstractString) == UTF8String
@test convert(Int64, test1) == 22
@test convert(Float32, test1) == Float32(22.)
@test convert(ASCIIString, vint) == "REF"


@testset "pointer" begin

@testset "unsafe_load & unsafe_store!" begin

@eval struct LoadableStruct
    a::Int64
    b::UInt8
end
Base.one(::Type{LoadableStruct}) = LoadableStruct(1,1)
Base.zero(::Type{LoadableStruct}) = LoadableStruct(0,0)

@testset for T in (Int8, UInt16, Int32, UInt32, Int64, UInt64, Int128,
                   Float32, Float64,
                   LoadableStruct),
             cached in (false, true)
    d_a = CuArray(ones(T))
    d_b = CuArray(zeros(T))

    ptr_a = CUDAnative.DevicePtr{T,AS.Global}(pointer(d_a))
    ptr_b = CUDAnative.DevicePtr{T,AS.Global}(pointer(d_b))
    @test Array(d_a) != Array(d_b)

    let ptr_a=ptr_a, ptr_b=ptr_b #JuliaLang/julia#15276
        if cached && capability(dev) >= v"3.2"
            @on_device unsafe_store!(ptr_b, unsafe_cached_load(ptr_a))
        else
            @on_device unsafe_store!(ptr_b, unsafe_load(ptr_a))
        end
    end
    @test Array(d_a) == Array(d_b)
end

@testset "indexing" begin
    function kernel(src, dst)
        unsafe_store!(dst, CUDAnative.unsafe_cached_load(src, 4))
        return
    end

    T = Complex{Int8}

    src = CuArray([T(1) T(9); T(3) T(4)])
    dst = CuArray([0])

    @cuda kernel(
        CUDAnative.DevicePtr{T,AS.Global}(pointer(src)),
        CUDAnative.DevicePtr{T,AS.Global}(pointer(dst))
    )

    @test Array(src)[4] == Array(dst)[1]
end

end

end