const GAEVF = JuMP.GenericAffExpr{Float64, VariableRef}
const GQEVF = JuMP.GenericQuadExpr{Float64, VariableRef}

function moi_tests(
    model::DecisionModel,
    params::Bool,
    vars::Int,
    interval::Int,
    lessthan::Int,
    greaterthan::Int,
    equalto::Int,
    binary::Bool,
)
    JuMPmodel = PSI.get_jump_model(model)
    @test (:ParameterJuMP in keys(JuMPmodel.ext)) == params
    @test JuMP.num_variables(JuMPmodel) == vars
    @test JuMP.num_constraints(JuMPmodel, GAEVF, MOI.Interval{Float64}) == interval
    @test JuMP.num_constraints(JuMPmodel, GAEVF, MOI.LessThan{Float64}) == lessthan
    @test JuMP.num_constraints(JuMPmodel, GAEVF, MOI.GreaterThan{Float64}) == greaterthan
    @test JuMP.num_constraints(JuMPmodel, GAEVF, MOI.EqualTo{Float64}) == equalto
    @test ((JuMP.VariableRef, MOI.ZeroOne) in JuMP.list_of_constraint_types(JuMPmodel)) ==
          binary

    return
end

function psi_constraint_test(
    model::DecisionModel,
    constraint_keys::Vector{<:PSI.ConstraintKey},
)
    constraints = PSI.get_constraints(model)
    for con in constraint_keys
        @test get(constraints, con, nothing) !== nothing
    end
    return
end

function psi_aux_variable_test(
    model::DecisionModel,
    constraint_keys::Vector{<:PSI.AuxVarKey},
)
    op_container = PSI.get_optimization_container(model)
    vars = PSI.get_aux_variables(op_container)
    for key in constraint_keys
        @test get(vars, key, nothing) !== nothing
    end
    return
end

function psi_checkbinvar_test(
    model::DecisionModel,
    bin_variable_keys::Vector{<:PSI.VariableKey},
)
    container = PSI.get_optimization_container(model)
    for variable in bin_variable_keys
        for v in PSI.get_variable(container, variable)
            @test JuMP.is_binary(v)
        end
    end
    return
end

function psi_checkobjfun_test(model::DecisionModel, exp_type)
    model = PSI.get_jump_model(model)
    @test JuMP.objective_function_type(model) == exp_type
    return
end

function moi_lbvalue_test(model::DecisionModel, con_key::PSI.ConstraintKey, value::Number)
    for con in PSI.get_constraints(model)[con_key]
        @test JuMP.constraint_object(con).set.lower == value
    end
    return
end

function psi_checksolve_test(model::DecisionModel, status)
    model = PSI.get_jump_model(model)
    JuMP.optimize!(model)
    @test termination_status(model) in status
end

function psi_checksolve_test(model::DecisionModel, status, expected_result, tol=0.0)
    res = solve!(model)
    model = PSI.get_jump_model(model)
    @test termination_status(model) in status
    obj_value = JuMP.objective_value(model)
    @test isapprox(obj_value, expected_result, atol=tol)
end

function psi_ptdf_lmps(res::ProblemResults, ptdf)
    cp_duals = read_dual(res, PSI.ConstraintKey(CopperPlateBalanceConstraint, PSY.System))
    λ = Matrix{Float64}(cp_duals[:, propertynames(cp_duals) .!= :DateTime])

    flow_duals = read_dual(res, PSI.ConstraintKey(NetworkFlowConstraint, PSY.Line))
    μ = Matrix{Float64}(flow_duals[:, ptdf.axes[1]])

    buses = get_components(Bus, get_system(res))
    lmps = OrderedDict()
    for bus in buses
        lmps[get_name(bus)] = μ * ptdf[:, get_number(bus)]
    end
    lmp = λ .+ DataFrames.DataFrame(lmps)
    return lmp[!, sort(propertynames(lmp))]
end

function check_variable_unbounded(
    model::DecisionModel,
    ::Type{T},
    ::Type{U},
) where {T <: PSI.VariableType, U <: PSY.Component}
    return check_variable_unbounded(model::DecisionModel, PSI.VariableKey(T, U))
end

function check_variable_unbounded(model::DecisionModel, var_key::PSI.VariableKey)
    psi_cont = PSI.get_optimization_container(model)
    variable = PSI.get_variable(psi_cont, var_key)
    for var in variable
        if JuMP.has_lower_bound(var) || JuMP.has_upper_bound(var)
            return false
        end
    end
    return true
end

function check_variable_bounded(
    model::DecisionModel,
    ::Type{T},
    ::Type{U},
) where {T <: PSI.VariableType, U <: PSY.Component}
    return check_variable_bounded(model, PSI.VariableKey(T, U))
end

function check_variable_bounded(model::DecisionModel, var_key::PSI.VariableKey)
    psi_cont = PSI.get_optimization_container(model)
    variable = PSI.get_variable(psi_cont, var_key)
    for var in variable
        if !JuMP.has_lower_bound(var) || !JuMP.has_upper_bound(var)
            return false
        end
    end
    return true
end

function check_flow_variable_values(
    model::DecisionModel,
    ::Type{T},
    ::Type{U},
    device_name::String,
    limit::Float64,
) where {T <: PSI.VariableType, U <: PSY.Component}
    psi_cont = PSI.get_optimization_container(model)
    variable = PSI.get_variable(psi_cont, T(), U)
    for var in variable[device_name, :]
        if !(JuMP.value(var) <= (limit + 1e-2))
            return false
        end
    end
    return true
end

function check_flow_variable_values(
    model::DecisionModel,
    ::Type{T},
    ::Type{U},
    device_name::String,
    limit_min::Float64,
    limit_max::Float64,
) where {T <: PSI.VariableType, U <: PSY.Component}
    psi_cont = PSI.get_optimization_container(model)
    variable = PSI.get_variable(psi_cont, T(), U)
    for var in variable[device_name, :]
        if !(JuMP.value(var) <= (limit_max + 1e-2)) ||
           !(JuMP.value(var) >= (limit_min - 1e-2))
            return false
        end
    end
    return true
end

function check_flow_variable_values(
    model::DecisionModel,
    ::Type{T},
    ::Type{U},
    ::Type{V},
    device_name::String,
    limit_min::Float64,
    limit_max::Float64,
) where {T <: PSI.VariableType, U <: PSI.VariableType, V <: PSY.Component}
    psi_cont = PSI.get_optimization_container(model)
    time_steps = PSI.get_time_steps(psi_cont)
    pvariable = PSI.get_variable(psi_cont, T(), V)
    qvariable = PSI.get_variable(psi_cont, U(), V)
    for t in time_steps
        fp = JuMP.value(pvariable[device_name, t])
        fq = JuMP.value(qvariable[device_name, t])
        flow = sqrt((fp)^2 + (fq)^2)
        if !(flow <= (limit_max + 1e-2)^2) || !(flow >= (limit_min - 1e-2)^2)
            return false
        end
    end
    return true
end

function check_flow_variable_values(
    model::DecisionModel,
    ::Type{T},
    ::Type{U},
    ::Type{V},
    device_name::String,
    limit::Float64,
) where {T <: PSI.VariableType, U <: PSI.VariableType, V <: PSY.Component}
    psi_cont = PSI.get_optimization_container(model)
    time_steps = PSI.get_time_steps(psi_cont)
    pvariable = PSI.get_variable(psi_cont, T(), V)
    qvariable = PSI.get_variable(psi_cont, U(), V)
    for t in time_steps
        fp = JuMP.value(pvariable[device_name, t])
        fq = JuMP.value(qvariable[device_name, t])
        flow = sqrt((fp)^2 + (fq)^2)
        if !(flow <= (limit + 1e-2)^2)
            return false
        end
    end
    return true
end

function PSI.jump_value(int::Int)
    @warn("This is for testing purposes only.")
    return int
end

function _check_constraint_bounds(bounds::PSI.ConstraintBounds, valid_bounds::NamedTuple)
    @test bounds.coefficient.min == valid_bounds.coefficient.min
    @test bounds.coefficient.max == valid_bounds.coefficient.max
    @test bounds.rhs.min == valid_bounds.rhs.min
    @test bounds.rhs.max == valid_bounds.rhs.max
end

function _check_variable_bounds(bounds::PSI.VariableBounds, valid_bounds::NamedTuple)
    @test bounds.bounds.min == valid_bounds.min
    @test bounds.bounds.max == valid_bounds.max
end

function check_duration_on_initial_conditions_values(
    model,
    ::Type{T},
) where {T <: PSY.Component}
    initial_conditions_data =
        PSI.get_initial_conditions_data(PSI.get_optimization_container(model))
    duration_on_data = PSI.get_initial_condition(
        PSI.get_optimization_container(model),
        InitialTimeDurationOn(),
        T,
    )
    for ic in duration_on_data
        name = PSY.get_name(ic.component)
        on_var = PSI.get_initial_condition_value(initial_conditions_data, OnVariable(), T)[
            1,
            name,
        ]
        duration_on = JuMP.value(PSI.get_value(ic))
        if on_var == 1.0 && PSY.get_status(ic.component)
            @test duration_on == PSY.get_time_at_status(ic.component)
        elseif on_var == 1.0 && !PSY.get_status(ic.component)
            @test duration_on == 0.0
        end
    end
end

function check_duration_off_initial_conditions_values(
    model,
    ::Type{T},
) where {T <: PSY.Component}
    initial_conditions_data =
        PSI.get_initial_conditions_data(PSI.get_optimization_container(model))
    duration_off_data = PSI.get_initial_condition(
        PSI.get_optimization_container(model),
        InitialTimeDurationOff(),
        T,
    )
    for ic in duration_off_data
        name = PSY.get_name(ic.component)
        on_var = PSI.get_initial_condition_value(initial_conditions_data, OnVariable(), T)[
            1,
            name,
        ]
        duration_off = JuMP.value(PSI.get_value(ic))
        if on_var == 0.0 && !PSY.get_status(ic.component)
            @test duration_off == PSY.get_time_at_status(ic.component)
        elseif on_var == 0.0 && PSY.get_status(ic.component)
            @test duration_off == 0.0
        end
    end
end

function check_energy_initial_conditions_values(model, ::Type{T}) where {T <: PSY.Component}
    ic_data = PSI.get_initial_condition(
        PSI.get_optimization_container(model),
        InitialEnergyLevel(),
        T,
    )
    for ic in ic_data
        name = PSY.get_name(ic.component)
        e_value = JuMP.value(PSI.get_value(ic))
        @test PSY.get_initial_energy(ic.component) == e_value
    end
end

function check_energy_initial_conditions_values(model, ::Type{T}) where {T <: PSY.HydroGen}
    ic_data = PSI.get_initial_condition(
        PSI.get_optimization_container(model),
        InitialEnergyLevel(),
        T,
    )
    for ic in ic_data
        name = PSY.get_name(ic.component)
        e_value = JuMP.value(PSI.get_value(ic))
        @test PSY.get_initial_storage(ic.component) == e_value
    end
end

function check_status_initial_conditions_values(model, ::Type{T}) where {T <: PSY.Component}
    initial_conditions =
        PSI.get_initial_condition(PSI.get_optimization_container(model), DeviceStatus(), T)
    initial_conditions_data =
        PSI.get_initial_conditions_data(PSI.get_optimization_container(model))
    for ic in initial_conditions
        name = PSY.get_name(ic.component)
        status = PSI.get_initial_condition_value(initial_conditions_data, OnVariable(), T)[
            1,
            name,
        ]
        @test JuMP.value(PSI.get_value(ic)) == status
    end
end

function check_active_power_initial_condition_values(
    model,
    ::Type{T},
) where {T <: PSY.Component}
    initial_conditions =
        PSI.get_initial_condition(PSI.get_optimization_container(model), DevicePower(), T)
    initial_conditions_data =
        PSI.get_initial_conditions_data(PSI.get_optimization_container(model))
    for ic in initial_conditions
        name = PSY.get_name(ic.component)
        power = PSI.get_initial_condition_value(
            initial_conditions_data,
            ActivePowerVariable(),
            T,
        )[
            1,
            name,
        ]
        @test JuMP.value(PSI.get_value(ic)) == power
    end
end

function check_active_power_abovemin_initial_condition_values(
    model,
    ::Type{T},
) where {T <: PSY.Component}
    initial_conditions = PSI.get_initial_condition(
        PSI.get_optimization_container(model),
        PSI.DeviceAboveMinPower(),
        T,
    )
    initial_conditions_data =
        PSI.get_initial_conditions_data(PSI.get_optimization_container(model))
    for ic in initial_conditions
        name = PSY.get_name(ic.component)
        power = PSI.get_initial_condition_value(
            initial_conditions_data,
            PSI.PowerAboveMinimumVariable(),
            T,
        )[
            1,
            name,
        ]
        @test JuMP.value(PSI.get_value(ic)) == power
    end
end

function check_initialization_variable_count(
    model,
    ::S,
    ::Type{T},
) where {S <: PSI.VariableType, T <: PSY.Component}
    container = PSI.get_optimization_container(model)
    initial_conditions_data = PSI.get_initial_conditions_data(container)
    no_component = length(PSY.get_components(T, model.sys, x -> x.available))
    variable = PSI.get_initial_condition_value(initial_conditions_data, S(), T)
    rows, cols = size(variable)
    @test rows * cols == no_component * PSI.INITIALIZATION_PROBLEM_HORIZON
end

function check_variable_count(
    model,
    ::S,
    ::Type{T},
) where {S <: PSI.VariableType, T <: PSY.Component}
    no_component = length(PSY.get_components(T, model.sys, x -> x.available))
    time_steps = PSI.get_time_steps(PSI.get_optimization_container(model))[end]
    variable = PSI.get_variable(PSI.get_optimization_container(model), S(), T)
    @test length(variable) == no_component * time_steps
end

function check_initialization_constraint_count(
    model,
    ::S,
    ::Type{T};
    filter_func=x -> x.available,
    meta=PSI.CONTAINER_KEY_EMPTY_META,
) where {S <: PSI.ConstraintType, T <: PSY.Component}
    container = model.internal.ic_model_container
    no_component = length(PSY.get_components(T, model.sys, filter_func))
    time_steps = PSI.get_time_steps(container)[end]
    constraint = PSI.get_constraint(container, S(), T, meta)
    @test length(constraint) == no_component * time_steps
end

function check_constraint_count(
    model,
    ::S,
    ::Type{T};
    filter_func=x -> x.available,
    meta=PSI.CONTAINER_KEY_EMPTY_META,
) where {S <: PSI.ConstraintType, T <: PSY.Component}
    no_component = length(PSY.get_components(T, model.sys, filter_func))
    time_steps = PSI.get_time_steps(PSI.get_optimization_container(model))[end]
    constraint = PSI.get_constraint(PSI.get_optimization_container(model), S(), T, meta)
    @test length(constraint) == no_component * time_steps
end

function check_constraint_count(
    model,
    ::PSI.RampConstraint,
    ::Type{T},
) where {T <: PSY.Component}
    container = PSI.get_optimization_container(model)
    set_name =
        PSY.get_name.(
            PSI._get_ramp_constraint_devices(
                container,
                get_components(T, model.sys, x -> x.available),
            ),
        )
    check_constraint_count(
        model,
        PSI.RampConstraint(),
        T;
        meta="up",
        filter_func=x -> x.name in set_name,
    )
    check_constraint_count(
        model,
        PSI.RampConstraint(),
        T;
        meta="dn",
        filter_func=x -> x.name in set_name,
    )
    return
end

function check_constraint_count(
    model,
    ::PSI.DurationConstraint,
    ::Type{T},
) where {T <: PSY.Component}
    container = PSI.get_optimization_container(model)
    resolution = PSI.get_resolution(container)
    steps_per_hour = 60 / Dates.value(Dates.Minute(resolution))
    fraction_of_hour = 1 / steps_per_hour
    duration_devices = filter!(
        x -> !(
            PSY.get_time_limits(x).up <= fraction_of_hour &&
            PSY.get_time_limits(x).down <= fraction_of_hour
        ),
        collect(get_components(T, model.sys, x -> x.available)),
    )
    set_name = PSY.get_name.(duration_devices)
    check_constraint_count(
        model,
        PSI.DurationConstraint(),
        T;
        meta="up",
        filter_func=x -> x.name in set_name,
    )
    return check_constraint_count(
        model,
        PSI.DurationConstraint(),
        T;
        meta="dn",
        filter_func=x -> x.name in set_name,
    )
end

function check_constraint_count(
    model,
    ::PSI.MustRunConstraint,
    ::Type{T},
) where {T <: PSY.Component}
    container = PSI.get_optimization_container(model)
    _devices =
        filter!(x -> x.must_run, collect(get_components(T, model.sys, x -> x.available)))
    set_name = PSY.get_name.(_devices)
    return check_constraint_count(
        model,
        PSI.MustRunConstraint(),
        T,
        filter_func=x -> x.name in set_name,
    )
end


""" Misc non-math functions used in FreeFermionGutzwiller.jl"""

count_bonds(graph::SimpleGraph{Int64}) = ne(graph)

function get_updated_bonds(chain::GutzwillerChain,move::ExchangeMove)::SimpleGraph{Int64}
    """ Returns a new graph object with the bonds of the updated state"""
    state = get_State(chain)
    model = get_Model(chain)
    lattice = model["lattice"]

    sc = deepcopy(state.spin_config.sc)
    bonds = deepcopy(state.bonds)
    sc[move.sites[1]], sc[move.sites[2]] = sc[move.sites[2]], sc[move.sites[1]]

    for site in move.sites
        for neighbor in neighbors(lattice.graph,site)
            if sc[site] == sc[neighbor]
                rem_edge!(bonds,site,neighbor)
            else
                add_edge!(bonds,site,neighbor)
            end
        end
    end
    return bonds
end

function get_conditioned_state(
    chain, n_up,n_down, conditioning_tol=10^6)
    sites = collect(1:n_up+n_down)
    cc_up = 2*conditioning_tol
    cc_down = 2*conditioning_tol
    while (cc_up > conditioning_tol  || cc_down > conditioning_tol)
        # vectors to store the location of the up and down electrons
        R_up = zeros(Int64, n_up)
        R_down = zeros(Int64, n_down)
        sample!(sites,R_up,replace=false)
        leftover_sites = collect(setdiff(Set(sites),Set(R_up)))
        sample!(leftover_sites,R_down,replace=false)

                # get occupied wavefunctions from lattice and fermi energy
        filled_states = get_Wavefunctions(chain)

        cc_up = cond(filled_states[R_up,:])
        cc_down = cond(filled_states[R_down,:])

        if (cc_up < conditioning_tol && cc_down < conditioning_tol)
            return R_up, R_down, filled_states
        end
    end
end

function update_Rs!(state::GutzwillerState,move::ExchangeMove)
    r_up = state.r_up
    r_down = state.r_down
    bd = state.business_directory
    # use bd to find what electrons are being exchanged
    l1, l2 = move.sites[1], move.sites[2]
    electron_A = bd[l1]
    electron_B = bd[l2]

    if electron_A.spin=="up" # then A is spin up and B is spin down
        r_up[electron_A.label] = l2
        r_down[electron_B.label] = l1
    else # A is down and B is up
        r_down[electron_A.label] = l2
        r_up[electron_B.label] = l1
    end
end

function update_Spin_config!(state::GutzwillerState,move::ExchangeMove)
    sc = state.spin_config.sc
    l1 = move.sites[1]
    l2 = move.sites[2]
    sc[l1], sc[l2] = sc[l2], sc[l1]
end

function update_Bonds!(chain::GutzwillerChain,move::ExchangeMove)
    state = get_State(chain)
    tmp = get_updated_bonds(chain,move)
    state.bonds = tmp
end

function update_Determinants!(chain::GutzwillerChain,move::ExchangeMove,extras)
    # extras contains det_ratio_up and det_ratio_down

    state = get_State(chain)

    if isnothing(extras)
        trash, extras = compute_ratio(chain,move)
    end
    det_ratio_up = extras[1]
    det_ratio_down = extras[2]
    state.det_A_up *= det_ratio_up
    state.det_A_down *= det_ratio_down
end

function update_Inverses!(state::GutzwillerState,move::ExchangeMove)
    u1,v1,u2,v2 = get_update_vectors(state,move)
    # recall u1 v1 are for up, u2 v2 are for down
    t = sherman_morrison_inverse_update(state.A_inv_up,u1,v1)
    r = sherman_morrison_inverse_update(state.A_inv_down,u2,v2)

    state.A_inv_up = t
    state.A_inv_down = r
end

function update_Business_directory!(state::GutzwillerState,move::ExchangeMove)
    bd = state.business_directory
    l1 = move.sites[1]
    l2 = move.sites[2]
    bd[l1], bd[l2] = bd[l2], bd[l1]
end

function make_business_directory(r_up::Array,r_down::Array)::Array
    """ Makes the directory for the lattice. The i'th element
    is the index of the electron living on site i. indices <= N/2
    refer to up spins and indices > N/2 refer to down spins """
    ns = length([r_up ; r_down])
    bd = Array{Electron}(undef, ns)

    for i in 1:length(r_up) # assume len(r_up) = len(r_down)
        bd[r_up[i]] = Electron(i,"up")
        bd[r_down[i]] = Electron(i,"down")
    end

    return bd
end

function get_ExchangeMove(bonds::SimpleGraph{Int64})::ExchangeMove
    """ draw a random bond and return the two sites attached to that bond """
    index = rand(1:ne(bonds))
    edge = nth(edges(bonds),index) #nth is from IterTools
    move_sites = (src(edge),dst(edge))
    return ExchangeMove(move_sites)
end

function get_update_vectors(state::GutzwillerState,move::ExchangeMove)
    """ Gets vectors which perform rank 1 row update to the slater determinant
    specifically, u is zero except at column i and v_i = ϕ_i(r') - ϕ_i(r)
    u1 and v1 are for up spin, and u2 and v2 are for down spin"""
    # get electron indices from business directory
    bd = state.business_directory

    site_A = move.sites[1]
    site_B = move.sites[2]

    electron_A = bd[site_A]
    electron_B = bd[site_B]

    u1 = zeros(length(state.r_up))
    v1 = zeros(length(state.r_up))
    u2 = zeros(length(state.r_down))
    v2 = zeros(length(state.r_down))

    if electron_A.spin=="up"
        # then A is up (1) and B is down (2)
        # up electron is moving from site A to site B
        u1[electron_A.label] = 1
        v1 = state.wavefunctions[site_B,:] - state.wavefunctions[site_A,:]

        # down electron is moving from site B to site A
        u2[electron_B.label] = 1
        v2 = state.wavefunctions[site_A,:] - state.wavefunctions[site_B,:]

    else
        # A is down (2) and B is up (1)
        u2[electron_A.label] = 1
        v2 = state.wavefunctions[site_B,:] - state.wavefunctions[site_A,:]

        u1[electron_B.label] = 1
        v1 = state.wavefunctions[site_A,:] - state.wavefunctions[site_B,:]
    end

    return u1,v1,u2,v2
end


for jy in ("j","y"), nu in (0,1)
    jynu = Expr(:quote, symbol(string(jy,nu)))
    jynuf = Expr(:quote, symbol(string(jy,nu,"f")))
    bjynu = symbol(string("bessel",jy,nu))
    if jy == "y"
        @eval begin
            $bjynu(x::Float64) = nan_dom_err(ccall(($jynu,libm),  Float64, (Float64,), x), x)
            $bjynu(x::Float32) = nan_dom_err(ccall(($jynuf,libm), Float32, (Float32,), x), x)
        end
    else
        @eval begin
            $bjynu(x::Float64) = ccall(($jynu,libm),  Float64, (Float64,), x)
            $bjynu(x::Float32) = ccall(($jynuf,libm), Float32, (Float32,), x)
        end
    end
    @eval begin
        $bjynu(x::Real) = $bjynu(float(x))
        $bjynu(x::Complex) = $(symbol(string("bessel",jy)))($nu,x)
        @vectorize_1arg Number $bjynu
    end
end

type AmosException <: Exception
    info::Int32
end

let
    const ai::Array{Float64,1} = Array(Float64,2)
    const ae::Array{Int32,1} = Array(Int32,2)
    global _airy, _biry
    function _airy(z::Complex128, id::Int32, kode::Int32)
        ccall((:zairy_,openspecfun), Void,
              (Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32},
               Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32}),
              &real(z), &imag(z),
              &id, &kode,
              pointer(ai,1), pointer(ai,2),
              pointer(ae,1), pointer(ae,2))
        if ae[2] == 0 || ae[2] == 3
            return complex(ai[1],ai[2])
        else
            throw(AmosException(ae[2]))
        end
    end
    function _biry(z::Complex128, id::Int32, kode::Int32)
        ccall((:zbiry_,openspecfun), Void,
              (Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32},
               Ptr{Float64}, Ptr{Float64}, Ptr{Int32}),
              &real(z), &imag(z),
              &id, &kode,
              pointer(ai,1), pointer(ai,2),
              pointer(ae,1))
        if ae[1] == 0 || ae[1] == 3  # ignore underflow
            return complex(ai[1],ai[2])
        else
            throw(AmosException(ae[2]))
        end
    end
end

function airy(k::Int, z::Complex128)
    id = int32(k==1 || k==3)
    if k == 0 || k == 1
        return _airy(z, id, int32(1))
    elseif k == 2 || k == 3
        return _biry(z, id, int32(1))
    else
        error("invalid argument")
    end
end

airy(z) = airy(0,z)
@vectorize_1arg Number airy
airyprime(z) = airy(1,z)
@vectorize_1arg Number airyprime
airyai(z) = airy(0,z)
@vectorize_1arg Number airyai
airyaiprime(z) = airy(1,z)
@vectorize_1arg Number airyaiprime
airybi(z) = airy(2,z)
@vectorize_1arg Number airybi
airybiprime(z) = airy(3,z)
@vectorize_1arg Number airybiprime

airy(k::Number, x::FloatingPoint) = oftype(x, real(airy(k, complex(x))))
airy(k::Number, x::Real) = airy(k, float(x))
airy(k::Number, z::Complex64) = complex64(airy(k, complex128(z)))
airy(k::Number, z::Complex) = airy(convert(Int,k), complex128(z))
@vectorize_2arg Number airy

function airyx(k::Int, z::Complex128)
    id = int32(k==1 || k==3)
    if k == 0 || k == 1
        return _airy(z, id, int32(2))
    elseif k == 2 || k == 3
        return _biry(z, id, int32(2))
    else
        error("invalid argument")
    end
end

airyx(z) = airyx(0,z)
@vectorize_1arg Number airyx

airyx(k::Number, x::FloatingPoint) = oftype(x, real(airyx(k, complex(x))))
airyx(k::Number, x::Real) = airyx(k, float(x))
airyx(k::Number, z::Complex64) = complex64(airyx(k, complex128(z)))
airyx(k::Number, z::Complex) = airyx(convert(Int,k), complex128(z))
@vectorize_2arg Number airyx

const cy = Array(Float64,2)
const ae = Array(Int32,2)
const wrk = Array(Float64,2)

function _besselh(nu::Float64, k::Int32, z::Complex128, kode::Int32)
    ccall((:zbesh_,openspecfun), Void,
          (Ptr{Float64}, Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32},
           Ptr{Int32}, Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32}),
          &real(z), &imag(z), &nu, &kode, &k, &1,
          pointer(cy,1), pointer(cy,2),
          pointer(ae,1), pointer(ae,2))
    if ae[2] == 0 || ae[2] == 3
        return complex(cy[1],cy[2])
    else
        throw(AmosException(ae[2]))
    end
end

function _besseli(nu::Float64, z::Complex128, kode::Int32)
    ccall((:zbesi_,openspecfun), Void,
          (Ptr{Float64}, Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32},
           Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32}),
          &real(z), &imag(z), &nu, &kode, &1,
          pointer(cy,1), pointer(cy,2),
          pointer(ae,1), pointer(ae,2))
    if ae[2] == 0 || ae[2] == 3
        return complex(cy[1],cy[2])
    else
        throw(AmosException(ae[2]))
    end
end

function _besselj(nu::Float64, z::Complex128, kode::Int32)
    ccall((:zbesj_,openspecfun), Void,
          (Ptr{Float64}, Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32},
           Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32}),
          &real(z), &imag(z), &nu, &kode, &1,
          pointer(cy,1), pointer(cy,2),
          pointer(ae,1), pointer(ae,2))
    if ae[2] == 0 || ae[2] == 3
        return complex(cy[1],cy[2])
    else
        throw(AmosException(ae[2]))
    end
end

function _besselk(nu::Float64, z::Complex128, kode::Int32)
    ccall((:zbesk_,openspecfun), Void,
          (Ptr{Float64}, Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32},
           Ptr{Float64}, Ptr{Float64}, Ptr{Int32}, Ptr{Int32}),
          &real(z), &imag(z), &nu, &kode, &1,
          pointer(cy,1), pointer(cy,2),
          pointer(ae,1), pointer(ae,2))
    if ae[2] == 0 || ae[2] == 3
        return complex(cy[1],cy[2])
    else
        throw(AmosException(ae[2]))
    end
end

function _bessely(nu::Float64, z::Complex128, kode::Int32)
    ccall((:zbesy_,openspecfun), Void,
          (Ptr{Float64}, Ptr{Float64}, Ptr{Float64}, Ptr{Int32},
           Ptr{Int32}, Ptr{Float64}, Ptr{Float64}, Ptr{Int32},
           Ptr{Float64}, Ptr{Float64}, Ptr{Int32}),
          &real(z), &imag(z), &nu, &kode, &1,
          pointer(cy,1), pointer(cy,2),
          pointer(ae,1), pointer(wrk,1),
          pointer(wrk,2), pointer(ae,2))
    if ae[2] == 0 || ae[2] == 3
        return complex(cy[1],cy[2])
    else
        throw(AmosException(ae[2]))
    end
end

function besselh(nu::Float64, k::Integer, z::Complex128)
    if nu < 0
        s = (k == 1) ? 1 : -1
        return _besselh(-nu,int32(k),z,int32(1)) * complex(cospi(nu),-s*sinpi(nu))
    end
    return _besselh(nu,int32(k),z,int32(1))
end

function besselhx(nu::Float64, k::Integer, z::Complex128)
    if nu < 0
        s = (k == 1) ? 1 : -1
        return _besselh(-nu,int32(k),z,int32(2)) * complex(cospi(nu),-s*sinpi(nu))
    end
    return _besselh(nu,int32(k),z,int32(2))
end

function besseli(nu::Float64, z::Complex128)
    if nu < 0
        return _besseli(-nu,z,int32(1)) - 2_besselk(-nu,z,int32(1))*sinpi(nu)/pi
    else
        return _besseli(nu,z,int32(1))
    end
end

function besselix(nu::Float64, z::Complex128)
    if nu < 0
        return _besseli(-nu,z,int32(2)) - 2_besselk(-nu,z,int32(2))*exp(-abs(real(z))-z)*sinpi(nu)/pi
    else
        return _besseli(nu,z,int32(2))
    end
end

function besselj(nu::Float64, z::Complex128)
    if nu < 0
        return _besselj(-nu,z,int32(1))*cospi(nu) + _bessely(-nu,z,int32(1))*sinpi(nu)
    else
        return _besselj(nu,z,int32(1))
    end
end

besselj(nu::Integer, x::FloatingPoint) = typemin(Int32) <= nu <= typemax(Int32) ?
    oftype(x, ccall((:jn, libm), Float64, (Cint, Float64), nu, x)) :
    besselj(float64(nu), x)

besselj(nu::Integer, x::Float32) = typemin(Int32) <= nu <= typemax(Int32) ?
    ccall((:jnf, libm), Float32, (Cint, Float32), nu, x) :
    besselj(float64(nu), x)

function besseljx(nu::Float64, z::Complex128)
    if nu < 0
        return _besselj(-nu,z,int32(2))*cospi(nu) + _bessely(-nu,z,int32(2))*sinpi(nu)
    else
        return _besselj(nu,z,int32(2))
    end
end

besselk(nu::Float64, z::Complex128) = _besselk(abs(nu), z, int32(1))

besselkx(nu::Float64, z::Complex128) = _besselk(abs(nu), z, int32(2))

function bessely(nu::Float64, z::Complex128)
    if nu < 0
        return _bessely(-nu,z,int32(1))*cospi(nu) - _besselj(-nu,z,int32(1))*sinpi(nu)
    else
        return _bessely(nu,z,int32(1))
    end
end

function besselyx(nu::Float64, z::Complex128)
    if nu < 0
        return _bessely(-nu,z,int32(2))*cospi(nu) - _besselj(-nu,z,int32(2))*sinpi(nu)
    else
        return _bessely(nu,z,int32(2))
    end
end

besselh(nu, z) = besselh(nu, 1, z)
besselh(nu::Real, k::Integer, z::Complex64) = complex64(besselh(float64(nu), k, complex128(z)))
besselh(nu::Real, k::Integer, z::Complex) = besselh(float64(nu), k, complex128(z))
besselh(nu::Real, k::Integer, x::Real) = besselh(float64(nu), k, complex128(x))
@vectorize_2arg Number besselh

hankelh1(nu, z) = besselh(nu, 1, z)
@vectorize_2arg Number hankelh1

hankelh2(nu, z) = besselh(nu, 2, z)
@vectorize_2arg Number hankelh2

besselhx(nu::Real, k::Integer, z::Complex64) = complex64(besselhx(float64(nu), k, complex128(z)))
besselhx(nu::Real, k::Integer, z::Complex) = besselhx(float64(nu), k, complex128(z))
besselhx(nu::Real, k::Integer, x::Real) = besselhx(float64(nu), k, complex128(x))

hankelh1x(nu, z) = besselhx(nu, 1, z)
@vectorize_2arg Number hankelh1x

hankelh2x(nu, z) = besselhx(nu, 2, z)
@vectorize_2arg Number hankelh2x

function besseli(nu::Real, x::FloatingPoint)
    if x < 0 && !isinteger(nu)
        throw(DomainError())
    end
    oftype(x, real(besseli(float64(nu), complex128(x))))
end

function besselix(nu::Real, x::FloatingPoint)
    if x < 0 && !isinteger(nu)
        throw(DomainError())
    end
    oftype(x, real(besselix(float64(nu), complex128(x))))
end

function besselj(nu::FloatingPoint, x::FloatingPoint)
    if isinteger(nu)
        if typemin(Int32) <= nu <= typemax(Int32)
            return besselj(int(nu), x)
        end
    elseif x < 0
        throw(DomainError())
    end
    oftype(x, real(besselj(float64(nu), complex128(x))))
end

function besseljx(nu::Real, x::FloatingPoint)
    if x < 0 && !isinteger(nu)
        throw(DomainError())
    end
    oftype(x, real(besseljx(float64(nu), complex128(x))))
end

function besselk(nu::Real, x::FloatingPoint)
    if x < 0
        throw(DomainError())
    end
    if x == 0
        return oftype(x, Inf)
    end
    oftype(x, real(besselk(float64(nu), complex128(x))))
end

function besselkx(nu::Real, x::FloatingPoint)
    if x < 0
        throw(DomainError())
    end
    if x == 0
        return oftype(x, Inf)
    end
    oftype(x, real(besselkx(float64(nu), complex128(x))))
end

function bessely(nu::Real, x::FloatingPoint)
    if x < 0
        throw(DomainError())
    end
    if isinteger(nu) && typemin(Int32) <= nu <= typemax(Int32)
        return bessely(int(nu), x)
    end
    oftype(x, real(bessely(float64(nu), complex128(x))))
end
function bessely(nu::Integer, x::FloatingPoint)
    if x < 0
        throw(DomainError())
    end
    return oftype(x, ccall((:yn, libm), Float64, (Cint, Float64), nu, x))
end
function bessely(nu::Integer, x::Float32)
    if x < 0
        throw(DomainError())
    end
    return ccall((:ynf, libm), Float32, (Cint, Float32), nu, x)
end

function besselyx(nu::Real, x::FloatingPoint)
    if x < 0
        throw(DomainError())
    end
    oftype(x, real(besselyx(float64(nu), complex128(x))))
end

for f in ("i", "ix", "j", "jx", "k", "kx", "y", "yx")
    bfn = symbol(string("bessel", f))
    @eval begin
        $bfn(nu::Real, z::Complex64) = complex64($bfn(float64(nu), complex128(z)))
        $bfn(nu::Real, z::Complex) = $bfn(float64(nu), complex128(z))
        $bfn(nu::Real, x::Integer) = $bfn(nu, float64(x))
        @vectorize_2arg Number $bfn
    end
end


module Fields

export
    Face, Cell,
    AbstractField, Field, CellField, XFaceField, YFaceField, ZFaceField,
    ReducedField, AveragedField, ComputedField,
    interior, interiorparent, data,
    xnode, ynode, znode, location,
    set!, compute!, @compute,
    VelocityFields, TracerFields, tracernames, PressureFields, TendencyFields

using Oceananigans.Architectures
using Oceananigans.Grids
using Oceananigans.BoundaryConditions

Base.zeros(FT, ::CPU, Nx, Ny, Nz) = zeros(FT, Nx, Ny, Nz)
Base.zeros(FT, ::GPU, Nx, Ny, Nz) = zeros(FT, Nx, Ny, Nz) |> CuArray
Base.zeros(arch, grid, Nx, Ny, Nz) = zeros(eltype(grid), arch, Nx, Ny, Nz)

include("new_data.jl")
include("abstract_field.jl")
include("field.jl")
include("reduced_field.jl")
include("averaged_field.jl")
include("computed_field.jl")
include("pressure_field.jl")
include("function_field.jl")
include("set!.jl")
include("field_tuples.jl")
include("show_fields.jl")

end


using SCIP
using QuadraticToBinary

SCIPopt = SCIP.Optimizer()
MOI.set(SCIPopt, MOI.Silent(), true)
MOI.set(SCIPopt, MOI.TimeLimitSec(), 5)
SCIPopt_CACHED = MOIU.CachingOptimizer(MOIU.UniversalFallback(MOIU.Model{Float64}()), SCIPopt)

push!(solvers, (opt = SCIPopt, mode = BilevelJuMP.SOS1Mode()))
push!(solvers_sos, (opt = SCIPopt, mode = BilevelJuMP.SOS1Mode()))
push!(solvers_unit, (opt = SCIPopt, mode = BilevelJuMP.SOS1Mode()))
push!(solvers_indicator, (opt = SCIPopt, mode = BilevelJuMP.IndicatorMode()))

push!(solvers_fa, (opt = SCIPopt, mode = BilevelJuMP.FortunyAmatMcCarlMode(with_slack = false)))
push!(solvers_fa, (opt = SCIPopt, mode = BilevelJuMP.FortunyAmatMcCarlMode(with_slack = true)))
push!(solvers_fa2, (opt = SCIPopt, mode = BilevelJuMP.FortunyAmatMcCarlMode(primal_big_M = 100, dual_big_M = 100)))

push!(solvers_cached, (opt = SCIPopt_CACHED, mode = BilevelJuMP.SOS1Mode()))

push!(solvers_bin_exp, (
    opt = QuadraticToBinary.Optimizer{Float64}(SCIPopt_CACHED),
    mode = BilevelJuMP.StrongDualityMode(1e-9, inequality = true)))
MOI.set(solvers_bin_exp[end].opt, QuadraticToBinary.GlobalVariablePrecision(), 1e-5)

push!(solvers_bin_exp, (
    opt = QuadraticToBinary.Optimizer{Float64}(SCIPopt_CACHED),
    mode = BilevelJuMP.StrongDualityMode(inequality = false)))
MOI.set(solvers_bin_exp[end].opt, QuadraticToBinary.GlobalVariablePrecision(), 1e-5)

QTB_SCIPopt_BRIDGED = MOI.Bridges.full_bridge_optimizer(
    QuadraticToBinary.Optimizer{Float64}(SCIPopt_CACHED, lb=-100,ub=100),
    Float64)
MOI.Bridges.add_bridge(QTB_SCIPopt_BRIDGED, MOI.Bridges.Constraint.SOCtoNonConvexQuadBridge{Float64})
MOI.set(QTB_SCIPopt_BRIDGED, QuadraticToBinary.GlobalVariablePrecision(), 1e-5)

push!(solvers_sos_quad_bin, (
    opt = QTB_SCIPopt_BRIDGED,
    mode = BilevelJuMP.SOS1Mode()))

push!(solvers_fa_quad_bin, (
    opt = QTB_SCIPopt_BRIDGED,
    mode = BilevelJuMP.FortunyAmatMcCarlMode(primal_big_M = 100, dual_big_M = 100)))

QTB_SCIPopt = QuadraticToBinary.Optimizer{Float64}(SCIPopt_CACHED, lb=-5, ub=5)
QTB_SCIPopt_B = MOI.Bridges.Constraint.SOCtoNonConvexQuad{Float64}(QTB_SCIPopt)
push!(solvers_fa_quad_bin_mixed, (
    opt = QTB_SCIPopt_B,
    mode = BilevelJuMP.MixedMode(default=
        BilevelJuMP.FortunyAmatMcCarlMode(primal_big_M = 10, dual_big_M = 10))))
MOI.set(QTB_SCIPopt_B, QuadraticToBinary.GlobalVariablePrecision(), 1e-3)


function thermalize(filepath::String, r::MolecularDynamicsTrial, N::Int64,
        n::Int64 )
    for i in 1:n
        println("Thermalizing, step $i of $n, with $N verlet steps per relaxation")
        verlet!(r,N)
        save(filepath,r)
        relax!(r)
        save(filepath,r)
    end
end


module Adaptation

import Base: string
import LinearAlgebra, Statistics
using ..AdvancedHMC: DEBUG

abstract type AbstractAdaptor end

include("stepsize.jl")
include("precond.jl")

abstract type AbstractCompositeAdaptor <: AbstractAdaptor end

# TODO: generalise this to a list of adaptors
struct NaiveCompAdaptor <: AbstractCompositeAdaptor
    pc  :: AbstractPreConditioner
    ssa :: StepSizeAdaptor
end

function adapt!(tp::NaiveCompAdaptor, θ::AbstractVector{<:Real}, α::AbstractFloat)
    adapt!(tp.ssa, θ, α)
    adapt!(tp.pc, θ, α)
end

function getM⁻¹(ca::AbstractCompositeAdaptor)
    return getM⁻¹(ca.pc)
end

function getϵ(ca::AbstractCompositeAdaptor)
    return getϵ(ca.ssa)
end

include("stan_adaption.jl")

export adapt!, getϵ, getM⁻¹,
       NesterovDualAveraging,
       UnitPreConditioner, DiagPreConditioner, DensePreConditioner,
       NaiveCompAdaptor, StanNUTSAdaptor

end # module


#!/usr/bin/julia --color=yes

# THIS IS OLD - USE COMBINED INSTEAD

using ArgParse

using JLD
using Multilane
using POMDPs
using MCTS
using DataFrames
using DataFramesMeta
using PGFPlots

function print_tex(p, name)
    print_tex(plot(p), name)
end

function print_tex(p::TikzPictures.TikzPicture, name)
    println("""

        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

        % $name
        """)
    println(p.data)
    println("""

        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

        """)
end

function show_svg(p)
    f = string(tempname(), ".svg")
    PGFPlots.save(f, plot(p))
    @spawn run(`display -alpha off $f`)
end

bg = standard_uniform(1.0, correlation=0.75)

N = 1000

rng = MersenneTwister(123)

bs = Array(BehaviorModel, N)
for i in 1:N
    bs[i] = rand(rng, bg)
end

v0s_75 = Float64[bs[i].p_idm.v0 for i in 1:N]
Ts_75 = Float64[bs[i].p_idm.T for i in 1:N]

bg = standard_uniform(1.0, correlation=true)

rng = MersenneTwister(123)

bs = Array(BehaviorModel, N)
for i in 1:N
    bs[i] = rand(rng, bg)
end

v0s_1 = Float64[bs[i].p_idm.v0 for i in 1:N]
Ts_1 = Float64[bs[i].p_idm.T for i in 1:N]

bg = standard_uniform(1.0, correlation=false)

rng = MersenneTwister(123)

bs = Array(BehaviorModel, N)
for i in 1:N
    bs[i] = rand(rng, bg)
end

v0s_0 = Float64[bs[i].p_idm.v0 for i in 1:N]
Ts_0 = Float64[bs[i].p_idm.T for i in 1:N]

axis_args = Dict{Symbol, Any}(
    # :width=>"0.5\\columnwidth"
    :width=>"5cm"
)

scatter_args = Dict{Symbol, Any}(
    :markSize=>1
)

pushPGFPlotsPreamble("\\usepackage{siunitx}")

g = GroupPlot(2, 2, groupStyle = "horizontal sep = 0.075\\columnwidth, vertical sep = 0.15\\columnwidth")
push!(g, Axis(Plots.Scatter(v0s_0, Ts_0; scatter_args...); title="Scenario 1, \$\\rho=0\$", ylabel="\$T\$ (\\si{s})", axis_args...))
push!(g, Axis(Plots.Scatter(v0s_1, Ts_1; scatter_args...); title="Scenario 2, \$\\rho=1\$", xlabel="\$v_0\$ (\\si{m/s})", axis_args...))
push!(g, Axis(Plots.Scatter(v0s_75, Ts_75; scatter_args...); title="Scenario 3, \$\\rho=0.75\$", xlabel="\$v_0\$ (\\si{m/s})", ylabel="\$T\$ (\\si{s})", axis_args...))

# print_tex(g, "SCATTER")

PGFPlots.save("/home/zach/Devel/Behavior-aware_lane_changing/acc_17_paper/fig/scatter.tex", g)

# show_svg(g)


export AbstractSolverType
export DiscreteSplittingType, HEVISplitting
export getdtmodel

"""
    AbstractSolverType

This is an abstract type representing a generic solver. By
a "solver," we mean an ODE solver together with any potential
implicit solver (linear solvers).
"""
abstract type AbstractSolverType end

"""
    DiscreteSplittingType

This is an abstract type representing a temporal splitting
in the discrete equations. For example, HEVI
(horizontally explicit, vertically implicit) type methods.
"""
abstract type DiscreteSplittingType end

"""
    SlowFastSplitting

The tendency is treated using a standard slow-fast splitting,
where the fast processes are purely related to acoustic/gravity
waves (ideal for a typical multirate or MIS method).
This splitting does not take into account the geometric
direction (vertical vs horizontal).
"""
struct SlowFastSplitting <: DiscreteSplittingType end

"""
    HEVISplitting

HEVI (horizontally explicit, vertically implicit) type method,
where vertical acoustic waves are treated implicitly. All other
dynamics are treated explicitly.

Note: Can potentially imagine several different types of
HEVI splittings (for example, include vertical momentum and/or
diffusion)
"""
struct HEVISplitting <: DiscreteSplittingType end

"""
    getdtmodel(ode_solver::AbstractSolverType, bl)

A function which returns a model representing the dynamics
with the most restrictive time-stepping requirements.
"""
getdtmodel(ode_solver::AbstractSolverType, bl) =
    throw(MethodError(solversetup, (ode_solver, bl)),)

"""
    solversetup(
        ode_solver::AbstractSolverType,
        dg,
        Q,
        dt,
        t0,
        diffusion_direction,
    )

A function which returns a configured ODE solver.
"""
solversetup(
    ode_solver::AbstractSolverType,
    dg,
    Q,
    dt,
    t0,
    diffusion_direction,
) = throw(MethodError(
    solversetup,
    (ode_solver, dg, Q, dt, t0, diffusion_direction),
))

include("ExplicitSolverType.jl")
include("ImplicitSolverType.jl")
include("IMEXSolverType.jl")
include("HEVISolverType.jl")
include("MultirateSolverType.jl")
include("MISSolverType.jl")
include("SplitExplicitSolverType.jl")

DefaultSolverType = IMEXSolverType
export DefaultSolverType


using DSGE, ModelConstructors, Dates, OrderedCollections, Test, CSV, DataFrames, Random, FileIO

regenerate_output = false

## Regime switching and full-distribution
# Initialize model objects
Random.seed!(1793 * 10)
m = Model1002("ss60"; custom_settings = Dict{Symbol, Setting}(:flexible_ait_policy_change =>
                                                              Setting(:flexible_ait_policy_change,
                                                                      false),
                                                              :add_pgap => Setting(:add_pgap, false),
                                                              :add_ygap => Setting(:add_ygap, false))) # Set to false unless you want to re-generate any saved output
df_full = CSV.read(joinpath(dirname(@__FILE__), "../reference/uncertain_altpolicy_data.csv"), DataFrame)
m <= Setting(:forecast_horizons, 20)
m <= Setting(:cond_full_names, [:obs_gdp, :obs_corepce, :obs_spread, # Have to add anticipated rates to conditional data
                                :obs_nominalrate, :obs_longrate,
                                :obs_nominalrate1, :obs_nominalrate2, :obs_nominalrate3,
                                :obs_nominalrate4, :obs_nominalrate5, :obs_nominalrate6])
m <= Setting(:cond_semi_names, [:obs_spread,
                                :obs_nominalrate, :obs_longrate,
                                :obs_nominalrate1, :obs_nominalrate2, :obs_nominalrate3,
                                :obs_nominalrate4, :obs_nominalrate5, :obs_nominalrate6])
m <= Setting(:date_forecast_start, Date(2020, 6, 30))
m <= Setting(:date_conditional_end, Date(2020, 6, 30))
m <= Setting(:forecast_ndraws, 5)
m <= Setting(:forecast_block_size, get_setting(m, :forecast_ndraws))
m <= Setting(:forecast_jstep, 1)
m <= Setting(:use_parallel_workers, false)
setup_regime_switching_inds!(m; cond_type = :full)

# For parameters, use modal ones, but also draw from prior for the COVID-19 shocks
mparas = repeat(ModelConstructors.get_values(m.parameters)', get_setting(m, :forecast_ndraws), 1)
j = length(m.parameters) # To help index for regime-switching parameters
biidc_keys = Dict()
ziid_keys = Dict()
φ_keys = Dict()
for p in m.parameters # Find the keys of the COVID-19 shocks
    if haskey(p.regimes, :value)
        for k in keys(p.regimes[:value])
            if k > 1
                global j += 1
                if p.key == :σ_φ
                    φ_keys[k] = j
                elseif p.key == :σ_biidc
                    biidc_keys[k] = j
                elseif p.key == :σ_ziid
                    ziid_keys[k] = j
                end
            end
        end
    end
end

for i in 1:size(mparas, 1)
    mparas[i, φ_keys[2]] = rand(get(m.parameters[m.keys[:σ_φ]].prior)) # draw from prior for second-regime value
    mparas[i, biidc_keys[2]] = rand(get(m.parameters[m.keys[:σ_biidc]].prior)) # draw from prior for second-regime value
    mparas[i, ziid_keys[2]] = rand(get(m.parameters[m.keys[:σ_ziid]].prior)) # draw from prior for second-regime value
    mparas[i, φ_keys[3]] = rand(get(m.parameters[m.keys[:σ_φ]].prior)) # draw from prior for third-regime value
    mparas[i, biidc_keys[3]] = rand(get(m.parameters[m.keys[:σ_biidc]].prior)) # draw from prior for third-regime value
    mparas[i, ziid_keys[3]] = rand(get(m.parameters[m.keys[:σ_ziid]].prior)) # draw from prior for third-regime value
end

# Function for adding new regimes to parameters
function covid_set_regime_vals(m::AbstractDSGEModel, n::Int)
    if n > 4
        for p in m.parameters
            if haskey(p.regimes, :value)
                for i in 5:n
                    ModelConstructors.set_regime_val!(p, i, ModelConstructors.regime_val(p, 4))
                end
            end
        end
    end
end

# Run forecast!
output_vars = [:forecastobs, :bddforecastobs]
forecast_one(m, :full, :full, output_vars, verbose = :none, params = mparas, df = df_full,
             zlb_method = :temporary_altpolicy, set_regime_vals_altpolicy = covid_set_regime_vals)

# Either test against saved output or re-generate output
output_files = get_forecast_output_files(m, :full, :full, output_vars)
if regenerate_output
    using JLD2, FileIO
    if (VERSION >= v"1.5")
        JLD2.jldopen(joinpath(dirname(@__FILE__), "../reference/automatic_tempalt_zlb_fulldist_v1p5.jld2"),
                     true, true, true, IOStream) do file
            for (k, v) in output_files
                write(file, string(k), load(v, "arr"))
            end
        end
    else
        JLD2.jldopen(joinpath(dirname(@__FILE__), "../reference/automatic_tempalt_zlb_fulldist.jld2"),
                     true, true, true, IOStream) do file
            for (k, v) in output_files
                write(file, string(k), load(v, "arr"))
            end
        end
    end
else
    refdata = (VERSION >= v"1.5") ?
        load(joinpath(dirname(@__FILE__), "../reference/automatic_tempalt_zlb_fulldist_v1p5.jld2")) :
        load(joinpath(dirname(@__FILE__), "../reference/automatic_tempalt_zlb_fulldist.jld2"))
    @testset "Automatic enforcement of ZLB as a temporary alternative policy during full-distribution forecast" begin
        for (k, v) in output_files # Test variables which don't have forward-looking measurement equations

            # measurement eqns for forward looking variables e.g. :obs_longrate are still being updated.
            if k == :bddforecastobs
                @test_matrix_approx_eq refdata[string(k)][:, # this test seems to work but perhaps due to changes in the way the automatic ZLB is
                                                                vcat(1:9, 12:13), :] load(v, "arr")[:, vcat(1:9, 12:13), :] # inferred, it no longer matches reference output. Leaving this test as "broken" for now.
                @test all(refdata[string(k)][:, m.observables[:obs_nominalrate], :] .> -1e-14)
            else
                @test @test_matrix_approx_eq refdata[string(k)][:,
                                                                vcat(1:9, 12:13), :] load(v, "arr")[:, vcat(1:9, 12:13), :]
                @test !all(refdata[string(k)][:, m.observables[:obs_nominalrate], :] .> -1e-14)
            end
        end
    end
    for v in values(output_files)
        rm(v)
    end
end


export AbstractSystem,
       AbstractSystemCache,
       SystemNullCache,
       cache,
       evaluate,
       evaluate!,
       jacobian,
       jacobian!,
       differentiate_parameters!,
       differentiate_parameters,
       evaluate_and_jacobian,
       evaluate_and_jacobian!

"""
    SystemNullCache

An empty cache if no cache is necessary.
"""
struct SystemNullCache <: AbstractSystemCache end

"""
    cache(F::AbstractSystem, x)::AbstractSystemCache

Create a cache for the evaluation (incl. Jacobian) of `F` with elements of the type
of `x`.

    cache(F::AbstractSystem, x, p)::AbstractSystemCache

Create a cache for the evaluation (incl. Jacobian) of `F` with elements of the type
of `x` and parameters `p`.
"""
cache(F::AbstractSystem, args...) = throw(MethodError(cache, tuple(F, args...)))

"""
    evaluate!(u, F::AbstractSystem, x, cache::AbstractSystemCache)

Evaluate the system `F` at `x` and store the result in `u`.

    evaluate!(u, F::AbstractSystem, x, p, cache::AbstractSystemCache)

Evaluate the system `F` at `x` and parameters `p` and store the result in `u`.
"""
evaluate!(u, F::AbstractSystem, args...) =
    error(MethodError(evaluate!, tuple(u, F, args...)))

"""
    evaluate(F::AbstractSystem, x::AbstractVector, cache=cache(F, x))

Evaluate the system `F` at `x`.

    evaluate(F::AbstractSystem, x::AbstractVector, p, cache=cache(F, x))

Evaluate the system `F` at `x` and parameters `p`.
"""
evaluate(F::AbstractSystem, x, c::AbstractSystemCache = cache(F, x)) = evaluate(F, x, c)
evaluate(F::AbstractSystem, x, p, c::AbstractSystemCache = cache(F, x, p)) =
    evaluate(F, x, p, c)

"""
    jacobian!(u, F::AbstractSystem, x , cache::AbstractSystemCache)

Evaluate the Jacobian of the system `F` at `x` and store the result in `u`.

    jacobian!(u, F::AbstractSystem, x , p, cache::AbstractSystemCache)

Evaluate the Jacobian of the system `F` at `x` and parameters `p` and store the result in `u`.
"""
jacobian!(u, F::AbstractSystem, args...) =
    error(MethodError(jacobian!, tuple(u, F, args...)))

"""
    jacobian(F::AbstractSystem, x, cache=cache(F, x))

Evaluate the Jacobian of the system `F` at `x`.

    jacobian(F::AbstractSystem, x , p, cache=cache(F, x))

Evaluate the Jacobian of the system `F` at `x` and parameters `p`.
"""
jacobian(F::AbstractSystem, x, c::AbstractSystemCache = cache(F, x)) = jacobian(F, x, c)
jacobian(F::AbstractSystem, x, p, c::AbstractSystemCache = cache(F, x, p)) =
    jacobian(F, x, p, c)

"""
    differentiate_parameters!(u, F::AbstractSystem, x, p, cache::AbstractSystemCache)

Evaluate the Jacobian of the system `F` at `x` and parameters `p` w.r.t. the parameters
and store the result in `u`.
"""
function differentiate_parameters! end

"""
    differentiate_parameters(F::AbstractSystem, x, p, cache=cache(F, x))

Evaluate the Jacobian of the system `F` at `x` and parameters `p` w.r.t. the parameters
"""
differentiate_parameters(F::AbstractSystem, x, p, c::AbstractSystemCache = cache(F, x, p)) =
    differentiate_parameters(F, x, p, c)

# Optional
"""
    evaluate_and_jacobian!(u, U, F, x , cache::AbstractSystemCache)

Evaluate the system `F` and its Jacobian at `x` and store the results in `u` (evalution)
and `U` (Jacobian).
"""
function evaluate_and_jacobian!(u, U, F::AbstractSystem, x, cache::AbstractSystemCache)
    evaluate!(u, F, x, cache)
    jacobian!(U, F, x, cache)
    nothing
end

"""
    evaluate_and_jacobian!(u, U, F, x, p, cache::AbstractSystemCache)

Evaluate the system `F` and its Jacobian at `x` and parameters `p` and store the results in `u` (evalution)
and `U` (Jacobian).
"""
function evaluate_and_jacobian!(u, U, F::AbstractSystem, x, p, cache::AbstractSystemCache)
    evaluate!(u, F, x, p, cache)
    jacobian!(U, F, x, p, cache)
    nothing
end

"""
    evaluate_and_jacobian(F::AbstractSystem, x, cache=cache(F, x))

Evaluate the system `F` and its Jacobian at `x`.
"""
function evaluate_and_jacobian(F::AbstractSystem, x, c::AbstractSystemCache = cache(F, x))
    u = evaluate(F, x, c)
    U = jacobian(F, x, c)
    u, U
end
"""
    evaluate_and_jacobian(F::AbstractSystem, x, p, cache=cache(F, x))

Evaluate the system `F` and its Jacobian at `x` and parameters `p`.
"""
function evaluate_and_jacobian(
    F::AbstractSystem,
    x,
    p,
    c::AbstractSystemCache = cache(F, x),
)
    u = evaluate(F, x, p, c)
    U = jacobian(F, x, p, c)
    u, U
end

"""
    Base.size(F::AbstractSystem)

Returns a tuple `(m, n)` indicating that `F` is a system of `m` polynomials `m` in `n` variables.
"""
Base.size(::AbstractSystem) = error("Mandatory to define `Base.size` for `AbstractSystem`s")
Base.size(F::AbstractSystem, i::Integer) = size(F)[i]
Base.length(F::AbstractSystem) = size(F, 1)
npolynomials(F::AbstractSystem) = length(F)

"""
    degrees(F::AbstractSystem)

Returns a the degrees of the polynomials.
"""
degrees(F::AbstractSystem) = nothing

include("systems/sp_system.jl")
include("systems/fixed_homotopy.jl")
include("systems/fp_system.jl")
include("systems/totaldegree_system.jl")
include("systems/multihom_totaldegree_system.jl")
include("systems/composition_system.jl")
include("systems/fixed_parameter_system.jl")
include("systems/squared_up_system.jl")
include("systems/trace_test_system.jl")
include("systems/trace_test_pencil.jl")


# Vector shuffles

function shufflevector_instrs(N, T, I, two_operands)
    typ = llvmtype(T)
    vtyp2 = vtyp1 = "<$N x $typ>"
    M = length(I)
    vtyp3 = "<$M x i32>"
    vtypr = "<$M x $typ>"
    mask = "<" * join(map(x->string("i32 ", x), I), ", ") * ">"
    instrs = String[]
    v2 = two_operands ? "%1" : "undef"
    push!(instrs, "%res = shufflevector $vtyp1 %0, $vtyp2 $v2, $vtyp3 $mask")
    push!(instrs, "ret $vtypr %res")
    return M, String[], instrs
end

@generated function shufflevector(
    v1::Vec{W,T}, v2::Vec{W,T}, ::Val{I}
) where {W,T,I}
    M, decls, instrs = shufflevector_instrs(W, T, I, true)
    quote
        $(Expr(:meta, :inline))
        Vec{$M,T}(Base.llvmcall($((join(decls, "\n"), join(instrs, "\n"))),
            Vec{$M,T},
            Tuple{Vec{W,T}, Vec{W,T}},
            v1, v2))
    end
end
@inline function shufflevector(v1::SVec{W,T}, v2::SVec{W,T}, ::Val{I}) where {W,T,I}
    SVec(shufflevector(extract_data(v1), extract_data(v2), Val(I)))
end

@generated function shufflevector(v1::Vec{W,T}, ::Val{I}) where {W,T,I}
    M, decls, instrs = shufflevector_instrs(W, T, I, false)
    quote
        $(Expr(:meta, :inline))
        Base.llvmcall(
            $((join(decls, "\n"), join(instrs, "\n"))),
            Vec{$M,T}, Tuple{Vec{$W,T}}, v1
        )
    end
end
@inline function shufflevector(v1::SVec{W,T}, ::Val{I}) where {W,T,I}
    SVec(shufflevector(extract_data(v1), Val(I)))
end

@inline rotate_vector_left(v::AbstractSIMDVector{W}) where {W} = shufflevector(v, Val(ntuple(i -> (i % W), Val(W))))


using DifferentialEquations
using ForwardDiff
using DiffEqBase: ODEProblem, SDEProblem, RODEProblem, DEIntegrator
using OrdinaryDiffEq: ODEIntegrator
using StochasticDiffEq: SDEIntegrator

""" Continuous-time problem. """
const CTProblem = Union{ODEProblem, SDEProblem, RODEProblem}
# TODO: maybe use Abstract*Problem; but note that
# AbstractDiscreteProblem <: AbstractODEProblem

function get_integrator(prob::CTProblem; save_everystep=false,
                        alg = nothing,
                        kwargs...)
    if alg === nothing
        alg, extra_kwargs = default_algorithm(prob; kwargs...)
    else
        extra_kwargs = []
    end
    return init(prob, alg; kwargs..., extra_kwargs...,
                save_everystep=save_everystep)
end
# See:
# - [[../../OrdinaryDiffEq/src/solve.jl::function init\b]]
# - http://docs.juliadiffeq.org/latest/basics/common_solver_opts.html

@inline function assert_success(integrator::DEIntegrator)
    if ! (integrator.sol.retcode in (:Default, :Success))
        throw(IntegratorError(integrator.sol.retcode))
    end
end
# http://docs.juliadiffeq.org/latest/basics/solution.html#Return-Codes-(RetCodes)-1

const ContinuousLEProblem = LEProblem{ODEProblem}

"""
    ContinuousLEProblem(phase_dynamics, u0 [, p];
                        t_attr=<number>, <keyword arguments>)

This is a short-hand notation for:

```julia
LEProblem(ODEProblem(phase_dynamics, u0 [, p]), t_attr)
```

For the list of usable keyword arguments, see [`LEProblem`](@ref).
"""
ContinuousLEProblem(phase_dynamics, u0, p=nothing;
                    tspan=(0.0, 100.0), kwargs...) =
    LEProblem(ODEProblem(phase_dynamics, u0, tspan, p);
              kwargs...)

init_phase_state(integrator::DEIntegrator) = integrator.sol.prob.u0[:, 1]
init_tangent_state(integrator::DEIntegrator) = integrator.sol.prob.u0[:, 2:end]

current_state(integrator::DEIntegrator) = integrator.u


using Statistics

export BVHAccelerator

"""
    PrimitiveInfo

A type holding information about a primitive (its index in an array, its centre point and
its bounding box). These are used when constructing the BVH accelerator tree structure.

"""
struct PrimitiveInfo
    number::Int64
    centroid::Point3
    BBox::BoundingBox
end

"""
    LinearBVHNode

A node of the flattened version of the BVH three. These nodes are used in the actual
ray-tracer acceleration.

# Fields

- `BBox::BoundingBox`: Bounding box of the primitive(s) contained in this node.
- `offset::Int64`: If this is a leaf node, contains the index of the corresponding primitive
  in the `primitives` array of the `BVHAccelerator`.
- `leaf::Bool`: Whether this is a leaf or branch node.
- `axis::Int64`: The axis along which the primitives are split if this is a branch node.
  Knowing this relative to the ray direction is relevant in the traversal of the tree.

"""
struct LinearBVHNode
    BBox::BoundingBox
    offset::Int64
    leaf::Bool
    axis::Int64
end

"""
    BVHBuildNode

A mutable type representing a node in the Bounding Volume Hierarchy tree. This is used when
creating the tree, which is then flattened into a more efficient structure using the
immutable `LinearBVHNode` types.

"""
mutable struct BVHBuildNode
    leaf::Bool
    split_axis::Int64
    first_offset::Int64
    BBox::BoundingBox
    childA::BVHBuildNode
    childB::BVHBuildNode
    BVHBuildNode() = new()
end

"""
    init_leaf!(node::BVHBuildNode, idx::Integer, BB::BoundingBox)

Initialize the `BVHBuildNode` as a leaf node of the BVH tree.

"""
function init_leaf!(node::BVHBuildNode, idx::Integer, BB::BoundingBox)
    node.leaf = true
    node.first_offset = idx
    node.BBox = BB
    node.split_axis = -1
    return node
end

"""
    init_interior!(node::BVHBuildNode, idx::Integer, BB::BoundingBox)

Initialize the `BVHBuildNode` as an interior node of the BVH tree.

"""

function init_interior!(node::BVHBuildNode, axis::Integer, childA::BVHBuildNode, childB::BVHBuildNode)
    node.leaf = false
    node.split_axis = axis
    node.BBox = BoundingBox(childA.BBox, childB.BBox)
    node.childA = childA
    node.childB = childB
    node.first_offset = -1
    return node
end

"""
    BVH_build_recursive(build_data, startN, endN, primitives, ordered_primitives,
                        total_nodes)

Recursively construct the BVH tree using `build_data` and `primitives`. This will populate
the array `ordered_primitives` with the primitives in the same order as to the tree leaf
nodes which refer to them.

"""
function BVH_build_recursive(build_data::Array{PrimitiveInfo,1}, startN::Integer,
                             endN::Integer, primitives::Array{T,1}, ordered_primitives::Array{T,1},
                             total_nodes::Integer) where T <: Primitive

    total_nodes += 1
    node = BVHBuildNode()

    if startN < 1 || endN < 1
        error()
    end

    if startN == endN
        # Only one primitive in the range, create leaf node
        n = build_data[startN].number
        push!(ordered_primitives, primitives[n])
        offset = length(ordered_primitives)
        init_leaf!(node, offset, build_data[startN].BBox)
    else
        # More than one primitive in the range, make partition and call recursively
        CBox = BoundingBox([build_data[i].centroid for i = startN:endN])
        dim = max_extent(CBox)
        pmid = mean(CBox.pMin, CBox.pMax)
        cmp(x,y) = x.centroid[dim] < y.centroid[dim]
        build_data[startN:endN] = sort(build_data[startN:endN] ; lt=cmp)
        mid = ceil(Int64, (startN+endN) / 2)

        # TODO: FIX THIS BETTER! THIS HAS SOME SUBTLE BUG
        #for i = startN:endN
        #    if (pmid[dim] <= build_data[i].centroid[dim])
        #        mid = i
        #        break
        #    end
        #end

        childA, total_nodes = BVH_build_recursive(build_data, startN, mid-1, primitives,
                                                  ordered_primitives, total_nodes)
        childB, total_nodes = BVH_build_recursive(build_data, mid, endN, primitives,
                                                  ordered_primitives, total_nodes)
        init_interior!(node, dim, childA, childB)
    end
    return node, total_nodes
end

"""
    flatten_BVH!(node::BVHBuildNode, linear::Array{LinearBVHNode,1}, idx::Integer)

Recursively traverse the BVH tree starting from `node`, and produce the flattened version
of the tree in the array `linear`.

"""
function flatten_BVH!(node::BVHBuildNode, linear::Array{LinearBVHNode,1}, idx::Integer)
    idx0 = idx
    if node.leaf
        new_node = LinearBVHNode(node.BBox, node.first_offset, true, -1)
    else
        idx1 = flatten_BVH!(node.childA, linear, idx+1)
        idx2 = flatten_BVH!(node.childB, linear, idx1+1)
        new_node = LinearBVHNode(node.BBox, idx1+1, false, node.split_axis)
        idx = idx2
    end
    linear[idx0] = new_node
    return idx
end

# LEFT: your number is my number, plus 1. Also, give me the biggest number in your subtree.
# RIGHT: your number is the biggest number in the LEFT subtree, plus 1
# I got to give upwards the biggest number in my subtree. But that's just the highest number
# in my right subtree.

"""
    BVHAccelerator{P<:Primitive} <: Aggregate

The Bounding Volume Hierarchy accelerator keeps the list of primitives, and a list of
`LinearBVHNode` objects, which contain all the information about the tree structure.

"""
struct BVHAccelerator{P<:GeometricPrimitive} <: Aggregate
    primitives::Array{P,1}
    nodes::Array{LinearBVHNode,1}
end

"""
    world_bounds(B::BVHAccelerator)

Return the bounding box containing all the primitives in the Accelerator.

"""
world_bounds(B::BVHAccelerator) = B.nodes[1].BBox

"""
    BVHAccelerator(prims::Array{T,1}) where T<:Primitive

Construct a `BVHAccelerator` from the list of primitives.

"""
function BVHAccelerator(prims::Array{T,1}) where T<:Primitive
    # Refine primitives to their fully intersectable components
    primtype = eltype(prims)
    primitives = primtype[]
    for p in prims
        fully_refine!(p, primitives)
    end

    # Extract relevant info from primitives and push into an array
    build_data = PrimitiveInfo[]
    for (i, obj) in enumerate(primitives)
        BB = world_bounds(obj)
        c = mean(BB.pMin, BB.pMax)
        info = PrimitiveInfo(i, c, BB)
        push!(build_data, info)
    end

    # Generate binary tree structure of bounding boxes
    ordered = primtype[]
    root_node, total_nodes = BVH_build_recursive(build_data, 1, length(primitives), primitives,
                                                 ordered, 0)

    # Transform the tree structure into a more efficient form
    linear = Array{LinearBVHNode}(undef,total_nodes)
    flatten_BVH!(root_node, linear, 1)
    BVHAccelerator(ordered, linear)
end

# Two arrays that are reused by the `intersect` and `intersectP` methods to
# avoid reallocating them every time those functions are called (which is very
# often). A significant optimization.
const TODO_ARRAY = zeros(Int64, (64, Threads.nthreads()))
const dir_is_neg = zeros(Bool, (3, Threads.nthreads()))

IntersectionTypes = Union{
                    Nothing,
                    Intersection{Lambert{SampledSpectrum}},
                    Intersection{SpecularDiffuse{SampledSpectrum}},
                    Intersection{AshikhminShirleySingle{SampledSpectrum}}
                    }

"""
    intersect(ray::Ray, BVH::BVHAccelerator)

Find the nearest intersection of the ray and the primitives in the `BVHAccelerator`. If you
only need to know whether a ray intersects something or not, use [`intersectP`](@ref)
instead, since `intersect` must find _all possible_ intersections to determine the nearest
one and is therefore much slower.

"""
function intersect(ray::Ray, BVH::BVHAccelerator)
    if length(BVH.nodes) == 0
        return nothing
    end
    thread_id = Threads.threadid()
    nodeN = 1
    todo_offset = 0
    todo = TODO_ARRAY
    for i = 1:64
        todo[i,thread_id] = 0
    end
    for i = 1:3
        dir_is_neg[i,thread_id] = ray.direction[i] < 0.0
    end

    tmin = Inf
    best_isect = nothing

    while true
        # Check the node given by the index nodeN
        node = BVH.nodes[nodeN]
        if intersectP(ray, node.BBox)
            # We hit the bounding box of this node.
            if node.leaf
                # This is a leaf node.
                # Check intersection with the primitive in the node and keep best intersection.
                isect = intersect(ray, BVH.primitives[node.offset])::IntersectionTypes
                best_isect = update_isect(isect, best_isect)::IntersectionTypes

                # If there's nothing in the todo queue, we exit the loop.
                if todo_offset == 0
                    break
                end
                # Othewise, set nodeN based on current todo value and shift todo queue left.
                nodeN = todo[todo_offset,thread_id]
                todo_offset -= 1
            else
                # This is not a leaf node.
                # We point nodeN to either the left or right child node, depending on the ray
                # direction. We shift right in the todo queue and put the other node there.
                todo_offset += 1
                if dir_is_neg[node.axis,thread_id]
                    # Put left child in todo queue and set nodeN to right child.
                    todo[todo_offset,thread_id] = nodeN + 1
                    nodeN = node.offset
                else
                    # Put right child in todo queue and set nodeN to left child.
                    todo[todo_offset,thread_id] = node.offset
                    nodeN = nodeN + 1
                end
            end
        else
            # We did not hit a bounding box.
            # If there's nothing in the todo queue, we exit.
            if todo_offset == 0
                break
            end
            # Otherwise, set nodeN based on current todo value and shift left in the todo queue.
            nodeN = todo[todo_offset,thread_id]
            todo_offset -= 1

        end
    end
    return best_isect
end

"""
    intersectP(ray::Ray, BVH::BVHAccelerator)

Returns `true` if the given ray intersects any primitive in the `BVHAccelerator`, else
returns false. This is much faster to do than [`intersect`](@ref), since it's enough to
find any intersection, not specifically the nearest one.

Possibly the most often called function in the entire ray-tracer.

"""
function intersectP(ray::Ray, BVH::BVHAccelerator)
    if length(BVH.nodes) == 0
        return false
    end
    thread_id = Threads.threadid()
    nodeN = 1
    todo_offset = 0
    todo = TODO_ARRAY
    for i = 1:64
        todo[i,thread_id] = 0
    end
    for i = 1:3
        dir_is_neg[i,thread_id] = ray.direction[i] < 0.0
    end
    while true
        node = BVH.nodes[nodeN]
        if intersectP(ray, node.BBox)
            if node.leaf
                if intersectP(ray, BVH.primitives[node.offset])
                    return true
                end
                if todo_offset == 0
                    break
                end
                nodeN = todo[todo_offset,thread_id]
                todo_offset -= 1
            else
                todo_offset += 1
                if dir_is_neg[node.axis,thread_id]
                    todo[todo_offset,thread_id] = nodeN + 1
                    nodeN = node.offset
                else
                    todo[todo_offset,thread_id] = node.offset
                    nodeN = nodeN + 1
                end
            end
        else
            if todo_offset == 0
                break
            end
            nodeN = todo[todo_offset,thread_id]
            todo_offset -= 1

        end
    end
    return false
end


function LVGNA_alignments(testing=true)

    files = readdir(sparse_matrices_path)

    if testing
        files = files[6:7]
        files =  ["worm_PHY1.smat","worm_Y2H1.smat"]
        result_path = all_results_path*"LVGNA_alignments/test/"
        alphas = [1.0]
        betas = [0.0]
        samples = 300000
    else
        result_path = all_results_path*"LVGNA_alignments/"
        alphas = [.5,1.0]
        betas = [0.0,1.0,1e1,1e2]
        samples = 3000000
    end

    orders = [3]
        # generates the tensors using TuranShadow as smat
        # is needed for post processing. Use ssten files for
        # previously generated triangle adjacency tensors, but
        # post processing isn't supported.

        # _MultiMotif_M() flag is used to generate tensors,
        # replace with _M() for non-generating routines.

    if !isdir(result_path)
        mkpath(result_path)
    end

    #
    #  --  LowRankTAME  --  #
    #
    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=KlauAlgo(),method=LowRankTAME_MultiMotif_M(),iter=15,
                                    alphas,betas,profile=true,no_matching=false,motif=Clique(),orders,samples=samples)
    filename =  filter(x-> !isspace(x),"LVGNA_pairwiseAlignment_LowRankTAME_alphas:$(alphas)_betas:$(betas)_iter:15_order:3_postProcessing:KlauAlgo_profile:true_samples:$(samples)_tol:1e-6_results.json")
    save_results(result_path*filename,X)

    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=LocalSearch(),method=LowRankTAME_MultiMotif_M(),iter=15,
                                    alphas,betas,profile=true,no_matching=false,motif=Clique(),orders,samples=samples)
    filename =  filter(x-> !isspace(x),"LVGNA_pairwiseAlignment_LowRankTAME_alphas:$(alphas)_betas:$(betas)_iter:15_order:$(orders)_postProcessing:LocalSearch_profile:true_tol:1e-6_results.json")
    save_results(result_path*filename,X)

    #=   Low Rank Matching Routines
    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=KlauAlgo(),method=LowRankTAME_MultiMotif_M(),iter=15,
                                    alphas,betas,profile=true,no_matching=false,motif=Clique(),orders,samples=samples,low_rank_matching = true)
    filename =  filter(x-> !isspace(x),LVGNA_pairwiseAlignment_LowRankTAME_lrm_alphas:$(alphas)_betas:$(betas)_iter:15_order:3_postProcessing:KlauAlgo_profile:true_samples:$(samples)_tol:1e-6_results.json")
    save_results(result_path*filename,X)

    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=LocalSearch(),method=LowRankTAME_MultiMotif_M(),iter=15,
                                    alphas,betas,profile=true,no_matching=false,motif=Clique(),orders,samples=samples,low_rank_matching = true)
    filename =  filter(x-> !isspace(x),"LVGNA_pairwiseAlignment_LowRankTAME_lrm_alphas:$(alphas)_betas:$(betas)_iter:15_order:3_postProcessing:LocalSearch_profile:true_samples:$(samples)_tol:1e-6_results.json")
    save_results(result_path*filename,X)
    =#

    #
    #  --  Lambda - TAME  --  #
    #
    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=KlauAlgo(),method=ΛTAME_MultiMotif_M(),iter=15,matchingMethod=ΛTAME_GramMatching(),
                                    alphas,betas,profile=true,no_matching=false,orders,samples=samples)
    filename =  filter(x-> !isspace(x),"LVGNA_pairwiseAlignment_LambdaTAME_alphas:$(alphas)_betas:$(betas)_iter:15_matchingMethod:GramMatching_orders:$(orders)_postProcessing:KlauAlgo_profile:true_tol:1e-6_results.json")
    save_results(result_path*filename,X)

    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=LocalSearch(),method=ΛTAME_MultiMotif_M(),iter=15,matchingMethod=ΛTAME_GramMatching(),
                                    alphas,betas,profile=true,no_matching=false,motif=Clique(),orders,samples=samples)
    filename =  filter(x-> !isspace(x),"LVGNA_pairwiseAlignment_LambdaTAME_alphas:$(alphas)_betas:$(betas)_iter:15_matchingMethod:GramMatching_orders:$(orders)_postProcessing:LocalSearch_profile:true_samples:$(samples)_tol:1e-6_results.json")
    save_results(result_path*filename,X)

    #= Low Rank Matching Routines
    #02/11/22 TODO: must fix samples and orders being passed in for calling rankOneMatching

    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=LocalSearch(),method=ΛTAME_M(),iter=15,matchingMethod=ΛTAME_rankOneMatching(),
                                    alphas,betas,profile=true,no_matching=false,motif=Clique(),orders=[3],samples=samples)
    filename =  filter(x-> !isspace(x)"LVGNA_pairwiseAlignment_LambdaTAME_alphas:$(alphas)_betas:$(betas)_iter:15_matchingMethod:rankOneMatching_order:3_postProcessing:LocalSearch_profile:true_samples:$(samples)_tol:1e-6_results.json")
    save_results(result_path*filename,X)
    =#

    #
    #  -- LowRankEigenAlign --  #
    #
    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=KlauAlgo(),method=LowRankEigenAlign_M(),profile=true)
    filename =  filter(x-> !isspace(x),"LVGNA_pairwiseAlignment_LREA_postProcessing:KlauAlgo_profile:true_results.json")
    save_results(result_path*filename,X)

    X = distributed_pairwise_alignment(files,sparse_matrices_path;postProcessing=LocalSearch(),method=LowRankEigenAlign_M(),profile=true)
    filename =  filter(x-> !isspace(x),"LVGNA_pairwiseAlignment_LREA_postProcessing:LocalSearch_profile:true_results.json")
    save_results(result_path*filename,X)

    #=
    #
    #   (Julia) TAME implementation
    #
    X = distributed_pairwise_alignment(files,sparse_matrices_path;method=TAME_M(),iter=15,
                                       alphas = [1.0,.5],betas = [0.0,1e1,1e2,1e3],
                                       profile=true,no_matching=false)
    =#

end


argtypes(x::Combined{T, A}) where {T, A} = A
argtypes(x) = Any

function data_limits(x)
    error("No datalimits for $(typeof(x)) and $(argtypes(x))")
end

function data_limits(x::Atomic)
    isempty(x.plots) ? atomic_limits(x) : data_limits(x.plots)
end

"""
Data limits calculate a minimal boundingbox from the data points in a plot.
This doesn't include any transformations, markers etc.
"""
function atomic_limits(x::Atomic{<: Tuple{Arg1}}) where Arg1
    return xyz_boundingbox(transform_func(x), to_value(x[1]))
end

function atomic_limits(x::Atomic{<: Tuple{X, Y, Z}}) where {X, Y, Z}
    return xyz_boundingbox(transform_func(x), to_value.(x[1:3])...)
end

function atomic_limits(x::Atomic{<: Tuple{X, Y}}) where {X, Y}
    return xyz_boundingbox(transform_func(x), to_value.(x[1:2])...)
end

_isfinite(x) = isfinite(x)
_isfinite(x::VecTypes) = all(isfinite, x)
scalarmax(x::Union{Tuple, AbstractArray}, y::Union{Tuple, AbstractArray}) = max.(x, y)
scalarmax(x, y) = max(x, y)
scalarmin(x::Union{Tuple, AbstractArray}, y::Union{Tuple, AbstractArray}) = min.(x, y)
scalarmin(x, y) = min(x, y)

extrema_nan(itr::Pair) = (itr[1], itr[2])
extrema_nan(itr::ClosedInterval) = (minimum(itr), maximum(itr))

function extrema_nan(itr)
    vs = iterate(itr)
    vs === nothing && return (NaN, NaN)
    v, s = vs
    vmin = vmax = v
    # find first finite value
    while vs !== nothing && !_isfinite(v)
        v, s = vs
        vmin = vmax = v
        vs = iterate(itr, s)
    end
    while vs !== nothing
        x, s = vs
        vs = iterate(itr, s)
        _isfinite(x) || continue
        vmax = scalarmax(x, vmax)
        vmin = scalarmin(x, vmin)
    end
    return (vmin, vmax)
end

function xyz_boundingbox(transform_func, mesh::GeometryBasics.Mesh)
    xyz_boundingbox(transform_func, coordinates(mesh))
end

function xyz_boundingbox(transform_func, xyz)
    isempty(xyz) && return FRect3D()
    mini, maxi = extrema_nan((apply_transform(transform_func, point) for point in xyz))
    w = maxi .- mini
    return FRect3D(to_ndim(Vec3f0, mini, 0), to_ndim(Vec3f0, w, 0))
end

const NumOrArray = Union{AbstractArray, Number}

function xyz_boundingbox(transform_func, x::AbstractVector, y::AbstractVector, z::NumOrArray=0)
    # use lazy variant of broadcast!
    points = Base.broadcasted(Point3, x, y', z)
    return xyz_boundingbox(transform_func, points)
end

function xyz_boundingbox(transform_func, x::NumOrArray, y::NumOrArray, z::NumOrArray = 0)
    # use lazy variant of broadcast!
    points = Base.broadcasted(Point3, x, y, z)
    return xyz_boundingbox(transform_func, points)
end

function xyz_boundingbox(transform_func, x, y, z = 0)
    isempty(x) && return FRect3D()
    minmax = extrema_nan.(apply_transform.((transform_func,), (x, y, z)))
    mini, maxi = Vec(first.(minmax)), Vec(last.(minmax))
    w = maxi .- mini
    return FRect3D(to_ndim(Vec3f0, mini, 0), to_ndim(Vec3f0, w, 0))
end

const ImageLike{Arg} = Union{Heatmap{Arg}, Image{Arg}}
function data_limits(x::ImageLike{<: Tuple{X, Y, Z}}) where {X, Y, Z}
    xyz_boundingbox(transform_func(x), to_value.((x[1], x[2]))...)
end

function data_limits(x::Volume)
    xyz_boundingbox(transform_func(x), to_value.((x[1], x[2], x[3]))...)
end

function text_limits(x::VecTypes)
    p = to_ndim(Vec3f0, x, 0.0)
    return FRect3D(p, p)
end

function text_limits(x::AbstractVector)
    return FRect3D(x)
end

function atomic_limits(x::Text{<: Tuple{Arg1}}) where Arg1
    return boundingbox(x)
end

function data_limits(x::Annotations)
    # data limits is supposed to not include any transformation.
    # for the annotation, we use the model matrix directly, so we need to
    # to inverse that transformation for the correct limits
    bb = data_limits(x.plots[1])
    return inv(modelmatrix(x)) * bb
end

Base.isfinite(x::Rect) = all(isfinite.(minimum(x))) &&  all(isfinite.(maximum(x)))

function data_limits(plots::Vector)
    isempty(plots) && return
    bb = FRect3D()
    plot_idx = iterate(plots)
    while plot_idx !== nothing
        plot, idx = plot_idx
        plot_idx = iterate(plots, idx)
        # axis shouldn't be part of the data limit
        isaxis(plot) && continue
        isa(plot, Legend) && continue
        bb2 = data_limits(plot)::FRect3D
        isfinite(bb) || (bb = bb2)
        isfinite(bb2) || continue
        bb = union(bb, bb2)
    end
    bb
end

data_limits(s::Scene) = data_limits(plots_from_camera(s))
data_limits(plot::Combined) = data_limits(plot.plots)


mutable struct NatureHook10 <: AbstractHook
    step::Int
    max_steps::Int
    food_counts::Vector{Float32}
    food_balance::Vector{Float32}
    act_counts::Vector{Int}
    act_probs::Vector{Float32}
    player_acts::Vector{Vector{Int}}
    total_rewards::Vector{Float32}
    trial_id::String
end
function NatureHook10(env::NatureEnv, trial_id::String, max_steps::Int)
    NatureHook(0,
        max_steps,
        zeros(Float32, env.food_types),
        [],
        zeros(length(action_space(env, 1))),
        Vector{Float32}(),
        [zeros(length(action_space(env, 1))) for i in 1:length(env.players)],
        zeros(length(env.players)),
        trial_id,
       )
end
NatureHook = NatureHook10

function (hook::NatureHook)(::PreActStage, policy, env, action)
    for p in 1:length(env.players)
        hook.act_counts[action[p].action] += 1
        push!(hook.act_probs, exp(action[p].meta.action_log_prob[1]))
        hook.player_acts[p][action[p].action] += 1
    end
end

function (hook::NatureHook)(::PostActStage, policy, env)
    for p in 1:length(env.players)
        hook.total_rewards[p] += reward(env, p)
        hook.food_counts = hook.food_counts .+ env.players[p].food_counts
        push!(hook.food_balance, max(env.players[p].food_counts...) / min(env.players[p].food_counts...))
    end
    hook.step += 1
    steps_per_checkpoint = 1_000
    if hook.step % steps_per_checkpoint == 0
        log_step = if hasproperty(global_logger(), :global_step)
            global_logger().global_step
        else -1 end
        serialize("checkpoints/$(hook.trial_id)/$(Int(hook.step / steps_per_checkpoint)).jls",
                  Dict(:policy => policy,
                       :tb_step => log_step,
                       :hook_step => hook.step))
    end

end

function (hook::NatureHook)(::PostEpisodeStage, policy, env)

    @info "episode"     len=env.step                         log_step_increment=0
    @info "total_food"  total_food=Tuple(hook.food_counts)   log_step_increment=0
    @info "actions"     act=Tuple(hook.act_counts)           log_step_increment=0
    @info "rewards"     reward=Tuple(hook.total_rewards)     log_step_increment=0
    @info "rewards"     total_reward=sum(hook.total_rewards) log_step_increment=0
    @info "act_prob"    act_prob=mean(hook.act_probs)        log_step_increment=0
    @info "player_acts" player_acts=Tuple([Tuple(hook.player_acts[p]) for p in 1:length(env.players)]) log_step_increment=0
    @info "exchanges"   ex=Tuple(env.exchanges)              log_step_increment=0
    @info "food_balance" food_balance=mean(hook.food_balance)

    hook.act_counts    = zeros(length(action_space(env, 1)))
    hook.total_rewards = zeros(length(env.players))
    hook.food_counts   = zeros(env.food_types)
    hook.act_probs     = Vector{Float32}()
    hook.food_balance  = []

    hook.player_acts = [zeros(length(action_space(env, 1))) for _ in env.players]
end


### SOME CONVERGENCE REQUIRED ---

"""
$(TYPEDEF)

Condensed representation of KernelDensityEstimate, by saving points and bandwidth
"""
mutable struct EasyMessage{T <: Tuple}
  pts::Array{Float64,2}
  bws::Array{Float64,1}
  manifolds::T
  inferdim::Float64
  EasyMessage{T}() where {T <: Tuple} = new{T}()
  EasyMessage{T}(a::Array{Float64,2}, b::Array{Float64,1}, manis::T, inferdim::Union{Float64, Int32, Int64}=0.0) where {T <: Tuple} = new{T}(a,b, manis, Float64(inferdim))
  EasyMessage{T}(p::BallTreeDensity, manis::T, inferdim::Union{Float64, Int32, Int64}=0.0) where {T <: Tuple}  = new{T}(getPoints(p), getBW(p)[:,1], manis, Float64(inferdim))
end
EasyMessage(a::Array{Float64,2}, b::Array{Float64,1}, manis::T, inferdim::Union{Float64, Int32, Int64}=0) where {T <: Tuple} = EasyMessage{T}(a, b, manis, inferdim)
EasyMessage(p::BallTreeDensity, manis::T, inferdim::Union{Float64, Int32, Int64}=0) where {T <: Tuple} = EasyMessage{T}(p, manis, inferdim)

const TempBeliefMsg = Dict{Symbol, Tuple{BallTreeDensity, Float64}}

# Dict{Symbol,   -- is for variable label
#  Vector{       -- multiple msgs for the same variable
#   Symbol,      -- Clique index
#   Int,         -- Depth in tree
#   BTD          -- Belief estimate
#   inferredDim  -- Information count
#  }
const TempUpMsgPlotting = Dict{Symbol,Vector{Tuple{Symbol, Int, BallTreeDensity, Float64}}}

"""
$(TYPEDEF)

DESPARATELY NEEDS TO BE UPDATED TO USE TempBeliefMsg DEFINITION (start of refactor).
"""
mutable struct NBPMessage <: Singleton
  p::Dict{Symbol, EasyMessage}
end

### SOME CONVERGENCE REQUIRED ^^^


type ExampleIndividual <: Individual
  genome::Array{Int64, 1}
  fitness::Float64
  code

  function ExampleIndividual(size::Int64, max_value::Int64)
    genome = rand(1:max_value, size)
    return new(genome, -1.0, nothing)
  end

  ExampleIndividual(genome::Array{Int64, 1}) = new(genome, -1.0, nothing)
end

type ExamplePopulation <: Population
  individuals::Array{ExampleIndividual, 1}

  function ExamplePopulation(population_size::Int64, genome_size::Int64)
    individuals = Array(ExampleIndividual, 0)
    for i=1:population_size
      push!(individuals, ExampleIndividual(genome_size, 1000))
    end

    return new(individuals)
  end

  function ExamplePopulation(individuals::Array{ExampleIndividual, 1})
    return new(copy(individuals))
  end
end


# Base.:+(a::Nothing,b::Nothing) = nothing
# Base.:-(a::Nothing,b::Nothing) = nothing

function simple_allowed_reactions(
    dgs::Vector{<:Union{Missing,Real}}, #this shouldn't contain `nothing`s though
    threshold::Real,
    allow_missing::Bool=false)

    allowed_reactions = dgs .<= threshold
    if allow_missing==true
        allowed_reactions[ismissing.(dgs)] .= true
    else
        allowed_reactions[ismissing.(dgs)] .= false
    end

    Vector{Bool}(allowed_reactions)
end

"""
Indicates which reactions can proceed based on the dg threshold.

Returns two Vector{Bool} of length equal to rids, one for forward reactions and one for backwards.

Note: `allow_unbalanced==false` will only allow reactions with `is_balanced==nothing`
      when `allow_nothings==true`.
      `allow_unbalanced==true` will only allow reactions with `is_balanced==nothing`
      when `allow_nothings==true`
"""
function filter_reactions_by_dg(
    threshold::Real,
    env_key::Any,
    rids::IDs,
    rstructs::Reactions;
    allow_nothings::Bool=false,
    allow_unbalanced::Bool=false,
    allow_within_ci::Bool=true)

    mean_dgs = Union{Nothing,Missing,Real}[rstructs[rid].metadata["dg"][env_key]["standard_dg_prime_value"] for rid in rids]
    cis = Union{Nothing,Missing,Real}[rstructs[rid].metadata["dg"][env_key]["standard_dg_prime_ci"] for rid in rids]
    ## convert to missing because it can handle operators
    mean_dgs[isnothing.(mean_dgs)] .= missing
    cis[isnothing.(cis)] .= missing

    ## identify best case scenario dg values
    if allow_within_ci==true
        forward_dgs = Vector{Union{Missing,Real}}(mean_dgs - cis)
        backward_dgs = Vector{Union{Missing,Real}}(-mean_dgs - cis)
    else
        forward_dgs = Vector{Union{Missing,Real}}(mean_dgs)
        backward_dgs = Vector{Union{Missing,Real}}(-mean_dgs)
    end

    forward_allowed = simple_allowed_reactions(forward_dgs,threshold,allow_nothings)
    backward_allowed = simple_allowed_reactions(backward_dgs,threshold,allow_nothings)

    ## gather balance data
    balances = Union{Nothing,Missing,Bool}[rstructs[rid].metadata["dg"][env_key]["is_balanced"] for rid in rids]
    ## convert to missing because it can handle operators
    balances[isnothing.(balances)] .= missing

    if (allow_unbalanced == true) & (allow_nothings == true)
        ## don't use balance data
        return forward_allowed, backward_allowed

    elseif (allow_unbalanced == true) & (allow_nothings == false)
        forward_allowed[ismissing.(balances)] .= false
        backward_allowed[ismissing.(balances)] .= false
        return forward_allowed, backward_allowed

    elseif (allow_unbalanced == false) & (allow_nothings == true)
        balances[ismissing.(balances)] .= true

        forward_allowed[balances .== false] .= false
        backward_allowed[balances .== false] .= false
        return forward_allowed, backward_allowed

    elseif (allow_unbalanced == false) & (allow_nothings == false)
        balances[ismissing.(balances)] .= false

        forward_allowed[balances .== false] .= false
        backward_allowed[balances .== false] .= false
        return forward_allowed, backward_allowed
    end
end


using MeshArrays, NCDatasets, NCTiles
grid = GridSpec("LLC90","grids/")

exampledir = joinpath("data","ex4")
datadir = joinpath(exampledir,"model_output")
saveloc = joinpath(exampledir,"tiles")
fnames = joinpath.(Ref(datadir),filter(x->occursin("data",x),readdir(datadir)))

metafile = parsemeta(joinpath(datadir,"ptr_3d_set1.0000000732.meta"))
README = readlines(joinpath(datadir,"README"))

timeunits = "days since 1992-1-1 0:0:0"
nsteps = 12
timeinterval = 30
time_steps = timeinterval/2:timeinterval:timeinterval*nsteps
dep = -1 .* collect([-5 -15 -25 -35 -45 -55 -65 -75.0049972534180 -85.0250015258789 -95.0950012207031 -105.309997558594 -115.870002746582 -127.150001525879 -139.740005493164 -154.470001220703 -172.399993896484 -194.735000610352 -222.710006713867 -257.470001220703 -299.929992675781 -350.679992675781 -409.929992675781 -477.470001220703 -552.710021972656 -634.734985351563 -722.400024414063 -814.469970703125 -909.739990234375 -1007.15502929688 -1105.90502929688 -1205.53503417969 -1306.20495605469 -1409.15002441406  -1517.09497070313 -1634.17504882813 -1765.13500976563 -1914.15002441406 -2084.03491210938 -2276.22509765625 -2491.25000000000 -2729.25000000000 -2990.25000000000 -3274.25000000000 -3581.25000000000 -3911.25000000000 -4264.25000000000 -4640.25000000000 -5039.25000000000 -5461.25000000000 -5906.25000000000]')
prec = Float32
iosize = (90,1170,50)
tilesize = (90,90)
dims = [
    NCvar("i_c","1",tilesize[1],1:tilesize[1],Dict("long_name" => "Cartesian coordinate 1"),NCDatasets),
    NCvar("j_c","1",tilesize[2],1:tilesize[2],Dict("long_name" => "Cartesian coordinate 2"),NCDatasets),
    NCvar("dep_c","m",size(dep),dep,Dict("long_name" => "depth","standard_name" => "depth","positive" => "down"),NCDatasets),
    NCvar("tim",timeunits,Inf,time_steps,Dict(("long_name" => "time","standard_name" => "time")),NCDatasets)
]

gridvars = GridLoad(grid)

land = gridvars["hFacC"]
for f in land.fIndex
    for d in 1:size(land,2)
        land[f,d][land[f,d].==0] .= NaN
        land[f,d][land[f,d].>0] .= 1
    end
end

tilarea = TileData(gridvars["RAC"],tilesize,grid)
tilland = TileData(land,tilesize,grid)
thic = gridvars["DRC"]

fldidx = 1:106
for fidx in fldidx
    @time begin
        fldname = metafile["fldList"][fidx]
        println("Processing "*fldname)
        diaginfo = readAvailDiagnosticsLog(joinpath(exampledir,"available_diagnostics.log"),fldname)
        fielddata = BinData(fnames,prec,iosize,fidx)
        tilfld = TileData(fielddata,tilesize,grid)
        tillat = TileData(gridvars["YC"],tilfld.tileinfo,tilfld.tilesize,tilfld.precision,tilfld.numtiles)
        tillon = TileData(gridvars["XC"],tilfld.tileinfo,tilfld.tilesize,tilfld.precision,tilfld.numtiles)
        flds = Dict([fldname => NCvar(fldname,diaginfo["units"],dims,tilfld,Dict(),NCDatasets),
                    "lon" => NCvar("lon","degrees_east",dims[1:2],tillon,Dict("long_name" => "longitude"),NCDatasets),
                    "lat" => NCvar("lat","degrees_north",dims[1:2],tillat,Dict("long_name" => "latitude"),NCDatasets),
                    "area" => NCvar("area","m^2",dims[1:2],tilarea,Dict(["long_name" => "grid cell area", "standard_name" => "cell_area"]),NCDatasets),
                    "land" => NCvar("land","1",dims[1:3],tilland,Dict(["long_name" => "land mask", "standard_name" => "land_binary_mask"]),NCDatasets),
                    "thic" => NCvar("thic","m",dims[3],thic,Dict("standard_name" => "cell_thickness"),NCDatasets)
        ])
        savepath = joinpath(saveloc,fldname)
        if !isdir(savepath); mkpath(savepath); end

        numtiles = flds[fldname].values.numtiles
        savenames = joinpath.(Ref(savepath),fldname*".".*lpad.(string.(1:numtiles),4,"0").*".nc")
        rm.(savenames, force=true)

        datasets = [createfile(savenames[tidx],flds,README, itile = tidx, ntile = length(savenames)) for tidx in 1:length(savenames)]

        ds = [x[1] for x in datasets]
        fldvars = [x[2] for x in datasets]
        #dims = [x[3] for x in datasets]

        for k in keys(flds)
            if isa(flds[k].values,TileData)
                addData(fldvars,flds[k])
            else
                tmpfldvars = [fv[findfirst(isequal(k),name.(fv))] for fv in fldvars]
                addData.(tmpfldvars,Ref(flds[k]))
            end
        end

        for dim in dims
            addDimData.(ds,Ref(dim))
        end

        close.(ds)
    end
end


"""
Basic types and functions to support `Ballot`s, a fundamental unit of data exchange
for successful Paxos execution.
"""
module Ballots

export Ballot,
  BallotNumber, Command, InstanceID, Operation, Request, RequestID, SequenceNumber, Promise

using UUIDs

using ..Nodes
using ..Configurations

"""
An `InstanceID` uniquely names a specific instance of the Paxos algorithm.
Ballots occur within a specific instance, with the ultimate goal of
choosing a ballot by consensus.

`InstanceID`s are ordered, with "earlier" instances coming before later ones.
The ordering is intended to support ordering instances (and their chosen ballot)
into an ordered ledger of chosen commands.
"""
struct InstanceID
  value::UInt128
end

"""
Return `true` if the `left` id refers to an instance that occurs earlier
than the `right`; otherwise, return `false`
"""
function before(left::InstanceID, right::InstanceID)
  left.value < right.value
end

"""
  Return `true` if the left id refers to an instance that occurs later than
  the `right`; otherwise, return true
"""
function after(left, right)
  before(right, left)
end

struct SequenceNumber
  value::UInt128
end

struct Operation
  action::Any
end

Command = Union{Operation,Configuration}

struct RequestID
  id::UUID
  clientID::NodeID
  clientSequenceNumber::SequenceNumber
end

struct Request
  id::RequestID
  command::Command
end

struct BallotNumber
  instanceID::InstanceID
  sequenceNumber::SequenceNumber
  leaderID::NodeID
end
BallotNumber(original::BallotNumber) =
  BallotNumber(original.instanceID, original.sequenceNumber + 1, original.leaderID)

struct Ballot
  number::BallotNumber
  request::Request
end

"""
A `Promise` is a message from a member committing to a specific
`Ballot`, and agreeing not to make a similar promise for any ballot
in the same instance but with a lower sequence number.
"""
struct Promise
  memberID::NodeID
  ballotNumber::BallotNumber
end

Ballot(original::Ballot) =
  Ballot(BallotNumber(original.number), original.request)

end