mutable struct Simulation
    system::PSY.System
    reset::Bool
    problem::DiffEqBase.DAEProblem
    perturbations::Vector{<:Perturbation}
    x0_init::Vector{Float64}
    initialized::Bool
    tstops::Vector{Float64}
    callbacks::DiffEqBase.CallbackSet
    solution::Union{Nothing, DiffEqBase.DAESolution}
    simulation_folder::String
    ext::Dict{String, Any}
end

function Simulation(
    simulation_folder::String,
    system::PSY.System,
    tspan::NTuple{2, Float64},
    perturbations::Vector{<:Perturbation} = Vector{Perturbation}();
    kwargs...,
)
    check_folder(simulation_folder)
    simulation_system = deepcopy(system)
    PSY.set_units_base_system!(simulation_system, "device_base")
    check_kwargs(kwargs, SIMULATION_ACCEPTED_KWARGS, "Simulation")
    initialized = false
    DAE_vector = _index_dynamic_system!(simulation_system)
    var_count = get_variable_count(simulation_system)

    flat_start = zeros(var_count)
    bus_count = length(PSY.get_components(PSY.Bus, simulation_system))
    flat_start[1:bus_count] .= 1.0
    x0_init = get(kwargs, :initial_guess, flat_start)

    initialize_simulation = get(kwargs, :initialize_simulation, true)
    if initialize_simulation
        @info("Initializing Simulation States")
        _add_aux_arrays!(simulation_system, Real)
        initialized = calculate_initial_conditions!(simulation_system, x0_init)
    end

    dx0 = zeros(var_count)
    callback_set, tstops = _build_perturbations(simulation_system, perturbations)
    _add_aux_arrays!(simulation_system, Float64)
    prob = DiffEqBase.DAEProblem(
        system!,
        dx0,
        x0_init,
        tspan,
        simulation_system,
        differential_vars = DAE_vector;
        kwargs...,
    )

    if get(kwargs, :system_to_file, false)
        PSY.to_json(
            simulation_system,
            joinpath(simulation_folder, "initialized_system.json"),
        )
    end
    return Simulation(
        simulation_system,
        false,
        prob,
        perturbations,
        x0_init,
        initialized,
        tstops,
        callback_set,
        nothing,
        simulation_folder,
        Dict{String, Any}(),
    )
end

function Simulation(
    simulation_folder::String,
    system::PSY.System,
    tspan::NTuple{2, Float64},
    perturbation::Perturbation;
    initialize_simulation::Bool = true,
    kwargs...,
)
    return Simulation(
        simulation_folder,
        system,
        tspan,
        [perturbation];
        initialize_simulation = initialize_simulation,
        kwargs...,
    )
end

function _add_aux_arrays!(system::PSY.System, T)
    bus_count = get_bus_count(system)
    aux_arrays = Dict(
        1 => collect(zeros(T, bus_count)),                       #I_injections_r
        2 => collect(zeros(T, bus_count)),                       #I_injections_i
        3 => collect(zeros(T, get_n_injection_states(system))),  #injection_ode
        4 => collect(zeros(T, get_n_branches_states(system))),   #branches_ode
        5 => collect(zeros(Complex{T}, bus_count)),              #I_bus
        6 => collect(zeros(T, 2 * bus_count)),                   #I_balance
    )
    system.internal.ext[AUX_ARRAYS] = aux_arrays
    return
end

function _build_perturbations(system::PSY.System, perturbations::Vector{<:Perturbation})
    isempty(perturbations) && return DiffEqBase.CallbackSet(), [0.0]
    perturbations_count = length(perturbations)
    callback_vector = Vector{DiffEqBase.DiscreteCallback}(undef, perturbations_count)
    tstops = Vector{Float64}(undef, perturbations_count)
    for (ix, pert) in enumerate(perturbations)
        condition = (x, t, integrator) -> t in [pert.time]
        affect = get_affect(system, pert)
        callback_vector[ix] = DiffEqBase.DiscreteCallback(condition, affect)
        tstops[ix] = pert.time
    end
    callback_tuple = Tuple(cb for cb in callback_vector)
    callback_set = DiffEqBase.CallbackSet((), callback_tuple)
    return callback_set, tstops
end

function _index_local_states!(
    component_state_index::Vector{Int64},
    local_states::Vector{Symbol},
    component::PSY.DynamicComponent,
)
    for (ix, s) in enumerate(component.states)
        component_state_index[ix] = findfirst(x -> x == s, local_states)
    end
    return
end

function _attach_ports!(component::PSY.DynamicComponent)
    component.ext[PORTS] = Ports(component)
    return
end

function _attach_inner_vars!(
    device::PSY.DynamicGenerator,
    ::Type{T} = Real,
) where {T <: Real}
    device.ext[INNER_VARS] = zeros(T, 8)
    return
end

function _attach_inner_vars!(
    device::PSY.DynamicInverter,
    ::Type{T} = Real,
) where {T <: Real}
    device.ext[INNER_VARS] = zeros(T, 14)
    return
end

function _attach_control_refs!(device::PSY.DynamicInjection)
    device.ext[CONTROL_REFS] = [
        PSY.get_V_ref(device),
        PSY.get_ω_ref(device),
        PSY.get_P_ref(device),
        PSY.get_reactive_power(PSY.get_static_injector(device)),
    ]
    return
end

function _index_port_mapping!(
    index_component_inputs::Vector{Int64},
    local_states::Vector{Symbol},
    component::PSY.DynamicComponent,
)
    _attach_ports!(component)
    for i in component.ext[PORTS].state
        tmp = [(ix, var) for (ix, var) in enumerate(local_states) if var == i]
        isempty(tmp) && continue
        push!(index_component_inputs, tmp[1][1])
    end

    return
end

function _get_Ybus(sys::PSY.System)
    n_buses = length(PSY.get_components(PSY.Bus, sys))
    dyn_lines = PSY.get_components(PSY.DynamicBranch, sys)
    if !isempty(PSY.get_components(PSY.ACBranch, sys))
        Ybus_ = PSY.Ybus(sys)
        Ybus = Ybus_[:, :]
        lookup = Ybus_.lookup[1]
        if !isempty(dyn_lines)
            for br in dyn_lines
                ybus!(Ybus, br, lookup, -1.0)
            end
        end
    else
        Ybus = SparseMatrixCSC{Complex{Float64}, Int64}(zeros(n_buses, n_buses))
        lookup = Dict{Int.Int}()
    end
    return Ybus, lookup
end

function _make_device_index!(device::PSY.DynamicInjection, sys_basepower::Float64)
    states = PSY.get_states(device)
    device_state_mapping = Dict{Type{<:PSY.DynamicComponent}, Vector{Int64}}()
    input_port_mapping = Dict{Type{<:PSY.DynamicComponent}, Vector{Int64}}()
    _attach_inner_vars!(device)
    _attach_control_refs!(device)

    for c in PSY.get_dynamic_components(device)
        device_state_mapping[typeof(c)] = Vector{Int64}(undef, length(c.states))
        input_port_mapping[typeof(c)] = Vector{Int64}()
        _index_local_states!(device_state_mapping[typeof(c)], states, c)
        _index_port_mapping!(input_port_mapping[typeof(c)], states, c)
        device.ext[LOCAL_STATE_MAPPING] = device_state_mapping
        device.ext[INPUT_PORT_MAPPING] = input_port_mapping
    end

    return
end

function _add_states_to_global!(
    global_state_index::Dict{String, Dict{Symbol, Int64}},
    state_space_ix::Vector{Int64},
    device::PSY.Device,
)
    device_state_ix = Dict{Symbol, Int}()
    for s in PSY.get_states(device)
        state_space_ix[1] += 1
        device_state_ix[s] = state_space_ix[1]
    end
    global_state_index[PSY.get_name(device)] = device_state_ix
    return
end

function _index_dynamic_system!(sys::PSY.System)
    n_buses = length(PSY.get_components(PSY.Bus, sys))
    DAE_vector = collect(falses(n_buses * 2))
    global_state_index = Dict{String, Dict{Symbol, Int64}}()
    state_space_ix = [n_buses * 2]
    current_buses_ix = collect(1:n_buses)
    static_bus_var_count = 2 * length(current_buses_ix)
    voltage_buses_ix = Vector{Int}()
    total_states = 0
    first_dyn_branch_point = -1
    branches_n_states = 0
    global_vars = Dict{Symbol, Number}(
        :ω_sys => 1.0,
        :ω_sys_index => -1, #To define 0 if infinite source, bus_number otherwise,
    )
    total_shunts = Dict{Int, Float64}()
    found_ref_bus = false
    sys_basepower = PSY.get_base_power(sys)

    Ybus, lookup = _get_Ybus(sys)
    dyn_branches = PSY.get_components(PSY.DynamicBranch, sys)

    if !(isempty(dyn_branches))
        first_dyn_branch_point = state_space_ix[1] + 1
        for br in dyn_branches
            arc = PSY.get_arc(br)
            n_states = PSY.get_n_states(br)
            from_bus_number = PSY.get_number(arc.from)
            to_bus_number = PSY.get_number(arc.to)
            bus_ix_from = lookup[from_bus_number]
            bus_ix_to = lookup[to_bus_number]
            merge!(
                +,
                total_shunts,
                Dict(
                    bus_ix_from => 1 / PSY.get_b(br).from,
                    bus_ix_to => 1 / PSY.get_b(br).to,
                ),
            )
            push!(voltage_buses_ix, bus_ix_from, bus_ix_to)
            DAE_vector[bus_ix_from] = DAE_vector[bus_ix_from + n_buses] = true
            DAE_vector[bus_ix_to] = DAE_vector[bus_ix_to + n_buses] = true
            DAE_vector = push!(DAE_vector, collect(trues(n_states))...)
            total_states += n_states
            _add_states_to_global!(global_state_index, state_space_ix, br)
        end

        for (ix, val) in enumerate(DAE_vector[1:n_buses])
            if val
                #This V_ix should be V_number.
                global_state_index["V_$(ix)"] = Dict(:R => ix, :I => ix + n_buses)
                total_states += 2
                static_bus_var_count -= 2
                push!(voltage_buses_ix, ix)
                @assert static_bus_var_count >= 0
            end
        end
        branches_n_states = state_space_ix[1] - n_buses * 2
    else
        @debug("System doesn't contain Dynamic Branches")
    end
    unique!(voltage_buses_ix)
    sources = PSY.get_components(PSY.Source, sys)
    for s in sources
        btype = PSY.get_bustype(PSY.get_bus(s))
        if (btype == PSY.BusTypes.REF) && found_ref_bus
            throw(IS.ConflictingInputsError("The system can't have more than one source or generator in the REF Bus"))
        end
        btype != PSY.BusTypes.REF && continue
        global_vars[:ω_sys_index] = 0 #To define 0 if infinite source, bus_number otherwise,
        found_ref_bus = true
    end
    dynamic_injection = PSY.get_components(PSY.DynamicInjection, sys)
    isempty(dynamic_injection) &&
        error("System doesn't contain any DynamicInjection devices")
    for d in dynamic_injection
        if !(:states in fieldnames(typeof(d)))
            continue
        end
        device_bus = PSY.get_bus(d)
        btype = PSY.get_bustype(device_bus)
        if (btype == PSY.BusTypes.REF) && found_ref_bus
            throw(IS.ConflictingInputsError("The system can't have more than one source or generator in the REF Bus"))
        end
        _make_device_index!(d, sys_basepower)
        device_n_states = PSY.get_n_states(d)
        DAE_vector = push!(DAE_vector, collect(trues(device_n_states))...)
        total_states += device_n_states
        _add_states_to_global!(global_state_index, state_space_ix, d)
        btype != PSY.BusTypes.REF && continue
        global_vars[:ω_sys_index] = global_state_index[PSY.get_name(d)][:ω] #To define 0 if infinite source, bus_number otherwise,
        found_ref_bus = true
    end
    injection_n_states = state_space_ix[1] - branches_n_states - n_buses * 2
    @assert total_states == state_space_ix[1] - static_bus_var_count
    @debug total_states
    setdiff!(current_buses_ix, voltage_buses_ix)
    sys_ext = Dict{String, Any}()
    counts = Base.ImmutableDict(
        :total_states => total_states,
        :injection_n_states => injection_n_states,
        :branches_n_states => branches_n_states,
        :first_dyn_injection_pointer => 2 * n_buses + branches_n_states + 1,
        :first_dyn_branch_point => first_dyn_branch_point,
        :total_variables => total_states + static_bus_var_count,
        :bus_count => n_buses,
    )

    sys_ext[LITS_COUNTS] = counts
    sys_ext[GLOBAL_INDEX] = global_state_index
    sys_ext[VOLTAGE_BUSES_IX] = voltage_buses_ix
    sys_ext[CURRENT_BUSES_IX] = current_buses_ix
    sys_ext[YBUS] = Ybus
    sys_ext[TOTAL_SHUNTS] = total_shunts
    sys_ext[GLOBAL_VARS] = global_vars
    sys_ext[DYN_LINES] = !isempty(PSY.get_components(PSY.DynamicBranch, sys))
    sys_ext[LOOKUP] = lookup
    @assert sys_ext[GLOBAL_VARS][:ω_sys_index] != -1
    sys.internal.ext = sys_ext
    return DAE_vector
end

get_injection_pointer(sys::PSY.System) =
    PSY.get_ext(sys)[LITS_COUNTS][:first_dyn_injection_pointer]
get_branches_pointer(sys::PSY.System) =
    PSY.get_ext(sys)[LITS_COUNTS][:first_dyn_branch_point]
get_n_injection_states(sys::PSY.System) = PSY.get_ext(sys)[LITS_COUNTS][:injection_n_states]
get_n_branches_states(sys::PSY.System) = PSY.get_ext(sys)[LITS_COUNTS][:branches_n_states]
get_system_state_count(sys::PSY.System) = PSY.get_ext(sys)[LITS_COUNTS][:total_states]
get_variable_count(sys::PSY.System) = PSY.get_ext(sys)[LITS_COUNTS][:total_variables]
get_device_index(sys::PSY.System, device::D) where {D <: PSY.DynamicInjection} =
    PSY.get_ext(sys)[GLOBAL_INDEX][device.name]
get_inner_vars(device::PSY.DynamicInjection) = device.ext[INNER_VARS]
get_ω_sys(sys::PSY.System) = PSY.get_ext(sys)[GLOBAL_VARS][:ω_sys]
get_current_bus_ix(sys::PSY.System) = PSY.get_ext(sys)[CURRENT_BUSES_IX]
get_voltage_bus_ix(sys::PSY.System) = PSY.get_ext(sys)[VOLTAGE_BUSES_IX]
get_total_shunts(sys::PSY.System) = PSY.get_ext(sys)[TOTAL_SHUNTS]
get_bus_count(sys::PSY.System) = PSY.get_ext(sys)[LITS_COUNTS][:bus_count]
get_lookup(sys::PSY.System) = PSY.get_ext(sys)[LOOKUP]

function _get_internal_mapping(
    device::PSY.DynamicInjection,
    key::AbstractString,
    ty::Type{T},
) where {T <: PSY.DynamicComponent}
    device_index = PSY.get_ext(device)[key]
    val = get(device_index, ty, nothing)
    @assert !isnothing(val)
    return val
end

function get_local_state_ix(
    device::PSY.DynamicInjection,
    ty::Type{T},
) where {T <: PSY.DynamicComponent}
    return _get_internal_mapping(device, LOCAL_STATE_MAPPING, ty)
end

function get_input_port_ix(
    device::PSY.DynamicInjection,
    ty::Type{T},
) where {T <: PSY.DynamicComponent}
    return _get_internal_mapping(device, INPUT_PORT_MAPPING, ty)
end

function run_simulation!(sim::Simulation, solver; kwargs...)
    if sim.reset
        @error("Reset the simulation")
    end

    sim.solution = DiffEqBase.solve(
        sim.problem,
        solver;
        callback = sim.callbacks,
        tstops = sim.tstops,
        kwargs...,
    )
    return
end

function _change_vector_type(sys::PSY.System)
    for d in PSY.get_components(PSY.DynamicInjection, sys)
        _attach_inner_vars!(d, Real)
    end
end

function _determine_stability(vals::Vector{Complex{Float64}})
    stable = true
    for real_eig in real(vals)
        real_eig > 0.0 && return false
    end
    return true
end

function small_signal_analysis(sim::Simulation; kwargs...)
    if sim.reset
        @error("Reset the simulation")
    end
    _change_vector_type(sim.system)
    _add_aux_arrays!(sim.system, Real)
    var_count = LITS.get_variable_count(sim.system)
    dx0 = zeros(var_count) #Define a vector of zeros for the derivative
    bus_count = length(PSY.get_components(PSY.Bus, sim.system))
    sysf! = (out, x) -> LITS.system!(
        out,            #output of the function
        dx0,            #derivatives equal to zero
        x,              #states
        sim.system,     #Parameters
        0.0,            #time equals to zero.
    )
    out = zeros(var_count) #Define a vector of zeros for the output
    x_eval = get(kwargs, :operating_point, sim.x0_init)
    jacobian = ForwardDiff.jacobian(sysf!, out, x_eval)
    n_buses = length(PSY.get_components(PSY.Bus, sim.system))
    diff_states = collect(trues(var_count))
    diff_states[1:(2 * n_buses)] .= false
    for b_ix in get_voltage_bus_ix(sim.system)
        diff_states[b_ix] = true
        diff_states[b_ix + n_buses] = true
    end
    alg_states = .!diff_states
    fx = @view jacobian[diff_states, diff_states]
    gy = jacobian[alg_states, alg_states]
    fy = @view jacobian[diff_states, alg_states]
    gx = @view jacobian[alg_states, diff_states]
    # TODO: Make operation using BLAS!
    reduced_jacobian = fx - fy * inv(gy) * gx
    vals, vect = LinearAlgebra.eigen(reduced_jacobian)
    sources = collect(PSY.get_components(PSY.Source, sim.system))
    if isempty(sources)
        @warn("No Infinite Bus found. Confirm stability directly checking eigenvalues.\nIf all eigenvalues are on the left-half plane and only one eigenvalue is zero, the system is small signal stable.")
        info_evals = "Eigenvalues are:\n"
        for i in vals
            info_evals = info_evals * string(i) * "\n"
        end
        @info(info_evals)
    end
    return SmallSignalOutput(
        reduced_jacobian,
        vals,
        vect,
        _determine_stability(vals),
        x_eval,
    )
end


function updateT0!(allvar::AllVar)
    @extract allvar : alg pbf
    @extract pbf : L
    @extract alg : verbose nprint epsconv maxiter mindec μext μint
    en = Inf
    assign = zeros(Int,L,2)
    ndec = 0
    initialize_msg0(allvar)

    for it in 1:maxiter
        onesweepT0!(allvar)
        ndec, assign = assignment!(allvar,ndec,assign)
        en = compute_en(allvar)
        if mod(it, nprint) == 0 && verbose
            @printf("it = %d -- en %.3f μext %.2f μint %.2f ndec %d\n", it, en,μext, μint,ndec)
        end
        if ndec >= mindec
            if verbose
                @printf("it = %d -- en %.3f μext %.2f μint %.2f ndec %d\n", it, en,μext, μint,ndec)
            end
            return it, :converged, en
        end
    end
    if verbose
        @printf("it = %d -- en %.3f μext %.2f μint %.2f ndec %d\n", maxiter, en,μext, μint,ndec)
    end
    return maxiter, :unconverged, en
end

function initialize_msg0(allvar::AllVar)
    @extract allvar : pbf
    @extract pbf : L N P B F newP newB newF

    for marg in (newP, newB, newF)
        marg[1,0] = -Inf
        marg[1,N+1] = -Inf
        value, idx = findmax(marg)
        marg .= marg .- marg[idx]
    end
    for i in 1:L
        for marg in (P[i], F[i], B[i])
            marg[1,0] = - Inf
            marg[1,N+1] = -Inf
            if i == 1
                for n = 1:N+1
                    marg[0,n] = -Inf
                end
            end
            if i == L
                for n = 0:N
                    marg[0,n] = -Inf
                end
            end
            value, idx = findmax(marg)
            marg .= marg .- marg[idx]
        end
    end

end

function onesweepT0!(allvar::AllVar)
    @extract allvar : pbf alg
    @extract pbf : L P B F newP newB newF
    @extract alg : damp

    v = randperm(L)
    @inbounds for idx in 1:L # i-th marginal 1 ≤ i ≤ L
        i = v[idx]
        newPBFiT0!(allvar, i::Int)
        p,f,b = P[i], F[i], B[i]
        p .= newP; f .= newF; b .= newB;
        #p .= damp .* p .+ (1.0 - damp) .* newP # inplace sum
        #f .= damp .* f .+ (1.0 - damp) .* newF
        #b .= damp .* b .+ (1.0 - damp) .* newB
    end
end

function newPBFiT0!(allvar::AllVar, i::Int)
    @extract allvar : pbf
    @extract pbf : L N forward central bckward newP newB newF

    for fcb in (forward,central,bckward)
        fill!(fcb, 0.0)
    end

    centralT0!(central, allvar, i)
    i > 1 &&  forwardT0!(forward, allvar, i)
    i < L && backwardT0!(bckward, allvar, i)

    if i==1 # case site = 1 first site
        newF .= central
        newP .= central .+ bckward
        newB .= newP
    elseif 1 < i < L #internal sites
        newF .= forward .+ central
        newP .= newF .+ bckward
        newB .= central .+ bckward
    elseif i == L# case site = L
        newF .= forward .+ central
        newB .= central
        newP .= newF
    else
        error("the end of the world has come ...")
    end
    newP[1,0] = -Inf
    newB[1,0] = -Inf
    newF[1,0] = -Inf
    newP[1,N+1] = -Inf
    newB[1,N+1] = -Inf
    newF[1,N+1] = -Inf
    if i == 1
        for n in 1:N+1
            newP[0,n] = -Inf
            newB[0,n] = -Inf
            newF[0,n] = -Inf
        end
    end
    if i == L
        for n in 0:N
            newP[0,n] = -Inf
            newB[0,n] = -Inf
            newF[0,n] = -Inf
        end
    end
    for marg in (newP,newB,newF)
        v, idx = findmax(marg)
        marg .= marg .- marg[idx]
    end
end

# function myargmax(Pj,nbeg,nend)
#     #argm = -Inf
#     val = (-1,-1,-Inf)
#     @inbounds for n = nbeg:nend
#         for x = 0:1
#             val = ifelse(Pj[x, n]>val[3],(x,n,Pj[x,n]),val)
#         end
#     end
#     @assert val[1] > -1
#     @assert val[2] > -1
#     return val[1],val[2]
# end

function myargmax(Pj,nbeg,nend)
    argm = -Inf
    xj = -1
    nj = -1
    @inbounds for n = nbeg:nend
        for x = 0:1
            if Pj[x, n] > argm
                argm = Pj[x, n]
                xj = x
                nj = n
            end
        end
    end
    @assert(xj > -1)
    @assert(nj > -1)
    return xj,nj
end

function centralT0!(central::Marginal, allvar::AllVar, i::Int)
    @extract allvar:pbf jh seq alg
    @extract pbf:N P L q
    @extract jh:J h
    @extract seq:intseq
    @extract alg:μext μint

    A0 = q
    # case xᵢ = 1 (match)
    @inbounds for ni = 1:N
        Ani = intseq[ni]
        scra = h[Ani, i]
        for j = 1:i-2
            xj, nj = myargmax(P[j], 0, ni - 1)
            Anj = (nj == 0 || xj == 0) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        for j = i+2:L
            xj, nj = myargmax(P[j], ni + 1, N + 1)
            Anj = (xj == 0 || nj == N + 1) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        central[1, ni] = scra
    end
    # case xᵢ = 0 (gap)
    @inbounds for ni = 0:N+1
        scra = 0
        Ani = A0
        for j = 1:i-2
            xj, nj = myargmax(P[j], 0, ni)
            Anj = (nj == 0 || nj == N + 1 || xj == 0) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        for j = i+2:L
            xj, nj = myargmax(P[j], ni, N + 1)
            Anj = (nj == 0 || nj == N + 1 || xj == 0) ? A0 : intseq[nj]
            scra += J[Ani, Anj, i, j]
        end
        central[0, ni] = scra
    end
    @inbounds for ni = 0:N+1
        central[0, ni] =
            (ni == 0 || ni == N + 1) ? central[0, ni] - μext + h[A0, i] :
            central[0, ni] - μint + h[A0, i]
    end
    #cannot match n = 0 pointer
    #central[1,0] = -Inf
    #mcentral = maximum(central)
    #central = central .- mcentral
end

function forwardT0!(forward::Marginal,allvar::AllVar,i::Int)
    @extract allvar : pbf jh seq alg
    @extract pbf : N P F q
    @extract jh : J
    @extract seq : intseq
    @extract alg : λo λe
    fwdm1 = F[i-1]
    A0 = q
    # case xᵢ = 1 (match)
    @inbounds for ni in 1:N
        Ani = intseq[ni]
        scra = -Inf
        @simd for nim in 0:ni-1
           Anim = nim == 0 ? A0 : intseq[nim]
           Δn = ni - nim - 1
           pen = (Δn > 0)*(nim > 0)*(- λo[i] - λe[i]*(Δn - 1))
           scra = max(scra, fwdm1[0,nim] + J[Ani, A0,   i, i-1] + pen)
           scra = max(scra, fwdm1[1,nim] + J[Ani, Anim, i, i-1] + pen)
        end
        forward[1, ni] = scra
    end
    # case xᵢ = 0 gap
    @inbounds @simd for ni in 1:N
        Ani = intseq[ni]
        forward[0,ni] = max(fwdm1[1,ni]+J[A0,Ani,i,i-1],fwdm1[0,ni]+J[A0,A0,i,i-1])
    end
    forward[0,0] = fwdm1[0,0] + J[A0,A0,i,i-1]
    scra = -Inf
    @inbounds @simd for nim = 1:N
        Anim = intseq[nim]
        scra = max(scra, fwdm1[1,nim] + J[A0,Anim,i,i-1])
    end
    forward[0,N+1] = max(scra, fwdm1[0,N+1] + J[A0,A0,i,i-1])
end

function backwardT0!(backward::Marginal,allvar::AllVar,i::Int)
    @extract allvar : pbf jh seq alg
    @extract pbf : N P B q
    @extract jh : J
    @extract seq : intseq
    @extract alg : λo λe

    bckp1 = B[i+1]
    A0 = q

    # case xᵢ = 1 (match)
    @inbounds for ni in 1:N
        # case xᵢp = 0 (gap)
        Ani = intseq[ni]
        scra = bckp1[0,ni] + J[A0,Ani,i+1,i]
        scra = max(scra, bckp1[0,N+1] + J[A0,Ani,i+1,i])
        for nip in ni+1:N
            Anip = intseq[nip]
            Δn = nip - ni- 1
	        pen = (Δn > 0)*(- λo[i+1] - λe[i+1]*(Δn - 1))
            scra = max(scra, bckp1[1,nip] + J[Anip, Ani, i+1,i] + pen)
        end
        backward[1,ni] = scra
    end
    # case xᵢ = 0 (gap)
    @inbounds for ni in 0:N+1
        scra = bckp1[0,ni] + J[A0,A0,i+1,i]
        for nip in ni+1:N
            Anip = intseq[nip]
            Δn = nip - ni - 1
	        pen = (ni > 0)*(Δn > 0)*(-λo[i+1] - λe[i+1]*(Δn - 1))
            scra = max(scra, bckp1[1,nip], J[Anip,A0,i+1,i] + pen)
        end
        backward[0,ni] = scra
    end
    #mbackward = maximum(backward)
    #backward = backward .- mbackward
end

function assignment!(allvar::AllVar, ndec::Int, assign::Matrix{Int})

    @extract allvar : pbf
    @extract pbf : L N P
    count = 0
    @inbounds for i in 1:L
        Pi = P[i]
        #v, idx = findmax(Pi)
        new_a = argmax(Pi)
        #display(new_a)
        #count how many nodes change state
        if (new_a[1] != assign[i,1]) || (new_a[2] != assign[i,2])
            count += 1
        end
        assign[i,1] = new_a[1]
        assign[i,2] = new_a[2]
    end

    if count == 0
        ndec += 1
    else
        ndec = 0
    end

    return ndec, assign
end


function FieldTransformGenerator(arg0::Field, arg1::jint, arg2::TransformProvider, arg3::jdouble)
    return FieldTransformGenerator((Field, jint, TransformProvider, jdouble), arg0, arg1, arg2, arg3)
end

function generate(obj::FieldTransformGenerator, arg0::AbsoluteDate, arg1::AbsoluteDate)
    return jcall(obj, "generate", List, (AbsoluteDate, AbsoluteDate), arg0, arg1)
end


# module Hangul

# 자음 모음
const Consonant = :Consonant
const Vowel     = :Vowel

# Hangul Compatibility Letter
primitive type Letter{T} <: AbstractChar 32 end

# Hangul Syllables
struct Syllable
    initial::Union{Nothing,Letter{Consonant}}
    medial::Union{Nothing,Letter{Vowel}}
    final::Union{Nothing,Letter{Consonant}}
end

struct ComposeError <: Exception
    message::String
end

struct DecomposeError <: Exception
    message::String
end

struct LetterError <: Exception
    message::String
end

Syllable(i::Union{Nothing,Letter{Consonant}}, m::Union{Nothing,Letter{Vowel}}) = Syllable(i, m, nothing)

Letter{T}(c::UInt32) where T = reinterpret(Letter{T}, c)
Base.codepoint(l::Letter{T}) where T = reinterpret(UInt32, l)

"""
    Jamo
"""
function Jamo(c::Char)::Union{Letter,YetJamo} # LetterError
    Jamo(codepoint(c))
end

function Jamo(c::Union{UInt16,UInt32})::Union{Letter,YetJamo} # LetterError
    # 0x3130 : undef
    # 0x318f : undef
    if 0x3131 <= c <= 0x314e || # ㄱ   ... ㅎ  # Consonant letters
       0x3165 <= c <= 0x3186    # ㄴㄴ ... ㆆ  # Old consonant letters
        Letter{Consonant}(c)
    elseif 0x314f <= c <= 0x3163 || # ㅏ ... ㅣ   # Vowel letters
           0x3187 <= c <= 0x318e    # ㆇ ... ㆎ   # Old vowel letters
        Letter{Vowel}(c)
    elseif is_yetjamo(c)
        YetJamo(c)
    else
        throw(LetterError(string(c)))
    end
end

macro _w_str(s::String) # Vector{Char}
    Expr(:call, ()->first.(split(s, ' ')))
end

const 모음채움 = Letter{Vowel}(0x3164)     # HANGUL FILLER
const 자음채움 = Letter{Consonant}(0x318f) # ?

const 초성표 = Letter{Consonant}.(_w"ㄱ ㄲ ㄴ ㄷ ㄸ ㄹ ㅁ ㅂ ㅃ ㅅ ㅆ ㅇ ㅈ ㅉ ㅊ ㅋ ㅌ ㅍ ㅎ")
const 중성표 = Letter{Vowel}.(_w"ㅏ ㅐ ㅑ ㅒ ㅓ ㅔ ㅕ ㅖ ㅗ ㅘ ㅙ ㅚ ㅛ ㅜ ㅝ ㅞ ㅟ ㅠ ㅡ ㅢ ㅣ")
const 종성표 = Letter{Consonant}.(vcat([자음채움], _w"ㄱ ㄲ ㄳ ㄴ ㄵ ㄶ ㄷ ㄹ ㄺ ㄻ ㄼ ㄽ ㄾ ㄿ ㅀ ㅁ ㅂ ㅄ ㅅ ㅆ ㅇ ㅈ ㅊ ㅋ ㅌ ㅍ ㅎ"))

const 중성오프셋 = 28  # length(종성표)
const 초성오프셋 = 588 # length(중성표) * 중성오프셋  # 21 * 28
const 가 = 0xac00 # codepoint('가')
const 힣 = 0xd7a3 # codepoint('힣')

indexof(순서::Vector{Letter{T}}, 소리::Letter{T}) where T = findall(in(소리), 순서)[1]

function throw_compose_error(syl::Syllable) # ComposeError
    throw(ComposeError(string(syl)))
end

function throw_decompose_error(c::Char) # DecomposeError
    throw(DecomposeError(string(c)))
end

function to_char(syl::Syllable)::Char # ComposeError
    초성, 중성, 종성 = syl.initial, syl.medial, syl.final
    if 초성 isa Nothing
        if 중성 isa Nothing
            if 종성 isa Nothing
                throw_compose_error(syl)
            else
                Char(종성)
            end
        else # 초성x, 중성o
            if 종성 isa Nothing
                Char(중성)
            else
                throw_compose_error(syl)
            end
        end
    else # 초성o
        if 중성 isa Nothing
            if 종성 isa Nothing
                Char(초성)
            else
                throw_compose_error(syl)
            end
        else # 초성o, 중성o
            초성인덱스 = (indexof(초성표, 초성) - 1) * 초성오프셋
            중성인덱스 = (indexof(중성표, 중성) - 1) * 중성오프셋
            value = 가 + 초성인덱스 + 중성인덱스
            if 종성 isa Nothing
            else
                종성인덱스 = indexof(종성표, 종성) - 1
                value += 종성인덱스
            end
            Char(value)
        end
    end
end

function to_syllable(c::Char)::Syllable # DecomposeError
    code = codepoint(c)
    if 가<= code <= 힣
        value::UInt32 = code - 가
        초성인덱스 = Int(round(value / 초성오프셋, RoundToZero)) + 1
        n = value % 초성오프셋
        중성인덱스 = Int(round(n / 중성오프셋, RoundToZero)) + 1
        종성인덱스 = Int(n % 중성오프셋) + 1
        Syllable(초성표[초성인덱스], 중성표[중성인덱스], 1 == 종성인덱스 ? nothing : 종성표[종성인덱스])
    else
        if c in 초성표
            Syllable(Letter{Consonant}(c), nothing, nothing)
        elseif c in 중성표
            Syllable(nothing, Letter{Vowel}(c), nothing)
        else
            throw_decompose_error(c)
        end
    end
end

# module Hangul


#	MIT License
#
#  Copyright © 2020 Michael J Simms. All rights reserved.
#
#	Permission is hereby granted, free of charge, to any person obtaining a copy
#	of this software and associated documentation files (the "Software"), to deal
#	in the Software without restriction, including without limitation the rights
#	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#	copies of the Software, and to permit persons to whom the Software is
#	furnished to do so, subject to the following conditions:
#
#	The above copyright notice and this permission notice shall be included in all
#	copies or substantial portions of the Software.
#
#	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#	SOFTWARE.

module IsolationForest

# Tree node, used internally.
mutable struct Node
    featureName::String
    splitValue::Float64
    left::Any
    right::Any
end

# This struct represents a sample. Each sample has a name and list of features.
mutable struct Sample
    name::String
    features::Dict{String,Float64}
end

# Isolation Forest.
mutable struct Forest
    numTrees::UInt64
    subSamplingSize::UInt64
    featureValues::Dict{String,Array} # Dictionary that maps feature names to a sorted array of feature values
    trees::Array
end

# Adds the features to the specified sample.
function add_features_to_sample(sample::Sample, features::Dict{String,Float64})
    merge!(sample.features, features)
end

# Adds each of the sample's features to the list of known features with the corresponding set of unique values.
function add_sample_to_forest(forest::Forest, sample::Sample)

    # We don't store the sample directly, just the features.
    for feature_name in keys(sample.features)
        feature_value = sample.features[feature_name] # Value to insert

        try
            existing_feature_values = forest.featureValues[feature_name]
            push!(existing_feature_values, feature_value)
            sort(existing_feature_values)
            forest.featureValues[feature_name] = existing_feature_values
        catch error
            if isa(error, KeyError)
                existing_feature_values = []
                push!(existing_feature_values, feature_value)
                forest.featureValues[feature_name] = existing_feature_values
            end
        end
    end
end

# Creates and returns a single tree. As this is a recursive function, depth indicates the current depth of the recursion.
function create_tree(forest::Forest, feature_values::Dict{String,Array}, depth::UInt64)

    # Sanity check
    feature_values_len = length(feature_values)
    if feature_values_len <= 1
        return Nothing
    end

    # If we've exceeded the maximum desired depth, then stop.
    if (forest.subSamplingSize > 0) && (depth >= forest.subSamplingSize)
        return Nothing
    end

    # Randomly select a feature.
    selected_feature_index = rand(1:feature_values_len)
    all_feature_names = collect(keys(feature_values))
    selected_feature_name = all_feature_names[selected_feature_index]

    # Randomly select a split value.
    feature_value_set = feature_values[selected_feature_name]
    feature_value_set_len = length(feature_value_set)
    if feature_value_set_len <= 1
        return Nothing
    end
    split_value_index = rand(1:feature_value_set_len)
    split_value = feature_value_set[split_value_index]

    # Create a tree node to hold the split value.
    tree = Node(selected_feature_name, split_value, Nothing, Nothing)

    # Create two versions of the feature value set that we just used,
    # one for the left side of the tree and one for the right.
    temp_feature_values = feature_values

    # Create the left subtree.
    left_features = feature_value_set[1:split_value_index]
    temp_feature_values[selected_feature_name] = left_features
    tree.left = IsolationForest.create_tree(forest, temp_feature_values, depth + 1)

    # Create the right subtree.
    if split_value_index + 1 < feature_value_set_len
        right_features = feature_value_set[split_value_index + 1:feature_value_set_len]
        temp_feature_values[selected_feature_name] = right_features
        tree.right = IsolationForest.create_tree(forest, temp_feature_values, depth + 1)
    end

    return tree
end

# Creates a forest containing the number of trees specified to the constructor.
function create_forest(forest::Forest)
    for i = 1:forest.numTrees
        temp_feature_values = deepcopy(forest.featureValues)
        tree = create_tree(forest, temp_feature_values, UInt64(0))
        if tree != Nothing
            push!(forest.trees, tree)
        end
    end
end

# Scores the sample against the specified tree.
function score_sample_against_tree(tree::Node, sample::Sample)
    depth = 0.0
    current_node = tree

    while current_node != Nothing
        found_feature = false

        # Find the next feature in the sample.
        for current_feature in sample.features
            current_feature_name = current_feature.first

            # If the current node has the feature in question.
            if current_feature_name == current_node.featureName
                current_feature_value = current_feature.second
                if current_feature_value < current_node.splitValue
                    current_node = current_node.left
                else
                    current_node = current_node.right
                end

                depth = depth + 1.0
                found_feature = true
                break
            end
        end

        # If the tree contained a feature not in the sample then take
        # both sides of the tree and average the scores together.
        if found_feature == false
            left_depth = depth + score_sample_against_tree(sample, current_node.left)
            right_depth = depth + score_sample_against_tree(sample, current_node.right)
            return (left_depth + right_depth) / 2.0
        end
    end

    return depth
end

# Scores the sample against the entire forest of trees. Result is the average path length.
function score_sample_against_forest(forest::Forest, sample::Sample)
    num_trees = 0
    avg_path_len = 0.0

    for tree in forest.trees
        path_len = score_sample_against_tree(tree, sample)
        if path_len > 0
            avg_path_len = avg_path_len + path_len
            num_trees = num_trees + 1
        end
    end

    if num_trees > 0
        avg_path_len = avg_path_len / num_trees
    end

    return avg_path_len
end

# Scores the sample against the entire forest of trees. Result is normalized so that values
# close to 1 indicate anomalies and values close to zero indicate normal values.
function H(i)
    return log(i) + 0.5772156649
end
function C(n)
    return 2 * H(n - 1) - (2 * (n - 1) / n)
end
function score_sample_against_forest_normalized(forest::Forest, sample::Sample)

    # Compute the average path length for all valid trees.
    num_trees = 0
    avg_path_len = 0.0

    for tree in forest.trees
        path_len = score_sample_against_tree(tree, sample)
        if path_len > 0
            avg_path_len = avg_path_len + path_len
            num_trees = num_trees + 1
        end
    end

    if num_trees > 0
        avg_path_len = avg_path_len / num_trees
    end

    # Normalize, per the original paper.
    score = 0.0
    if num_trees > 1.0
        exponent = -1.0 * (avg_path_len / C(num_trees))
        score = 2 ^ exponent
    end

    return score
end

# Destroys a single tree.
function destroy_tree(tree::Node)
end

# Destroys the entire forest of trees.
function destroy()
    for tree in self.trees
        self.destroy_tree(tree)
    end
end

# Returns the specified node as a dictionary object.
function dump_node(node::Node)
    data = Dict()
    if node != Nothing
        data["Feature Name"] = node.featureName
        data["Split Value"] = node.splitValue
        if node.left == Nothing
            data["Left"] = Dict()
        else
            data["Left"] = dump_node(node.left)
        end
        if node.right == Nothing
            data["Right"] = Dict()
        else
            data["Right"] = dump_node(node.right)
        end
    end
    return data
end

# Loads/creates a node from a JSON object.
function load_node(data::Dict)
    try
        node = Node(data["Feature Name"], data["Split Value"])
        node.left = load_node(data["Left"])
        node.right = load_node(data["Right"])
        return node
    catch error
    end

    return Nothing
end

# Returns the specified tree as a dictionary object.
function dump_tree(tree::Node)
    data = dump_node(tree)
    return data
end

# Loads/creates a tree from a dictionary object.
function load_tree(forest::Forest, data::Dict)
    tree = load_node(data)
    if tree != Nothing
        push!(forest.trees, tree)
    end
end

# Returns the forest as a JSON object.
function dump(forest::Forest)
    data = Dict()
    data["Sub Sampling Size"] = forest.subSamplingSize
    data["Feature Values"] = forest.featureValues
    tree_data = []
    for tree in forest.trees
        push!(tree_data, dump_tree(tree))
    end
    data["Trees"] = tree_data
    return data
end

# Loads the forest from a JSON object.
function load(data::Dict)
    forest = Forest(0, 0, Dict(), [])
    forest.subSamplingSize = data["Sub Sampling Size"]
    forest.featureValues = data["Feature Values"]
    for tree_data in data["Trees"]
        load_tree(forest, tree_data)
    end
    forest.numTrees = length(forest.trees)
    return forest
end

end


#!/home/fng514/bin/julia
# Christian Theil Have, 2017.
# Merging of imputed data.
#
# Merges a selected set of individuals from a pair of imputations on the same panel.
# If the same individual is present in more both imputations, then the script will
# select the best-possible imputed genotypes for the individuals from either imputation
# on a per genotype basis. INFO scores as well as AN (allele number) and (AC) allele
# count is recalculated and the output VCF file is written with the same fields.
#
# It is not as generic as I would like it to be:
# Currently, designed to work with the format produced by Sangers imputation server.
# Adaptation will be need for other formats.

using ArgParse
function main(args)
	s = ArgParseSettings("Example for merge.jl: " *  # description
		"flags, options help, " *
		"required arguments.")

	@add_arg_table s begin
		"--vcfs"
		nargs = '*'
		help = "A list of VCF files to merge"
		"--merge-mode"
		help = "One of best,mean,weightedmean. Default is best."
		default = "best"
		"--out"
	end

	parsed_args = parse_args(s)
	println("Parsed args:")
	for (key,val) in parsed_args
		println("  $key  =>  $(repr(val))")
	end

	@assert length(parsed_args["vcfs"]) > 0

	# Merge method:
	if parsed_args["merge-mode"] == "best"
		mergefun = best_gp
	elseif parsed_args["merge-mode"] == "mean"
		mergefun = mean_gp
	elseif parsed_args["merge-mode"] == "weightedmean"
		mergefun = confidence_weighted_mean_gp
	elseif parsed_args["merge-mode"] == "bestinfo"
		mergefun = best_info
	else
		throw("Invalid merge-mode : $(parsed_args["merge-mode"])")
	end

	merged_imputed(parsed_args["vcfs"], parsed_args["out"], mergefun)
end

main(ARGS)


"""
This is a terrible function. But is there any other way to reliable get the
abstract supertype of an arbitrary type hierarchy without loosing performance?
"""
@generated function go_abstract{T<:AbstractGeometry}(::Type{T})
    ff = T
    while ff.name.name != :AbstractGeometry
       ff = supertype(ff)
    end
    :($ff)
end

@generated function eltype_or{T<:GeometryPrimitive}(::Type{T}, OR)
    ET = go_abstract(T).parameters[2]
    if isa(ET, TypeVar)
        return :(OR)
    else
        return :($ET)
    end
end
@generated function ndims_or{T<:GeometryPrimitive}(::Type{T}, OR)
    N = go_abstract(T).parameters[1]
    if isa(N, TypeVar)
        return :(OR)
    else
        return :($N)
    end
end

function Base.eltype{T<:AbstractGeometry}(x::Type{T})
    eltype_or(T, Any)
end
@generated function Base.ndims{T<:AbstractGeometry}(x::Type{T})
    N = go_abstract(T).parameters[1]
    isa(N, TypeVar) && error("Ndims not given for type $T")
    :($N)
end

Base.eltype{N, T}(x::AbstractGeometry{N, T}) = T
Base.ndims{N, T}(x::AbstractGeometry{N, T}) = N

Base.eltype{T}(::Type{AFG{T}}) = T
Base.eltype{FG <: AFG}(::Type{FG}) = eltype(supertype(FG))


let
    @test AutomotiveDrivingModels.cyclic_shift_left!([1,2,3,4], 1, 4) == [2,3,4,1]
    @test AutomotiveDrivingModels.cyclic_shift_left!([1,2,3,4], 2, 4) == [3,4,1,2]
    @test AutomotiveDrivingModels.cyclic_shift_left!([1,2,3,4,5,6,7], 1, 4) == [2,3,4,1,5,6,7]
    @test AutomotiveDrivingModels.cyclic_shift_left!([1,2,3,4,5,6,7], 2, 4) == [3,4,1,2,5,6,7]

    td = get_test_trajdata()
    scene = Scene()
#    print(`scene=`)
#    println(scene)

    g = get!(scene, td, 1)
#    print(`g=`)
#    println(g)
    col = get_first_collision(g)
#    print(`col=`)
#    println(col)
    @test col.A == 1
    @test col.B == 2
    @test col.is_colliding == true

    @test get_first_collision(get!(scene, td, 2), CPAMemory()).is_colliding == true
end


"""
    function randomAdapt_(p, code, platform)

Function that adapt the random function invocations of the code to be executable in the different platforms.
"""
function randomAdapt_(p::Program_, code::Expr, platform::String)

    if platform == "cpu"
        for i in VALIDDISTRIBUTIONS
            code = postwalk(x -> @capture(x,$i(v__)) ? :(rand($i($(v...)))) : x, code)
        end
    elseif platform == "gpu"
        for i in VALIDDISTRIBUTIONS

            if inexpr(code,i) && !(i in VALIDDISTRIBUTIONSCUDA)
                error(i," random distribution valid for cpu but still not implemented in gpu.")
            else

                s = Meta.parse(string("AgentBasedModels.",i,"CUDA"))
                code = postwalk(x -> @capture(x,$i(v__)) ? :($s(rand(),$(v...))) : x, code)

            end

        end
    end

    return code
end


################################################################################
###################### DEFINE MASTER PARAMETERS FOR THIS FILE ##################
################################################################################

using Mimi
using Distributions
using CSVFiles
using DataFrames
using CSV
using Random
using StatsBase
using Statistics

include("main_model_growth.jl")

# define the model regions in the order that Mimi returns stuff
myregions = ["EU", "USA", "Other OECD","Former USSR","China","Southeast Asia","Africa","Latin America"]
myyears = [2020, 2030, 2040, 2050, 2075, 2100, 2150, 2200, 2250, 2300]

# define the output directory
dir_output = joinpath(@__DIR__, "../../output/")

# define number of Monte Carlo runs
if !@isdefined samplesize
    samplesize = 50000
end

# define the seed
masterseed = 22081994

# define the pulse size
scc_pulse_size = 75000.

################################################################################
################### MC MODEL RUNS    ###########################################
################################################################################

### PART I: results for fixed persistence values

# create a DataFrame where results will be stored
df_sccMC_singleGE = DataFrame(damagePAGE09=false, permafr=false, seaice=false, ge=-999., scen="-999", pulse_size=-999.,
                                 civvalue=-999.,
                                 ge_adapt = -999.,
                                  mean=-999., median=-999., min=-999., max=-999., perc25=-999.,
                                  perc75=-999., sd=-999., varcoeff=-999., perc05=-999., perc95=-999.,
                                  perc10=-999., perc90=-999.,
                                  share_zeroSCC = -999.)

# loop through all relevant parameter combinations and save out results and SCC summary stats
for jj_page09damages in [false] # whether or not to use the market damage function from PAGE09 instead of the new PAGE-ICE function using Burke et al. (2015)
    for jj_permafr in [true] # whether or not to include the permafrost carbon feedback, see Yumashev et al. (2019)
        for jj_seaice in [true] # whether or not to include the surface albedo feedback, see Yumashev et al. (2019)
            for jj_scen in ["RCP4.5 & SSP2"]
                for jj_civvalue in [1., 10.0^20] # whether or not to hold the civilization value damage cap at its default value

                    # skip undesired combinations of parameters
                    if jj_page09damages != false && (jj_permafr != true  || jj_seaice != true || jj_civvalue != 1.)
                        continue
                    elseif jj_permafr != true && (jj_seaice != true || jj_civvalue != 1.)
                        continue
                    elseif jj_permafr != jj_seaice
                        continue
                    end

                    # print the parameter combination to track progress
                    print(string("page09_", jj_page09damages, " Permafr_", jj_permafr, " Scen_", jj_scen, " Seaice_", jj_seaice
                                     ))

                    # loop through the different values for persistence and its decay rate (= adaptation)
                    for jj_ge in [0.0:0.1:1.0;]
                        for jj_geadapt in [0.0:0.005:0.04;]

                            # skip redundant iterations with non-zero adaptation and zero persistence
                            if jj_ge == 0. && jj_geadapt > 0.
                                continue
                            elseif jj_geadapt != 0. && jj_civvalue != 1.
                                continue
                            end

                            # print out the growth effects to track progress
                            print(string("Rho = ", jj_ge, " ge_adapt = ", jj_geadapt))

                            # define the output directory for the Monte Carlo files
                            dir_MCoutput = string(dir_output, "montecarlo-singleGE/scen", jj_scen, "_permafr", jj_permafr, "_seaice", jj_seaice,
                                                            "_page09", jj_page09damages, "/", "ge", jj_ge,
                                                            "_geadapt", jj_geadapt,
                                                            "_civ", jj_civvalue, "/")

                            # fix the seed and calculate the SCC using a triangular distribution collapsing to a single value
                            Random.seed!(masterseed)
                            global scc_mcs_object = get_scc_mcs(samplesize, 2020, dir_MCoutput,
                                                                    scenario=jj_scen,
                                                                    pulse_size=scc_pulse_size,
                                                                    use_permafrost=jj_permafr,
                                                                    use_seaice=jj_seaice,
                                                                    use_page09damages=jj_page09damages,
                                                                    ge_minimum=jj_ge,
                                                                    ge_maximum=jj_ge + 10^(-30),
                                                                    ge_mode=jj_ge,
                                                                    civvalue_multiplier=jj_civvalue,
                                                                    geadrate = jj_geadapt)

                            # write out the full distribution
                            writedlm(string(dir_MCoutput, "SCC_MC.csv"), scc_mcs_object, ",")

                            # write results into the data frame
                            push!(df_sccMC_singleGE, [jj_page09damages, jj_permafr, jj_seaice, jj_ge,
                                                          jj_scen, scc_pulse_size, jj_civvalue,
                                                          jj_geadapt,
                                                          mean(scc_mcs_object[:, 1]),
                                                          median(scc_mcs_object[:, 1]),
                                                          minimum(scc_mcs_object[:, 1]),
                                                          maximum(scc_mcs_object[:, 1]),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 25),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 75),
                                                          Statistics.std(scc_mcs_object[:, 1]),
                                                          Statistics.std(scc_mcs_object[:, 1]) / mean(scc_mcs_object[:, 1]),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 5),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 95),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 10),
                                                          StatsBase.percentile(scc_mcs_object[:, 1], 90),
                                                          count(scc_mcs_object .== 0.) / samplesize])

                          # clean out the MCS objects
                            scc_mcs_object = nothing

                        end
                    end
                end
            end
        end
    end
end

# remove the first placeholder row
df_sccMC_singleGE = df_sccMC_singleGE[df_sccMC_singleGE[!, :scen] .!= "-999", :]

# export the results
CSV.write(string(dir_output, "MimiPageGrowthEffectsResults_SCC_fixedGE.csv"), df_sccMC_singleGE)

### PART II: results for the EMPIRICAL persistence distribution

# create a DataFrame where results will be stored
df_sccMC = DataFrame(permafr=false, seaice=false, ge_string="-999", scen="-999",
                                  convergence=-999.,  bound=-999., eqwshare=-999.,
                                  civvalue=-999., pulse=-999., ge_adapt = -999.,
                                  emfeed = -999., switch = -999.,
                                  mean=-999., median=-999., min=-999., max=-999., perc25=-999.,
                                  perc75=-999., sd=-999., varcoeff=-999.,
                                  perc05=-999., perc95=-999., perc10=-999., perc90=-999.,
                                  share_zeroSCC = -999., count_NaN = -999)

# loop through all relevant parameter/scenario combinations and save out results and SCC summary stats
for jj_scen in ["RCP4.5 & SSP2", "RCP2.6 & SSP1", "RCP8.5 & SSP5", "RCP1.9 & SSP1"]
    for jj_gestring in ["EMPIRICAL"] # which persistence distribution to use (use "EMPIRICAL+" for a version where negative persistence draws are trimmed to zero)
        for jj_permafr in [true]
            for jj_seaice in [true]
                for jj_civvalue in [1., 10.0^20]
                    for jj_cbabs in [740.65] # the level of the subsistence boundary for per capita consumption
                        for jj_eqwshare in [0.99, 0.95, 0.999] # the share of per capita consumption until which damages are equity-weighted
                            for jj_convergence in [1., 0.] # whether or not to use the convergence system for the subsistence and equity-weighting thresholds
                                for jj_pulse in [scc_pulse_size] # the size of the CO2 pulse used for SCC calculations
                                    for jj_geadapt in [0.0:0.01:0.05;] # the decay rate of persistence over time
                                        for jj_emfeed in [1., 0.] # whether or not persistence-related GDP reductions cause proportional reductions in emissions
                                            for jj_regionswitch in [0., 1.] # whether or not persistence is only limited to Global South regions

                                        # overwrite jj_geadapt with nothing if its zero (which will triger the deterministic default value of zero)
                                        if jj_geadapt == 0.
                                            jj_geadapt = nothing
                                        end

                                        # skip undesired or infeasible combinations of parameters/scenarios
                                        if jj_scen != "RCP4.5 & SSP2" && (jj_gestring != "EMPIRICAL" || jj_permafr != true ||
                                                                            jj_seaice != true || jj_civvalue != 1. || jj_cbabs != 740.65 ||
                                                                            jj_eqwshare != 0.99 || jj_convergence != 1. || jj_pulse != scc_pulse_size ||
                                                                            jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_permafr != jj_seaice
                                            continue
                                        elseif jj_gestring != "EMPIRICAL" && (jj_cbabs != 740.65 || jj_eqwshare != 0.99 || jj_convergence != 1. ||
                                                                                jj_pulse != scc_pulse_size || jj_geadapt != nothing  || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_civvalue != 1. && (jj_cbabs != 740.65 || jj_eqwshare != 0.99 || jj_convergence != 1. ||
                                                                    jj_pulse != scc_pulse_size || jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_cbabs != 740.65 && (jj_civvalue != 1. || jj_gestring != "EMPIRICAL" || jj_eqwshare != 0.99 ||
                                                                    jj_convergence != 1. || jj_pulse != scc_pulse_size || jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_eqwshare != 0.99 && (jj_civvalue != 1. || jj_gestring != "EMPIRICAL" || jj_convergence != 1. ||
                                                                        jj_pulse != scc_pulse_size || jj_geadapt != nothing || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_convergence != 1. && (jj_civvalue != 1. || jj_gestring != "EMPIRICAL" || jj_pulse != scc_pulse_size ||
                                                                        jj_geadapt != nothing  || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_pulse != scc_pulse_size && (jj_geadapt != nothing  || jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_geadapt != nothing  && (jj_emfeed != 1. || jj_regionswitch != 0.)
                                            continue
                                        elseif jj_emfeed != 1. && jj_regionswitch != 0.
                                            continue
                                        end

                                        # set the ge_string parameters
                                        if jj_gestring == "MILD"
                                            ge_string_min = 0.
                                            ge_string_mode = 0.
                                            ge_string_max = 0.5280681 # value based on converging ratio of marginal impacts from Burke et al regression for 1Lag (temperature and precipitation)
                                            ge_use_empirical = 0.
                                        elseif jj_gestring == "MEDIUM"
                                            ge_string_min = 0.
                                            ge_string_mode = 0.5280681
                                            ge_string_max = 1.
                                            ge_use_empirical = 0.
                                        elseif jj_gestring == "EMPIRICAL" # note: for EMPIRICAL and EMPIRICAL+, the triangular will be overwritten by draws from the empirical distribution
                                            ge_string_min = 0.
                                            ge_string_mode = 0.
                                            ge_string_max = 0.0001
                                            ge_use_empirical = 1.
                                        elseif jj_gestring == "EMPIRICAL+"
                                            ge_string_min = 0.
                                            ge_string_mode = 0.
                                            ge_string_max = 0.0001
                                            ge_use_empirical = 2.
                                        end

                                        # print out the parameters to track progress
                                        print(string("Scen_", jj_scen, " Ge-mode_", jj_gestring,
                                                     " Permafr_", jj_permafr, " Seaice_", jj_seaice, " Civvalue_", jj_civvalue,
                                                     " cbabs_", jj_cbabs, " eqwshare_", jj_eqwshare,
                                                     " convergence", jj_convergence, "pulse_", jj_pulse,
                                                     " geadapt", jj_geadapt, " emfeed", jj_emfeed,
                                                     " switch", jj_regionswitch))

                                        # define the output directory for the Monte Carlo files
                                        dir_MCoutput = string(dir_output, "mc_diGE/ge", jj_gestring,
                                                                        "_scen", jj_scen,
                                                                        "_pf", jj_permafr,
                                                                        "_se", jj_seaice,
                                                                        "_co", jj_convergence, "_bd", jj_cbabs, "_eq", jj_eqwshare,
                                                                        "_ci", jj_civvalue,
                                                                        "_p", jj_pulse,
                                                                        "_gead", jj_geadapt,
                                                                        "_emfe", jj_emfeed,
                                                                        "_sw", jj_regionswitch,
                                                                         "/")

                                        # calculate the MC SCC
                                        Random.seed!(masterseed)
                                        global scc_mcs_object = get_scc_mcs(samplesize, 2020, dir_MCoutput,
                                                                            scenario=jj_scen,
                                                                            pulse_size=jj_pulse,
                                                                            use_permafrost=jj_permafr,
                                                                            use_seaice=jj_seaice,
                                                                            use_page09damages=false,
                                                                            ge_minimum=ge_string_min,
                                                                            ge_maximum=ge_string_max,
                                                                            ge_mode=ge_string_mode,
                                                                            ge_use_empirical=ge_use_empirical,
                                                                            civvalue_multiplier=jj_civvalue,
                                                                            use_convergence=jj_convergence,
                                                                            cbabs=jj_cbabs,
                                                                            eqwbound=jj_eqwshare,
                                                                            geadrate = jj_geadapt,
                                                                            emfeedback = jj_emfeed,
                                                                            ge_use_switch = jj_regionswitch
                                                                            )

                                        # write out the full distribution
                                        writedlm(string(dir_MCoutput, "SCC_MC.csv"),
                                                                scc_mcs_object, ",")

                                        # create a NaN-free version of the SCC distribution
                                        scc_mcs_object = filter(!isnan, scc_mcs_object)

                                        # push the results into the data frame
                                        push!(df_sccMC, [jj_permafr, jj_seaice, jj_gestring,  jj_scen,
                                                        jj_convergence,  jj_cbabs, jj_eqwshare,
                                                        jj_civvalue, jj_pulse,
                                                        ifelse(jj_geadapt == nothing, 0., jj_geadapt),
                                                        jj_emfeed, jj_regionswitch,
                                                        mean(scc_mcs_object[:, 1]),
                                                        median(scc_mcs_object[:, 1]),
                                                        minimum(scc_mcs_object[:, 1]),
                                                        maximum(scc_mcs_object[:, 1]),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 25),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 75),
                                                        Statistics.std(scc_mcs_object[:, 1]),
                                                        Statistics.std(scc_mcs_object[:, 1]) / mean(scc_mcs_object[:, 1]),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 5),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 95),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 10),
                                                        StatsBase.percentile(scc_mcs_object[:, 1], 90),
                                                        count(scc_mcs_object .== 0.) / length(scc_mcs_object),
                                                        samplesize - length(scc_mcs_object)])

                                        # clean out the scc_mcs_object
                                        global scc_mcs_object = nothing
                                        ge_string_min = nothing
                                        ge_string_max = nothing
                                        ge_string_mode = nothing

                                            end
                                        end
                                    end
                                end
                            end
                        end
                    end
                end
            end
        end
    end
end

# remove the first placeholder row
df_sccMC = df_sccMC[df_sccMC[!, :scen] .!= "-999", :]

# export the results
CSV.write(string(dir_output, "MimiPageGrowthEffectsResultsMonteCarlo.csv"), df_sccMC)

### PART III: run the default scenario/parameter settings with an empirical persistence distribution derived from low-pass filtering climate vars

# rerun the main settings for an empirical distribution that is derived from low-pass filtering climate variables
include("main_model_growth_lowpass.jl")

# set the output directory for MC results
dir_MCoutput = string(dir_output, "lowpassfiltered_persistence/")

# calculate the stochastic mean SCC using the default parameter & scenario settings
Random.seed!(masterseed)
global scc_mcs_object = get_scc_mcs(samplesize, 2020, dir_MCoutput,
                                    scenario="RCP4.5 & SSP2",
                                    pulse_size=scc_pulse_size,
                                    use_permafrost=true,
                                    use_seaice=true,
                                    use_page09damages=false,
                                    ge_minimum=0.,
                                    ge_maximum=0.0001,
                                    ge_mode=0.,
                                    ge_use_empirical=1.,
                                    civvalue_multiplier=1.,
                                    use_convergence= 1.,
                                    cbabs=740.65,
                                    eqwbound=0.99,
                                    geadrate = 0.,
                                    emfeedback = 1.,
                                    ge_use_switch = 0.
                                    )

# # write out the full distribution
writedlm(string(dir_MCoutput, "SCC_MC.csv"),
                        scc_mcs_object, ",")

# clean out
global scc_mcs_object = nothing


function place_sell_order_command(k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    t = try Pinguu.examine_order(closed_orders[id_str], id_str, client, config)
    catch
        return nothing
    end
    qty = floor(t["use_vol"], digits=config["precision"])
    price = round((abs(t["current_profit"]) / qty *
        (1 + fees["maker"] +
            parse(Float64, config["take_profit"])/100)
            * t["price_correction"]), digits=config["sell_precision"])
    price = Pinguu.price2string(price, config["sell_precision"])

    account = client.get_account()["balances"];
    new = 0
    for bal in account
        if bal["asset"] == config["farm_coin"]
            if parse(Float64, bal["free"]) <= qty
                new = parse(Float64, bal["free"])
            end
        end
    end
    if qty > new && new != 0
        reference = qty * parse(Float64, price)
        qty = floor(new, digits=config["precision"])
        price = Pinguu.price2string(reference / qty, config["sell_precision"])
    end
    @info("Try to place sell")
    @info("Qty  = $qty")
    @info("Price = " * price)

    order = try client.order_limit_sell(
        symbol=config["symbol"],
        quantity=qty,
        price=price);
        catch
        nothing
    end
    if order !== nothing
        order["time"] = string(now())
        open_orders["SELL"][k] = order
        @info("Success")
    else
        v["fail"] = 1
        @info("Failed.. Placing another trigger")
    end
    open(joinpath(bot_path, "open_orders.json"), "w") do f
        JSON.print(f, open_orders, 4)
    end
    open(joinpath(bot_path, "trigger.json"), "w") do f
        JSON.print(f, Dict(), 4)
    end
end
export place_sell_order_command

function sell_filled(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    #
    ## Check if id is in closed_orders
    !(id_str in keys(closed_orders)) ? closed_orders[id_str] = Dict() : ""
    tmp["time"] = string(now())
    #
    ## Delete Sell from open_orders
    delete!(open_orders["SELL"], k)
    #
    ## Cancel all related open BUYs
    for (k2, v2) in open_orders["BUY"]
        if split(k2, "_")[end] == id_str
            sleep(0.25)
            #
            ## Load order status
            tmp_buy = try client.get_order(
                symbol=config["symbol"],
                orderId=v2["orderId"])
            catch
                nothing
            end
            #
            ## Update open/closed orders
            if tmp_buy !== nothing
                if !(tmp_buy["status"] in [client.ORDER_STATUS_FILLED, client.ORDER_STATUS_CANCELED])
                    sleep(0.25)
                    result = try client.cancel_order(
                        symbol=config["symbol"],
                        orderId=v2["orderId"])
                    catch
                        nothing
                    end
                    if result !== nothing
                        delete!(open_orders["BUY"], k2)
                    else
                        open_orders["BUY"][k2]["status"] = "cancel"
                    end
                else
                    delete!(open_orders["BUY"], k2)
                end
                if parse(Float64, tmp_buy["executedQty"]) > 0
                    closed_orders[id_str][k2] = tmp_buy
                end
            else
                open_orders["BUY"][k2]["status"] = "cancel"
            end
        end
    end
    #
    ## Write to files
    open(joinpath(bot_path, "closed_orders.json"), "w") do f
        JSON.print(f, closed_orders, 4)
    end
    open(joinpath(bot_path, "open_orders.json"), "w") do f
        JSON.print(f, open_orders, 4)
    end
    qty = 0     # Totel volume
    net_qty = 0 # Total worth
    val = 0     # Value in base coin
    for (k2,v2) in closed_orders[id_str]
        v2["side"] == "BUY" ? qty += parse(Float64, v2["executedQty"]) : ""
        v2["side"] == "BUY" ? net_qty += parse(Float64, v2["executedQty"]) : net_qty -= parse(Float64, v2["executedQty"])
        v2["side"] == "BUY" ? val -= parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 + fees["taker"]) : val += parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 - fees["maker"])
    end
    #
    ## Check if this was the only SELL
    if val < 0 && val + parse(Float64, tmp["cummulativeQuoteQty"]) < 0
        #
        ## If there was an extra SELL executed
        rnd_id = "SELL-" * lpad(rand(1:1:1000), 6, "0")
        while rnd_id in keys(closed_orders[id_str])
            rnd_id = "SELL-" * lpad(rand(1:1:1000), 6, "0")
        end
        closed_orders[id_str][rnd_id] = deepcopy(tmp);
        open(joinpath(bot_path, "closed_orders.json"), "w") do f
            JSON.print(f, closed_orders, 4)
        end
        #
        ## Create new sell
        if abs(val) > config["min_qty"]
            #
            ## qty is high enough to be sold
            new_qty = floor(net_qty, digits=config["precision"])
            new_price = Pinguu.price2string(abs(val)/net_qty * (1 + parse(Float64, config["take_profit"])/100), config["sell_precision"], round_up=true)
            sleep(0.2)
            order = try client.order_limit_sell(
                symbol=config["symbol"],
                quantity=new_qty,
                price=new_price);
                catch
                nothing
            end
            new_id = "SELL_" * id_str
            if order !== nothing
                order["time"] = string(now())
                open_orders["SELL"][new_id] = order
                open(joinpath(bot_path, "open_orders.json"), "w") do f
                    JSON.print(f, open_orders, 4)
                end
                #
                ## Place new buy
                extra_iteration = 0
                for (k2,v2) in closed_orders[id_str]
                    if length(split(k2, "SELL")) < 2
                        if parse(Int64, split(split(k2, "_")[1], "-")[end]) > extra_iteration
                            extra_iteration = parse(Int64, split(split(k2, "_")[1], "-")[end])
                        end
                    end
                end
                extra_iteration += 1
                if extra_iteration <= parse(Float64, config["extra_order_count"])
                    new_price = Pinguu.price2string(parse(Float64, new_price) * (1 - config["extra_order_step"]), config["sell_precision"])
                    quant = floor(config["order_volume"] * (config["extra_order_martingale"])^(extra_iteration + 1) / parse(Float64, new_price), digits=config["precision"])

                    new_buy = Dict()
                    new_buy["status"] = "place order"
                    new_buy["origQty"] = quant
                    new_buy["price"] = new_price
                    new_buy_id = "EXTRA-" * string(extra_iteration) * "_" * id_str
                    open_orders["BUY"][new_buy_id] = new_buy
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end
                end
            else
                #
                ## If new sell could not be placed
                open_orders["SELL"][new_id] = Dict()
                open_orders["SELL"][new_id]["origQty"] = new_quant
                open_orders["SELL"][new_id]["price"]   = new_price
                open_orders["SELL"][new_id]["status"]  = "place order"
                open_orders["SELL"][new_id]["time"]    = string(now())
                open_orders["SELL"][new_id]["fail"]    = 0
                open_orders["SELL"][new_id]["if_fail"] = new_quant - 1/10^config["precision"]

                open(joinpath(bot_path, "open_orders.json"), "w") do f
                    JSON.print(f, open_orders, 4)
                end
            end
        else
            #
            ## qty is NOT high enough to be sold
            new_price = Pinguu.price2string(abs(val)/net_qty * (1 + parse(Float64, config["take_profit"])/100), digits=config["sell_precision"])

            extra_iteration = 0
            for (k2,v2) in closed_orders[id_str]
                if length(split(k2, "SELL")) < 2
                    if parse(Int64, split(split(k2, "_")[1], "-")[end]) > extra_iteration
                        extra_iteration = parse(Int64, split(split(k2, "_")[1], "-")[end])
                    end
                end
            end
            extra_iteration += 1
            if extra_iteration <= parse(Float64, config["extra_order_count"])
                new_price = string(round(parse(Float64, new_price) * (1 - config["extra_order_step"]), digits=config["sell_precision"]))
                quant = floor(config["order_volume"] * (config["extra_order_martingale"])^(extra_iteration + 1) / parse(Float64, new_price), digits=config["precision"])

                new_buy = Dict()
                new_buy["status"] = "place order"
                new_buy["origQty"] = quant
                new_buy["price"] = new_price
                new_buy_id = "EXTRA-" * string(extra_iteration) * "_" * id_str
                open_orders["BUY"][new_buy_id] = new_buy
                open(joinpath(bot_path, "open_orders.json"), "w") do f
                    JSON.print(f, open_orders, 4)
                end
            end
        end
    else
        #
        ## If everything went well and the profit is positive
        sell_id = !haskey(closed_orders[id_str], "SELL") ? "SELL" : "SELL-" * lpad(rand(1:1:100000), 6, "0")
        closed_orders[id_str][sell_id] = deepcopy(tmp);

        open(joinpath(bot_path, "closed_order-" * id_str * ".json"), "w") do f
            JSON.print(f, closed_orders[id_str], 4)
        end

        try for (k5,v5) in closed_orders
                if DateTime(split(k5, "--")[end]) < now() - Day(42)
                    delete!(closed_orders, k5)
                end
            end
        catch;
        end

        open(joinpath(bot_path, "closed_orders.json"), "w") do f
            JSON.print(f, closed_orders, 4)
        end
    end
end
export sell_filled

function sell_canceled(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    delete!(open_orders["SELL"], k)
    open(joinpath(bot_path, "open_orders.json"), "w") do f
        JSON.print(f, open_orders, 4)
    end
    if parse(Float64, tmp["executedQty"]) > 0
        closed_orders[id_str]["SELL-" * lpad(rand(1:1:100000), 6, "0")] = tmp
        open(joinpath(bot_path, "closed_orders.json"), "w") do f
            JSON.print(f, closed_orders, 4)
        end
    end
end
export sell_canceled

function sell_status_changed(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    open_orders["SELL"][k] = tmp
    open_orders["SELL"][k]["time"] = string(now())
    open(joinpath(bot_path, "open_orders.json"), "w") do f
        JSON.print(f, open_orders, 4)
    end
end
export sell_status_changed

function cancel_sell(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    tmp_sell = try client.cancel_order(
        symbol=config["symbol"],
        orderId=v["orderId"])
    catch
        nothing
    end
    if tmp_sell !== nothing || tmp["status"] == client.ORDER_STATUS_FILLED || tmp["status"] == client.ORDER_STATUS_CANCELED
        if parse(Float64, v2["executedQty"]) > 0
            closed_orders[id_str]["SELL-" * lpad(rand(1:1:10000), 5, "0")] = result
            open(joinpath(bot_path, "closed_orders.json"), "w") do f
                JSON.print(f, closed_orders, 4)
            end
        end
        delete!(open_orders["SELL"], k2)
    else
        @info("Cancel SELL failed")
    end
end
export cancel_sell


module MusicXML

using LightXML

@enum STEP C=1 D=3 E=5 F=6 G=8 A=10 B=12
@enum TYPE whole=1 half=2 quarter=3 eighth=4 sixteenth=5 thirtysecond=6

type Pitch
  step::STEP
  octave::Int
  alter::Int
end

type Note
  pitch::Pitch
  duration::Int
  voice::Int
  notetype::TYPE
  isrest::Bool
end

type Measure
  notes::Array{Note, 1}
end

type Part
  measures::Array{Measure, 1}
end

type MXMLTree
  parts::Array{Part, 1}
end

function parseMXMLFile(fname::String)
  xmlfile = parse_file(fname)
  return parseMXMLTree(root(xmlfile))
end

function parseMXMLTree(root::XMLElement)
  parts = Part[]
  for c1 in child_elements(root)
    if LightXML.name(c1) == "part"
      measures = Measure[]
      for c2 in child_elements(c1)
        if LightXML.name(c2) == "measure"
          notes = Note[]
          for c3 in child_elements(c2)
            if LightXML.name(c3) == "note"
              c4 = find_element(c3, "pitch")
              step = A
              octave = 0
              alter = 0
              isrest = true
              if typeof(c4) != Void
                step = eval(parse(content(find_element(c4, "step"))))
                octave = parse(content(find_element(c4, "octave")))
                alternode = find_element(c4, "alter")
                alter = typeof(alternode) == Void ? 0 : parse(content(alternode))
                isrest = false
              end
              p = Pitch(step, octave, alter)
              duration = parse(content(find_element(c3, "duration")))
              voice = parse(content(find_element(c3, "voice")))
              notetype = eval(parse(content(find_element(c3, "type"))))

              push!(notes, Note(p, duration, voice, notetype, isrest))
            end
          end
          push!(measures, Measure(notes))
        end
      end
      push!(parts, Part(measures))
    end
  end
  return MXMLTree(parts)
end

end


# type definition
immutable GenoMatrix
    x          :: SharedVector{Int8}
    xt         :: SharedVector{Int8}
    n          :: Int
    p          :: Int
    blocksize  :: Int
    tblocksize :: Int
    snpids     :: Vector{String}

    GenoMatrix(x,xt,n,p,blocksize,tblocksize,snpids) = new(x,xt,n,p,blocksize,tblocksize,snpids)
end

# additional constructors, all of which will infer snpids field from BIM file
# consequently the snpids field never appers as an argument
# first constructor uses all remaining fields of GenoMatrix as arguments
function GenoMatrix(
    filename   :: String,
    tfilename  :: String,
    n          :: Int,
    p          :: Int,
    blocksize  :: Int,
    tblocksize :: Int;
    pids       :: Vector{Int} = procs(),
)
    # find SNP ids from the corresponding BIM file
    # first create filepath to BIM file
    bimfile = filename[1:(endof(filename)-3)] * "bim"

    # specify column element types of BIM file
    #eltypes = [Int, String, Int, Int, String, String]

    # load BIM
    #df = readtable(bimfile, header = false, separator = '\t', eltypes = eltypes)

    # second column of BIM file contains the SNP ids
    #snpids = convert(Vector{String}, df[:x2]) :: Vector{String}
    snpids = convert(Vector{String}, split(chomp(readstring(`awk -F " " '{print $2}' $bimfile`)), '\n', keep=false))

    x = GenoMatrix(
        read_bedfile(filename, pids=pids),
        read_bedfile(tfilename, transpose = true, pids=pids),
        n, p, blocksize, tblocksize, snpids)

    return x
end

# another constructor for when blocksizes are not precomputed
function GenoMatrix(
    filename  :: String,
    tfilename :: String,
    n         :: Int,
    p         :: Int;
    pids      :: Vector{Int} = procs()
)
    blocksize  = ( (n-1) >>> 2) + 1
    tblocksize = ( (p-1) >>> 2) + 1
    return GenoMatrix(filename, tfilename, n, p, blocksize, tblocksize, pids=pids) :: PLINK.GenoMatrix
end

# constructor to load entirely from file
function GenoMatrix(
    filename  :: String,
    tfilename :: String;
    pids      :: Vector{Int} = procs()
)
    # find n from the corresponding FAM file
    famfile = filename[1:(endof(filename)-3)] * "fam"
    n       = countlines(famfile)

    # find p from the corresponding BIM file
    bimfile = filename[1:(endof(filename)-3)] * "bim"
    p       = countlines(bimfile)

    # blocksizes are easy to calculate
    blocksize  = ((n-1) >>> 2) + 1
    tblocksize = ((p-1) >>> 2) + 1

    # can now create GenoMatrix
    return GenoMatrix(filename, tfilename, n, p, blocksize, tblocksize, pids=pids) :: PLINK.GenoMatrix
end

# subroutines
Base.size(x::GenoMatrix) = (x.n, x.p)
Base.size(x::GenoMatrix, d::Int) = d == 1 ? x.n : x.p

Base.length(x::GenoMatrix) = x.n * x.p

Base.eltype(x::GenoMatrix) = Int8

Base.ndims(x::GenoMatrix) = 2

Base.copy(x::GenoMatrix) = GenoMatrix(x.x,x.xt,x.n,x.p,x.blocksize,x.tblocksize)

Base.IndexStyle(x::Type{GenoMatrix}) = Base.IndexLinear()

Base.procs(x::Type{GenoMatrix}) = procs(x.x)

function ==(x::GenoMatrix, y::GenoMatrix)
    x.x  == y.x  &&
    x.xt == y.xt &&
    x.n  == y.n  &&
    x.p  == y.p  &&
    x.blocksize  == y.blocksize &&
    x.tblocksize == y.tblocksize
end

isequal(x::GenoMatrix, y::GenoMatrix) = x == y

# important matrix indexing! returns the Int8
function getindex(
    X   :: GenoMatrix,
    x   :: SharedVector{Int8},
    row :: Int,
    col :: Int
)
    genotype_block = x[(col-1)*X.blocksize + ((row - 1) >>> 2) + 1]
    k = 2*((row-1) & 3)
    genotype = (genotype_block >>> k) & THREE8
    return genotype
end

# internal routine to index the transposed matrix
# used for dott()
function getindex_t(
    X   :: GenoMatrix,
    x   :: SharedVector{Int8},
    row :: Int,
    col :: Int
)
    genotype_block = x[(col-1)*X.tblocksize + ((row - 1) >>> 2) + 1]
    k = 2*((row-1) & 3)
    genotype = (genotype_block >>> k) & THREE8
    return genotype
end

# default indexing is to column-major matrix
# can use x.xt to index the transpose
getindex(x::GenoMatrix, row::Int, col::Int) = getindex(x, x.x, row, col)

"""
    read_bedfile(filename [, transpose=false, pids=procs]) -> SharedVector{Int8}

This function reads a PLINK binary file (BED) and returns a `SharedArray` of `Int8` numbers.
It discards the first three bytes ("magic numbers") since they are not needed here.

Arguments:

- `filename` is the path to the BED file

Optional Arguments:

- `transpose` indicates if the compressed matrix is transposed. Defaults to `false`.
- `pids` indicates the processes over which to distribute the output `SharedArray`. Defaults to `procs()` (all available processes).

Output:

- A vector of `Int8` numbers. For a BED file encoding `n` cases and `p` SNPs,
  there should be *at least* `(n*p/4)` numbers. The scaling factor of `4` comes from the
  compression of four genotypes into each byte. But PLINK stores each column in blocks
  of bytes instead of a continuous bitstream, which sometimes entails extra unused bits
  at the end of each block.
"""
function read_bedfile(
    filename  :: String;
    transpose :: Bool = false,
    pids      :: Vector{Int} = procs()
)

    # check that file is BED file
    @assert contains(filename, ".bed") "Filename must explicitly point to a PLINK \".bed\" file."

#    # check magic numbers and mode
#   isequal(read(xstream, Int8), MNUM1) || throw(error("Problem with first byte of magic number, is this a true BED file?"))
#   isequal(read(xstream, Int8), MNUM2) || throw(error("Problem with second byte of magic number, is this a true BED file?"))
#   (transpose && isequal(read(xstream, Int8), ONE8)) && throw(error("For transposed matrix, third byte of BED file must indicate individual-major format."))

    # how many bytes do we have?
    nbytes = filesize(filename)

    # file seems to be a true BED file
    # will close filestream and slurp entire file into SharedArray
    x = SharedArray{Int8}(abspath(filename), (nbytes-3,), 3, pids=pids) :: SharedVector{Int8}

    # return the genotypes
    return x
end

function display(x::GenoMatrix)
    println("A compressed GenoMatrix object with the following features:")
    println("\tnumber of cases        = $(x.n)")
    println("\tgenetic covariates     = $(x.p)")
end