function QRDecomposition(arg0::RealMatrix)
    return QRDecomposition((RealMatrix,), arg0)
end

function QRDecomposition(arg0::RealMatrix, arg1::jdouble)
    return QRDecomposition((RealMatrix, jdouble), arg0, arg1)
end

function get_h(obj::QRDecomposition)
    return jcall(obj, "getH", RealMatrix, ())
end

function get_q(obj::QRDecomposition)
    return jcall(obj, "getQ", RealMatrix, ())
end

function get_qt(obj::QRDecomposition)
    return jcall(obj, "getQT", RealMatrix, ())
end

function get_r(obj::QRDecomposition)
    return jcall(obj, "getR", RealMatrix, ())
end

function get_solver(obj::QRDecomposition)
    return jcall(obj, "getSolver", DecompositionSolver, ())
end


using Test
using UltraDark: Grids, PencilGrids, output_grids, OutputConfig

for grid_type in [Grids, ]
    grids = grid_type(1.0, 16)

    for (name, output_config) in [
              ("default", OutputConfig(mktempdir(), [])),
              ("nothing", OutputConfig(mktempdir(), []; box=false, slice=false, psi=true, rho=true)),
              ("box", OutputConfig(mktempdir(), []; box=true, slice=false, psi=true, rho=true)),
              ("slice", OutputConfig(mktempdir(), []; box=true, slice=true, psi=true, rho=true)),
              ("box & slice", OutputConfig(mktempdir(), []; box=true, slice=true, psi=true, rho=true)),
                         ]
        mkpath(output_config.directory)
        @test output_grids(grids, output_config, 0) == nothing
    end
end

for grid_type in [PencilGrids, ]
    grids = grid_type(1.0, 16)

    for (name, output_config) in [
              ("default", OutputConfig(mktempdir(), [])),
              ("nothing", OutputConfig(mktempdir(), []; box=false, slice=false, psi=true, rho=true)),
              ("box", OutputConfig(mktempdir(), []; box=true, slice=false, psi=true, rho=true)),
                         ]
        mkpath(output_config.directory)
        @test output_grids(grids, output_config, 0) == nothing
    end

    for (name, output_config) in [
              ("slice", OutputConfig(mktempdir(), []; box=true, slice=true, psi=true, rho=true)),
              ("box & slice", OutputConfig(mktempdir(), []; box=true, slice=true, psi=true, rho=true)),
                         ]
        mkpath(output_config.directory)
        @test_broken output_grids(grids, output_config, 0) == nothing
    end
end


# DOCTODO  add docstring for TechRenewable
struct TechRenewable <: TechnicalParams
    installedcapacity::Float64 # [MW]
    reactivepowerlimits::Union{NamedTuple{(:min, :max),Tuple{Float64,Float64}},Nothing} # [MVar]
    powerfactor::Union{Float64,Nothing} # [-1. -1]
end

# DOCTODO document this constructor for TechRenewable
TechRenewable(; InstalledCapacity = 0,
                reactivepowerlimits = nothing,
                powerfactor = nothing
              ) = TechRenewable(InstalledCapacity, reactivepowerlimits,
                                powerfactor)

"""
    TechThermal(activepower::Float64,
            activepowerlimits::NamedTuple{(:min, :max),Tuple{Float64,Float64}},
            reactivepower::Union{Float64,Nothing},
            reactivepowerlimits::Union{(min::Float64,max::Float64),Nothing},
            ramplimits::Union{NamedTuple{(:up, :down),Tuple{Float64,Float64}},Nothing},
            timelimits::Union{NamedTuple{(:min, :max),Tuple{Float64,Float64}},Nothing})

Data Structure for the economical parameters of thermal generation technologies.
    The data structure can be called calling all the fields directly or using named fields.
    Two examples [DOCTODO only one example exists below] are provided one with minimal data definition and a more comprenhensive one

    # Examples

    ```jldoctest

    julia> Tech = TechThermal(activepower = 100.0, activepowerlimits = (min = 50.0, max = 200.0))
   [ Info: 'Reactive Power' limits defined as nothing
    TechThermal:
       activepower: 100.0
       activepowerlimits: (min = 50.0, max = 200.0)
       reactivepower: nothing
       reactivepowerlimits: nothing
       ramplimits: nothing
       timelimits: nothing

"""
struct TechThermal <: TechnicalParams
    activepower::Float64 # [MW]
    activepowerlimits::NamedTuple{(:min, :max),Tuple{Float64,Float64}} # [MW]
    reactivepower::Union{Float64,Nothing} # [MVAr]
    reactivepowerlimits::Union{NamedTuple{(:min, :max),Tuple{Float64,Float64}},Nothing} # [MVAr]
    ramplimits::Union{NamedTuple{(:up, :down),Tuple{Float64,Float64}},Nothing} #MW/Hr
    timelimits::Union{NamedTuple{(:up, :down),Tuple{Float64,Float64}},Nothing} #Hr
    function TechThermal(activepower, activepowerlimits, reactivepower, reactivepowerlimits, ramplimits, timelimits)

        new(activepower, PowerSystems.orderedlimits(activepowerlimits, "Real Power"), reactivepower, PowerSystems.orderedlimits(reactivepowerlimits, "Reactive Power"), ramplimits, timelimits)

    end
end

# DOCTODO document this constructor
TechThermal(; activepower = 0.0,
            activepowerlimits = (min = 0.0, max = 0.0),
            reactivepower = nothing,
            reactivepowerlimits = nothing,
            ramplimits = nothing,
            timelimits = nothing
            ) = TechThermal(activepower, activepowerlimits, reactivepower,
                            reactivepowerlimits, ramplimits, timelimits)


"""
    Interface to SNOPT. Modified from official repository
    (https://github.com/snopt/SNOPT7.jl) to have increased workspace size.
    SNOPT license is required.
"""

const libsnopt7 = "libsnopt7"

mutable struct snoptWorkspace
    status::Int

    # Workspace
    leniw::Int
    lenrw::Int
    iw::Vector{Int32}
    rw::Vector{Float64}

    leniu::Int
    lenru::Int
    iu::Vector{Int32}
    ru::Vector{Float64}

    x::Vector{Float64}
    lambda::Vector{Float64}
    obj_val::Float64

    num_inf::Int
    sum_inf::Float64

    iterations::Int
    major_itns::Int
    run_time::Float64

    function snoptWorkspace(leniw::Int,lenrw::Int)
        prob = new(0,leniw, lenrw,
                   zeros(Int,leniw), zeros(Float64,lenrw), 0, 0)
        finalizer(freeWorkspace!,prob)
        prob
    end
end

function freeWorkspace!(prob::snoptWorkspace)
    ccall((:f_snend, libsnopt7),
          Cvoid, (Ptr{Cint}, Cint, Ptr{Float64}, Cint),
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
end

# Exit codes
SNOPT_status = Dict(
    1=>:Solve_Succeeded,
    2=>:Feasible_Point_Found,
    3=>:Solved_To_Acceptable_Level,
    4=>:Solved_To_Acceptable_Level,
    5=>:Solved_To_Acceptable_Level,
    6=>:Solved_To_Acceptable_Level,
    11=>:Infeasible_Problem_Detected,
    12=>:Infeasible_Problem_Detected,
    13=>:Infeasible_Problem_Detected,
    14=>:Infeasible_Problem_Detected,
    15=>:Infeasible_Problem_Detected,
    16=>:Infeasible_Problem_Detected,
    21=>:Unbounded_Problem_Detected,
    22=>:Unbounded_Problem_Detected,
    31=>:Maximum_Iterations_Exceeded,
    32=>:Maximum_Iterations_Exceeded,
    34=>:Maximum_CpuTime_Exceeded,
    41=>:Numerical_Difficulties,
    42=>:Numerical_Difficulties,
    43=>:Numerical_Difficulties,
    44=>:Numerical_Difficulties,
    45=>:Numerical_Difficulties,
    51=>:User_Supplied_Function_Error,
    52=>:User_Supplied_Function_Error,
    56=>:User_Supplied_Function_Error,
    61=>:User_Supplied_Function_Undefined,
    62=>:User_Supplied_Function_Undefined,
    63=>:User_Supplied_Function_Undefined,
    71=>:User_Requested_Stop,
    72=>:User_Requested_Stop,
    73=>:User_Requested_Stop,
    74=>:User_Requested_Stop,
    81=>:Insufficient_Memory,
    82=>:Insufficient_Memory,
    83=>:Insufficient_Memory,
    91=>:Invalid_Problem_Definition,
    92=>:Invalid_Problem_Definition,
    999=>:Internal_Error)

# Callbacks
function obj_wrapper!(mode_::Ptr{Cint}, nnobj::Cint, x_::Ptr{Float64},
                      f_::Ptr{Float64}, g_::Ptr{Float64},
                      status::Cint)
    x    = unsafe_wrap(Array, x_, Int(nnobj))
    mode = unsafe_load(mode_)

    if mode == 0 || mode == 2
        obj = convert(Float64, eval_f(x)) :: Float64
        unsafe_store!(f_,obj)
    end

    if mode == 1 || mode == 2
        g = unsafe_wrap(Array,g_,Int(nnobj))
        eval_grad_f(x,g)
    end
    return
end

function con_wrapper!(mode_::Ptr{Cint}, nncon::Cint, nnjac::Cint, negcon::Cint,
                      x_::Ptr{Float64}, c_::Ptr{Float64}, J_::Ptr{Float64},
                      status::Cint)

    x    = unsafe_wrap(Array, x_, Int(nnjac))
    mode = unsafe_load(mode_)

    if mode == 0 || mode == 2
        c = unsafe_wrap(Array, c_, Int(nncon))
        eval_g(x, c)
    end

    if mode == 1 || mode == 2
        J = unsafe_wrap(Array,J_,Int(negcon))
        eval_jac_g(x, J)
    end
    return
end

# SNOPT7 routines
function initialize(printfile::String, summfile::String)
    prob = snoptWorkspace(30500,3000)

    ccall((:f_sninitx, libsnopt7), Cvoid,
          (Ptr{UInt8}, Cint, Ptr{UInt8}, Cint,
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          printfile, length(printfile), summfile, length(summfile),
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
    return prob
end

function initialize(printfile::String, summfile::String,
                    leniw::Int, lenrw::Int)
    prob = snoptWorkspace(leniw, lenrw)

    ccall((:f_sninitx, libsnopt7), Cvoid,
          (Ptr{UInt8}, Cint, Cint,
           Ptr{UInt8}, Cint, Cint,
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          printfile, plen, prob.iprint, summfile, slen, prob.isumm,
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
    return prob
end

# make larger workspace
function initialize(printfile::String, summfile::String,ws::Tuple)
    prob = snoptWorkspace(ws[1],ws[2])

    ccall((:f_sninitx, libsnopt7), Cvoid,
          (Ptr{UInt8}, Cint, Ptr{UInt8}, Cint,
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          printfile, length(printfile), summfile, length(summfile),
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
    return prob
end

function readOptions(prob::snoptWorkspace, specsfile::String)
    status = [0]
    ccall((:f_snspecf, libsnopt7), Cvoid,
          (Ptr{UInt8}, Cint, Ptr{Cint},
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          specsfile, length(specsfile), status,
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
    prob.status = status[1]
    return Int(prob.status)
end

function snopt!(prob::snoptWorkspace, start::String, name::String,
                m::Int, n::Int, nnCon::Int, nnObj::Int, nnJac::Int,
                fObj::Float64, iObj::Int,
                confun::Function, objfun::Function,
                #eval_f::Function, eval_grad_f::Function,
                #eval_g::Function, eval_jac_g::Function,
                J::SparseMatrixCSC, bl::Vector{Float64}, bu::Vector{Float64},
                hs::Vector{Int}, x::Vector{Float64})

    @assert n+m == length(x) == length(bl) == length(bu)
    @assert n+m == length(hs)

    prob.iu = [0]
    prob.ru = [0.]

    prob.x      = copy(x)
    prob.lambda = zeros(Float64,n+m)
    pi          = zeros(Float64,m)

    obj_callback = @cfunction($objfun, Cvoid,
                             (Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble},
                              Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cint}))
    con_callback = @cfunction($confun, Cvoid,
                             (Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble},
                              Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cint}))

    valJ = J.nzval
    indJ = convert(Array{Cint}, J.rowval)
    locJ = convert(Array{Cint}, J.colptr)
    neJ  = length(valJ)

    status  = [0]
    nS      = [0]
    nInf    = [0]
    sInf    = [0.0]
    obj_val = [0.0]
    miniw   = [0]
    minrw   = [0]

    ccall((:f_snoptb, libsnopt7), Cvoid,
          (Ptr{UInt8}, Ptr{UInt8},
           Cint, Cint, Cint, Cint, Cint, Cint,
           Cint, Cdouble,
           Ptr{Cvoid}, Ptr{Cvoid},
           Ptr{Cdouble}, Ptr{Cint}, Ptr{Cint},
           Ptr{Float64}, Ptr{Float64}, Ptr{Cint},
           Ptr{Float64}, Ptr{Float64}, Ptr{Float64},
           Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble}, Ptr{Cdouble},
           Ptr{Cint}, Ptr{Cint},
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint,
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          start, name, m, n, neJ, nnCon, nnObj, nnJac,
          iObj, fObj,
          con_callback, obj_callback,
          valJ, indJ, locJ,
          bl, bu, hs, prob.x, pi, prob.lambda,
          status, nS, nInf, sInf, obj_val,
          miniw, minrw,
          prob.iu, prob.leniu, prob.ru, prob.lenru,
          prob.iw, prob.leniw, prob.rw, prob.lenrw)

    prob.status  = status[1]
    prob.obj_val = obj_val[1]

    prob.num_inf = nInf[1]
    prob.sum_inf = sInf[1]

    prob.iterations = prob.iw[421]
    prob.major_itns = prob.iw[422]

    prob.run_time   = prob.rw[462]

    return Int(prob.status)
end

function setOption!(prob::snoptWorkspace, optstring::String)
    # Set SNOPT7 option via string
    if !isascii(optstring)
        error("SNOPT7: Non-ASCII parameters not supported")
    end

    errors = [0]
    ccall((:f_snset, libsnopt7), Cvoid,
          (Ptr{UInt8}, Cint, Ptr{Cint},
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          optstring, length(optstring), errors,
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
    return errors[1]
end

function setOption!(prob::snoptWorkspace, keyword::String, value::Int)
    # Set SNOPT7 integer option
    if !isascii(keyword)
        error("SNOPT7: Non-ASCII parameters not supported")
    end

    errors = [0]
    ccall((:f_snseti, libsnopt7), Cvoid,
          (Ptr{UInt8}, Cint, Cint, Ptr{Cint},
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          optstring, length(optstring), value, errors,
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
    return errors[1]
end

function setOption!(prob::snoptWorkspace, keyword::String, value::Float64)
    # Set SNOPT7 real option
    if !isascii(keyword)
        error("SNOPT7: Non-ASCII parameters not supported")
    end

    errors = [0]
    ccall((:f_snseti, libsnopt7), Cvoid,
          (Ptr{UInt8}, Cint, Cdouble, Ptr{Cint},
           Ptr{Cint}, Cint, Ptr{Cdouble}, Cint),
          optstring, length(optstring), value, errors,
          prob.iw, prob.leniw, prob.rw, prob.lenrw)
    return errors[1]
end

mutable struct VariableData
    lower::Float64
    upper::Float64
    x::Float64
    g::Vector{Float64}
end
VariableData() = VariableData(-Inf, Inf, 0.0, [])

mutable struct Optimizer <: MOI.AbstractOptimizer
    sense::MOI.OptimizationSense
    objective::Union{MOI.SingleVariable,MOI.ScalarAffineFunction{Float64},MOI.ScalarQuadraticFunction{Float64},Nothing}
    variables::Vector{VariableData}
    linear_constraints::Vector{VariableData}
    nonlin::Union{MOI.NLPBlockData,Nothing}

    n::Int
    nncon::Int
    nlcon::Int

    workspace::Union{snoptWorkspace,Nothing}
    options::Dict{String, Any}
end
Optimizer(;options...) = Optimizer(MOI.MIN_SENSE, nothing, [], [], nothing, 0, 0, 0, nothing, options)

function MOI.empty!(model::Optimizer)
    model.sense = MOI.MIN_SENSE
    model.objective = nothing
    empty!(model.variables)
    empty!(model.linear_constraints)
    model.nonlin = nothing
    model.n = 0
    model.nncon = 0
    model.nlcon = 0
end

function MOI.is_empty(model::Optimizer)
    return model.objective == nothing &&
           isempty(model.variables) &&
           isempty(model.linear_constraints) &&
           model.nonlin == nothing &&
           model.n == 0 && model.nncon == 0 && model.nlcon == 0
end

function MOI.copy_to(model::Optimizer, src::MOI.ModelLike; copy_names = false)
    return MOI.Utilities.default_copy_to(model, src, copy_names)
end

# MOI objects that SNOPT supports
MOI.supports(::Optimizer, ::MOI.NLPBlock) = true
MOI.supports(::Optimizer, ::MOI.ObjectiveFunction{MOI.SingleVariable}) = true
MOI.supports(::Optimizer, ::MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}) = true
MOI.supports(::Optimizer, ::MOI.ObjectiveFunction{MOI.ScalarQuadraticFunction{Float64}}) = true
MOI.supports(::Optimizer, ::MOI.ObjectiveSense) = true
MOI.supports(::Optimizer, ::MOI.VariablePrimalStart,::Type{MOI.VariableIndex}) = true  #?
MOI.supports(::Optimizer, ::MOI.RawParameter) = true

MOI.supports_constraint(::Optimizer, ::Type{MOI.SingleVariable}, ::Type{MOI.LessThan{Float64}}) = true
MOI.supports_constraint(::Optimizer, ::Type{MOI.SingleVariable}, ::Type{MOI.GreaterThan{Float64}}) = true
MOI.supports_constraint(::Optimizer, ::Type{MOI.SingleVariable}, ::Type{MOI.EqualTo{Float64}}) = true
MOI.supports_constraint(::Optimizer, ::Type{MOI.SingleVariable}, ::Type{MOI.Interval{Float64}}) = true

MOI.supports_constraint(::Optimizer, ::Type{MOI.ScalarAffineFunction{Float64}}, ::Type{MOI.LessThan{Float64}}) = true
MOI.supports_constraint(::Optimizer, ::Type{MOI.ScalarAffineFunction{Float64}}, ::Type{MOI.GreaterThan{Float64}}) = true
MOI.supports_constraint(::Optimizer, ::Type{MOI.ScalarAffineFunction{Float64}}, ::Type{MOI.EqualTo{Float64}}) = true
MOI.supports_constraint(::Optimizer, ::Type{MOI.ScalarAffineFunction{Float64}}, ::Type{MOI.Interval{Float64}}) = true

MOI.get(model::Optimizer, ::MOI.NumberOfVariables) = length(model.variables)

function MOI.set(model::Optimizer, p::MOI.RawParameter, value)
    model.options[p.name] = value
    return
end

function MOI.set(model::Optimizer, ::MOI.ObjectiveSense, sense::MOI.OptimizationSense)
    model.sense = sense
    return
end

# Linear/quadratic objective
function MOI.set(model::Optimizer, ::MOI.ObjectiveFunction,
                 func::Union{MOI.SingleVariable, MOI.ScalarAffineFunction, MOI.ScalarQuadraticFunction})
    model.objective = func
    return
end

# Variables
function MOI.set(model::Optimizer, ::MOI.VariablePrimalStart, vi::MOI.VariableIndex, value::Float64)
    model.variables[vi.value].x = value
    return
end

function MOI.add_variable(model::Optimizer)
    push!(model.variables, VariableData())
    return MOI.VariableIndex(length(model.variables))
end

function MOI.add_variables(model::Optimizer, n::Int)
    return [MOI.add_variable(model) for i in 1:n]
end

function MOI.add_constraint(model::Optimizer, v::MOI.SingleVariable, lt::MOI.LessThan{Float64})
    # Upper bound on x
    vi = v.variable
    model.variables[vi.value].upper = lt.upper
    return MOI.ConstraintIndex{MOI.SingleVariable, MOI.LessThan{Float64}}(vi.value)
end

function MOI.add_constraint(model::Optimizer, v::MOI.SingleVariable, gt::MOI.GreaterThan{Float64})
    # Lower bound on x
    vi = v.variable
    model.variables[vi.value].lower = gt.lower
    return MOI.ConstraintIndex{MOI.SingleVariable, MOI.GreaterThan{Float64}}(vi.value)
end

function MOI.add_constraint(model::Optimizer, v::MOI.SingleVariable, eq::MOI.EqualTo{Float64})
    # Fixed variable
    vi = v.variable
    model.variables[vi.value].lower = eq.value
    model.variables[vi.value].upper = eq.value
    return MOI.ConstraintIndex{MOI.SingleVariable, MOI.EqualTo{Float64}}(vi.value)
end

function MOI.add_constraint(model::Optimizer, v::MOI.SingleVariable, bds::MOI.Interval{Float64})
    # Lower/upper bounds on x
    vi = v.variable
    model.variables[vi.value].lower = bds.lower
    model.variables[vi.value].upper = bds.upper
    return MOI.ConstraintIndex{MOI.SingleVariable, MOI.Interval{Float64}}(vi.value)
end

# Linear constraints
macro define_add_constraint(set_type)
    quote
        function MOI.add_constraint(model::Optimizer, v::MOI.ScalarAffineFunction{Float64},
                                    set::$set_type)
            # Bounds on constraint
            x = VariableData()
            if typeof(set) == MOI.LessThan{Float64}
                x.upper = set.upper - v.constant
            elseif typeof(set) == MOI.GreaterThan{Float64}
                x.lower = set.lower - v.constant
            elseif typeof(set) == MOI.EqualTo{Float64}
                x.lower = set.value - v.constant
                x.upper = set.value - v.constant
            elseif typeof(set) == MOI.Interval{Float64}
                x.lower = set.lower - v.constant
                x.upper = set.upper - v.constant
            end

            x.g = [ t.coefficient for t in v.terms ]

            push!(model.linear_constraints, x)
            return MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64},$set_type}(length(model.linear_constraints))
        end
    end
end
@define_add_constraint(MOI.LessThan{Float64})
@define_add_constraint(MOI.GreaterThan{Float64})
@define_add_constraint(MOI.EqualTo{Float64})
@define_add_constraint(MOI.Interval{Float64})

# Nonlinear constraints
function MOI.set(model::Optimizer, ::MOI.NLPBlock, nonlin::MOI.NLPBlockData)
    model.nonlin = nonlin
    return
end

# Solve
function MOI.optimize!(model::Optimizer;
        ws::Tuple=(),outfile::String="snopt.out")

    n     = length(model.variables)
    nncon = model.nonlin !== nothing ? length(model.nonlin.constraint_bounds) : 0
    nlcon = length(model.linear_constraints)

    m     = nncon + length(model.linear_constraints)

    model.n = n
    model.nncon = nncon
    model.nlcon = nlcon

    # Bounds/initial point
    #    Variables
    bl = [v.lower for v in model.variables]
    bu = [v.upper for v in model.variables]
    x  = [v.x     for v in model.variables]
    #    Nonlinear constraints
    if nncon > 0
        for bound in model.nonlin.constraint_bounds
            push!(bl, bound.lower)
            push!(bu, bound.upper)
            push!(x,  0.0)
        end
    end

    #    Linear constraints
    for v in model.linear_constraints
        push!(bl, v.lower)
        push!(bu, v.upper)
        push!( x, v.x)
    end

    # Nonlinear data
    if nncon > 0
        features = MOI.features_available(model.nonlin.evaluator)
        has_objgrad = (:Grad in features)
        has_jacobian = (:Jac in features)

        MOI.initialize(model.nonlin.evaluator, features)

        # Jacobian
        sparseJ = MOI.jacobian_structure(model.nonlin.evaluator)  # returns tuple of coordinates
        nnjac = n

        j_rows = [sparseJ[i][1] for i in 1:length(sparseJ)]
        j_cols = [sparseJ[i][2] for i in 1:length(sparseJ)]
        j_vals = zeros(length(sparseJ))
    else
        nnjac  = 0
        j_rows = []
        j_cols = []
        j_vals = []
    end

    #   Add linear constraint Jacobian
    i = nncon
    for v in model.linear_constraints
        i = i+1
        for j in 1:n
            push!(j_rows, i)
            push!(j_cols, j)
            push!(j_vals, v.g[j])
        end
    end
    J = sparse(j_rows, j_cols, j_vals)

    # Minimize or maximize?

    if model.nonlin !== nothing
        # Nonlinear constraints/objective
        nnobj = n

        # Callbacks
        function eval_obj(mode_::Ptr{Cint}, n_::Ptr{Cint}, x_::Ptr{Float64},
                          f_::Ptr{Float64}, g_::Ptr{Float64}, status::Ptr{Cint})
            mode = unsafe_load(mode_)
            n    = unsafe_load(n_)

            x    = unsafe_wrap(Array, x_, Int(n))

            if mode == 0 || mode == 2
                if model.nonlin.has_objective
                    obj = MOI.eval_objective(model.nonlin.evaluator, x)
                elseif model.objective !== nothing
                    obj = eval_function(model.objective, x)
                else
                    obj = 0.0
                end
                unsafe_store!(f_,obj)
            end

            if mode == 1 || mode == 2
                g = unsafe_wrap(Array,g_, Int(n))
                if model.nonlin.has_objective
                    MOI.eval_objective_gradient(model.nonlin.evaluator, g, x)
                end
            end
            return
        end

        function eval_con(mode_::Ptr{Cint},
                          nncon_::Ptr{Cint}, nnjac_::Ptr{Cint}, negcon_::Ptr{Cint},
                          x_::Ptr{Float64}, c_::Ptr{Float64}, J_::Ptr{Float64},
                          status::Ptr{Cint})
            mode   = unsafe_load(mode_)
            nncon  = unsafe_load(nncon_)
            nnjac  = unsafe_load(nnjac_)
            negcon = unsafe_load(negcon_)

            x = unsafe_wrap(Array, x_, Int(nnjac))
            if mode == 0 || mode == 2
                c = unsafe_wrap(Array, c_, Int(nncon))
                MOI.eval_constraint(model.nonlin.evaluator, c, x)
            end

            if mode == 1 || mode == 2
                J = unsafe_wrap(Array,J_, Int(negcon))

                # MOI returns Jacobian in order specified by j_rows, j_cols
                # SNOPT needs it in sparse-by-col format
                JJ = zeros(negcon)
                MOI.eval_constraint_jacobian(model.nonlin.evaluator, JJ, x)
                vals = sparse(j_rows,j_cols,JJ).nzval
                for k in 1:negcon
                    J[k] = vals[k]
                end
            end
            return
        end

        # SNOPT
        fobj = 0.0
        iobj = 0
        hs   = zeros(Int,n+m)

        # make larger workspace
        if ws == ()
            # set workspace size (10x recommended by manual)
            ws = (1000*(n+m),2000*(n+m))
        end
        model.workspace = SNOPT7.initialize(outfile,"screen",ws)

        # model.workspace = SNOPT7.initialize("snopt.out","screen")

        for (name,value) in model.options
            # Replace underscore with space
            keyword = replace(String(name), "_" => " ")
            SNOPT7.setOption!(model.workspace, "$keyword $value")
        end
        SNOPT7.setOption!(model.workspace, "Summary frequency 1")
        SNOPT7.setOption!(model.workspace, "Print frequency 1")

        start = "Cold"
        name  = "prob"
        status = SNOPT7.snopt!(model.workspace, start, name,
                              m, n, nncon, nnobj, nnjac,
                              fobj, iobj, eval_con, eval_obj,
                              J, bl, bu, hs, x)
        SNOPT7.freeWorkspace!(model.workspace)

    else
        # Linear/quadratic objective

    end

end

# Return info
function MOI.get(model::Optimizer, ::MOI.TerminationStatus)
    if model.workspace === nothing
        return MOI.NO_SOLUTION
    end

    status = SNOPT_status[model.workspace.status]
    if status in (:Solve_Succeeded,
                  :Feasible_Point_Found)
        return MOI.LOCALLY_SOLVED
    elseif status ==:Infeasible_Problem_Detected
        return MOI.LOCALLY_INFEASIBLE
    elseif status ==:Unbounded_Problem_Detected
        return MOI.INFEASIBLE_OR_UNBOUNDED
    elseif status == :Solved_To_Acceptable_Level
        return MOI.ALMOST_LOCALLY_SOLVED
    elseif status == :User_Requested_Stop
        return MOI.INTERRUPTED
    elseif status == :Maximum_Iterations_Exceeded
        return MOI.ITERATION_LIMIT
    elseif status == :Maximum_CpuTime_Exceeded
        return MOI.TIME_LIMIT
    elseif status == (:Invalid_Problem_Definition,
                      :User_Supplied_Function_Error,
                      :User_Supplied_Function_Undefined)
        return MOI.INVALID_MODEL
    elseif status == :Insufficient_Memory
        return MOI.MEMORY_LIMIT
    elseif status == :Numerical_Difficulties
        return MOI.NUMERICAL_ERROR
    else
        error("Unrecognized SNOPT7 status $status")
    end
end

function MOI.get(model::Optimizer, ::MOI.ResultCount)
    return (model.workspace !== nothing) ? 1 : 0
end

function MOI.get(model::Optimizer, ::MOI.PrimalStatus)
    if model.workspace === nothing
        return MOI.NO_SOLUTION
    end
    status = SNOPT_status[model.workspace.status]
    if status == :Solve_Succeeded
        return MOI.FEASIBLE_POINT
    elseif status == :Feasible_Point_Found
        return MOI.FEASIBLE_POINT
    elseif status == :Solved_To_Acceptable_Level
        return MOI.NEARLY_FEASIBLE_POINT
    elseif status == :Infeasible_Problem_Detected
        return MOI.INFEASIBLE_POINT
    else
        return MOI.UNKNOWN_RESULT_STATUS
    end
end

function MOI.get(model::Optimizer, ::MOI.DualStatus)
    if model.workspace === nothing
        return MOI.NO_SOLUTION
    end
    status = SNOPT_status[model.workspace.status]
    if status == :Solve_Succeeded
        return MOI.FEASIBLE_POINT
    elseif status == :Feasible_Point_Found
        return MOI.FEASIBLE_POINT
    elseif status == :Solved_To_Acceptable_Level
        return MOI.NEARLY_FEASIBLE_POINT
    elseif status == :Infeasible_Problem_Detected
        return MOI.UNKNOWN_RESULT_STATUS # unbounded?
    else
        return MOI.UNKNOWN_RESULT_STATUS
    end
end

function MOI.get(model::Optimizer, ::MOI.ObjectiveValue)
    if model.workspace === nothing
        @error("ObjectiveValue not available.")
    end
    return model.workspace.obj_val
end

function MOI.get(model::Optimizer, ::MOI.VariablePrimal, vi::MOI.VariableIndex)
    if model.workspace === nothing
        @error("VariablePrimal not available.")
    end
    return model.workspace.x[vi.value]
end

function MOI.get(model::Optimizer, ::MOI.ConstraintPrimal,
                 ci::MOI.ConstraintIndex{MOI.SingleVariable,S}) where {S}
    if model.workspace === nothing
        @error("ConstraintPrimal not available.")
    end
    vi = MOI.VariableIndex(ci.value)
    return model.workspace.x[vi.value]
end

macro define_constraint_primal(set_type)
    quote
        function MOI.get(model::Optimizer, ::MOI.ConstraintPrimal,
                         ci::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}, $set_type})
            if model.workspace === nothing
                @error("ConstraintPrimal not available.")
            end
            if !(1 <= ci.value <= model.nlcon)
                @error("Invalid constraint index ", ci.value)
            end
            return model.workspace.x[ci.value + model.n+model.nncon]
        end
    end
end
@define_constraint_primal(MOI.LessThan{Float64})
@define_constraint_primal(MOI.GreaterThan{Float64})
@define_constraint_primal(MOI.EqualTo{Float64})
@define_constraint_primal(MOI.Interval{Float64})

function MOI.get(model::Optimizer, ::MOI.ConstraintDual,
                 ci::MOI.ConstraintIndex{MOI.SingleVariable,S}) where {S}
    if model.workspace === nothing
        @error("ConstraintDual not available.")
    end
    vi = MOI.VariableIndex(ci.value)
    return model.workspace.lambda[vi.value]
end

function MOI.get(model::Optimizer, ::MOI.ConstraintDual,
                 ci::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}, S}) where {S}
    if model.workspace === nothing
        @error("ConstraintDual not available.")
    end
    @assert 1 <= ci.value <= model.nlcon
    return model.workspace.lambda[ci.value + model.n + model.nncon]
end

function MOI.get(model::Optimizer, ::MOI.NLPBlockDual)
    if model.workspace === nothing
        @error("NLPBlockDual not available.")
    end
    return model.workspace.lambda[model.n+1:model.n+model.nncon]
end


## Defines all hashing functionality
## As usual, check out http://www.lysator.liu.se/~nisse/nettle/nettle.html#Hash-functions

import Base: show
export Hasher, update!, digest, digest!, hexdigest!, hexdigest

immutable Hasher
    hash_type::HashType
    state::Vector{UInt8}
end

# Constructor for Hasher
function Hasher(name::AbstractString)
    hash_types = get_hash_types()
    name = uppercase(name)
    if !haskey(hash_types, name)
        throw(ArgumentError("Invalid hash type $name: call Nettle.get_hash_types() to see available list"))
    end

    # Construct Hasher object for this type and initialize using Nettle's init functions
    hash_type = hash_types[name]
    state = Vector{UInt8}(hash_type.context_size)
    ccall(hash_type.init, Void, (Ptr{Void},), state)
    return Hasher(hash_type, state)
end

# Update hash state with new data
function update!(state::Hasher, data)
    ccall(state.hash_type.update, Void, (Ptr{Void},Csize_t,Ptr{UInt8}), state.state, sizeof(data), pointer(data))
    return state
end

# Spit out a digest of the current hash state and reset it
function digest!(state::Hasher)
    digest = Vector{UInt8}(state.hash_type.digest_size)
    ccall(state.hash_type.digest, Void, (Ptr{Void},UInt32,Ptr{UInt8}), state.state, sizeof(digest), pointer(digest))
    return digest
end

# Take a digest, and convert it to a printable hex representation
hexdigest!(state::Hasher) = bytes2hex(digest!(state))

# The one-shot functions that makes this whole thing so easy.
digest(hash_name::AbstractString, data) = digest!(update!(Hasher(hash_name), data))
digest(hash_name::AbstractString, io::IO) = digest(hash_name, readall(io))
hexdigest(hash_name::AbstractString, data) = hexdigest!(update!(Hasher(hash_name), data))
hexdigest(hash_name::AbstractString, io::IO) = hexdigest(hash_name, readall(io))

# Custom show overrides make this package have a little more pizzaz!
function show(io::IO, x::HashType)
    write(io, "$(x.name) Hash\n")
    write(io, "  Context size: $(x.context_size) bytes\n")
    write(io, "  Digest size: $(x.digest_size) bytes\n")
    write(io, "  Block size: $(x.block_size) bytes")
end
show(io::IO, x::Hasher) = write(io, "$(x.hash_type.name) Hash state")


using FactoredValueMCTS
using Test

using POMDPs
using MultiAgentSysAdmin
using MultiUAVDelivery

@testset "FactoredValueMCTS.jl" begin

    @testset "varel" begin
        @testset "sysadmin" begin
            @testset "local" begin
                mdp = BiSysAdmin{false}()
                solver = FVMCTSSolver()
                planner = solve(solver, mdp)
                s = rand(initialstate(mdp))
                a = action(planner, s)
                @test a isa actiontype(mdp)
            end

            @testset "global" begin
                mdp = BiSysAdmin{true}()
                solver = FVMCTSSolver()
                planner = solve(solver, mdp)
                s = rand(initialstate(mdp))
                a = action(planner, s)
                @test a isa actiontype(mdp)
            end
        end

    end

    @testset "maxplus" begin
         @testset "sysadmin" begin
            @testset "local" begin
                mdp = BiSysAdmin{false}()
                solver = FVMCTSSolver(;coordination_strategy=MaxPlus())
                planner = solve(solver, mdp)
                s = rand(initialstate(mdp))
                a = action(planner, s)
                @test a isa actiontype(mdp)
            end

            @testset "global" begin
                mdp = BiSysAdmin{true}()
                solver = FVMCTSSolver(;coordination_strategy=MaxPlus())
                planner = solve(solver, mdp)
                s = rand(initialstate(mdp))
                a = action(planner, s)
                @test a isa actiontype(mdp)
            end
        end

        @testset "uav" begin
            mdp = FirstOrderMultiUAVDelivery()
            solver = FVMCTSSolver(;coordination_strategy=MaxPlus())
            planner = solve(solver, mdp)
            s = rand(initialstate(mdp))
            a = action(planner, s)
            @test a isa actiontype(mdp)
        end

    end

end


function BNB_control(user, balances_all;
        settings_path=joinpath(Pinguu.base_dir, "BNB_control/BNB_settings.json"))
    acc = JSON.parsefile(settings_path)
    prices = Dict()
    for (user_id, v) in user
        if haskey(acc, user_id)
            client = user[user_id]["client"]
            if !haskey(acc[user_id], "order")
                acc[user_id]["order"] = Dict()
                open(settings_path, "w") do f
                    JSON.print(f, acc, 4)
                end
            end
            if acc[user_id]["active"] == 1
                @info("---")
                @info("User: " * user_id)
                sleep(0.25)
                if !haskey(prices, acc[user_id]["symbol"])
                    ticker = client.get_ticker(symbol=acc[user_id]["symbol"])["lastPrice"]
                    price = parse(Float64, ticker)
                    prices[acc[user_id]["symbol"]] = Dict()
                    prices[acc[user_id]["symbol"]]["price"] = price
                    prices[acc[user_id]["symbol"]]["ticker"] = ticker
                else
                    price  = prices[acc[user_id]["symbol"]]["price"]
                    ticker = prices[acc[user_id]["symbol"]]["ticker"]
                end

                if balances_all[user_id]["BNB"]["estimated_free"] * price < acc[user_id]["low_limit"] && length(acc[user_id]["order"]) == 0
                    #
                    ##
                    ###
                    # If BNB level is low & no order ongoing => Check if enough base money is available
                    if balances_all[user_id][acc[user_id]["base_coin"]]["estimated_free"] <= acc[user_id]["val"]
                        @info(string("Base volume not sufficient: ", round(balances_all[user_id][acc[user_id]["base_coin"]]["estimated_free"], digits=2), " ", acc[user_id]["base_coin"]))
                        continue
                    end
                    qty = ceil(acc[user_id]["val"]/price, digits=3)
                    sleep(0.5)
                    result = try client.order_market_buy(
                        symbol=acc[user_id]["symbol"],
                        quantity=qty);
                        catch
                        @info("order_market_buy failed")
                        nothing
                    end
                    if result !== nothing
                        acc[user_id]["order"] = result
                        open(settings_path, "w") do f
                            JSON.print(f, acc, 4)
                        end
                    end
                    if user_id == "1"
                        telegram = JSON.parsefile(joinpath(Pinguu.base_dir, "telegram.json"))
                        msg = "[BNB Control] Kauf getätigt!"
                        TelegramClient(Pinguu.telegram; chat_id=telegram[user_id])
                        sendMessage(text = msg)
                    end
                elseif length(acc[user_id]["order"]) != 0
                    #
                    ##
                    ###
                    # Check ongoing order
                    order = try client.get_order(
                        symbol=acc[user_id]["symbol"],
                        orderId=acc[user_id]["order"]["orderId"])
                    catch
                        @info("get_order failed")
                        nothing
                    end
                    if order !== nothing
                        if order["status"] == client.ORDER_STATUS_FILLED
                            yr,m,d = split(string(today()), "-")
                            filename = joinpath(Pinguu.base_dir * "BNB_control/users/" * user_id * "/closed_orders/" * yr * "/" * m * "/", "closed_" * string(now()) * ".json")
                            !isdir(dirname(filename)) ? mkpath(dirname(filename)) : ""
                            open(filename, "w") do f
                                JSON.print(f, order, 4)
                            end
                            acc[user_id]["order"] = Dict()
                            open(settings_path, "w") do f
                                JSON.print(f, acc, 4)
                            end
                            if user_id == "1"
                                telegram = JSON.parsefile(joinpath(Pinguu.base_dir, "telegram.json"))
                                msg = "[BNB Control] Kauf abgeschlossen!"
                                TelegramClient(Pinguu.telegram; chat_id=telegram[user_id])
                                sendMessage(text = msg)
                            end
                        else
                            @info("Order status: " * order["status"])
                        end
                    end
                else
                    #
                    ##
                    ###
                    # Nothing to do
                    @info(string("BNB level: ", round(balances_all[user_id]["BNB"]["estimated_free"] * price, digits=2), " ", acc[user_id]["base_coin"]))
                    @info(string("BNB low limit: ", round(acc[user_id]["low_limit"], digits=2), " ", acc[user_id]["base_coin"]))
                end
            end
        end
    end
end
export BNB_control


using Oceananigans.Models: AbstractModel
using Oceananigans.Advection: WENO5
using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractFreeSurface
using Oceananigans.TimeSteppers: AbstractTimeStepper, QuasiAdamsBashforth2TimeStepper
using Oceananigans.Models: PrescribedVelocityFields
using Oceananigans.TurbulenceClosures: VerticallyImplicitTimeDiscretization
using Oceananigans.Advection: AbstractAdvectionScheme

import Oceananigans.Simulations: new_time_step
import Oceananigans.Diagnostics: accurate_cell_advection_timescale
import Oceananigans.Advection: WENO5
import Oceananigans.Models.HydrostaticFreeSurfaceModels: build_implicit_step_solver, validate_tracer_advection
import Oceananigans.TurbulenceClosures: implicit_diffusion_solver

const MultiRegionModel = HydrostaticFreeSurfaceModel{<:Any, <:Any, <:AbstractArchitecture, <:Any, <:MultiRegionGrid}

# Utility to generate the inputs to complex `getregion`s
function getregionalproperties(T, inner=true)
    type = eval(T)
    names = fieldnames(type)
    args  = Vector(undef, length(names))
    for (n, name) in enumerate(names)
        args[n] = inner ? :(_getregion(t.$name, r)) : :(getregion(t.$name, r))
    end
    return args
end

Types = (:HydrostaticFreeSurfaceModel,
         :ImplicitFreeSurface,
         :ExplicitFreeSurface,
         :QuasiAdamsBashforth2TimeStepper,
         :PrescribedVelocityFields)

for T in Types
    @eval begin
        # This assumes a constructor of the form T(arg1, arg2, ...) exists,
        # # which is not the case for all types.
        @inline  getregion(t::$T, r) = $T($(getregionalproperties(T, true)...))
        @inline _getregion(t::$T, r) = $T($(getregionalproperties(T, false)...))
    end
end

@inline isregional(pv::PrescribedVelocityFields) = isregional(pv.u) | isregional(pv.v) | isregional(pv.w)
@inline devices(pv::PrescribedVelocityFields)    = devices(pv[findfirst(isregional, (pv.u, pv.v, pv.w))])

validate_tracer_advection(tracer_advection::MultiRegionObject, grid::MultiRegionGrid) = tracer_advection, NamedTuple()

@inline isregional(mrm::MultiRegionModel)        = true
@inline devices(mrm::MultiRegionModel)           = devices(mrm.grid)
@inline getdevice(mrm::MultiRegionModel, d)      = getdevice(mrm.grid, d)
@inline switch_region!(mrm::MultiRegionModel, d) = switch_region!(mrm.grid, d)

implicit_diffusion_solver(time_discretization::VerticallyImplicitTimeDiscretization, mrg::MultiRegionGrid) =
      construct_regionally(implicit_diffusion_solver, time_discretization, mrg)

WENO5(mrg::MultiRegionGrid, args...; kwargs...) = construct_regionally(WENO5, mrg, args...; kwargs...)

function accurate_cell_advection_timescale(grid::MultiRegionGrid, velocities)
    Δt = construct_regionally(accurate_cell_advection_timescale, grid, velocities)
    return minimum(Δt.regions)
end

function new_time_step(old_Δt, wizard, model::MultiRegionModel)
    Δt = construct_regionally(new_time_step, old_Δt, wizard, model)
    return minimum(Δt.regions)
end


Network = NetLib.SimpleNet

netparams = NetLib.SimpleNetHP(
  width=100,
  depth_common=4,
  use_batch_norm=false)

self_play = SelfPlayParams(
  sim=SimParams(
    num_games=1000,
    num_workers=4,
    use_gpu=false,
    reset_every=16,
    flip_probability=0.,
    alternate_colors=false),
  mcts=MctsParams(
    num_iters_per_turn=50,
    cpuct=1.0,
    temperature=ConstSchedule(0.),
    dirichlet_noise_ϵ=0.,
    dirichlet_noise_α=1.))

arena = ArenaParams(
  sim=SimParams(
    num_games=100,
    num_workers=10,
    use_gpu=false,
    reset_every=1,
    flip_probability=0.,
    alternate_colors=true),
  mcts = self_play.mcts,
  update_threshold=0.00)

learning = LearningParams(
  use_gpu=false,
  use_position_averaging=false,
  samples_weighing_policy=CONSTANT_WEIGHT,
  rewards_renormalization=10,
  l2_regularization=1e-4,
  optimiser=Adam(lr=5e-3),
  batch_size=64,
  loss_computation_batch_size=2048,
  nonvalidity_penalty=1.,
  min_checkpoints_per_epoch=1,
  max_batches_per_checkpoint=5_000,
  num_checkpoints=1)

params = Params(
  arena=arena,
  self_play=self_play,
  learning=learning,
  num_iters=5,
  memory_analysis=nothing,
  ternary_rewards=false,
  use_symmetries=false,
  mem_buffer_size=PLSchedule(80_000))

benchmark_sim = SimParams(
  arena.sim;
  num_games=500,
  num_workers=10)

benchmark = [
  Benchmark.Single(
    Benchmark.Full(self_play.mcts),
    benchmark_sim),
  Benchmark.Single(
    Benchmark.NetworkOnly(),
    benchmark_sim)]

experiment = Experiment(
  "grid-world", GameSpec(), params, Network, netparams, benchmark)


# mbe_demo_c_test.jl
#
# This script tests the Julia implementation against the C implementation.

# Include our custom models.
include("models.jl")

# We work inside a module because it's more flexible than the main workspace.
module MBEDemoCTest

# Use/import the modules we'll need.
using ..MBEModels  # The MBEModels module included above (and "above" this module)
using Random
import Libdl

# Set some constants (we could randomize these too...).
I  = 5.
κc = 0.5
μc = 1.
ρ  = 2.
α  = 10.

# We'll do this many random tests.
n = 100

# Connect to the library containing our C version of the controller.
c_lib = Libdl.dlopen("windows/lib-reuc.dll")
c_fcn = Libdl.dlsym(c_lib, :reuc)

try

    # Creat a place to store the differences.
    Δτ = zeros(2, n)

    # Seed the random number generator so that this test is repeatable.
    Random.seed!(1)

    for k = 1:n

        # Create a random state and target.
        q_TI   = normalize(randn(4))
        q_BI   = normalize(randn(4))
        ω_BI_B = [randn(1); randn(1); 0.]

        # Run the Julia implementation.
        τ_B = MBEModels.rate_controller(I, κc, μc, ρ, α, q_TI, q_BI, ω_BI_B)

        # Run the C implementation.
        τ_B_c = [0.; 0.; 0.]
        ccall(c_fcn, # Function to call
            Nothing, # No return value
            # Function interface:
            (Float64, Float64, Float64, Float64, Float64, # Parameters
            Ptr{Float64}, Ptr{Float64}, Ptr{Float64}, # Quaternions and rate
            Ptr{Float64}), # Output torque
            # Actual values to pass:
            I, κc, μc, ρ, α,
            q_TI, q_BI, ω_BI_B,
            τ_B_c)

        # Store the differences.
        Δτ[:,k] = τ_B[1:2] - τ_B_c[1:2]

    end

    display(Δτ)
    println("Maximum difference in torque: ", maximum(Δτ[:]), " Nm")

finally
    Libdl.dlclose(c_lib) # Close out the library.
end

end # module


# This is loaded conditionally via @require if PyPlot is loaded

using .PyPlot: plt, PyCall

export pgdsgui

get_bystr(@nospecialize(by)) = by === inclusive ? "Inclusive" :
                               by === exclusive ? "Exclusive" : error("unknown ", by)

"""
    methodref, ax = pgdsgui(tinf::InferenceTimingNode; consts::Bool=true, by=inclusive)
    methodref     = pgdsgui(ax, tinf::InferenceTimingNode; kwargs...)

Create a scatter plot comparing:
    - (vertical axis) the inference time for all instances of each Method, as captured by `tinf`;
    - (horizontal axis) the run time cost, as estimated by capturing a `@profile` before calling this function.

Each dot corresponds to a single method. The face color encodes the number of times that method was inferred,
and the edge color corresponds to the fraction of the runtime spent on runtime dispatch (black is 0%, bright red is 100%).
Clicking on a dot prints the method (or location, if inlined) to the REPL, and sets `methodref[]` to
that method.

`ax` is the pyplot axis of the scatterplot.

!!! compat
    `pgdsgui` depends on PyPlot via the Requires.jl package. You must load both SnoopCompile and PyPlot for this function to be defined.
"""
function pgdsgui(ax::PyCall.PyObject, ridata; bystr, consts, markersz=25, linewidth=0.5, t0 = 0.001, interactive::Bool=true, kwargs...)
    methodref = Ref{Union{Method,MethodLoc}}()
    function onclick(event)
        xc, yc = event.xdata, event.ydata
        idx = argmin((log.(rts .+ t0) .- log(xc)).^2 + (log.(its .+ t0) .- log(yc)).^2)
        m = meths[idx]
        methodref[] = m
        println(m, " ($(nspecs[idx]) specializations)")
    end

    meths, rts, its, nspecs, ecols = Union{Method,MethodLoc}[], Float64[], Float64[], Int[], Tuple{Float64,Float64,Float64}[]
    for (m, d) in ridata  # (rt, trtd, it, nspec)
        push!(meths, m)
        push!(rts, d.trun)
        push!(its, d.tinf)
        push!(nspecs, d.nspec)
        push!(ecols, (d.trun > 0 ? d.trtd/d.trun : 0.0, 0.0, 0.0))
    end
    sp = sortperm(nspecs)
    meths, rts, its, nspecs, ecols = meths[sp], rts[sp], its[sp], nspecs[sp], ecols[sp]

    # Add t0 to each entry to handle times of zero in the log-log plot
    smap = ax.scatter(rts .+ t0, its .+ t0, markersz, nspecs; norm=plt.matplotlib.colors.LogNorm(), edgecolors=ecols, linewidths=linewidth, kwargs...)
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_xlabel("Run time (self) + $t0 (s)")
    ax.set_ylabel("$bystr inference time + $t0 (s)")
    # ax.set_aspect("equal")
    ax.axis("square")
    constsmode = consts ? "incl." : "excl."
    plt.colorbar(smap, label = "# specializations ($constsmode consts)", ax=ax)
    if interactive
        ax.get_figure().canvas.mpl_connect("button_press_event", onclick)
    end
    return methodref
end

function pgdsgui(ridata; kwargs...)
    fig, ax = plt.subplots()
    return pgdsgui(ax, ridata; kwargs...), ax
end

function pgdsgui(ax::PyCall.PyObject, tinf::InferenceTimingNode; consts::Bool=true, by=inclusive, kwargs...)
    pgdsgui(ax, runtime_inferencetime(tinf; consts, by); bystr=get_bystr(by), consts, kwargs...)
end
function pgdsgui(tinf::InferenceTimingNode; consts::Bool=true, by=inclusive, kwargs...)
    pgdsgui(runtime_inferencetime(tinf; consts, by); bystr=get_bystr(by), consts, kwargs...)
end

@deprecate specialization_plot pgdsgui


#=
Metis is used to partition the mesh.
This uses METIS_jll: libmetis
=#
const METIS_NOPTIONS = 40

# codes returned by metis functions
const METIS_OK = Cint(1)
const METIS_ERROR_INPUT = Cint(-2)
const METIS_ERROR_MEMORY = Cint(-3)
const METIS_ERROR = Cint(-4)

# location in options vector
const METIS_OPTION_PTYPE     = 1
const METIS_OPTION_OBJTYPE   = 2
const METIS_OPTION_CTYPE     = 3
const METIS_OPTION_IPTYPE    = 4
const METIS_OPTION_RTYPE     = 5
const METIS_OPTION_DBGLVL    = 6
const METIS_OPTION_NITER     = 7
const METIS_OPTION_NCUTS     = 8
const METIS_OPTION_SEED      = 9
const METIS_OPTION_NO2HOP    = 10
const METIS_OPTION_MINCONN   = 11
const METIS_OPTION_CONTIG    = 12
const METIS_OPTION_COMPRESS  = 13
const METIS_OPTION_CCORDER   = 14
const METIS_OPTION_PFACTOR   = 15
const METIS_OPTION_NSEPS     = 16
const METIS_OPTION_UFACTOR   = 17
const METIS_OPTION_NUMBERING = 18
const METIS_OPTION_HELP      = 19
const METIS_OPTION_TPWGTS    = 20
const METIS_OPTION_NCOMMON   = 21
const METIS_OPTION_NOOUTPUT  = 22
const METIS_OPTION_BALANCE   = 23
const METIS_OPTION_GTYPE     = 24
const METIS_OPTION_UBVEC     = 25

# Partitioning method
const METIS_PTYPE_RB   = Cint(0)
const METIS_PTYPE_KWAY = Cint(1)
# objectives
const METIS_OBJTYPE_CUT  = Cint(0)
const METIS_OBJTYPE_VOL  = Cint(1)
const METIS_OBJTYPE_NODE = Cint(2)
# Coarsening method
const METIS_CTYPE_RM   = Cint(0)
const METIS_CTYPE_SHEM = Cint(1)
# Initial partitioning schemes
const METIS_IPTYPE_GROW    = Cint(0)
const METIS_IPTYPE_RANDOM  = Cint(1)
const METIS_IPTYPE_EDGE    = Cint(2)
const METIS_IPTYPE_NODE    = Cint(3)
const METIS_IPTYPE_METISRB = Cint(4)
# Refinement schemes
const METIS_RTYPE_FM        = Cint(0)
const METIS_RTYPE_GREEDY    = Cint(1)
const METIS_RTYPE_SEP2SIDED = Cint(2)
const METIS_RTYPE_SEP1SIDED = Cint(3)
# Debug levels (bit positions)
const METIS_DBG_INFO       = Cint(1)    # Shows various diagnostic messages
const METIS_DBG_TIME       = Cint(2)    # Perform timing analysis
const METIS_DBG_COARSEN    = Cint(4)    # Show the coarsening progress
const METIS_DBG_REFINE     = Cint(8)    # Show the refinement progress
const METIS_DBG_IPART      = Cint(16)   # Show info on initial partitioning
const METIS_DBG_MOVEINFO   = Cint(32)   # Show info on vertex moves during refinement
const METIS_DBG_SEPINFO    = Cint(64)   # Show info on vertex moves during sep refinement
const METIS_DBG_CONNINFO   = Cint(128)  # Show info on minimization of subdomain connectivity
const METIS_DBG_CONTIGINFO = Cint(256)  # Show info on elimination of connected components
const METIS_DBG_MEMORY     = Cint(2048) # Show info related to wspace allocation
# Graph types
const METIS_GTYPE_DUAL  = Cint(0)
const METIS_GTYPE_NODAL = Cint(1)

#=
This is the primary function used to partition a mesh into np partitions.
It returns a list of partition numbers for each element.
=#
function get_element_partitions(mesh, np)
    # number of common nodes between neighbors
    if config.dimension == 1
        common_nodes = 1;
    elseif config.dimension == 2
        common_nodes = 2;
    elseif config.dimension == 3
        if mesh.nv[1] == 4
            common_nodes = 3;
        elseif mesh.nv[1] == 8
            common_nodes = 4;
        end
    end

    # options
    options = fill(Cint(-1), METIS_NOPTIONS);
    # It turns out the default values are all -1
    # r = ccall((:METIS_SetDefaultOptions, libmetis), Cint, (Ptr{Cint},), options);
    # These can now be set as desired

    # Input arguments for metis
    ne = Cint(mesh.nel);# number of elements
    nn = Cint(mesh.nx); # number of nodes
    (eptr, eind) = make_eptr_eind(mesh); # connectivity in a particular format
    vwgt = C_NULL;      # vertex weights (not used)
    vsize = C_NULL;     # vertex size (not used)
    ncommon = Cint(common_nodes);  # number of common nodes to between neighbors
    nparts = Cint(np);  # number of partitions
    tpwgts = C_NULL;    # partition weights (not used)
    # options = C_NULL; # options vector
    # Output arguments
    objval = fill(Cint(0),1);  # objective value
    epart = fill(Cint(0), ne); # partitioned elements
    npart = fill(Cint(0), nn); # partitioned nodes

    r = ccall((:METIS_PartMeshDual, libmetis), Cint,
        (Ref{Cint}, Ref{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ref{Cint},
         Ref{Cint}, Ptr{Cfloat}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}),
        ne, nn, eptr, eind, vwgt, vsize, ncommon, nparts, tpwgts, options, objval, epart, npart);

    if !(r == METIS_OK)
        #const METIS_ERROR_INPUT = Cint(-2)
        #const METIS_ERROR_MEMORY = Cint(-3)
        #const METIS_ERROR = Cint(-4)
        err_msg = ["Input arugment error","Memory allocation error","Unknown error"];
        printerr("METIS returned error: "*err_msg[-r-1], fatal=true);
    end

    return epart;
end

# Writes the element data in a special format for metis.
function make_eptr_eind(mesh)
    nodes_per_element = mesh.nv[1];
    eptr = fill(Cint(0), mesh.nel+1);
    eind = fill(Cint(0), mesh.nel * nodes_per_element);

    #eptr: the start index in eind for this element's nodes
    #eind: the indices of nodes
    for ei=1:mesh.nel
        offset = (ei-1)*nodes_per_element; # for 0-based index
        eptr[ei] = Cint(offset);
        for ni=1:nodes_per_element
            eind[offset+ni] = Cint(mesh.elements[ni,ei] - 1);
        end
    end
    eptr[end] = Cint(length(eind));

    return (eptr, eind);
end


# generateur d'Instances pour la RODD projet 3

N = 8
Nm = 4
Nf = 4
C = 1
G = 5
A = 2
T = 50
init = 0.001

# Ouvrir le fichier "output.txt" dans lequel on pourra écrire
fout = open("output.txt", "w")

println(fout, "title")
println(fout, "title")
println(fout, "title")
println(fout, "title")

println(fout, "N = " * string(N))
println(fout, "Nm = " * string(Nm))
println(fout, "Nf = " * string(Nf))
println(fout, "C = " * string(C))
println(fout, "G = " * string(G))
println(fout, "A = " * string(A))
println(fout, "T = " * string(T))
println(fout, "init = " * string(init))

for i in 1:N
  for j in 1:G
    myLine = string(i) * " " * string(1) * " " * string(j) * " " * string(1) * " " * string(rand(1:A))
    println(fout, myLine)
    myLine = string(i) * " " * string(1) * " " * string(j) * " " * string(2) * " " * string(rand(1:A))
    println(fout, myLine)
  end
end

close(fout)


@testset "Ancestor Tests" begin
    @assert ALifeBenchmark.MUTATE == false

    model = ALifeBenchmark.TierraModel(ALifeBenchmark.LARGE_ANCESTOR)

    for _ in 1:900
        ALifeBenchmark.execute_slice!(model, slice_size = 1)
    end

    len_ancestor = length(ALifeBenchmark.LARGE_ANCESTOR)
    @test model.memory[1:len_ancestor] == model.memory[len_ancestor+1:2*len_ancestor] || model.memory[1:len_ancestor] == model.memory[2^16 - len_ancestor + 1:2^16]

    model = ALifeBenchmark.TierraModel(ALifeBenchmark.SMALL_ANCESTOR)

    for _ in 1:300
        ALifeBenchmark.execute_slice!(model, slice_size = 1)
    end

    len_ancestor = length(ALifeBenchmark.SMALL_ANCESTOR)
    @test model.memory[1:len_ancestor] == model.memory[len_ancestor+1:2*len_ancestor] || model.memory[1:len_ancestor] == model.memory[2^16 - len_ancestor + 1:2^16]
end


export extract_amp, extract_vis, extract_lcamp, extract_cphase

function getvisfield(obs)
    obsamps = obs.data::PyObject
    u = deepcopy((get(obsamps, Vector{Float64}, "u")))
    v = deepcopy((get(obsamps, Vector{Float64}, "v")))
    err = deepcopy((get(obsamps, Vector{Float64}, "sigma")))
    vis = deepcopy((get(obsamps, Vector{Complex{Float64}}, "vis")))
    t1 = Symbol.(deepcopy((get(obsamps, Vector{String}, "t1"))))
    t2 = Symbol.(deepcopy((get(obsamps, Vector{String}, "t2"))))
    baseline = tuple.(t1, t2)
    time = deepcopy((get(obsamps, Vector{Float64}, "time")))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return  StructArray{Comrade.EHTVisibilityDatum{Float64}}(
        visr = real.(vis),
        visi = imag.(vis),
        u = u,
        v = v,
        error = err,
        time = time,
        frequency = freq,
        bandwidth = bw,
        baseline = baseline
    )
end

function getampfield(obs)
    obsamps = obs.amp::PyObject
    uamp = deepcopy(get(obsamps, Vector{Float64}, "u"))
    vamp = deepcopy(get(obsamps, Vector{Float64}, "v"))
    erramp = deepcopy(get(obsamps, Vector{Float64}, "sigma"))
    amps = deepcopy(get(obsamps, Vector{Float64}, "amp"))
    t1 = Symbol.(deepcopy(get(obsamps, Vector{String}, "t1")))
    t2 = Symbol.(deepcopy(get(obsamps, Vector{String}, "t2")))
    baseline = tuple.(t1, t2)
    time = deepcopy(get(obsamps, Vector{Float64}, "time"))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return  StructArray{Comrade.EHTVisibilityAmplitudeDatum{Float64}}(
        amp = amps,
        u = uamp,
        v = vamp,
        error = erramp,
        time = time,
        frequency = freq,
        bandwidth = bw,
        baseline = baseline
    )
end

function getcpfield(obs)
    obscp = obs.cphase::PyObject
    u1 = deepcopy((get(obscp, Vector{Float64}, "u1")))
    v1 = deepcopy((get(obscp, Vector{Float64}, "v1")))
    u2 = deepcopy((get(obscp, Vector{Float64}, "u2")))
    v2 = deepcopy((get(obscp, Vector{Float64}, "v2")))
    u3 = deepcopy((get(obscp, Vector{Float64}, "u3")))
    v3 = deepcopy((get(obscp, Vector{Float64}, "v3")))
    cp = deg2rad.(deepcopy((get(obscp, Vector{Float64}, "cphase"))))
    errcp = deg2rad.(deepcopy((get(obscp, Vector{Float64}, "sigmacp"))))

    t1 = Symbol.(deepcopy((get(obscp, Vector{String}, "t1"))))
    t2 = Symbol.(deepcopy((get(obscp, Vector{String}, "t2"))))
    t3 = Symbol.(deepcopy((get(obscp, Vector{String}, "t3"))))
    baseline = tuple.(t1, t2, t3)
    time = deepcopy(get(obscp, Vector{Float64}, "time"))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return StructArray{Comrade.EHTClosurePhaseDatum{Float64}}(
        phase = cp,
        u1 = u1,
        v1 = v1,
        u2 = u2,
        v2 = v2,
        u3 = u3,
        v3 = v3,
        error = errcp,
        time = time,
        frequency = freq,
        bandwidth = bw,
        triangle = baseline
    )

end

function getlcampfield(obs)
    obslcamp = obs.logcamp::PyObject
    u1 = get(obslcamp, Vector{Float64}, "u1")
    v1 = get(obslcamp, Vector{Float64}, "v1")
    u2 = get(obslcamp, Vector{Float64}, "u2")
    v2 = get(obslcamp, Vector{Float64}, "v2")
    u3 = get(obslcamp, Vector{Float64}, "u3")
    v3 = get(obslcamp, Vector{Float64}, "v3")
    u4 = get(obslcamp, Vector{Float64}, "u4")
    v4 = get(obslcamp, Vector{Float64}, "v4")
    camp = ((get(obslcamp, Vector{Float64}, "camp")))
    errcamp = ((get(obslcamp, Vector{Float64}, "sigmaca")))

    t1 = Symbol.(((get(obslcamp, Vector{String}, "t1"))))
    t2 = Symbol.(((get(obslcamp, Vector{String}, "t2"))))
    t3 = Symbol.(((get(obslcamp, Vector{String}, "t3"))))
    t4 = Symbol.(((get(obslcamp, Vector{String}, "t4"))))
    baseline = tuple.(t1, t2, t3, t4)
    time = (get(obslcamp, Vector{Float64}, "time"))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return StructArray{Comrade.EHTLogClosureAmplitudeDatum{Float64}}(
        amp = camp,
        u1 = u1,
        v1 = v1,
        u2 = u2,
        v2 = v2,
        u3 = u3,
        v3 = v3,
        u4 = u4,
        v4 = v4,
        error = errcamp,
        time = time,
        frequency = freq,
        bandwidth = bw,
        quadrangle = baseline
    )

end

function getradec(obs)::Tuple{Float64, Float64}
    return (float(obs.ra), float(obs.dec))
end

function getmjd(obs)::Int
    return Int(obs.mjd)
end

function getsource(obs)::Symbol
    return Symbol(obs.source)
end

"""
    extract_amp(obs)
Extracts the visibility amplitudes from an ehtim observation object.

Any valid keyword arguments to `add_amp` in ehtim can be passed through extract_amp.

Returns an EHTObservation with visibility amplitude data
"""
function extract_amp(obsc; kwargs...)
    obs = obsc.copy()
    obs.reorder_tarr_snr()
    obs.add_amp(;kwargs...)
    data = getampfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf
    ac = _arrayconfig(data, bw, rf)
    return Comrade.EHTObservation(data = data, mjd = mjd,
                   ra = ra, dec= dec,
                   config = ac,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )
end

"""
    extract_vis(obs)
Extracts the complex visibilities from an ehtim observation object

This grabs the raw `data` object from the obs object. Any keyword arguments are ignored.

Returns an EHTObservation with complex visibility data
"""
function extract_vis(obsc; kwargs...)
    obs = obsc.copy()
    obs.reorder_tarr_snr()

    data = getvisfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf
    ac = _arrayconfig(data, bw, rf)
    return Comrade.EHTObservation(
                   data = data, mjd = mjd,
                   config=ac,
                   ra = ra, dec= dec,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )
end

function closure_designmat(scancp::Scan{A,B,C}, scanvis) where {A,B,C<:StructArray{<:EHTClosurePhaseDatum}}
    ant1, ant2, ant3 = baselines(scancp)
    antvis1, antvis2 = baselines(scanvis)
    design_mat = zeros(length(scancp), length(scanvis))
    #throw("here")
    # fill in each row of the design matrix
    for i in axes(design_mat, 2), j in axes(design_mat,1)
        a1 = ant1[j]
        a2 = ant2[j]
        a3 = ant3[j]
        # check leg 1
        ((antvis1[i] == a1) & (antvis2[i] == a2)) && (design_mat[j,i] = 1.0)
        ((antvis1[i] == a2) & (antvis2[i] == a1)) && (design_mat[j,i] = -1.0)

        #check leg 2
        ((antvis1[i] == a2) & (antvis2[i] == a3)) && (design_mat[j,i] = 1.0)
        ((antvis1[i] == a3) & (antvis2[i] == a2)) && (design_mat[j,i] = -1.0)

        #check leg 3
        ((antvis1[i] == a3) & (antvis2[i] == a1)) && (design_mat[j,i] = 1.0)
        ((antvis1[i] == a1) & (antvis2[i] == a3)) && (design_mat[j,i] = -1.0)
    end
    return design_mat
end

function closure_designmat(scanlca::Scan{A,B,C}, scanvis) where {A,B,C<:StructArray{<:EHTLogClosureAmplitudeDatum}}
    ant1, ant2, ant3, ant4 = baselines(scanlca)
    antvis1, antvis2 = baselines(scanvis)
    design_mat = zeros(length(scanlca), length(scanvis))
    # fill in each row of the design matrix
    for i in axes(design_mat, 2), j in axes(design_mat,1)
        a1 = ant1[j]
        a2 = ant2[j]
        a3 = ant3[j]
        a4 = ant4[j]

        av1 = antvis1[i]
        av2 = antvis2[i]
        # check leg 1
        (((av1 == a1)&(av2 == a2)) || (av1 == a2)&(av2 == a1))&& (design_mat[j,i] = 1.0)

        #check leg 2
        (((av1 == a3)&(av2 == a4)) || (av1 == a4)&(av2 == a3))&& (design_mat[j,i] = 1.0)

        #check leg 3
        (((av1 == a1)&(av2 == a4)) || (av1 == a4)&(av2 == a1))&& (design_mat[j,i] = -1.0)

        #check leg 4
        (((av1 == a2)&(av2 == a3)) || (av1 == a3)&(av2 == a2))&& (design_mat[j,i] = -1.0)

    end
    return design_mat
end

function minimal_lcamp(obsc; kwargs...)

    obs = obsc.copy()
    # reorder to maximize the snr
    obs.reorder_tarr_snr()

    lcamp = _ehtim_lcamp(obs; count="max", kwargs...)
    stlca = scantable(lcamp)

    #Now make the vis obs
    dvis = extract_vis(obsc)
    st = scantable(dvis)

    minset, dmat = _minimal_closure(stlca, st)

    # now create EHTObservation
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf
    ac = arrayconfig(dvis)
    clac = ClosureConfig(ac, dmat)
    return EHTObservation(data = minset, mjd = mjd,
                          config=clac,
                          ra = ra, dec= dec,
                          bandwidth=bw, frequency=rf,
                          source = source,
                        )

end

function _ehtim_cphase(obsc; count="max", cut_trivial=false, uvmin=0.1e9, kwargs...)
    obs = obsc.copy()

    # cut 0 baselines since these are trivial triangles
    if cut_trivial
        obs = obs.flag_uvdist(uv_min=uvmin)
    end

    obs.reorder_tarr_snr()

    obs.add_cphase(;count=count, kwargs...)
    data = getcpfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf

    cphase = EHTObservation(data = data, mjd = mjd,
                   config=nothing,
                   ra = ra, dec= dec,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )

    stcp = scantable(cphase)

    #Now make the vis obs
    dvis = extract_vis(obsc)
    st = scantable(dvis)
    S = eltype(dvis[:visr])

    dmat = Matrix{S}[]
    for i in 1:length(st)
        scanvis = st[i]
        ind = findfirst(==(scanvis.time), stcp.times)
        if isnothing(ind)
            inow = length(dmat)
            # I want to construct block diagonal matrices for efficienty but we need
            # to be careful with scan that can't form closures. This hack moves these
            # scans into the preceding scan but fills it with zeros
            if i > 1
                dmat[inow] = hcat(dmat[inow], zeros(S, size(dmat[inow],1), length(scanvis.scan)))
            else
                push!(dmat, zeros(S, 1, length(scanvis.scan)))
            end
            continue
        end
        dmatscan = closure_designmat(stcp[ind], st[i])
        push!(dmat, dmatscan)
    end

    if iszero(dmat[1])
        dmat[2] = hcat(zeros(S, size(dmat[2],1), size(dmat[1],2)), dmat[2])
        dmat = dmat[2:end]
    end

    ac = arrayconfig(dvis)
    clac = ClosureConfig(ac, dmat)
    return  EHTObservation(data = data, mjd = mjd,
                           config=clac,
                           ra = ra, dec= dec,
                           bandwidth=bw, frequency=rf,
                           source = source
                          )
end

function _make_lcamp(obsc, count="max"; kwargs...)
    obs = obsc.copy()

    obs.reorder_tarr_snr()

    obs.add_logcamp(;count=count, kwargs...)
    data = getlcampfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf

    return EHTObservation(data = data, mjd = mjd,
                   config=nothing,
                   ra = ra, dec= dec,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )

end

function _ehtim_lcamp(obsc; count="max", kwargs...)

    lcamp = _make_lcamp(obsc; count, kwargs...)

    stlca = scantable(lcamp)

    #Now make the vis obs
    dvis = extract_vis(obsc)
    st = scantable(dvis)
    S = eltype(dvis[:visr])

    dmat = Matrix{S}[]
    for i in 1:length(st)
        scanvis = st[i]
        ind = findfirst(==(scanvis.time), stlca.times)
        if isnothing(ind)
            inow = length(dmat)
            # I want to construct block diagonal matrices for efficienty but we need
            # to be careful with scan that can't form closures. This hack moves these
            # scans into the preceding scan but fills it with zeros
            if i > 1
                dmat[inow] = hcat(dmat[inow], zeros(S, size(dmat[inow],1), length(scanvis.scan)))
            else
                push!(dmat, zeros(S, 1, length(scanvis.scan)))
            end
            continue
        end
        dmatscan = closure_designmat(stlca[ind], st[i])
        push!(dmat, dmatscan)
    end

    if iszero(dmat[1])
        dmat[2] = hcat(zeros(S, size(dmat[2],1), size(dmat[1],2)), dmat[2])
        dmat = dmat[2:end]
    end

    ac = arrayconfig(dvis)
    clac = ClosureConfig(ac, dmat)
    return  EHTObservation(data = lcamp.data, mjd = lcamp.mjd,
                           config=clac,
                           ra = lcamp.ra, dec= lcamp.dec,
                           bandwidth=lcamp.bandwidth, frequency=lcamp.frequency,
                           source = lcamp.source
                          )
end

function minimal_cphase(obsc; kwargs...)
    # compute a maximum set of closure phases
    obs = obsc.copy()

    # reorder to maximize the snr
    obs.reorder_tarr_snr()

    cphase = _ehtim_cphase(obs; count="max", kwargs...)
    stcp = scantable(cphase)

    #Now make the vis obs
    dvis = extract_vis(obsc)
    st = scantable(dvis)

    minset, dmat = _minimal_closure(stcp, st)
    # now create EHTObservation
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf
    ac = arrayconfig(dvis)
    clac = ClosureConfig(ac, dmat)
    return EHTObservation(data = minset, mjd = mjd,
                          config=clac,
                          ra = ra, dec= dec,
                          bandwidth=bw, frequency=rf,
                          source = source,
                        )
end

function _minimal_closure(stcl, st)

    # Determine the number of timestamps containing a closure triangle
    T = typeof(stcl[1][1])
    S = typeof(stcl[1][1].u1)
    # loop over all timestamps
    minset = StructVector{T}(undef, 0)
    dmat = Matrix{S}[]
    for i in 1:length(st.times)

        scanvis = st[i]
        # find closure scan that matches time
        ind = findfirst(==(scanvis.time), stcl.times)
        if isnothing(ind)
            inow = length(dmat)
            # I want to construct block diagonal matrices for efficiency but we need
            # to be careful with scans that can't form closures. This hack moves these
            # scans into the preceding scan but fills it with zeros
            if i > 1
                dmat[inow] = hcat(dmat[inow], zeros(S, size(dmat[inow],1), length(scanvis.scan)))
            else
                push!(dmat, zeros(S, 1, length(scanvis.scan)))
            end
            continue
        end

        # Get our cphase scan
        scancl = stcl[ind]

        # sort by closure SNR so we form a nice minimal set
        snr = inv.(scancl.scan.error)
        ind_snr = sortperm(snr)
        scancl = scancl[ind_snr]
        snr = snr[ind_snr]

        # now find the visibility scan that matches the closure one based on timestamps

        # initialize the design matrix
        design_mat = closure_designmat(scancl, scanvis)

        # determine the expected size of the minimal set
        # this is needed to make sure we aren't killing too many triangles
        nmin = rank(design_mat)

        # print some info
        #println("For timestamp $(stcl.times[i]):")

        # get the current stations
        #println("Observing stations are $(stations(scancl))")

        #println("Size of maximal set of closure products = $(length(scancl))")
        #println("Size of minimal set of closure products = $(nmin)")
        #println("...")

        ##########################################################
        # start of loop to recover minimal set
        ##########################################################
        minset_scan, dmat_min = minimal_closure_scan(scancl, scanvis, nmin)
        minset = append!!(minset, minset_scan.scan)
        push!(dmat, dmat_min)
    end

    # Now if the first scan can't form a closure we will have an extra row of zeros
    # kill this
    if iszero(dmat[1])
        dmat[2] = hcat(zeros(S, size(dmat[2],1), size(dmat[1],2)), dmat[2])
        dmat = dmat[2:end]
    end
    return minset, dmat
end

function minimal_closure_scan(scancl0, scanvis, nmin::Int)
    # make a mask to keep track of which clhases will stick around
    scancl = deepcopy(scancl0)
    keep = fill(true, length(scancl))

    # remember the original minimal set size
    nmin0 = nmin

    # perform the loop
    count = 1
    keep = fill(true, length(scancl))
    good = true
    while good
        # recreate the mask each time
        scankeep = scancl[keep]

        design_mat = closure_designmat(scankeep, scanvis)

        # determine the size of the minimal set
        nmin = rank(design_mat)
        #println(nmin0, " ", nmin, " ", size(design_mat, 1))

        # Now decide whether to continue pruning
        if (sum(keep) == nmin0) && (nmin == nmin0)
            good = false
        else
            if nmin == nmin0
                keep[count] = false
            else
                keep[count-1] = true
                count -= 1
            end
            #(nmin == nmin0) && (keep[count] = false)
        end
        count += 1
        count+1 > length(scancl) && break
    end

    # print out the size of the recovered set for double-checking
    scankeep = scancl[keep]
    dmat = closure_designmat(scankeep, scanvis)
    if length(scankeep) != nmin
        @error "minimal set not found $(length(scankeep)) $(nmin)"
        throw("No minimal set found at time $(scancl.time)")
    end
    return scankeep, dmat
end

"""
    extract_cphase(obs)
Extracts the closure phases from an ehtim observation object

Any valid keyword arguments to `add_cphase` in ehtim can be passed through extract_cphase.

Returns an EHTObservation with closure phases datums

# Keyword arguments:
 - count: How the closures are formed, the available options are "min-correct", "min", "max"
 - cut_trivial: Cut the trivial triangles from the closures
 - uvmin: The flag to decide what are trivial triangles. Any baseline with ||(u,v)|| < uvmin
          are removed.

# Warning

The `count` keyword argument is treated specially in `Comrade`. The default option
is "min-correct" and should almost always be used.
This option construct a minimal set of closure phases that is valid even when
the array isn't fully connected. For testing and legacy reasons we `ehtim` other count
options are also included. However, the current `ehtim` count="min" option is broken
and does construct proper minimal sets of closure quantities if the array isn't fully connected.

"""
function extract_cphase(obs; count="min-correct", cut_trivial=false, uvmin=0.1e9,  kwargs...)
    if count == "min-correct"
        return minimal_cphase(obs; cut_trivial, uvmin, kwargs...)
    else
        return _ehtim_cphase(obs; count=count, cut_trivial=cut_trivial, uvmin=uvmin, kwargs...)
    end
end

"""
    extract_lcamp(obs)
Extracts the log-closure amp. from an ehtim observation object

Any valid keyword arguments to `add_logcamp` in ehtim can be passed through extract_lcamp.

Returns an EHTObservation with log-closure amp. datums

# Warning
The `count` keyword argument is treated specially in `Comrade`. The default option
is "min-correct" and should almost always be used.
This option construct a minimal set of closure phases that is valid even when
the array isn't fully connected. For testing and legacy reasons we `ehtim` other count
options are also included. However, the current `ehtim` count="min" option is broken
and does construct proper minimal sets of closure quantities if the array isn't fully connected.

"""
function extract_lcamp(obs; count="min-correct", kwargs...)
    if count == "min-correct"
        return minimal_lcamp(obs; kwargs...)
    else
        return _ehtim_lcamp(obs; count=count, kwargs...)
    end
end


module  TestIO

using PopGen
using GeneticVariation, GZip
using Test

cats_gen = normpath(joinpath(@__DIR__,"..","data/source", "nancycats.gen"))
sharks_gen = normpath(joinpath(@__DIR__,"..","data/source", "gulfsharks.gen"))
oyster_vcf = normpath(joinpath(@__DIR__,"..","data/source", "filtered_oyster.vcf"))

@testset "Genepop io" begin
    @test typeof(read_from(cats_gen, silent = true)) == PopData
    @test typeof(read_from(sharks_gen, silent = true)) == PopData
    @test typeof(genepop(cats_gen, silent = true)) == PopData
    @test typeof(genepop(sharks_gen, silent = true)) == PopData
end

@testset "VCF io" begin
    @test typeof(read_from(oyster_vcf, silent = true)) == PopData
    @test typeof(vcf(oyster_vcf, silent = true)) == PopData
end

end # module


# codon_set.jl
# ============
#
# A tiny immutable set type for DNA and RNA codons.
#
# This file is a part of BioJulia.
# License is MIT: https://github.com/BioJulia/NaturalSelection.jl/blob/master/LICENSE.md

# Type definition
# ---------------

"""
CodonSet is a primitive type that represents a set of all possible 64 codons.

It is useful to compactly represent such a set in a single 64 bit word, when
one is working with variations between biological sequences.
"""
primitive type CodonSet{T<:NucleicAcid} 64 end

# Conversion and construction
# ---------------------------

CodonSet{T}() where T <: NucleicAcid = reinterpret(CodonSet{T}, UInt64(0))

# Base operators
# --------------

@inline function Base.:|(x::CodonSet{T}, y::UInt64) where T<:NucleicAcid
    return reinterpret(CodonSet{T}, reinterpret(UInt64, x) | y)
end

@inline function Base.:|(x::CodonSet{T}, y::Kmer{T,3}) where T<:NucleicAcid
    return x | UInt64(1) << UInt64(y)
end

@inline function Base.:&(x::CodonSet{T}, y::CodonSet{T}) where T <: NucleicAcid
    return reinterpret(CodonSet{T}, reinterpret(UInt64, x) & reinterpret(UInt64, y))
end

# Iteration interface
# -------------------

@inline function getnext(x::CodonSet, start::Integer)
    return trailing_zeros(reinterpret(UInt64, x) >> start) + start
end

Base.iteratorsize(::Type{CodonSet}) = HasLength()
Base.iteratoreltype(::Type{CodonSet}) = HasEltype()
Base.eltype(::Type{CodonSet{T}}) where T<:NucleicAcid = Kmer{T,3}
Base.length(x::CodonSet) = count_ones(reinterpret(UInt64, x))
Base.start(x::CodonSet) = getnext(x, UInt64(0))

@inline function Base.next(x::CodonSet{T}, state::UInt64) where T<:NucleicAcid
    return Kmer{T,3}(state), getnext(x, state + UInt64(1))
end

@inline function Base.done(x::CodonSet, state::UInt64)
    return state > UInt64(63)
end


"""
    animate(vis::MechanismVisualizer,
            times::Vector{Float64},
            configurations::Vector{Vector{Float64}};
            fps::Float64=60, realtimerate::Float64=1.)

Animate the given mechanism passing through a time-coded series of
configurations by linearly interpolating the configuration vectors.
"""
function animate(vis::MechanismVisualizer,
                 times::Vector{Float64},
                 configurations::AbstractVector{<:AbstractVector{Float64}};
                 fps::Float64 = 60., realtimerate::Float64 = 1.)
    @assert fps > 0
    @assert 0 < realtimerate < Inf

    state = vis.state
    interpolated_configurations = interpolate((times,), configurations, Gridded(Linear()))
    t0, tf = first(times), last(times)
    framenum = 0
    walltime0 = time()
    @throttle framenum while true
        t = min(tf, t0 + (time() - walltime0) * realtimerate)
        q = interpolated_configurations(t)
        set_configuration!(state, q)
        rbd.normalize_configuration!(state)
        _render_state!(vis)
        framenum += 1
        t == tf && break
    end max_rate = fps
end

function MeshCat.Animation(mvis::MechanismVisualizer,
        times::AbstractVector{<:Real},
        configurations::AbstractVector{<:AbstractVector{<:Real}};
        fps::Integer=30)
    @assert axes(times) == axes(configurations)
    interpolated_configurations = interpolate((times,), configurations, Gridded(Linear()))
    animation = Animation()
    num_frames = floor(Int, (times[end] - first(times)) * fps)
    for frame in 0 : num_frames
        time = first(times) + frame / fps
        let mvis = mvis, interpolated_configurations = interpolated_configurations, time=time
            atframe(animation,  frame) do
                set_configuration!(mvis, interpolated_configurations(time))
            end
        end
    end
    return animation
end

MeshCat.setanimation!(mvis::MechanismVisualizer, args...; kw...) =
    setanimation!(visualizer(mvis), args...; kw...)

function MeshCat.setanimation!(mvis::MechanismVisualizer,
                      times::AbstractVector{<:Real},
                      configurations::AbstractVector{<:AbstractVector{<:Real}};
                      fps::Integer=30,
                      play::Bool=true,
                      repetitions::Integer=1)
    Base.depwarn("""
    `setanimation!(mvis, times, configurations; ..)` is deprecated. Instead, you can construct an `Animation` and then call `setanimation!` with the result.

    For example, if you previously did:

    ```
    setanimation!(mvis, times, configurations, fps=30, play=true)
    ```

    You should now do:

    ```
    animation = Animation(mvis, times, configurations, fps=25)
    setanimation!(mvis, animation, play=true)
    ```
    """, :setanimation_t_q)
    animation = Animation(mvis, times, configurations, fps=fps)
    setanimation!(mvis, animation, play=play, repetitions=repetitions)
end


#!/bin/bash
export JULIA_NUM_THREADS=4

#good_orders = 55 .+ collect(1:14);
 #good_orders = 55 .+ collect(17:24)
 #good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41))
# julia -e 'idx_day_to_use=1; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=2; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=3; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=4; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=5; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=6; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=7; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=8; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=9; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=10; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=11; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
# julia -e 'idx_day_to_use=12; good_orders = 55 .+ collect(1:14); include("examples/calc_ccfs_subset_orders.jl")';
#julia -e 'idx_day_to_use=13; include("examples/calc_ccfs.jl")';
#julia -e 'idx_day_to_use=14; include("examples/calc_ccfs.jl")';

#good_orders = 55 .+ collect(17:24);
 julia -e 'idx_day_to_use=1; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=1; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';

 julia -e 'idx_day_to_use=2; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=2; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';

 julia -e 'idx_day_to_use=3; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=3; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=4; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=4; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';

 julia -e 'idx_day_to_use=5; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=5; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';

 julia -e 'idx_day_to_use=6; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=6; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';

  julia -e 'idx_day_to_use=7; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=8; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=9; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=10; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=11; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=12; good_orders = 55 .+ collect(17:24); include("examples/calc_ccfs_subset_orders.jl")';
#julia -e 'idx_day_to_use=13; include("examples/calc_ccfs.jl")';
#julia -e 'idx_day_to_use=14; include("examples/calc_ccfs.jl")';

#good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));
 julia -e 'idx_day_to_use=7; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=8; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=9; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=10; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=11; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';
 julia -e 'idx_day_to_use=12; good_orders = 55 .+ vcat(collect(27:29), collect(37:38),collect(40:41));  include("examples/calc_ccfs_subset_orders.jl")';
#julia -e 'idx_day_to_use=13; include("examples/calc_ccfs.jl")';
#julia -e 'idx_day_to_use=14; include("examples/calc_ccfs.jl")';


export read_non_aligned_sequence, write_mapped_contacts, score_BM, PPV

##############################################################
"""
    function read_non_aligned_sequence(seq::String)

    (seq[might be a path]) --> sequence

    USE:
    Given the path of a non aligned sequence (with '.' and insertions)
    returns the sequence. Takes as input .fasta format, or sequence itself,
    as well as the sequence without the path.

 	INPUT:
    `seq`: amino acid sequence or path to it

    OUTPUT:
    String

"""

function read_non_aligned_sequence(seq::String)
    isfasta(file_path::String) = readdlm(file_path)[1][1] == '>'
    seq =   if ispath(seq)
                if isfasta(seq)
                    readdlm(seq)[2]
                else
                    readdlm(seq)
                end

            else
                seq
            end
    return seq
end

##############################################################
"""
    function afa2model(seq::String)

    (seq) --> dictionary

    USE:
    Given a .afa non aligned sequence coming from a HMM
    returns a dictionary that maps its positions to the matched states (model).

    Example: yAC returns the following dictionary

    position       -->  position
    in the model   -->  in the original sequence

    2              -->  1
    3              -->  2

 	INPUT:
    - `seq`: amino acid sequence with insertions

    OUTPUT:
    Dictionary

"""

function afa2model(seq::String)
    seq = read_non_aligned_sequence(seq)
    p_new =  Dict{Int64,Int64}()
    k_lower_and_upper = 0
    k_gap_and_upper = 0
    for lettr in seq
        if islowercase(lettr)
            k_lower_and_upper += 1
        elseif lettr == '-'
            k_gap_and_upper +=1
        elseif isuppercase(lettr)
            k_gap_and_upper +=1
            k_lower_and_upper += 1
            p_new[k_lower_and_upper] = k_gap_and_upper
        end
    end

    if length(p_new) != length([lettr for lettr in seq if isuppercase(lettr)])
        error("Error: there is some inconsistency in the dictionary, verify input.")
    end
    return p_new
end

##############################################################
"""
    function write_mapped_contacts(afa::String, old_distances_path:String; path_out::String = "")
    function write_mapped_contacts(afa::String, old_distances::Array{Float64, 2}; path_out::String = "")

    (afa, old_distances_path, path_out) --> Nothing

    USE:
    Given a .afa sequence with insertions aligned to the distances file
    writes a new distance file with mapped position numbers.

 	INPUT:
    - `afa`: amino acid sequence with insertions

    - `old_distance_path`: path of amino acid distances from crystal data
    - `old_distances`: array with pairs of positions and residue distances

    - `path_out`: output path, if not declared will be the same of "old_distances_path"

    OUTPUT:
    Nothing

"""

function write_mapped_contacts(afa::String, old_distances_path::String; path_out::String = "")
    println("MAKE SURE THAT THE .afa FILE AND THE DISTANCE FILE REFER TO THE SAME SEQUENCE OR ARE ALIGNED.\n")

    afa_sequence = read_non_aligned_sequence(afa)
    l_afa = length(afa_sequence)
    l_gaps = length([lettr for lettr in afa_sequence if lettr == '-'])
    l_upper = length([lettr for lettr in afa_sequence if isuppercase(lettr)])
    full_length = l_afa - l_gaps

    println(
"The length of the full sequence is          $(full_length).
The length of the model-aligned sequence is $(l_upper).
The gaps in the model-aligned sequence are  $(l_gaps).")

    pdb2wt = afa2model(afa)
    old_distances = readdlm(old_distances_path)

    # count how many pairs with distances there will be in the new file a cool julia oneliner
    n_distances = length([i for i in 1:size(old_distances, 1) if haskey(pdb2wt, old_distances[i, 1]) && haskey(pdb2wt, old_distances[i, 2]) ])

    # new file for amino acid distances
    new_distances = zeros(Float64, n_distances, 3)

    # fill new matrix with positions mapped to our models
    l = 0
    for i in 1:size(old_distances, 1)
        if haskey(pdb2wt, old_distances[i, 1]) && haskey(pdb2wt, old_distances[i, 2])
            l +=1
            new_distances[l, :] = [pdb2wt[old_distances[i, 1]], pdb2wt[old_distances[i, 2]] , old_distances[i, 3]]
        end
    end

    path_out =  path_out == "" ? join(split(old_distances_file, '.')[1:end-1], '.')*"_mapped_model.txt" : path_out
    writedlm(path_out, new_distances)
end

function write_mapped_contacts(afa::String, old_distances::Array{Float64, 2}; path_out::String = "")
    println("MAKE SURE THAT THE .afa FILE AND THE DISTANCE FILE REFER TO THE SAME SEQUENCE OR ARE ALIGNED.\n")

    afa_sequence = read_non_aligned_sequence(afa)
    l_afa = length(afa_sequence)
    l_gaps = length([lettr for lettr in afa_sequence if lettr == '-'])
    l_upper = length([lettr for lettr in afa_sequence if isuppercase(lettr)])
    full_length = l_afa - l_gaps

    println(
"The length of the full sequence is          $(full_length).
The length of the model-aligned sequence is $(l_upper).
The gaps in the model-aligned sequence are  $(l_gaps).")

    pdb2wt = afa2model(afa)

    # count how many pairs with distances there will be in the new file a cool julia oneliner
    n_distances = length([i for i in 1:size(old_distances, 1) if haskey(pdb2wt, old_distances[i, 1]) && haskey(pdb2wt, old_distances[i, 2]) ])

    # new file for amino acid distances
    new_distances = zeros(Float64, n_distances, 3)

    # fill new matrix with positions mapped to our models
    l = 0
    for i in 1:size(old_distances, 1)
        if haskey(pdb2wt, old_distances[i, 1]) && haskey(pdb2wt, old_distances[i, 2])
            l +=1
            new_distances[l, :] = [pdb2wt[old_distances[i, 1]], pdb2wt[old_distances[i, 2]] , old_distances[i, 3]]
        end
    end

    path_out =  path_out == "" ? join(split(old_distances_file, '.')[1:end-1], '.')*"_mapped_model.txt" : path_out
    writedlm(path_out, new_distances)
end
;

##############################################################
"""
    function PPV(scores::Array{Float64,2}, distances::Array{Float64,2}; amino_dist = 4, atom_dist = 8)

    (scores, distances) --> PPV

    USE:
    Given a list of residue pairs with a score (arbitrarly defined) and a list of experimental residue distances
    returns the fraction of pairs - out of the highest scoring ones - that are considered "contacts".
    The definition of contacts depends on linear amino acid distance and physical distance between residues.

    INPUT:
    - `scores`: list of protein positions with a score
    - `distances`: list of protein positions with physical distance
    - `amino_dist`: threshold in terms of linear amino acid distance
    - `atom_dist`: threshold in terms of physical distance

    OUTPUT:
    PPV vector

"""

function PPV(scores::Array{Float64,2}, distances::Array{Float64,2}; amino_dist = 4, atom_dist = 8)
    if size(scores[1,:],1)!=3 || size(distances[1,:],1)!=3
        error("contactprediction.jl - PPV: `scores` and `distances` should be in the format `i j value`.")
    end

    scores = sortslices(scores, dims=1, rev=true, by=x->x[3])
    d = Dict{Tuple{Float64,Float64},Float64}()
    for l in 1:size(distances,1)
        d[(distances[l,1], distances[l,2])] = distances[l,3]
    end
    tp = 0
    np = 0
    TP = []
    cc=0
    for r in 1:size(scores,1)
        if abs(scores[r,1] - scores[r,2]) > amino_dist && get(d, (scores[r,1], scores[r,2]), -1.)>=0.
            np +=1
            if d[(scores[r,1], scores[r,2])]<atom_dist
                tp+=1
            else
                # cc<10?println("$(scores[r,1]), $(scores[r,2])"):Void
                cc+=1
            end
            push!(TP,tp/np)
        end
    end

    return TP
end

##############################################################

"""
    score_BM(J::Array{Float64,2}; APC::Bool = true, gap::Bool = false)
    score_BM(path_param::AbstractString; APC::Bool = true, gap::Bool = false)

    (J) --> scores
    (path_param) --> scores

    USE:
    Compute Frobenius norm of `q x q` blocks in matrix `J` and adds APC correction.
    This is to use the DCA model as a contact prediction predictor.

    INPUT:
    - `J`: coupling matrix in `q x q x L x L` format
    - `path_param`: path of fields and couplings parameters
    - `APC`: apply the famous APC correction. Default to `true`.
    - `gap`: Remove the state `21` from the Frobenius norm. Default to `false`.

    OUTPUT:
    scores

"""

#-----------------------------------------------

function score_BM(path_param::AbstractString; APC::Bool = true, gap::Bool = false)
    h, J = extract_params(path_param)
    q = size(J, 1)
    L = size(J, 4)

    println("Size of input matrix is ",size(J))

    S = zeros(Float64, L,L)
    for i in 1:L-1
        for j in i+1 : L
            S[i,j] = gap ? sqrt(sum(J[:, : ,i, j].^2)) : sqrt(sum(J[1:20, 1:20, i, j].^2))
            S[j,i] = S[i,j]
        end
    end

    F = zeros(Float64, convert(Int64, L*(L-1)/2),3)
    pos = 1
    for i in 1:L
        for j in i+1:L
            F[pos,:] = [i,j,S[i,j] - ( sum(S[:,j])/L * sum( S[:,i])/L ) / (sum(S)/(L^2)) * APC]
            pos += 1
        end
    end

    return F
end

function score_BM(J::Array{Float64,4}; APC::Bool = true, gap::Bool = false)
    q = size(J, 1)
    L = size(J, 4)

    println("Size of input matrix is ",size(J))

    S = zeros(Float64, L,L)
    for i in 1:L-1
        for j in i+1 : L
            S[i,j] = gap ? sqrt(sum(J[:, : ,i, j].^2)) : sqrt(sum(J[1:20, 1:20, i, j].^2))
            S[j,i] = S[i,j]
        end
    end

    F = zeros(Float64, convert(Int64, L*(L-1)/2),3)
    pos = 1
    for i in 1:L
        for j in i+1:L
            F[pos,:] = [i,j,S[i,j] - ( sum(S[:,j])/L * sum( S[:,i])/L ) / (sum(S)/(L^2)) * APC]
            pos += 1
        end
    end

    return F
end