"""
    solveSOPPeel(num::T,maxWidth::Int;peelSetting::PeelSetting=peellen, rrbSetting::Union{RRBSetting,Nothing}=sop,loggingOn::Bool=true, debugOn::Bool=false, timeLimit::Union{Nothing, T}=nothing, useN::Bool=false, useParallel::Bool=false,fileName::Union{Nothing,String}=nothing) where T<:Int

A method for testing various CVRP solver functions

# Arguments
- `num::T`: which problem from the set to get
- `maxWidth::Int`: The max width allowed in a decision diagram layer
- `peelSetting::PeelSetting`:How to pick the peel node
- `loggingOn::Bool`: An optional parameter that can be used to turn off logging statements in other functions
- `debugOn::Bool`: An optional parameter that can be used to turn on debug statements in other functions
- `timeLimit::Union{Nothing, U}`: An optional parameter that adds a time limit
- `useN::Bool`: An optional paramter to use a width of n
- `fileName::Union{Nothing,String}`: The file to write results to
"""
function solveSOPPeel(num::T,maxWidth::Int;peelSetting::PeelSetting=peellen, rrbSetting::Union{RRBSetting,Nothing}=sop,loggingOn::Bool=true, debugOn::Bool=false, timeLimit::Union{Nothing, T}=nothing, useN::Bool=false, useParallel::Bool=false,fileName::Union{Nothing,String}=nothing) where T<:Int
    model = readSOPFile(getSOPFilePath(num))

    if useN
        maxWidth = length(model)
    end
    start = time()
    rrbValues, rrbLastArc = precalculateSOPRRB(model)

    if !isnothing(fileName)
        f = open(fileName,"a")
            write(f, "__________________________________\n\n")
            write(f, string("Peel Setting: ", peelSetting, "\n"))
            write(f, string("Max Width: ", maxWidth, "\n"))
            if useParallel
                write(f, string("Processors: ", Threads.nthreads(), "\n"))
            end
            write(f, "Problem $num: \n")
        close(f)
    end
    if !useParallel
        bestKnownSolution, bestKnownValue, bestBound = peelAndBound(
            model,
            maxWidth,
            orderingType=soporder,
            peelSetting=peelSetting,
            rrbSetting=rrbSetting,
            rrbPacket=(rrbValues, rrbLastArc),
            loggingOn=loggingOn,
            debugOn=debugOn,
            timeElapsed=time()-start,
            timeLimit=timeLimit,
            fileName = fileName
        )
    else
        bestKnownSolution, bestKnownValue, bestBound = parallelPeelAndBound(
            model,
            maxWidth,
            orderingType=soporder,
            peelSetting=peelSetting,
            rrbSetting=rrbSetting,
            rrbPacket=(rrbValues, rrbLastArc),
            loggingOn=loggingOn,
            debugOn=debugOn,
            timeElapsed=time()-start,
            timeLimit=timeLimit,
            fileName = fileName
        )
    end
    if !isnothing(fileName)
        f = open(fileName,"a")
            writeSOPSolution(bestKnownSolution, bestKnownValue, bestBound,fileName)
            write(f, string(bestKnownSolution,",",bestKnownValue,",",bestBound, "\n"))
        close(f)
    end
    return bestKnownSolution, bestKnownValue, bestBound
end

"""
    solveSOPBnB(num::T,maxWidth::Int = 32; rrbSetting::Union{RRBSetting,Nothing}=sop,loggingOn::Bool=true, debugOn::Bool=false, timeLimit::Union{Nothing, T}=nothing, useN::Bool=false,fileName::Union{Nothing,String}=nothing) where T<:Int

A method for testing various CVRP solver functions

# Arguments
- `num::T`: which problem from the set to get
- `maxWidth::Int`: The max width allowed in a decision diagram layer
- `loggingOn::Bool`: An optional parameter that can be used to turn off logging statements in other functions
- `debugOn::Bool`: An optional parameter that can be used to turn on debug statements in other functions
- `timeLimit::Union{Nothing, U}`: An optional parameter that adds a time limit
- `useN::Bool`: An optional paramter to use a width of n
- `fileName::Union{Nothing,String}`: The file to write results to
"""
function solveSOPBnB(num::T,maxWidth::Int = 32; rrbSetting::Union{RRBSetting,Nothing}=sop,loggingOn::Bool=true, debugOn::Bool=false, timeLimit::Union{Nothing, T}=nothing, useN::Bool=false,fileName::Union{Nothing,String}=nothing) where T<:Int
    model = readSOPFile(getSOPFilePath(num))

    if useN
        maxWidth = length(model)
    end
    start = time()
    rrbValues, rrbLastArc = precalculateSOPRRB(model)

    if !isnothing(fileName)
        f = open(fileName,"a")
        write(f, "__________________________________\n\n")
            write(f, "Problem $num: \n")
            write(f, string("Max Width: ", maxWidth, "\n"))
        close(f)
    end
    bestKnownSolution, bestKnownValue, bestBound = branchAndBound(
        model,
        maxWidth,
        orderingType=soporder,
        rrbSetting=rrbSetting,
        rrbPacket=(rrbValues, rrbLastArc),
        loggingOn=loggingOn,
        debugOn=debugOn,
        timeElapsed=time()-start,
        timeLimit=timeLimit,
        fileName=fileName
    )
    if !isnothing(fileName)
        f = open(fileName,"a")
            writeSOPSolution(bestKnownSolution, bestKnownValue, bestBound,fileName)
            write(f, string(bestKnownSolution,",",bestKnownValue,",",bestBound, "\n"))
        close(f)
    end
    return bestKnownSolution, bestKnownValue, bestBound
end

function benchmarkSOPs(width::Int, fileName::String, timeLimit::Int;numStart=1,numEnd=41,usePeel::Bool=true, peelSetting::PeelSetting=peellen, useN::Bool=false, useParallel::Bool=false)
    if usePeel
        solveSOPPeel(8,64, loggingOn=true, debugOn=false)
    else
        solveSOPBnB(8,324, loggingOn=true, debugOn=false)
    end
    for i in numStart:numEnd
        if usePeel
            block = @timed solveSOPPeel(i,width,peelSetting=peelSetting, loggingOn=true, debugOn=false, timeLimit=timeLimit, useN=useN, useParallel=useParallel, fileName=fileName)
        else

            block = @timed solveSOPBnB(i,width, loggingOn=true, debugOn=false, timeLimit=timeLimit, fileName=fileName)
        end
        f = open(fileName,"a")
            write(f, string(block[2], " seconds \n"))
        close(f)
    end
end

"""
    getSOPFilePath(num::T;getSet::Bool=false) where T<:Int

The locations of the benchmark problems for convenience

# Arguments
- `num::T`: which problem from the selected set to get
- `getSet::Bool`: An optional parameter that can be used to get an entire set instead of just one
"""
function getSOPFilePath(num::T;getSet::Bool=false) where T<:Int
    #SOPLIB: http://comopt.ifi.uni-heidelberg.de/software/TSPLIB95/
    paths = String[
        "./examples/sop/ESC07.sop",#2125 (1)
        "./examples/sop/ESC11.sop",#2075 (2)
        "./examples/sop/ESC12.sop",#1675 (3)
        "./examples/sop/ESC25.sop",#1681 (4)
        "./examples/sop/ESC47.sop",#1288 (5)
        "./examples/sop/ESC63.sop",#62 (6)
        "./examples/sop/ESC78.sop",#18230 (7)
        "./examples/sop/br17.10.sop",#55 (8)
        "./examples/sop/br17.12.sop",#55 (9)
        "./examples/sop/ft53.1.sop",#7531 (10)
        "./examples/sop/ft53.2.sop",#8026 (11)
        "./examples/sop/ft53.3.sop",#10262 (12)
        "./examples/sop/ft53.4.sop",#14425 (13)
        "./examples/sop/ft70.1.sop",#39313 (14)
        "./examples/sop/ft70.2.sop",#[40101,40419]  (15)
        "./examples/sop/ft70.3.sop",#42535  (16)
        "./examples/sop/ft70.4.sop",#53530  (17)
        "./examples/sop/kro124p.1.sop",#[38762,39420]  (18)
        "./examples/sop/kro124p.2.sop",#[39841,41336]  (19)
        "./examples/sop/kro124p.3.sop",#[43904,49499]  (20)
        "./examples/sop/kro124p.4.sop",#[73021,76103]  (21)
        "./examples/sop/p43.1.sop",#28140  (22)
        "./examples/sop/p43.2.sop",#28480  (23)
        "./examples/sop/p43.3.sop",#28835 (24)
        "./examples/sop/p43.4.sop",#83005  (25)
        "./examples/sop/prob.42.sop",#243 (26)
        "./examples/sop/prob.100.sop",#[1045,1163]  (27)
        "./examples/sop/rbg048a.sop",#351 (28)
        "./examples/sop/rbg050c.sop",#467 (29)
        "./examples/sop/rbg109a.sop",#1038 (30)
        "./examples/sop/rbg150a.sop",#1750 (31)
        "./examples/sop/rbg174a.sop",#2033 (32)
        "./examples/sop/rbg253a.sop",#2950 (33)
        "./examples/sop/rbg323a.sop",#3140 (34)
        "./examples/sop/rbg341a.sop",#2568 (35)
        "./examples/sop/rbg358a.sop",#2545 (36)
        "./examples/sop/rbg378a.sop",#[2809,2816]  (37)
        "./examples/sop/ry48p.1.sop",#15805 (38)
        "./examples/sop/ry48p.2.sop",#[16074,16666]  (39)
        "./examples/sop/ry48p.3.sop",#[19490,19894]  (40)
        "./examples/sop/ry48p.4.sop"#31446  (41)
    ]
    if getSet
        return paths
    else
        return paths[num]
    end
end


mutable struct RandomPositionsXXZWithDegeneracyLifted <: Model
    posdatadescriptor
    interaction
    δs
    basis
end

# small constructor - useful for loading
RandomPositionsXXZWithDegeneracyLifted(posdatadescriptor, interaction) = RandomPositionsXXZWithDegeneracyLifted(posdatadescriptor, interaction, nothing, nothing)

function Base.getproperty(xxz::RandomPositionsXXZWithDegeneracyLifted, p::Symbol)
    if hasfield(typeof(xxz), p)
        return getfield(xxz, p)
    else
        return getproperty(getfield(xxz, :posdatadescriptor), p)
    end
end

model_name(::RandomPositionsXXZWithDegeneracyLifted) = "Jxxz_with_degeneracy_lifted"
model_fileprefix(xxz::RandomPositionsXXZWithDegeneracyLifted) = @sprintf("%s_%id_alpha_%.1f_N_%02i_lifted", xxz.geometry, xxz.dimension, xxz.interaction.α, xxz.system_size)

struct PreparedJXXZWithDegeneracyLifted{JT,SB}
    J::JT
    δs::Vector{Float64}
    basis::SB
end

# shot, field, rho
parameter_dims(model::PreparedJXXZWithDegeneracyLifted) = (size(model.J)[3], length(model.δs), size(model.J)[4])

function initialize_model!(model::RandomPositionsXXZWithDegeneracyLifted, array_initializer)
    posdata = load(model.posdatadescriptor)
    shots = posdata.shots
    N = posdata.system_size
    ρs = posdata.ρs
    J = fill!(array_initializer(Float64,N,N,shots, length(ρs)), 0)
    # see Jxxz_x.jl
    _compute_interactions!(J, posdata, model.interaction, nothing)

    return PreparedJXXZWithDegeneracyLifted(J, model.δs, model.basis)
end

_flat_to_indices(index, nshots) = fldmod1(index, nshots)

function split_workload(model::PreparedJXXZWithDegeneracyLifted, parts)
    nshots = size(model.J, 3)
    nρs = size(model.J, 4)
    return _workload_chunks(nshots*nρs, parts)
end

function do_parameters(diag_callback, model::PreparedJXXZWithDegeneracyLifted, parameter_chunk)
    N = size(model.J, 1)
    nshots = size(model.J, 3)
    for index in parameter_chunk
        logmsg("Starting index $(index) of $(parameter_chunk)")
        ρindex, shot = _flat_to_indices(index, nshots)

        J = @view model.J[:,:, shot, ρindex]
        H_int = symmetrize_operator(xxzmodel(J, -0.73), model.basis)

        for (k, δ) in enumerate(model.δs)
            logmsg("Doing #rho=$ρindex #shot=$shot #delta=$k")
            parameterI = CartesianIndex(shot, k, ρindex)
            z_fields = dropdims(sum(J; dims=1); dims=1)
            diag_callback(parameterI, H_int + δ * symmetrize_operator(z_field(z_fields), model.basis))
        end
    end
end


using Test, Onda, Dates

@testset "basic TimeSpan code paths" begin
    t = TimeSpan(Nanosecond(rand(UInt32)))
    @test t == TimeSpan(t)
    @test t == TimeSpan(first(t), last(t))
    @test contains(t, t)
    @test overlaps(t, t)
    @test shortest_timespan_containing([t]) == t
    @test shortest_timespan_containing((t,t,t)) == t
    @test duration(t) == Nanosecond(0)
    @test duration(first(t)) == first(t)
    @test_throws ArgumentError TimeSpan(4, 2)
end

@testset "contains(::TimeSpan...)" begin
    @test contains(TimeSpan(10, 20), TimeSpan(10, 20))
    @test contains(TimeSpan(10, 20), TimeSpan(11, 19))
    @test contains(TimeSpan(11, 20), TimeSpan(11, 19))
    @test contains(TimeSpan(10, 19), TimeSpan(11, 19))
    @test !contains(TimeSpan(10, 20), TimeSpan(11, 21))
    @test !contains(TimeSpan(11, 20), TimeSpan(10, 19))
    @test !contains(TimeSpan(10, 19), TimeSpan(10, 21))
    @test !contains(TimeSpan(11, 19), TimeSpan(10, 20))
end

@testset "overlaps(::TimeSpan...)" begin
    @test overlaps(TimeSpan(10, 20), TimeSpan(10, 20))
    @test overlaps(TimeSpan(10, 20), TimeSpan(11, 19))
    @test overlaps(TimeSpan(11, 20), TimeSpan(11, 19))
    @test overlaps(TimeSpan(10, 19), TimeSpan(11, 19))
    @test overlaps(TimeSpan(10, 20), TimeSpan(11, 21))
    @test overlaps(TimeSpan(11, 20), TimeSpan(10, 19))
    @test overlaps(TimeSpan(10, 19), TimeSpan(10, 21))
    @test overlaps(TimeSpan(11, 19), TimeSpan(10, 20))
    @test overlaps(TimeSpan(10, 20), TimeSpan(20, 30))
    @test overlaps(TimeSpan(20, 30), TimeSpan(10, 20))
    @test !overlaps(TimeSpan(10, 20), TimeSpan(21, 30))
    @test !overlaps(TimeSpan(21, 30), TimeSpan(10, 20))
end

@testset "shortest_timespan_containing(spans)" begin
    @test shortest_timespan_containing([TimeSpan(1, 2),
                                        TimeSpan(5, 10),
                                        TimeSpan(2, 3)]) == TimeSpan(1, 10)
    @test shortest_timespan_containing([TimeSpan(3, 7),
                                        TimeSpan(1, 10),
                                        TimeSpan(2, 5)]) == TimeSpan(1, 10)
end

@testset "time <--> index conversion" begin
    @test_throws ArgumentError time_from_index(200, 0)
    @test time_from_index(100, 1) == Nanosecond(0)
    @test time_from_index(100, 301:600) == TimeSpan(Second(3), Second(6))
    @test time_from_index(100, 101:101) == TimeSpan(Second(1), Second(1))
    @test_throws ArgumentError index_from_time(200, Nanosecond(-1))
    @test index_from_time(100, Nanosecond(0)) == 1
    @test index_from_time(100, TimeSpan(Second(3), Second(6))) == 301:600
    @test index_from_time(100, TimeSpan(Second(1), Second(1))) == 101:101
    # test non-integer sample rates
    rate = 100.66
    ns_per_sample = Onda.nanoseconds_per_sample(rate)
    for i in 1:1000
        t = Nanosecond(ceil(Int, (i - 1) * ns_per_sample))
        @test index_from_time(rate, t) == i
        @test time_from_index(rate, i) == t
    end
end


include("runtests_preamble.jl")

using Oceananigans.Units
using LESbrary.IdealizedExperiments: three_layer_constant_fluxes_simulation

@testset "Three layer constant flux simulation" begin
    simulation = three_layer_constant_fluxes_simulation(size = (1, 1, 32),
                                                        stop_time = 2.0,
                                                        snapshot_time_interval = 1.0,
                                                        averages_time_interval = 1.0,
                                                        averages_time_window = 1.0)
    run!(simulation)

    @test simulation.stop_time == 2.0
end


using ..Continuations: as,
    ContinuationSweep, ContinuationSolution,
    ContinuationCache, ContinuationSolver

abstract type AbstractSpecialPoint{tkind <: TimeKind, ptType} end
TimeKind(::Type{<: AbstractSpecialPoint{tkind}}) where tkind = tkind()

struct SpecialPoint{tkind, ptType, uType, JType,
                    } <: AbstractSpecialPoint{tkind, ptType}
    timekind::tkind
    point_type::ptType
    point_index::Int
    u::uType
    J::JType
    sweep::WeakRef
end
# TODO: put BifurcationProblem in SpecialPoint/Interval struct so that
# init(point) can initiate codim+1 bifurcation solver.

struct SpecialPointInterval{tkind, ptType, uType, JType,
                            } <: AbstractSpecialPoint{tkind, ptType}
    timekind::tkind
    point_type::ptType
    point_index::Int
    u0::uType
    u1::uType
    # J0::JType
    J1::JType
    sweep::WeakRef
end

struct BifurcationSweep{
        tkind <: TimeKind,
        ptType,
        S <: ContinuationSweep,
        JType <: AbstractArray,
        eType <: AbstractArray,
        siType <: SpecialPointInterval{tkind, ptType},
        spType <: SpecialPoint{tkind, ptType},
        }
    timekind::tkind
    super::S
    jacobians::Vector{JType}
    eigvals::Vector{eType}
    special_intervals::Vector{siType}
    special_points::Vector{spType}
end
# See: [[../continuations/solution.jl::ContinuationSweep]]

BifurcationSweep{tkind, ptType, S, JType, eType, siType, spType}(super::S) where{
    tkind <: TimeKind,
    ptType,
    S <: ContinuationSweep,
    JType <: AbstractArray,
    eType <: AbstractArray,
    siType <: SpecialPointInterval{tkind, ptType},
    spType <: SpecialPoint{tkind, ptType},
} = BifurcationSweep{tkind, ptType, S, JType, eType, siType, spType}(
    tkind(),
    super,
    JType[],
    eType[],
    siType[],
    spType[],
)

TimeKind(::Type{<: BifurcationSweep{tkind}}) where tkind = tkind()

Base.length(sweep::BifurcationSweep) = length(as(sweep, ContinuationSweep))

PointTypeType(::Type{<: BifurcationSweep{_tk, ptType}}) where {_tk, ptType} =
    ptType

point_type_type(::T) where {T <: BifurcationSweep} = PointTypeType(T)

contkind(sweep::BifurcationSweep) = contkind(problem_of(sweep)) # TODO: don't
# this is terrible!

function check_sweep_length(sweep)
    @assert length(sweep) == length(sweep.jacobians) == length(sweep.eigvals)
end

function sweeptype(prob::BifurcationProblem,
                   solver::ContinuationSolver,
                   JType::Type = typeof(solver.cache.J),
                   eType::Type = typeof(eigvals_prototype(prob, solver.cache)),
                   )
    tkind = typeof(timekind(solver.cache))
    S = eltype(solver.sol.sweeps)
    uType = eltype(S)
    ptType = point_type_type(prob)
    siType = SpecialPointInterval{tkind, ptType, uType, JType}
    spType = SpecialPoint{tkind, ptType, uType, JType}
    return BifurcationSweep{tkind, ptType, S, JType, eType, siType, spType}
end

function BifurcationSweep(super::ContinuationSweep, solver::ContinuationSolver)
    return sweeptype(solver.prob, solver)(super)
end

function push_special_point!(sweep::BifurcationSweep, point::SpecialPoint)
    @assert point.point_index < length(sweep)

    push!(sweep.special_points, point)

    # TODO: Add two new points rather than replacing the old one.
    # At the moment, just "move" the recorded points at the resolved
    # `point` so that there would be no discrepancy between the
    # plotted special points and stability of the lines.
    cont_sweep = as(sweep, ContinuationSweep)
    cont_sweep.u[point.point_index] = point.u
    cont_sweep.u[point.point_index + 1] = point.u

    return sweep
end

struct BifurcationSolution{S <: ContinuationSolution,
                           W <: BifurcationSweep}
    super::S
    sweeps::Vector{W}
end

BifurcationSolution(super::ContinuationSolution,
                    SweesiType::Type{<: BifurcationSweep}) =
    BifurcationSolution(super, SweesiType[])

TimeKind(::Type{<: BifurcationSolution{_, W}}) where {_, W} = TimeKind(W)


# Automatically generated
struct Params{T}
    z::Vector{Float64}
    footang::T
    g::T
    iff::T
    ihat::T
    irf::T
    ish::T
    ith::T
    k1::T
    k2::T
    k3::T
    k4::T
    k5::T
    k6::T
    k7::T
    k8::T
    lff::T
    lffo::T
    lhat::T
    lmtpxi::T
    lrf::T
    lrff::T
    lrffo::T
    lrfo::T
    lsh::T
    lsho::T
    lth::T
    ltho::T
    ltoexi::T
    mff::T
    mhat::T
    mrf::T
    msh::T
    mth::T
    mtpb::T
    mtpk::T
    rmtpxi::T
    rtoexi::T
    u4::T
    u5::T
    u6::T
    u7::T
    u8::T
    u9::T

end

# initialise with constant values
function Params(footang, g, iff, ihat, irf, ish, ith, k1, k2, k3, k4, k5, k6, k7, k8, lff, lffo, lhat, lmtpxi, lrf, lrff, lrffo, lrfo, lsh, lsho, lth, ltho, ltoexi, mff, mhat, mrf, msh, mth, mtpb, mtpk, rmtpxi, rtoexi)
    z = Vector{Float64}(undef, 827)
    u4 = 0.0
    u5 = 0.0
    u6 = 0.0
    u7 = 0.0
    u8 = 0.0
    u9 = 0.0
    z[380] = g * msh
    z[378] = g * mff
    z[725] = lffo * z[378]
    z[379] = g * mrf
    z[19] = cos(footang)
    z[20] = sin(footang)
    z[68] = lff - lffo
    z[69] = lrf - 0.5lrfo
    z[70] = lsh - lsho
    z[71] = lth - ltho
    z[72] = mhat + 2mff + 2mrf + 2msh + 2mth
    z[74] = (lff * mff + lff * mhat + lffo * mff + 2 * lff * mrf + 2 * lff * msh + 2 * lff * mth) / z[72]
    z[75] = (lrfo * mrf + 2 * lrf * mff + 2 * lrf * mhat + 2 * lrf * mrf + 4 * lrf * msh + 4 * lrf * mth) / z[72]
    z[76] = (lrffo * mrf) / z[72]
    z[77] = (lsh * mff + lsh * mhat + lsh * mrf + lsh * msh + lsho * msh + 2 * lsh * mth) / z[72]
    z[78] = (lth * mff + lth * mhat + lth * mrf + lth * msh + lth * mth + ltho * mth) / z[72]
    z[79] = (mff * z[68]) / z[72]
    z[80] = (lrf * mff + mrf * z[69]) / z[72]
    z[81] = (lsh * mff + lsh * mrf + msh * z[70]) / z[72]
    z[82] = (lth * mff + lth * mrf + lth * msh + mth * z[71]) / z[72]
    z[377] = g * mhat
    z[381] = g * mth
    z[411] = z[74] - lff
    z[412] = 0.5 * z[75] - lrf
    z[413] = z[77] - lsh
    z[414] = z[78] - lth
    z[499] = 0.5 * z[75] - 0.5lrfo
    z[500] = 0.5 * z[76] - 0.5lrffo
    z[524] = 2lrf - z[75]
    z[525] = lff - z[74]
    z[530] = lsh - z[77]
    z[545] = lsh - z[81]
    z[546] = lth - z[82]
    z[547] = lrf - z[80]
    z[548] = lth - z[78]
    z[553] = (lrf - 0.5lrfo) - z[80]
    z[576] = z[19] * z[76]
    z[586] = 2 * z[499] + 2 * z[19] * z[500]
    z[588] = 2 * z[500] + 2 * z[19] * z[499]
    z[596] = 2 * z[553] + 2 * z[19] * z[500]
    z[598] = 2 * z[500] + 2 * z[19] * z[553]
    z[608] = lff * mhat
    z[610] = lrf * mhat
    z[612] = lsh * mhat
    z[614] = lth * mhat
    z[620] = lffo * mff
    z[626] = lff * mrf
    z[628] = lrfo * mrf
    z[630] = lrffo * mrf
    z[672] = mrf * z[69]
    z[683] = lsh * mrf
    z[700] = msh * z[70]
    z[715] = mth * z[71]
    z[727] = z[71] * z[381]
    z[730] = z[70] * z[380]
    z[754] = ihat + 2iff + 2irf + 2ish + 2ith + mff * lffo^2
    z[755] = lrffo^2 + lrfo^2 + 4 * lff^2 + 2 * lrffo * lrfo * z[19]
    z[756] = lff * lrffo
    z[757] = lff * lrfo
    z[758] = lrffo^2 + 4 * z[69]^2
    z[759] = lrffo * z[19] * z[69]
    z[760] = lff^2 + lrf^2 + lsh^2 + lth^2
    z[761] = lff * lsh
    z[762] = lff * lth
    z[763] = lrf * lth
    z[764] = lff * lrf
    z[765] = lrf * lsh
    z[766] = lsh * lth
    z[768] = lrffo * z[19]
    z[769] = lrfo * z[19]
    z[770] = lrffo * z[20]
    z[771] = lrfo * z[20]
    z[772] = z[19] * z[69]
    z[773] = z[20] * z[69]
    z[775] = iff + irf + ish + ith
    z[776] = lrffo^2
    z[777] = z[69]^2
    z[782] = iff + irf + ish + ith + mff * lffo^2
    z[787] = iff + irf + ish
    z[788] = lsh * z[69]
    z[789] = lrffo * lsh
    z[794] = iff + irf + ish + mff * lffo^2
    z[795] = lff^2 + lrf^2 + lsh^2
    z[800] = iff + irf
    z[801] = (lrffo^2 + 4 * z[69]^2) - 4 * lrffo * z[19] * z[69]
    z[806] = iff + irf + mff * lffo^2
    z[807] = lff^2 + lrf^2

    return Params(z, footang, g, iff, ihat, irf, ish, ith, k1, k2, k3, k4, k5, k6, k7, k8, lff, lffo, lhat, lmtpxi, lrf, lrff, lrffo, lrfo, lsh, lsho, lth, ltho, ltoexi, mff, mhat, mrf, msh, mth, mtpb, mtpk, rmtpxi, rtoexi, u4, u5, u6, u7, u8, u9)
end


"""
This submodule contains ready-to-use Scatterer models of zooplankton.
"""
module Models

using SDWBA
using DelimitedFiles

export  krill_mcgeehee,
		krill_conti,
		calanoid_copepod,
		sandeel,
		daphnia,
		nauplius

DATA_DIR = joinpath(dirname(@__FILE__), "data/")

"""
Krill shape from McGeehee et al. 1998, 'Effects of orientation on acoustic scattering
from Antarctic krill at 120 kHz.' Deep-Sea Research II 45(7), 1273-1294.
"""
krill_mcgeehee = from_csv(joinpath(DATA_DIR, "generic_krill_McGeehee1998.csv"))

"""
Krill shape from Conti and Demer, 2006, 'Improved parameterization of the SDWBA for
estimating krill target strength.' ICES Journal of Marine Science 63(5), 928-935.
"""
krill_conti = from_csv(joinpath(DATA_DIR, "generic_krill_Conti2006.csv"))

"""
Generic sand eel/sand lance shape (Ammodytes spp.).  Shape is taken from illustration in
Bigelow and Schroeder, 'Fishes of the Gulf of Maine.'  Material properties from
Yasuma et al. (2009), 'Density and sound-speed contrasts, and target strength of
Japanese sandeel Ammodytes personatus.' Fisheries Science 75 (3) 545-552.
"""
sandeel = from_csv(joinpath(DATA_DIR, "sand_eel.csv"))

"""
Generic calanoid copepod shape, 1 mm long, with g and h for "typical" marine zooplankton
(both 1.04, as per Stanton and Chu 2000, 'Review and recommendations for the modelling
of acoustic scattering by fluid-like elongated zooplankton: euphausiids and copepods.'
ICES Journal of Marine Science 57, 793-807).
"""
calanoid_copepod = from_csv(joinpath(DATA_DIR, "generic_acartia.csv"))

"""
Generic Daphnia shape used in Warren et al. 2016, 'Measuring the distribution,
abundance, and biovolume of zooplankton in an oligotrophic freshwater lake
with a 710 kHz scientific echosounder.' Limnology and Oceanography: Methods
"""
daphnia = from_csv(joinpath(DATA_DIR, "daphnia.csv"))

"""
Alternative (more realistic?) daphnia shape
"""
daphnia2 = from_csv(joinpath(DATA_DIR, "daphnia2.csv"))

"""
Generic nauplius larva shape, 0.5 mm long, with g and h for "typical" marine zooplankton
(both 1.04, as per Stanton and Chu 2000, 'Review and recommendations for the modelling
of acoustic scattering by fluid-like elongated zooplankton: euphausiids and copepods.'
ICES Journal of Marine Science 57, 793-807).
"""
nauplius = from_csv(joinpath(DATA_DIR, "nauplius.csv"))

end


using MINLPTests, JuMP, Ipopt, AmplNLWriter, Test

const OPTIMIZER = MINLPTests.JuMP.with_optimizer(
    AmplNLWriter.Optimizer, Ipopt.amplexe, ["print_level=0"]
)

@testset "MINLPTests" begin
    ###
    ### src/nlp tests.
    ###

    MINLPTests.test_nlp(OPTIMIZER, exclude = [
        "005_011",  # Uses the function `\`
        "006_010",  # User-defined function
        "007_010"   # Infeasible model
    ], objective_tol = 1e-5, primal_tol = 1e-5, dual_tol = NaN)

    @testset "nlp_007_010" begin
        MINLPTests.nlp_007_010(OPTIMIZER, 1e-5, NaN, NaN;
            termination_target = MOI.INFEASIBLE,
            primal_target = MOI.NO_SOLUTION)
    end

    ###
    ### src/nlp-cvx tests.
    ###

    MINLPTests.test_nlp_cvx(OPTIMIZER, exclude = [
        "109_010"  # Ipopt fails to converge
    ])
end


export CSRGraph
export IndexType, LOCAL_INDICES, GLOBAL_INDICES
export StorageStatus, STORAGE_2D, STORAGE_1D_UNPACKED, STORAGE_1D_PACKED
export ProfileType, STATIC_PROFILE, DYNAMIC_PROFILE

"""
Can be used to differentiate global and local indices
"""
@enum IndexType LOCAL_INDICES GLOBAL_INDICES

"""
Status of the graph's or matrix's storage, when not in
a fill-complete state.
"""
@enum StorageStatus STORAGE_2D STORAGE_1D_UNPACKED STORAGE_1D_PACKED

"""
Allocation profile for matrix/graph entries
"""
@enum ProfileType STATIC_PROFILE DYNAMIC_PROFILE

#TODO document the type and constructors

mutable struct CSRGraph{GID <: Integer, PID <: Integer, LID <: Integer} <: RowGraph{GID, PID, LID}
    rowMap::BlockMap{GID, PID, LID}
    colMap::Union{BlockMap{GID, PID, LID}, Nothing}
    rangeMap::Union{BlockMap{GID, PID, LID}, Nothing}
    domainMap::Union{BlockMap{GID, PID, LID}, Nothing}

    #may be null if domainMap and colMap are the same
    importer::Union{Import{GID, PID, LID}, Nothing}
    #may be null if rangeMap and rowMap are the same
    exporter::Union{Export{GID, PID, LID}, Nothing}

    localGraph::LocalCSRGraph{LID, LID}

    #Local number of (populated) entries; must always be consistent
    nodeNumEntries::LID

    #Local number of (populated) diagonal entries.
    nodeNumDiags::LID

    #Local maximum of the number of entries in each row.
    nodeMaxNumRowEntries::LID

    #Global number of entries in the graph.
    globalNumEntries::GID

    #Global number of (populated) diagonal entries.
    globalNumDiags::GID

    #Global maximum of the number of entries in each row.
    globalMaxNumRowEntries::GID

    #Whether the graph was allocated with static or dynamic profile.
    pftype::ProfileType

    ## 1-D storage (Static profile) data structures ##
    localIndices1D::Array{LID, 1}
    globalIndices1D::Array{GID, 1}
    rowOffsets::Array{LID, 1}  #Tpetra: k_rowPts_

    ## 2-D storage (Dynamic profile) data structures ##
    localIndices2D::Array{Array{LID, 1}, 1}
    globalIndices2D::Array{Array{GID, 1}, 1}
    #may exist in 1-D storage if not packed
    numRowEntries::Array{LID, 1}

    storageStatus::StorageStatus

    indicesAllowed::Bool
    indicesType::IndexType
    fillComplete::Bool

    lowerTriangle::Bool
    upperTriangle::Bool
    indicesAreSorted::Bool
    noRedundancies::Bool
    haveLocalConstants::Bool
    haveGlobalConstants::Bool
    sortGhostsAssociatedWithEachProcessor::Bool

    plist::Dict

    nonLocals::Dict{GID, Array{GID, 1}}

    #Large ammounts of duplication between the constructors, so group it in an inner constructor
    function CSRGraph(
        rowMap::BlockMap{GID, PID, LID},
        colMap::Union{BlockMap{GID, PID, LID}, Nothing},
        rangeMap::Union{BlockMap{GID, PID, LID}, Nothing},
        domainMap::Union{BlockMap{GID, PID, LID}, Nothing},

        localGraph::LocalCSRGraph,

        nodeNumEntries::LID,

        pftype::ProfileType,
        storageStatus::StorageStatus,

        indicesType::IndexType,
        plist::Dict
    ) where {GID <: Integer, PID <: Integer, LID <: Integer}

        graph = new{GID, PID, LID}(
            rowMap,
            colMap,
            rangeMap,
            domainMap,

            nothing,
            nothing,

            localGraph,

            #Local number of (populated) entries; must always be consistent
            nodeNumEntries,

            #using 0 to indicate uninitiallized, since -1 isn't gareenteed to work
            0, #nodeNumDiags
            0, #nodeMaxNumRowEntries
            0, #globalNumEntries
            0, #globalNumDiags
            0, #globalMaxNumRowEntries

            #Whether the graph was allocated with static or dynamic profile.
            pftype,

            ## 1-D storage (Static profile) data structures ##
            LID[],
            GID[],
            LID[],

            ## 2-D storage (Dynamic profile) data structures ##
            Array{Array{LID, 1}, 1}(undef, 0),
            Array{Array{GID, 1}, 1}(undef, 0),
            LID[],

            storageStatus,

            false,
            indicesType,
            false,

            false,
            false,
            true,
            true,
            false,
            false,
            true,

            plist,

            Dict{GID, Array{GID, 1}}()
        )

        ## staticAssertions()
        #skipping sizeof checks
        #skipping max value checks related to size_t

        @assert(indicesType != LOCAL_INDICES || colMap != nothing,
            "Cannot have local indices with a null column Map")

        #ensure LID is a subset of GID (for positive numbers)
        if !(LID <: GID) && (GID != BigInt) && (GID != Integer)
            # all ints are assumed to be able to handle 1, up to their max
            if LID == BigInt || LID == Integer || typemax(LID) > typemax(GID)
                throw(InvalidArgumentError("The positive values of GID must "
                        * "be a superset of the positive values of LID"))
            end
        end

        graph
    end
end

#### Constructors #####

function CSRGraph(rowMap::BlockMap{GID, PID, LID}, maxNumEntriesPerRow::Integer,
        pftype::ProfileType; plist...) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, LID(maxNumEntriesPerRow),  pftype, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, maxNumEntriesPerRow::Integer,
        pftype::ProfileType, plist::Dict) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, nothing, LID(maxNumEntriesPerRow), pftype, plist)
end

#= Should be handled by the following constructors
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        maxNumEntriesPerRow::Integer, pftype::ProfileType; plist...) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, LID(maxNumEntriesPerRow), pftype, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        maxNumEntriesPerRow::Integer, pftype::ProfileType, plist::Dict) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, LID(maxNumEntriesPerRow), pftype, plist)
end
=#

function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::Union{BlockMap{GID, PID, LID}, Nothing},
        maxNumEntriesPerRow::Integer, pftype::ProfileType; plist...) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, maxNumEntriesPerRow, pftype, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::Union{BlockMap{GID, PID, LID}, Nothing},
        maxNumEntriesPerRow::Integer, pftype::ProfileType, plist::Dict) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    graph = CSRGraph(
        rowMap,
        colMap,
        nothing,
        nothing,

        LocalCSRGraph{LID, LID}(), #localGraph

        LID(0), #nodeNumEntries

        pftype,

        (pftype == STATIC_PROFILE ?
              STORAGE_1D_UNPACKED
            : STORAGE_2D),

        (colMap == nothing) ? GLOBAL_INDICES : LOCAL_INDICES,
        plist
    )
    allocateIndices(graph, graph.indicesType, LID(maxNumEntriesPerRow))

    resumeFill(graph, plist)
    checkInternalState(graph)

    graph
end

function CSRGraph(rowMap::BlockMap{GID, PID, LID}, numEntPerRow::AbstractArray{<:Integer, 1},
        pftype::ProfileType; plist...)  where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, numEntPerRow, pftype, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, numEntPerRow::AbstractArray{<:Integer, 1},
        pftype::ProfileType, plist::Dict)  where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, nothing, numEntPerRow, pftype, plist)
end

function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        numEntPerRow::AbstractArray{<:Integer, 1}, pftype::ProfileType;
        plist...)  where {GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, numEntPerRow, pftype, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        numEntPerRow::AbstractArray{<:Integer, 1}, pftype::ProfileType,
        plist::Dict)  where {GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, numEntPerRow, pftype, plist)
end

function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::Union{BlockMap{GID, PID, LID}, Nothing},
        numEntPerRow::AbstractArray{<:Integer, 1}, pftype::ProfileType;
        plist...)  where {GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, numEntPerRow, pftype, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::Union{BlockMap{GID, PID, LID}, Nothing},
        numEntPerRow::AbstractArray{<:Integer, 1}, pftype::ProfileType,
        plist::Dict)  where {GID <: Integer, PID <: Integer, LID <: Integer}
    graph = CSRGraph(
        rowMap,
        colMap,
        nothing,
        nothing,

        LocalCSRGraph{LID, LID}(), #localGraph

        LID(0), #nodeNumEntries

        #Whether the graph was allocated with static or dynamic profile.
        pftype,

        (pftype == STATIC_PROFILE ?
              STORAGE_1D_UNPACKED
            : STORAGE_2D),

        (colMap == nothing) ? GLOBAL_INDICES : LOCAL_INDICES,
        plist
    )

    localNumRows = numMyElements(rowMap)
    if length(numEntPerRow) != localNumRows
        throw(InvalidArgumentError("numEntPerRows has length $(length(numEntPerRow)) " *
                "!= the local number of rows $lclNumRows as spcified by the input row Map"))
    end

    allocateIndices(graph, graph.indicesType, Array{LID, 1}(numEntPerRow))

    resumeFill(graph, plist)
    checkInternalState(graph)

    graph
end

function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        rowPointers::AbstractArray{LID, 1}, columnIndices::Array{LID, 1};
        plist...) where {GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, rowPointers, columnIndices, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        rowPointers::AbstractArray{LID, 1}, columnIndices::Array{LID, 1},
        plist::Dict) where {GID <: Integer, PID <: Integer, LID <: Integer}
    graph = CSRGraph(
        rowMap,
        colMap,
        nothing,
        nothing,

        LocalCSRGraph{LID, LID}(), #localGraph

        LID(0), #nodeNumEntries

        STATIC_PROFILE,

        STORAGE_1D_PACKED,

        LOCAL_INDICES,
        plist
    )
    #seems to be already taken care of
    #allocateIndices(graph, LOCAL_INDICES, numEntPerRow)

    setAllIndicies(graph, rowPointers, columnIndicies)
    checkInternalState(graph)

    graph
end

function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        localGraph::LocalCSRGraph{LID, LID}; plist...) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    CSRGraph(rowMap, colMap, localGraph, Dict(plist))
end
function CSRGraph(rowMap::BlockMap{GID, PID, LID}, colMap::BlockMap{GID, PID, LID},
        localGraph::LocalCSRGraph{LID, LID}, plist::Dict) where {
        GID <: Integer, PID <: Integer, LID <: Integer}
    mapRowCount = numMyElements(rowMap)
    graph = CSRGraph(
        rowMap,
        colMap,
        rowMap,
        colMap,

        localGraph,

        localGraph.rowMap[mapRowCount+1], #nodeNumEntries

        STATIC_PROFILE,

        STORAGE_1D_PACKED,

        LOCAL_INDICES,
        plist
    )

    if numRows(localGraph) != numMyElements(rowMap)
        throw(InvalidArgumentError("input row map and input local "
                * "graph need to have the same number of rows.  The "
                * "row map claims $(numMyElements(rowMap)) row(s), "
                * "but the local graph claims $(numRows(localGraph)) "
                * "row(s)."))
    end

    #seems to be already taken care of
    #allocateIndices(graph, LOCAL_INDICES, numEntPerRow)

    makeImportExport(graph)

    d_inds = localGraph.entries
    graph.localIndices1D = d_inds

    d_ptrs = localGraph.rowMap
    graph.rowOffsets = d_ptrs

    #reset local properties
    graph.upperTriangle = true
    graph.lowerTriangle = true
    graph.nodeMaxNumRowEntries = 0
    graph.nodeNumDiags

    for localRow = 1:mapRowCount
        globalRow = gid(rowMap, localRow)
        rowLID = lid(colMap, globalRow)

        #possible that the local matrix has no entries in the column
        #corrisponding to the current row, in that case, the column map
        #might not contain that GID.  Hence, the index validity check
        if rowLID != 0
            if rowLID +1 > length(d_ptrs)
                throw(InvalidArgumentError("The given row Map and/or column Map "
                        * "is/are not compatible with the provided local graphs."))
            end
            if d_ptrs[rowLID] != d_ptr[rowLID+1]
                smallestCol = d_inds[d_ptrs[rowLID]]
                largestCol  = d_inds[d_ptrs[rowLID+1]-1]

                if smallestCol < localRow
                    graph.upperTriangle = false
                end
                if localRow < largestCol
                    graph.lowerTriangle = false
                end

                if rowLID in d_inds[d_ptrs[rowLID]:d_ptrs[rowLID]-1]
                    graph.nodeNumDiags += 1
                end
            end

            graph.nodeMaxNumRowEntries = max((d_ptrs[rowLID + 1] - d_ptrs[rowLID]),
                                            graph.nodeMaxNumRowEntries)
        end
    end

    graph.hasLocalConstants = true
    computeGlobalConstants(graph)

    graph.fillComplete = true
    checkInternalState(graph)

    graph
end


# -------------------------------------------------------------------------------------------------------------------------------------------------------------
# Baseline model
# -------------------------------------------------------------------------------------------------------------------------------------------------------------

using DataFrames, RData,  XLSX, LinearAlgebra, Statistics, CSV

dir = "X:/VIVES/1-Personal/Florian/git/IOTables/src/"
include(dir * "model/transform_data.jl") # Script with functions to import and transform raw data
include(dir * "model/price_hat.jl") # Script with function to obtain the price index
include(dir * "model/wage_hat.jl") # Script with function to obtain the wages and gross output
include(dir * "counterfactual/tariffs_function.jl") # Script with functions to create τ_hat_Z, τ_hat_F from tariff data
include(dir * "counterfactual/head_ries_index.jl") # Script with functions to create bilateral Head-Ries index (symmetric bilateral trade costs)
include(dir * "elasticity/elasticities_functions.jl") # Script with functions to compute statistics for estimating sectoral trade elasticities (method of Caliendo and Parro, 2015)

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

dir = "C:/Users/u0148308/Desktop/raw/" # location of raw data

# Data specification

# WIOD rev. 2013
source = "WIOD"
revision = "2013"
year = 1995 # specified year
N = 41 # number of countries
S = 35 # number of industries

# # WIOD rev. 2016
# source = "WIOD"
# revision = "2016"
# year = 2014 # specified year
# N = 44 # number of countries
# S = 56 # number of industries

# -------------------------------------------------------------------------------------------------------------------------------------------------------------
# prepare tariff data from WTO
# -------------------------------------------------------------------------------------------------------------------------------------------------------------

df_tariffs = DataFrame(reporter_lab=String[], year=Int64[], HS2017_code=String[],
    HS2017_lab=String[], duty_lab=String[], avg_tariff=Union{Float64,Missing}[], reporter=String[])

function tariff_data(reporter::String, id::String, year::String)

    col_names_old = ["Reporter", "Year", "Product Code", "Product Description", "Duty Description", "All items excl. NA - Simple Averages"]
    col_names_new = ["reporter_lab", "year", "HS2017_code", "HS2017_lab", "duty_lab", "avg_tariff"]

    path = dir * "MFN/" * reporter * "_" * year * "_TA/" * reporter * "_" * year * "_Duties_TA.txt"

    df = CSV.read(path, DataFrame)
    df = df[:, col_names_old]
    rename!(df, col_names_new)
    df[:, :reporter] .= id

    return df
end

ctry_names_2013 = ["AUS", "AUT", "BEL", "BGR", "BRA", "CAN", "CHN", "CYP", "CZE", "DEU", "DNK", "ESP", "EST", "FIN", "FRA", "GBR",
    "GRC", "HUN", "IDN", "IND", "IRL", "ITA", "JPN", "KOR", "LTU", "LUX", "LVA", "MEX", "MLT", "NLD", "POL", "PRT", "ROM", "RUS",
    "SVK", "SVN", "SWE", "TUR", "TWN", "USA"]

ctry_EU28 = ["AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK", "EST", "FIN", "FRA", "DEU", "GBR", "GRC", "HUN", "IRL", "ITA", "LVA",
    "LTU", "LUX", "MLT", "NLD", "POL", "PRT", "ROM", "SVK", "SVN", "ESP", "SWE"]

for i in ctry_names_2013

    if i in ctry_EU28
        df = tariff_data("EU", i, "2018") # for EU countries
    elseif i in ["RUS", "TUR"]
        df = tariff_data(i, i, "2019") # countries for which tariffs in 2018 were not available
    else
        df = tariff_data(i, i, "2018")
    end

    append!(df_tariffs, df)
end

# Correspondence table HS2017, ISIC rev. 3 and ISIC rev. 4
correspondence_HS_ISIC = DataFrame(XLSX.readtable(dir * "correspondence_HS_ISIC.xlsx", "Sheet1")...)
transform!(correspondence_HS_ISIC, [:ISIC3, :ISIC4, :HS2017] .=> ByRow(x-> lpad(x,2,'0')), renamecols=false)
rename!(correspondence_HS_ISIC, [:HS2017, :ISIC3, :ISIC4] .=> [:HS2017_code, :ISIC3_code, :ISIC4_code])

# join tables
transform!(df_tariffs, :HS2017_code => ByRow(x->replace(x, "'" => "")), renamecols=false) # remove ' from column otherwise we cannot join properly
df = leftjoin(df_tariffs, correspondence_HS_ISIC, on=:HS2017_code)

# -------

# WIOD rev. 2013
gdf = groupby(df, [:reporter, :ISIC3_code]) # group to compute average tariffs according to ISIC rev. 3 classification
df_tariffs = combine(gdf, :avg_tariff => mean, renamecols=false)
sort!(df_tariffs, [:reporter, :ISIC3_code])
dropmissing!(df_tariffs) # drop missing for using average later

# use average of all countries for RoW
gdf = groupby(df_tariffs, :ISIC3_code)
df_RoW = combine(gdf, :avg_tariff => mean, renamecols=false)
df_RoW[:, :reporter] .= "ZoW"

df_all = vcat(df_tariffs, df_RoW)

reporter = [ctry_names_2013; "ZoW"] # use "ZoW" instead of "RoW" so we can sort later on
industries = lpad.(1:S, 2, '0') # left pad with leading zeros so we can sort later on

gdf = groupby(df_all, :reporter)
df_ctry_avg = combine(gdf, :avg_tariff => mean, renamecols=false)
avg_ctry_tariff = df_ctry_avg.avg_tariff # use average country tariffs for sectors which are missing

df = DataFrame([repeat(reporter, inner=S) repeat(industries, outer=N) repeat(avg_ctry_tariff, inner=S)], [:reporter, :ISIC3_code, :ctry_avg])
df = leftjoin(df, df_all, on=[:reporter, :ISIC3_code])
df.avg_tariff = ifelse.(ismissing.(df.avg_tariff) .== true, df.ctry_avg, df.avg_tariff)
rename!(df, :ISIC3_code => :industry)
df_tariffs = df[:, [:reporter, :industry, :avg_tariff]]
sort!(df_tariffs, [:reporter, :industry])

CSV.write(dir * "MFN/df_tariffs.csv", df_tariffs)

# -------------------------------------------------------------------------------------------------------------------------------------------------------------
# format data to NS×N matrix
# -------------------------------------------------------------------------------------------------------------------------------------------------------------

vec_tariffs = df_tariffs.avg_tariff
matrix_tariffs = repeat(vec_tariffs, 1, N)
countries = [ctry_names_2013; "ZoW"]

τ_Z = ones(N*S,N)

for (i, ctry_i) in enumerate(countries)
    for (j, ctry_j) in enumerate(countries)

        if ctry_i == ctry_j
            τ_Z[1+(i-1)*S:i*S,j] .= 0

        elseif ctry_i in ctry_EU28 && ctry_j in ctry_EU28
            τ_Z[1+(i-1)*S:i*S,j] .= 0

        else
            τ_Z[1+(i-1)*S:i*S,j] = matrix_tariffs[1+(i-1)*S:i*S,j]
        end

    end
end

# export
rows = repeat(countries, inner=S) .* "__" .* repeat(industries, outer=N)
df_tariff_matrix = DataFrame([rows τ_Z], ["reporter"; countries])
XLSX.writetable(dir * "MFN/tariff_matrix.xlsx", df_tariff_matrix, overwrite=true)


using CBM
using Base.Test

function test_all()
    model = load_json(Pkg.dir() * "/CBM/Models/e_coli_core.json")

    println("test_add_reaction")
    test_add_reaction(model)
    println("test_change_objective")
    test_change_objective(model)
    println("test_change_reaction_bounds")
    test_change_reaction_bounds(model)
    println("test_fba")
    test_fba(model)
    println("test_find_blocked_reactions")
    test_find_blocked_reactions(model)
    println("test_find_deadend_metabolites")
    test_find_deadend_metabolites(model)
    println("test_find_exchange_reactions")
    test_find_exchange_reactions(model)
    println("test_find_reactions_from_gene")
    test_find_reactions_from_gene(model)
    println("test_find_reactions_from_metabolite")
    test_find_reactions_from_metabolite(model)
    println("test_fva")
    test_fva(model)
    println("test_gene_deletion")
    test_gene_deletion(model)
    println("test_remove_reaction")
    test_remove_reaction(model)
    println("test_robustness_analysis")
    test_robustness_analysis(model)
    println("test_solvers")
    test_solvers(model)
end

test_all()


const mac_gdrive = "/Volumes/GoogleDrive/My Drive"
const linux_gdrive = "/home/beatriz/mainen.flipping.5ht@gmail.com"
const files_dir = "Flipping/Datasets/FMRIOpto"
const figs_dir = "Flipping/Datasets/FMRIOpto/figures/"
## load
if ispath(linux_gdrive)
    ongoing_dir = linux_gdrive
else
    ongoing_dir = mac_gdrive
end

files_loc = joinpath(ongoing_dir,files_dir)
figs_loc = joinpath(ongoing_dir,figs_dir)

const columns_types = Dict(
    :Mouse => String,
    :Genotype => String,
    :Area => String,
    :Concentration => Float64,
    :Stim => Bool,
    :Repetition_nr => Int,
    :Trial_nr => Int,
    :Block => Int,
    :BOLD_90 => Float64,
    :BOLD_85 => Float64,
    :BOLD_0 => Float64,
    :Frame => Int,
    :Valid_frs => Bool,
    :Outlier_count_per_rep => Int
    )
using FMRIOpto
fulldata =  CSV.read(joinpath(files_loc,"odour_dose_data.csv"), DataFrame;
    types = columns_types,
    missingstring = "NaN")
mapcols!(Array, fulldata)
fulldata.Time = fulldata.Frame .+ 3
## average response per concentraition and stim over all presentation
gd0 = groupby(fulldata,[:Genotype, :Mouse, :Stim,:Area, :Concentration, :Time])
df0 = combine(gd0, [:BOLD_90, :BOLD_85,:BOLD_0] .=> mean .=> [:BOLD_90, :BOLD_85,:BOLD_0])

## define bold
fdata0 = filter(r -> r.Genotype == "sert" &&
    !(r.Area in ["pPIR", "OT", "NDB", "LHA", "COA"]) &&
    r.Concentration != 0.0001,
    # r.Mouse != "FM14",
    df0)
gd1 = groupby(fdata0, [:Mouse, :Area, :Stim, :Concentration])
df1 = combine(gd1, :BOLD_90 => mean => :BOLD_90)
plt, res, m, mOFC, mOB = analyze_bold(df1,:BOLD_90)
plt
res
m
mOFC
mOB
##
df2 = combine(groupby(fdata0, [:Genotype, :Mouse, :Stim,:Area, :Concentration]),
    [:BOLD_90, :BOLD_85,:BOLD_0] .=> max_bold .=> [:BOLD_90, :BOLD_85,:BOLD_0])
plt, res, m, mOFC, mOB, maPIR = analyze_bold(df2,:BOLD_90)
plt
res
m
mOFC
mOB


#
# Conditional Sampler
#
# chooses sampler based on values of a parent choice point
#
# implementation notes:
# 	* to facilitate EDAs using model-building (i.e. selection of parents), we implicitly constrain all the underlying samplers
# 	  to be of the same type (i.e. only the parameters differ)
# 	* we permit only one parent to be specified; to support models with multiple parents, the samplers can be nested:
#     ConditionalSampler(ConditionalSampler, ...)
#   * we permit parentcpid to be passed as "nothing" (which in effect makes the sampler unconditional as it simply calls the specified default sampler)
#     since often the parent cpid may not be known when the sampler is initially constructed.
#	* parentrecency indicates which invocation of the parent choice point to consider: 1 is the most recent, 2 is the second most recent etc.
#

type ConditionalSampler <: ModifyingSampler
	defaultsampler::Sampler
	parentcpid
	parentrecency::Int
	parentrecursiondepth # if nothing, then parent's recursion depth is ignored during sampling
	restricttoancestors::Bool
	conditionalsamplers::Dict{Any,Sampler}  # TODO: type of key is really Union{Real,Void} (since parent gns can be "nothing") rather than Any - is it worth specifying?
	function ConditionalSampler(defaultsampler::Sampler, parentcpid=nothing, parentrecency::Int=1, parentrecursiondepth=nothing, restricttoancestors=false, parentgns=Vector{Real}(), conditionalparams=Vector{Vector{Float64}}())
		# default parentcpid of nothing is for ease of construction as parent cpids may not be known at construction time
		# it turns the sampler into a non-conditional sampler
		(parentcpid != nothing) || isempty(parentgns) || error("Parent Gödel number cannot be specified if no parent cpid is given ")
		(parentrecency > 0) || error("Parent recency must be an integer 1 or more")
		(parentrecursiondepth == nothing) || ((typeof(parentrecursiondepth) <: Integer) && (parentrecursiondepth >= 1)) || error("Parent recursion must be nothing, or an integer 1 or more")
		(parentrecursiondepth == nothing) || warn("for parent recursion depth to take effect, ensure distinguishconditionalitybyrecursiondepth is set to true at model level")
		length(parentgns) >= length(conditionalparams) || error("More conditional parameter settings specified than parent Gödel numbers")
		conditionalsamplers = Dict{Any,Sampler}()
		for i in 1:length(parentgns)
			!haskey(conditionalsamplers, parentgns[i]) || error("Parent Gödel number is specified more than once")
			conditionalsampler = deepcopy(defaultsampler)
			# if params are specified, then set them, otherwise use default parameters
			if i <= length(conditionalparams)
				setparams!(conditionalsampler, conditionalparams[i])
			end
			conditionalsamplers[parentgns[i]] = conditionalsampler
		end
		new(defaultsampler, parentcpid, parentrecency, parentrecursiondepth, restricttoancestors, conditionalsamplers)
	end
end

# note: in the following methods, we could make assumption that numparams for each conditional sampler is the same as default sampler
# (which is what the constructor enforces), but where possible (generally, without too much of a performance impact) we avoid this assumption
# so that the code is a little more robust

# note: the code assumes that order of samplers returned by values(conditionalsamplers) is stable

# numparams(s::ConditionalSampler) = numparams(s.defaultampler) + sum(numparams, s.conditionalsamplers)

function paramranges(s::ConditionalSampler)
	pr = paramranges(s.defaultsampler)
	for conditionalsampler in values(s.conditionalsamplers)
		pr = [pr; paramranges(conditionalsampler)]
	end
	pr
end

function setparams!(s::ConditionalSampler, params)
	nparams = numparams(s)
	length(params) == nparams || error("expected $(nparams) parameters but got $(length(params))")
	paramstart = 1
	paramcount = numparams(s.defaultsampler)
	setparams!(s.defaultsampler, params[paramstart:(paramstart+paramcount-1)])
	for conditionalsampler in values(s.conditionalsamplers)
		paramstart += paramcount
		paramcount = numparams(conditionalsampler)
		setparams!(conditionalsampler, params[paramstart:(paramstart+paramcount-1)])
	end
end

function getparams(s::ConditionalSampler)
	ps = getparams(s.defaultsampler)
	for conditionalsampler in values(s.conditionalsamplers)
		ps = [ps; getparams(conditionalsampler)]
	end
	ps
end

function sample(s::ConditionalSampler, support, cc::ChoiceContext)
	# extract Gödel number of parent choice point at its most recent invocation
	parentgn = nothing
	if s.parentcpid != nothing
		recency = 0
		# if we are restricting conditionality to choice points in the same rule or its
		# ancestors in the execution tree, then we use ancestor cp sequence numbers to move
		# quickly through the ancestors in the trace
		# otherwise we simply step backwards through trace
		i = s.restricttoancestors ? getimmediateancestorcpseqnumber(cc.derivationstate) : length(cc.derivationstate.cmtrace)
		while (parentgn == nothing) && (i != nothing) && (i >= 1)
			(earliercpid, earliertrace) = cc.derivationstate.cmtrace[i]
			if (earliercpid == s.parentcpid) &&
				((s.parentrecursiondepth == nothing) || (earliertrace[:rcd] == s.parentrecursiondepth))
			# match parent based on cpid
			# AND if we are distinguishing conditionality based on the recursion depth of the parent choice point,
			# match recursion depth of that choice point in its trace
				recency += 1
				if recency == s.parentrecency
					parentgn = earliertrace[:gdl]
					@assert parentgn == cc.derivationstate.godelsequence[i]
				end
			end
			i = s.restricttoancestors ? earliertrace[:acs] : (i - 1)
		end
	end
	# if there is a sampler specified for the parent value, then use it, otherwise revert to default
	if (s.parentcpid != nothing) && (haskey(s.conditionalsamplers, parentgn))
		x, trace = sample(s.conditionalsamplers[parentgn], support, cc)
	else
		x, trace = sample(s.defaultsampler, support, cc)
	end
	x, Dict{Symbol, Any}(:par=>parentgn, :sub=>trace)
end

# TODO for consistency with other sub-sampler, we attempt re-estimation for all
# subsamplers, if we can tell then have too few (or no) traces here: we let the sub-sampler decide what
# to do in this situation.  This may be a (small?) performance overhead
#
# NOTE: if model is changed (i.e. a different parent is identified), then traces will be invalid
# and this method must NOT be used
#
function estimateparams!(s::ConditionalSampler, traces)
	# first divide out the traces according to which subsampler was used
	defaultsamplertraces = Vector{Dict}()
	conditionalsamplertraces = Dict{Any,Vector{Dict}}()
	for parentgn in keys(s.conditionalsamplers)
		conditionalsamplertraces[parentgn] = Vector{Dict}()
	end
	for trace in traces
		parentgn = trace[:par]
		if haskey(s.conditionalsamplers, parentgn)
			push!(conditionalsamplertraces[parentgn], trace[:sub])
		else
			push!(defaultsamplertraces, trace[:sub])
		end
	end
	# then re-estimate the subsamplers
	estimateparams!(s.defaultsampler, defaultsamplertraces)
	for parentgn in keys(s.conditionalsamplers)
		estimateparams!(s.conditionalsamplers[parentgn], conditionalsamplertraces[parentgn])
	end
end

function amendtrace(s::ConditionalSampler, trace, x)
	parentgn = trace[:par]
	if haskey(s.conditionalsamplers, parentgn)
		amendtrace(s.conditionalsamplers[parentgn], trace[:sub], x)
	else
		amendtrace(s.defaultsampler, trace[:sub], x)
	end
end

#
#
# K2 Metric: consider node A which has r distinct values
# par(A) are parents of A, which have q is distinct instantions of parents
# N_ij is number of cases where A takes i^th value and parents take j^th value (N_.j is sum of this over i)
# then:
#   log K2[A|par(A)] = sum_j=1^q log[(r-1)! / (N_.j + r -1)!] + sum_j=1^q sum_i=1^r log[N_ij!]
#
function determineparentusingK2(s::ConditionalSampler, gnhistories)

	numhistories = length(gnhistories)
	# the history for the current cp (i.e. "node A") which is the last value in each history
	ahistory = map(x->x[end], gnhistories)
	# A_values are values that current cp takes
	avalues = unique(ahistory)
	# println("A takes values: $(avalues)")
	r = length(avalues)
	# println("Therefore r is: $(r)")
	lfactrminus1 = lfact(r-1)

	# calculate value for no parent (effectively a parent taking all the same value)
	bestparindices = Any[nothing]
	bestlogK2 = lfactrminus1 - lfact(numhistories + r - 1)
	for i in 1:r
		Ni = count(x->x==avalues[i], ahistory)
		bestlogK2 += lfact(Ni)
	end
	# println("No parent gives a K2 value of: $(bestlogK2)")

	### TEMP
	# warn("removing calculation of sensible minimum K2")
	# bestlogK2 = -Inf
	###

	if numhistories > 0

		# consider each possible parent in turn
		for parindex in 1:(length(gnhistories[1]) - 1) # -1 since last entry is value of current cp
			# println("Considering potential parent #$(parindex) ...")

			# the history of the i^th possible parent is the i^th entry
			parhistory = map(x->x[parindex], gnhistories)
			parvalues = unique(parhistory)
			# println("par(A) takes values: $(parvalues)")
			q = length(parvalues)
			# println("Therefore q is: $(q)")

			logK2 = 0.0

			for j in 1:q
				Ndotj = count(x->x==parvalues[j], parhistory)
				# println("For j=$(j), N_.j is: $(Ndotj)")
				logK2 += lfactrminus1 - lfact(Ndotj + r - 1)
				# lfact is log factorial approximation
				# TODO: since current always integer, could do a lookup? will this be faster than Julia's lfact?

				for i in 1:r
					Nij = count(k->((ahistory[k]==avalues[i]) && (parhistory[k]==parvalues[j])), 1:numhistories)
					# println("For j=$(j), i=$(i), N_ij is: $(Nij)")
					logK2 += lfact(Nij)
				end

			end

			# println("Potential parent $(parindex) has logK2 of: $(logK2)")
			if logK2 > bestlogK2
				# strictly greater than so simplicitly of network is encouraged
				bestparindices = [parindex]
				bestlogK2 = logK2
			elseif logK2 == bestlogK2
				push!(bestparindices, parindex)
			end

		end

	end

	(nothing in bestparindices) ? nothing : bestparindices[rand(1:length(bestparindices))]

end

function estimateconditionalmodel!(s::ConditionalSampler, cplabels, gnhistories, traces)

	# assign new parent
	parindex = determineparentusingK2(s, gnhistories)
	if parindex != nothing
		cpid, recursiondepth, recency, restricttoancestors = cplabels[parindex]
	else
		cpid, recursiondepth, recency, restricttoancestors = nothing, nothing, 1, false
	end
	s.parentcpid = cpid
	s.parentrecency = recency
	s.parentrecursiondepth = recursiondepth
	s.restricttoancestors = restricttoancestors

	# println("******* New parent cpid is index $(parindex) : $(s.parentcpid)")

	# defaultsampler
	# TODO, if there is a parent, since we will account for all values of the parents, there is nothing left
	# to estimate the default: what to do? Currently we do this for all ...

	# we use the following to filter histories passed to any sub-sampler to remove the chosen cpid
	cplabelmask = convert(Vector{Bool}, map(c->c!=parindex, 1:length(cplabels)))
	historymask = [cplabelmask; true]
	# we add a true to the cpmask to retain final value which is the current cp Gödel number

	defaulttraces = map(t->t[:sub], traces)
	if supportsconditionalmodelestimation(s.defaultsampler)
		# pass all histories and traces to default sampler
		defaultgnhistories = map(h->h[historymask], gnhistories)
		estimateconditionalmodel!(s.defaultsampler, cplabels[cplabelmask], defaultgnhistories, defaulttraces)
	else
		estimateparams!(s.defaultsampler, defaulttraces)
	end

	# remove existing conditional samplers
	s.conditionalsamplers = Dict{Any,Sampler}()

	if parindex != nothing
		# if parent identified, create and estimate a sampler for each parent values

		parhistory = map(x->x[parindex], gnhistories)
		parvalues = unique(parhistory)

		for parvalue in parvalues

			conditionalsampler = deepcopy(s.defaultsampler)
			s.conditionalsamplers[parvalue] = conditionalsampler

			parentvaluemask = convert(Vector{Bool}, map(v->v==parvalue, parhistory))
			# we use the history to determine which histories are relevant
			# note: we can't use the parent value stored with the trace since the parent may have changed

			conditionaltraces = map(x->x[:sub], traces[parentvaluemask])
			if supportsconditionalmodelestimation(conditionalsampler)
				# pass histories and traces filtered by mask where parent value matches the condition on the sampler
				# we also filter the histories to remove parent index just chosen
				conditionalgnhistories = map(h->h[historymask], gnhistories[parentvaluemask])
				# println("<<<<<<< START nested")
				estimateconditionalmodel!(conditionalsampler, cplabels[cplabelmask], conditionalgnhistories, conditionaltraces)
				# println("<<<<<<< END nested")
			else
				estimateparams!(conditionalsampler, conditionaltraces)
			end

		end

	end

end

# pretty print the model
function show(io::IO, s::ConditionalSampler, indentdepth::Int=1)
	indent = " "^(10indentdepth)
	println(io, getsamplertypename(s) * " Parent: " * ((s.parentcpid == nothing) ? "(none)" : ("$(hex(s.parentcpid)) Recency: $(s.parentrecency) Recursion Depth: "
		* ((s.parentrecursiondepth == nothing) ?  "(any)" : "$(s.parentrecursiondepth)") * " Restrict to Ancestors: " * (s.restricttoancestors ?  "yes" : "no"))))
	for parentvalue in sort(collect(keys(s.conditionalsamplers)),lt=(x,y)->(x==nothing) ? false : (y==nothing) ? true : isless(x,y))
		print(io, indent * (parentvalue == nothing ? "(none)" : "$(parentvalue)") * ": ")
		show(io, s.conditionalsamplers[parentvalue], indentdepth+1)
	end
	print(io, indent * "default: ")
	show(io, s.defaultsampler, indentdepth+1)
end


"""
A `Robot` represents a mechanism, its state, and other relevant objects.

# Relevant fields in `Robot`
- The arrays `q_lo`, `q_hi`, `v_lo`, `v_hi`, `τ_lo`, `τ_hi` store the lower and upper bounds of the joint positions, velocities, and torques, respectively.
- The number of generalized coordinates, generalized velocities, and actuated joints are stored in `n_q`, `n_v`, `n_τ`, respectively.
- The end-effector frame is stored in `frame_ee`.
"""
struct Robot
    urdfpath::String
    mechanism::Mechanism
    state::MechanismState
    statecache::StateCache
    dynamicsresultcache::DynamicsResultCache
    mvis::MechanismVisualizer

    q_lo::Array{Float64}
    q_hi::Array{Float64}
    v_lo::Array{Float64}
    v_hi::Array{Float64}
    τ_lo::Array{Float64}
    τ_hi::Array{Float64}

    n_q::Int64  # Number of generalized coordinates
    n_v::Int64  # Number of generalized velocities
    n_τ::Int64  # Number of actuated joints

    frame_ee::CartesianFrame3D  # End-effector frame

    @doc """
        Robot(urdfpath, mechanism, frame_ee, mvis)

    Create a new `Robot`.
    """
    function Robot(urdfpath::String,
                   mechanism::Mechanism,
                   frame_ee::CartesianFrame3D,
                   mvis::MechanismVisualizer)
        state = MechanismState(mechanism)
        statecache = StateCache(mechanism)
        dynamicsresultcache = DynamicsResultCache(mechanism)

        q_lo = [lim for joint in joints(mechanism) for lim in map(x -> x.lower, position_bounds(joint))]
        q_hi = [lim for joint in joints(mechanism) for lim in map(x -> x.upper, position_bounds(joint))]
        v_lo = [lim for joint in joints(mechanism) for lim in map(x -> x.lower, velocity_bounds(joint))]
        v_hi = [lim for joint in joints(mechanism) for lim in map(x -> x.upper, velocity_bounds(joint))]
        τ_lo = [lim for joint in joints(mechanism) for lim in map(x -> x.lower,   effort_bounds(joint))]
        τ_hi = [lim for joint in joints(mechanism) for lim in map(x -> x.upper,   effort_bounds(joint))]

        new(
            urdfpath,
            mechanism,
            state,
            statecache,
            dynamicsresultcache,
            mvis,
            q_lo,
            q_hi,
            v_lo,
            v_hi,
            τ_lo,
            τ_hi,
            num_positions(mechanism),
            num_velocities(mechanism),
            num_velocities(mechanism),
            frame_ee
        )
    end
end

"""
    create_robot_kuka_iiwa_14(vis)

Create a new [KUKA LBR iiwa 14](https://www.kuka.com/en-gb/products/robotics-systems/industrial-robots/lbr-iiwa) robot.
"""
function create_robot_kuka_iiwa_14(vis::Visualizer)
    package_path = joinpath(@__DIR__, "..", "iiwa_stack")
    urdfpath = joinpath(@__DIR__, "..", "robots", "iiwa14.urdf")

    mechanism = parse_urdf(urdfpath, remove_fixed_tree_joints=false)
    frame_ee = default_frame(findbody(mechanism, "iiwa_link_pen_tip"))
    remove_fixed_tree_joints!(mechanism)

    urdfvisuals = URDFVisuals(urdfpath, package_path=[package_path])
    mvis = MechanismVisualizer(mechanism, urdfvisuals, vis["robot"])
    # setelement!(mvis, frame_ee)  # Visualize a triad at the end-effector

    Robot(urdfpath, mechanism, frame_ee, mvis)
end

"""
    create_robot_kinova_gen3_lite(vis)

Create a new [Kinova Gen3 lite](https://www.kinovarobotics.com/en/products/gen3-lite-robot) robot.
"""
function create_robot_kinova_gen3_lite(vis::Visualizer)
    package_path = joinpath(@__DIR__, "..", "ros_kortex")
    urdfpath = joinpath(@__DIR__, "..", "robots", "gen3_lite_gen3_lite_2f.urdf")

    mechanism = parse_urdf(urdfpath, remove_fixed_tree_joints=false)
    frame_ee = default_frame(findbody(mechanism, "tool_frame"))
    remove_fixed_tree_joints!(mechanism)

    urdfvisuals = URDFVisuals(urdfpath, package_path=[package_path])
    mvis = MechanismVisualizer(mechanism, urdfvisuals, vis["robot"])
    # setelement!(mvis, frame_ee)  # Visualize a triad at the end-effector

    Robot(urdfpath, mechanism, frame_ee, mvis)
end

export
    Robot,
    create_robot_kuka_iiwa_14,
    create_robot_kinova_gen3_lite


function assign_scenario(s::BasePriceScenario, model::JuMP.Model, config::RockyRoadOptions, params::RockyRoadParameters, vars::VariablesV2013)
    setvalue(params.ψ₂, 0.0);

    JuMP.solve(model);

    photel = getvalue(vars.CPRICE);

    for i in 1:config.N
        if i <= config.tnopol
            setupperbound(vars.CPRICE[i], max(photel[i],params.cpricebase[i]));
        end
    end

    setvalue(params.ψ₂, config.ψ₂₀);
end

function assign_scenario(s::OptimalPriceScenario, model::JuMP.Model, config::RockyRoadOptions, params::RockyRoadParameters, vars::VariablesV2013)
    #Coerce ICs first.
    JuMP.solve(model);
    setupperbound(vars.μ[1], config.μ₀);
end

function assign_scenario(s::Limit2DegreesScenario, model::JuMP.Model, config::RockyRoadOptions, params::RockyRoadParameters, vars::VariablesV2013)
    #Coerce ICs first.
    JuMP.solve(model);
    for i in 1:config.N
        setupperbound(vars.Tₐₜ[i], 2.0);
    end
end

function assign_scenario(s::SternScenario, model::JuMP.Model, config::RockyRoadOptions, params::RockyRoadParameters, vars::VariablesV2013)
    setvalue(params.α, 1.01);
    setvalue(params.ρ, 0.001);
    setvalue(params.optlrsav, (config.δk + .004)/(config.δk + .004*1.01 + 0.001)*config.γₑ);
    setvalue(params.rr[i=1:config.N], 1./((1+0.001).^(config.tstep*(i-1))));
end

function assign_scenario(s::SternCalibratedScenario, model::JuMP.Model, config::RockyRoadOptions, params::RockyRoadParameters, vars::VariablesV2013)
    setvalue(params.α, 2.1);
    setvalue(params.ρ, 0.001);
    setvalue(params.optlrsav, (config.δk + .004)/(config.δk + .004*2.1 + 0.001)*config.γₑ);
    setvalue(params.rr[i=1:config.N], 1./((1+0.001).^(config.tstep*(i-1))));

    for i in 1:config.N
        setlowerbound(vars.μ[i], 0.01);
    end

    JuMP.fix(vars.μ[1], 0.038976);
    JuMP.fix(vars.Tₐₜ[1], 0.83);

    JuMP.solve(model);
end

function assign_scenario(s::CopenhagenScenario, model::JuMP.Model, config::RockyRoadOptions, params::RockyRoadParameters, vars::VariablesV2013)
    #The Copenhagen participation fraction.
    imported_partfrac = ones(config.N);
    imported_partfrac[1:19] = [0.2,0.390423082,0.379051794,0.434731269,0.42272216,0.410416777,0.707776548,0.692148237,0.840306905,0.834064356,0.939658852,0.936731085,0.933881267,0.930944201,0.928088049,0.925153812,0.922301007,0.919378497,1.0];

    setvalue(params.partfract[i=1:config.N], imported_partfrac[i]);

    # The Emissions Control Rate Imported
    imported_μ = fill(0.9, config.N);
    imported_μ[1:27] = [0.02,0.055874801,0.110937151,0.163189757,0.206247482,0.241939219,0.30180914,0.364484979,0.423670192,0.478283881,0.534073643,0.588156847,0.633622,0.672457,0.705173102,0.733018573,0.756457118,0.776297581,0.794110815,0.822197128,0.839125811,0.854453754,0.868106413,0.880485825,0.891631752,0.901741794,0.9];

    for i in 1:config.N
        JuMP.fix(vars.μ[i], imported_μ[i]);
    end
end


module GraphIndexes

export
    GraphStrandPosition,
    UniqueMerIndex,
    isunmappable,
    find_unique_mer

using BioSequences: AbstractMer, position, ksize, fwmer, bwmer, each

using ..Graphs: NodeID, sequence, each_node_id, SequenceDistanceGraph

struct GraphStrandPosition
    node::NodeID
    position::UInt32
end

Base.summary(io::IO, pos::GraphStrandPosition) = print(io, "(Node: ", pos.node, ", Base position: ", pos.position, ')')
Base.show(io::IO, pos::GraphStrandPosition) = summary(io, pos)

"""
An index of all the unique kmers in a graph.

"""
struct UniqueMerIndex{M<:AbstractMer}
    mer_to_graphposition::Dict{M,GraphStrandPosition}
    unique_mers_per_node::Vector{UInt64}
    total_mers_per_node::Vector{UInt64}
end

@inline function isunmappable(idx::UniqueMerIndex, nodeid::NodeID)
    return idx.unique_mers_per_node[abs(nodeid)] == 0
end

function find_unique_mer(idx::UniqueMerIndex{M}, mer::M) where {M<:AbstractMer}
    pos = get(idx.mer_to_graphposition, mer, GraphStrandPosition(0, 0))
    exists = pos != GraphStrandPosition(0, 0)
    return exists, pos
end

function _discard_nonunique_mers!(mers::Vector{Pair{M,GraphStrandPosition}}) where {M<:AbstractMer}
    wi = 1
    ri = 1
    nri = 1
    stop = lastindex(mers) + 1
    while ri < stop
        while nri != stop && first(mers[nri]) == first(mers[ri])
            nri = nri + 1
        end
        if nri - ri == 1
            mers[wi] = mers[ri]
            wi = wi + 1
        end
        ri = nri
    end
    resize!(mers, wi - 1)
    return mers
end

function UniqueMerIndex{M}(graph::S) where {S<:SequenceDistanceGraph,M<:AbstractMer}
    t = 0
    total_mers_per_node = zeros(UInt64, n_nodes(graph))
    for nodeid in each_node_id(graph)
        seq = sequence(graph, nodeid)
        if length(seq) >= BioSequences.ksize(M)
            n = length(seq) + 1 - BioSequences.ksize(M)
            t = t + n
            total_mers_per_node[nodeid] = n
        end
    end
    kidxv = Vector{Pair{M,GraphStrandPosition}}(undef, t)
    index_i = 1
    for nodeid in each_node_id(graph)
        seq = sequence(graph, nodeid)
        if length(seq) >= BioSequences.ksize(M)
            for mer in each(M, seq)
                merpos = BioSequences.position(mer)
                mernode = ifelse(fwmer(mer) <= bwmer(mer), nodeid, -nodeid)
                mermer = ifelse(fwmer(mer) <= bwmer(mer), fwmer(mer), bwmer(mer))
                kidxv[index_i] = Pair(mermer, GraphStrandPosition(mernode, merpos))
                index_i = index_i + 1
            end
        end
    end

    sort!(kidxv, by = x -> first(x))

    _discard_nonunique_mers!(kidxv)

    mer_to_graphposition = Dict{M, GraphStrandPosition}()
    sizehint!(mer_to_graphposition, length(kidxv))
    unique_mers_per_node = zeros(UInt64, length(kidxv))

    for kidx in kidxv
        mer_to_graphposition[first(kidx)] = last(kidx)
        unique_mers_per_node[abs(last(kidx).node)] += 1
    end

    return UniqueMerIndex{M}(mer_to_graphposition, unique_mers_per_node, total_mers_per_node)
end

end # module


module MD5

using SHA
using SHA: lrot, SHA_CTX
import SHA: update!, digest!, transform!, pad_remainder!, buffer_pointer
import SHA: blocklen, state_type, digestlen, short_blocklen
export md5

include("constants.jl")
include("types.jl")
include("core.jl")

# Our basic function is to process arrays of bytes
function md5(data::T) where T<:Union{AbstractVector{UInt8}, NTuple{N,UInt8} where N}
    ctx = MD5_CTX()
    update!(ctx, data)
    return digest!(ctx)
end

# AbstractStrings are a pretty handy thing to be able to crunch through
md5(str::AbstractString) = md5(codeunits(str))

# Convenience function for IO devices, allows for things like:
# open("test.txt") do f
#     sha256(f)
# done
function md5(io::IO, chunk_size=4*1024)
    ctx = MD5_CTX()
    buff = Vector{UInt8}(undef, chunk_size)
    while !eof(io)
        num_read = readbytes!(io, buff)
        update!(ctx, buff[1:num_read])
    end
    return digest!(ctx)
end

end # module


# * BitFloats

module BitFloats

export Float80,  Inf80,  NaN80,
       Float128, Inf128, NaN128

import Base: !=, *, +, -, /, <, <=, ==, ^, abs, bswap, cos, decompose, eps, exp, exp2,
             exponent, exponent_half, exponent_mask, exponent_one, floatmax, floatmin,
             isequal, isless, issubnormal, ldexp, log, log10, log2, nextfloat, precision,
             promote_rule, reinterpret, rem, round, show, sign_mask, significand,
             significand_mask, sin, sqrt, trunc, typemax, typemin, uinttype, unsafe_trunc

import Base.Math: exponent_bits, exponent_raw_max, significand_bits

using Base: bswap_int, llvmcall, uniontypes

using Base.GMP: BITS_PER_LIMB, Limb

using Core: BuiltinInts

import BitIntegers

import Random: rand
using  Random: AbstractRNG, CloseOpen01, CloseOpen12, SamplerTrivial

# * definitions

primitive type Float80  <: AbstractFloat 80  end
primitive type Float128 <: AbstractFloat 128 end

const WBF = Union{Float80,Float128}

BitIntegers.@define_integers 80

uinttype(::Type{Float80}) = UInt80
uinttype(::Type{Float128}) = UInt128

const llvmvars = ((Float80, "x86_fp80", "i80", "f80"), (Float128, "fp128", "i128", "f128"))

# * traits

sign_mask(::Type{Float80})            = 0x8000_0000_0000_0000_0000 % UInt80
exponent_mask(::Type{Float80})        = 0x7fff_0000_0000_0000_0000 % UInt80
exponent_one(::Type{Float80})         = 0x3fff_8000_0000_0000_0000 % UInt80
exponent_half(::Type{Float80})        = 0x3ffe_8000_0000_0000_0000 % UInt80
significand_mask(::Type{Float80})     = 0x0000_ffff_ffff_ffff_ffff % UInt80
explicit_bit(::Type{Float80}=Float80) = 0x0000_8000_0000_0000_0000 % UInt80
# non-implicit most significand bit of significand is actually stored for Float80

sign_mask(::Type{Float128})           = 0x8000_0000_0000_0000_0000_0000_0000_0000
exponent_mask(::Type{Float128})       = 0x7fff_0000_0000_0000_0000_0000_0000_0000
exponent_one(::Type{Float128})        = 0x3fff_0000_0000_0000_0000_0000_0000_0000
exponent_half(::Type{Float128})       = 0x3ffe_0000_0000_0000_0000_0000_0000_0000
significand_mask(::Type{Float128})    = 0x0000_ffff_ffff_ffff_ffff_ffff_ffff_ffff

significand_bits(::Type{T}) where {T<:WBF} = trailing_ones(significand_mask(T))
exponent_bits(   ::Type{T}) where {T<:WBF} = sizeof(T)*8 - significand_bits(T) - 1
exponent_bias(   ::Type{T}) where {T<:WBF} = Int(exponent_one(T) >> significand_bits(T))
# exponent_bias is 16383 for both types
exponent_raw_max(::Type{T}) where {T<:WBF} = Int(exponent_mask(T) >> significand_bits(T))

eps(     ::Type{Float80})  = reinterpret(Float80,  0x3fc0_8000_0000_0000_0000 % UInt80)
floatmin(::Type{Float80})  = reinterpret(Float80,  0x0001_8000_0000_0000_0000 % UInt80)
floatmax(::Type{Float80})  = reinterpret(Float80,  0x7ffe_ffff_ffff_ffff_ffff % UInt80)
typemin( ::Type{Float80})  = -Inf80
typemax( ::Type{Float80})  =  Inf80

eps(     ::Type{Float128}) = reinterpret(Float128, 0x3f8f_0000_0000_0000_0000_0000_0000_0000)
floatmin(::Type{Float128}) = reinterpret(Float128, 0x0001_0000_0000_0000_0000_0000_0000_0000)
floatmax(::Type{Float128}) = reinterpret(Float128, 0x7ffe_ffff_ffff_ffff_ffff_ffff_ffff_ffff)
typemin( ::Type{Float128}) = -Inf128
typemax( ::Type{Float128}) =  Inf128

const Inf80 = reinterpret(Float80,   0x7fff_8000_0000_0000_0000 % UInt80)
const NaN80 = reinterpret(Float80,   0x7fff_c000_0000_0000_0000 % UInt80)

const Inf128 = reinterpret(Float128, 0x7fff_0000_0000_0000_0000_0000_0000_0000)
const NaN128 = reinterpret(Float128, 0x7fff_8000_0000_0000_0000_0000_0000_0000)

precision(::Type{Float80})  = 64
precision(::Type{Float128}) = 113

# * float functions

function exponent(x::T) where T<:WBF
    @noinline throw1(x) = throw(DomainError(x, "Cannot be NaN or Inf."))
    @noinline throw2(x) = throw(DomainError(x, "Cannot be subnormal converted to 0."))
    xs = reinterpret(Unsigned, x) & ~sign_mask(T)
    xs >= exponent_mask(T) && throw1(x)
    k = Int(xs >> significand_bits(T))
    if k == 0 # x is subnormal
        xs == 0 && throw2(x)
        m = leading_zeros(xs) - exponent_bits(T)
        if x isa Float80
            m -= 1 # non-implicit bit
        end
        k = 1 - m
    end
    return k - exponent_bias(T)
end

function significand(x::T) where T<:WBF
    xu = reinterpret(Unsigned, x)
    xs = xu & ~sign_mask(T)
    xs >= exponent_mask(T) && return x # NaN or Inf
    if xs <= significand_mask(T) # x is subnormal
        xs == 0 && return x # +-0
        m = unsigned(leading_zeros(xs) - exponent_bits(T)) - (x isa Float80)
        xs <<= m
        xu = xs | (xu & sign_mask(T))
        if x isa Float80
            xu |= explicit_bit() # set non-implicit bit to 1
        end
    end
    xu = (xu & ~exponent_mask(T)) | exponent_one(T)
    return reinterpret(T, xu)
end

function squeezeimplicit(u::UInt80)
    s = ((u & significand_mask(Float80)) << 1) & significand_mask(Float80)
    ((u & ~significand_mask(Float80)) | s) >> 1
end

function unsqueezeimplicit(u::UInt80)
    e = (u & ~(significand_mask(Float80) >> 1)) << 1
    u = e | (u & (significand_mask(Float80) >> 1))
    iszero(exponent_mask(Float80) & u) ? # zero or subnormal
        u :
        u | explicit_bit()
end

squeezeimplicit(f::UInt128) = f
unsqueezeimplicit(f::UInt128) = f

function nextfloat(f::WBF, d::Integer)
    F = typeof(f)
    fumax = squeezeimplicit(reinterpret(Unsigned, F(Inf)))
    U = typeof(fumax)

    isnan(f) && return f
    fi = reinterpret(Signed, f)
    fneg = fi < 0
    fu = squeezeimplicit(unsigned(fi & typemax(fi)))

    dneg = d < 0
    da = Base.uabs(d)
    if da > typemax(U)
        fneg = dneg
        fu = fumax
    else
        du = da % U
        if fneg ⊻ dneg
            if du > fu
                fu = min(fumax, du - fu)
                fneg = !fneg
            else
                fu = fu - du
            end
        else
            if fumax - fu < du
                fu = fumax
            else
                fu = fu + du
            end
        end
    end
    fu = unsqueezeimplicit(fu)
    if fneg
        fu |= sign_mask(F)
    end
    reinterpret(F, fu)
end

# for Float80, left-most (non-implicit) bit of significand should be 0
function issubnormal(x::T) where {T<:WBF}
    y = reinterpret(Unsigned, x)
    (y & exponent_mask(T) == 0) & (y & significand_mask(T) != 0)
end

function ldexp(x::T, e::Integer) where T<:WBF
    xu = reinterpret(Unsigned, x)
    xs = xu & ~sign_mask(T)
    xs >= exponent_mask(T) && return x # NaN or Inf
    k = Int(xs >> significand_bits(T))
    if k == 0 # x is subnormal
        xs == 0 && return x # +-0
        m = leading_zeros(xs) - exponent_bits(T) - (x isa Float80)
        ys = xs << unsigned(m)
        xu = ys | (xu & sign_mask(T))
        k = 1 - m
        # underflow, otherwise may have integer underflow in the following n + k
        e < -50000 && return flipsign(T(0.0), x)
    end
    # For cases where e of an Integer larger than Int make sure we properly
    # overflow/underflow; this is optimized away otherwise.
    if e > typemax(Int)
        return flipsign(T(Inf), x)
    elseif e < typemin(Int)
        return flipsign(T(0.0), x)
    end
    n = e % Int
    k += n
    # overflow, if k is larger than maximum possible exponent
    if k >= exponent_raw_max(T)
        return flipsign(T(Inf), x)
    end
    if k > 0 # normal case
        xu = (xu & ~exponent_mask(T)) | (rem(k, uinttype(T)) << significand_bits(T))
        if x isa Float80
            xu |= explicit_bit() # was
        end
        return reinterpret(T, xu)
    else # subnormal case
        expo = significand_bits(T) - (x isa Float80)
        if k <= -expo # underflow
            # overflow, for the case of integer overflow in n + k
            e > 50000 && return flipsign(T(Inf), x)
            return flipsign(T(0.0), x)
        end
        k += expo # k > 0
        z = T(2.0)^-expo
        xu = (xu & ~exponent_mask(T)) | (rem(k, uinttype(T)) << significand_bits(T))
        x isa Float80 && (xu |= explicit_bit())
        return z*reinterpret(T, xu)
    end
end

# * conversions

# ** signedness

reinterpret(::Type{Unsigned}, x::Float80)  = reinterpret(UInt80, x)
reinterpret(::Type{Unsigned}, x::Float128) = reinterpret(UInt128, x)
reinterpret(::Type{Signed},   x::Float80)  = reinterpret(Int80, x)
reinterpret(::Type{Signed},   x::Float128) = reinterpret(Int128, x)

Signed(  x::WBF) = Int(x)
Unsigned(x::WBF) = UInt(x)

trunc(::Type{Signed},   x::WBF) = trunc(Int, x)
trunc(::Type{Integer},  x::WBF) = trunc(Int, x)
trunc(::Type{Unsigned}, x::WBF) = trunc(UInt, x)

# ** from ints

# unsafe_trunc & F(::T)
for (F, f, i) = llvmvars
    for T = uniontypes(BuiltinInts)
        s = 8*sizeof(T)

        # T -> F
        itofp = T <: Signed ? :sitofp : :uitofp
        @eval begin
            (::Type{$F})(x::$T) = llvmcall(
                $"""
                %y = $itofp i$s %0 to $f
                %mi = bitcast $f %y to $i
                ret $i %mi
                """,
                $F, Tuple{$T}, x)

            promote_rule(::Type{$F}, ::Type{$T}) = $F
        end
        T === Bool && continue

        # F -> T
        fptoi = T <: Signed ? :fptosi : :fptoui
        @eval unsafe_trunc(::Type{$T}, x::$F) = llvmcall(
            $"""
            %x = bitcast $i %0 to $f
            %y = $fptoi $f %x to i$s
            ret i$s %y
            """,
            $T, Tuple{$F}, x)
    end
end

# trunc & T(::F)
for Tf in (Float80, Float128)
    for Ti in Base.BitInteger_types
        Ti == Int128 && continue
        @eval begin
            function trunc(::Type{$Ti},x::$Tf)
                # unlike in Base, these expressions can't be computed at compile time,
                # this segfaults otherwise
                if $Tf(typemin($Ti)) - one($Tf) < x < $Tf(typemax($Ti)) + one($Tf)
                    return unsafe_trunc($Ti, x)
                else
                    throw(InexactError(:trunc, $Ti, x))
                end
            end

            function (::Type{$Ti})(x::$Tf)
                if $Tf(typemin($Ti)) <= x < $Tf(typemax($Ti)) + one($Tf) && (round(x, RoundToZero) == x)
                    return unsafe_trunc($Ti, x)
                else
                    throw(InexactError($(Expr(:quote,Ti.name.name)), $Ti, x))
                end
            end
        end
    end

    Ti = Int128
    @eval begin
        function trunc(::Type{$Ti},x::$Tf)
            if $Tf(typemin($Ti)) <= x < $Tf(typemax($Ti))
                return unsafe_trunc($Ti,x)
            else
                throw(InexactError(:trunc, $Ti, x))
            end
        end

        function (::Type{$Ti})(x::$Tf)
            if ($Tf(typemin($Ti)) <= x < $Tf(typemax($Ti))) && (round(x, RoundToZero) == x)
                return unsafe_trunc($Ti,x)
            else
                throw(InexactError($(Expr(:quote,Ti.name.name)), $Ti, x))
            end
        end
    end
end

# ** from floats

for (F, f, i) = llvmvars
    for (S, s) = ((Float32, :float), (Float64, :double))
        @eval begin
            (::Type{$F})(x::$S) = llvmcall(
                $"""
                %y = fpext $s %0 to $f
                %yi = bitcast $f %y to $i
                ret $i %yi
                """,
                $F, Tuple{$S}, x)

            (::Type{$S})(x::$F) = llvmcall(
                $"""
                %x = bitcast $i %0 to $f
                %y = fptrunc $f %x to $s
                ret $s %y
                """,
                $S, Tuple{$F}, x)

            promote_rule(::Type{$F}, ::Type{$S}) = $F
        end
    end
    @eval begin
        (::Type{$F})(x::Float16) = $F(Float32(x))
        (::Type{Float16})(x::$F) = Float16(Float32(x))
        promote_rule(::Type{$F}, ::Type{Float16}) = $F
    end
end

# ** Float80 <-> Float128

promote_rule(::Type{Float128}, ::Type{Float80}) = Float128

if false # unimplemented by llvm
    @eval begin
        (::Type{Float128})(x::Float80) = llvmcall(
            """
            %x = bitcast i80 %0 to x86_fp80
            %y = fpext x86_fp80 %x to fp128
            %yi = bitcast fp128 %y to i128
            ret i128 %yi
            """,
            Float128, Tuple{Float80}, x)

        (::Type{Float80})(x::Float128) = llvmcall(
            """
            %x = bitcast i128 %0 to fp128
            %y = fptrunc fp128 %x to x86_fp80
            %yi = bitcast x86_fp80 %y to i80
            ret i80 %yi
            """,
            Float80, Tuple{Float128}, x)
    end
else
    function (::Type{Float128})(x::Float80)
        u = reinterpret(Unsigned, x)
        se = ((u & ~significand_mask(Float80)) % UInt128) << 48
        u &= significand_mask(Float80) >> 1 # >> 1 for explicit bit
        reinterpret(Float128, se | ((u % UInt128) << 49))
    end

    function (::Type{Float80})(x::Float128)
        u = reinterpret(Unsigned, x)
        se = ((u & ~significand_mask(Float128)) >> 48) % UInt80
        v = ((u & significand_mask(Float128)) >> 49) % UInt80
        if se & exponent_mask(Float80) != 0
            v |= explicit_bit()
        end
        reinterpret(Float80, se | v)
    end
end

# ** BigFloat

function (::Type{BigFloat})(x::WBF; precision::Integer=Base.MPFR.DEFAULT_PRECISION[])
    if isnan(x)
        BigFloat(NaN, precision=precision)
    elseif isinf(x)
        BigFloat(x < 0 ? -Inf : Inf, precision=precision)
    elseif iszero(x)
        BigFloat(x < 0 ? -0.0 : 0.0, precision=precision)
    else
        z = BigFloat(precision=precision)
        z.exp = exponent(x) + 1
        z.sign = x >= zero(x) ? 1 : -1
        y = significand(x)
        nlimbs = div(precision + BITS_PER_LIMB-1, BITS_PER_LIMB)
        mask = ~(Limb(0)) << mod(BITS_PER_LIMB - precision, BITS_PER_LIMB)

        if x isa Float80
            u = reinterpret(Unsigned, y) % UInt64
            n = nlimbs - div(64, BITS_PER_LIMB) + 1
        else
            u = reinterpret(Unsigned, y) % UInt128
            u = (u << 16) >> 1
            u |= sign_mask(Float128) # explicit 1 for the msb
            n = nlimbs - div(128, BITS_PER_LIMB) + 1
        end

        for i = 1:nlimbs
            l = (u >> ((i-n) * BITS_PER_LIMB)) % Limb
            i == 1 && (l &= mask)
            GC.@preserve z unsafe_store!(z.d, l, i)
        end
        z
    end
end

# alternative for Float128, useful to compare accuracy
function BigFloat_mpfr(x::Float128)
    z = BigFloat()
    ccall((:mpfr_set_float128, :libmpfr), Int32, (Ref{BigFloat}, Float128, Int32), z, x, Base.MPFR.ROUNDING_MODE[])
    if isnan(x) && signbit(x) != signbit(z)
        z.sign = -z.sign
    end
    return z
end

function Float128(x::BigFloat)
    ispos = x.sign == 1
    if isnan(x)
        NaN128
    elseif iszero(x) # should not be necessary (i.e. it works in the else branch too)
       x.sign * zero(Float128)
    else
        expo = x.exp - 1 # exponent bias for mpfr is -1
        # comparing x > floatmax(Float128) allocates a BigFloat, so we look at
        # expo to check for infinity
        if expo > 16383 || isinf(x)
            return x.sign * Inf128
        end
        e = Int(expo + exponent_bias(Float128))
        @assert e <= Int(0x7fff)
        n = ((x.prec-1) >> ( 6 - (BITS_PER_LIMB === 32))) + 1 # should be the number of limbs
        @assert n >= 1
        GC.@preserve x if BITS_PER_LIMB === 64
            u = (unsafe_load(x.d, n) % UInt128) << 64
            if n > 1
                u |= unsafe_load(x.d, n-1)
            end
        else
            u = zero(UInt128)
            shift = 96
            while n > 0 && shift >= 0
                u |= (unsafe_load(x.d, n) % UInt128) << shift
                n -= 1
                shift -= 32
            end
        end
        if e <= 0 # subnormal
            u >>= -e
            e = 0
        else
            u <<= 1 # remove explicit bit
        end
        u >>= exponent_bits(Float128) + 1
        u |= UInt128(e) << significand_bits(Float128)
        u |= sign_mask(Float128) * !ispos
        reinterpret(Float128, u)
    end
end

# doesn't work properly :(
# Float128_mpfr(x::BigFloat) =
#    ccall((:mpfr_get_float128, :libmpfr), Float128, (Ref{BigFloat}, Int32), x, Base.MPFR.ROUNDING_MODE[])

Float80(x::BigFloat) = Float80(Float128(x))

# ** round

for (F, f, i, fn) = llvmvars
    # TODO: can be broken for Float128
    for (mode, llvmfun) = ((:ToZero, :trunc), (:Down, :floor),
                           (:Up, :ceil), (:Nearest, :rint))
        fun = "@llvm.$llvmfun.$fn"
        @eval round(x::$F, r::$(RoundingMode{mode})) = llvmcall(
            ($"""declare $f $fun($f %Val)""",
             $"""
             %x = bitcast $i %0 to $f
             %y = call $f $fun($f %x)
             %z = bitcast $f %y to $i
             ret $i %z
             """), $F, Tuple{$F}, x)
    end
end

# * comparisons

for (F, f, i) = llvmvars
    for (op, fop) = ((:(==), :oeq), (:!=, :une), (:<, :olt), (:<=, :ole))
        @eval $op(x::$F, y::$F) = llvmcall(
            $"""
            %x = bitcast $i %0 to $f
            %y = bitcast $i %1 to $f
            %b = fcmp $fop $f %x, %y
            %c = zext i1 %b to i8
            ret i8 %c
            """,
            Bool, Tuple{$F,$F}, x, y)
    end

    # adapted from fpislt in intrinsics.cpp
    # could as well be written in Julia
    @eval fpislt(x::$F, y::$F) = llvmcall(
        $"""
        ; %xi = %0
        ; %yi = %1
        %xf = bitcast $i %0 to $f
        %yf = bitcast $i %1 to $f
        %c11 = fcmp ord $f %xf, %xf ; !isnan(xf)
        %c12 = fcmp uno $f %yf, %yf ; isnan(yf)
        %c1  = or i1 %c11, %c12
        %c21 = fcmp ord $f %xf, %yf ; !(isnan(xf) | isnan(yf))
        %c2211 = icmp sge $i %0, 0 ; xi >= 0
        %c2212 = icmp slt $i %0, %1 ; xi < yi
        %c221 = and i1 %c2211, %c2212
        %c2221 = icmp slt $i %0, 0 ; xi < 0
        %c2222 = icmp ugt $i %0, %1 ; unsigned(xi) > unsigned(yi)
        %c222 = and i1 %c2221, %c2222
        %c22 = or i1 %c221, %c222
        %c2 = and i1 %c21, %c22
        %c = or i1 %c1, %c2
        %cbool = zext i1 %c to i8
        ret i8 %cbool
        """,
        Bool, Tuple{$F,$F}, x, y)
end

isless(x::T, y::T) where {T<:WBF} = fpislt(x, y)

# implements Base.fpiseq
isequal(x::T, y::T) where {T<:WBF} =
    (isnan(x) & isnan(y)) | (reinterpret(Unsigned, x) === reinterpret(Unsigned, y))

# * arithmetic

for (F, f, i, fn) = llvmvars
    for (op, fop) = ((:*, :fmul), (:/, :fdiv), (:+, :fadd), (:-, :fsub), (:rem, :frem))
        @eval $op(x::$F, y::$F) = llvmcall(
            $"""
            %x = bitcast $i %0 to $f
            %y = bitcast $i %1 to $f
            %m = $fop $f %x, %y
            %mi = bitcast $f %m to $i
            ret $i %mi
            """, $F, Tuple{$F,$F}, x, y)
    end
    for (op, fop) = (:abs => :fabs, :log2 => :log2, :exp2 => :exp2, :sqrt => :sqrt,
                     :sin => :sin, :cos => :cos, :exp => :exp, :log => :log, :log10 => :log10)
        if F === Float128 && op ∈ (:log2, :exp2, :sqrt, :sin, :cos, :exp, :log, :log10)
            @eval $op(x::$F) = $F($op(big(x)))
        else
            @eval $op(x::$F) = llvmcall(
                ($"""declare $f  @llvm.$fop.$fn($f %Val)""",
                 $"""
                 %x = bitcast $i %0 to $f
                 %y = call $f @llvm.$fop.$fn($f %x)
                 %z = bitcast $f %y to $i
                 ret $i %z
                 """), $F, Tuple{$F}, x)
        end
    end
    F == Float128 && continue # broken
    @eval ^(x::$F, y::$F) = llvmcall(
        ($"""declare $f @llvm.pow.$fn($f %Val, $f %Power)""",
         $"""
         %x = bitcast $i %0 to $f
         %y = bitcast $i %1 to $f
         %m = call $f @llvm.pow.$fn($f %x, $f %y)
         %mi = bitcast $f %m to $i
         ret $i %mi
         """), $F, Tuple{$F,$F}, x, y)
end

^(x::Float128, y::Float128) = Float128(big(x)^big(y))

^(x::WBF, n::Integer) = x^(oftype(x, n))
-(x::F) where {F<:WBF} = F(-0.0) - x

# * hashing

# could return UInt64 instead of Int128 for Float80
function decompose(x::WBF)::Tuple{Int128,Int,Int}
    T = typeof(x)
    isnan(x) && return 0, 0, 0
    isinf(x) && return ifelse(x < 0, -1, 1), 0, 0
    n = reinterpret(Unsigned, x)
    s = (n & significand_mask(T)) % Int128
    e = ((n & exponent_mask(T)) >> significand_bits(T)) % Int
    if T === Float128 # add explicit bit
        s |= Int128(e != 0) << significand_bits(T)
    end
    d = ifelse(signbit(x), -1, 1)
    # for Float80, significand_bits(T) includes the explicit bit
    E = exponent_bias(T) + significand_bits(T) - (T === Float80)
    s, e - E + (e == 0), d
end

# * rand

rand(rng::AbstractRNG, ::SamplerTrivial{CloseOpen12{Float80}}) =
    reinterpret(Float80, rand(rng, UInt64) | exponent_one(Float80) | explicit_bit())

rand(rng::AbstractRNG, ::SamplerTrivial{CloseOpen12{Float128}}) =
    reinterpret(Float128, rand(rng, UInt128) & significand_mask(Float128) | exponent_one(Float128))

rand(rng::AbstractRNG, ::SamplerTrivial{CloseOpen01{T}}) where {T<:WBF} =
    rand(rng, CloseOpen12(T)) - one(T)

# * misc

bswap(x::WBF) = bswap_int(x)

show(io::IO, x::WBF) =
    if isnan(x)
        print(io, "NaN", sizeof(x)*8)
    elseif isinf(x)
        print(io, x > 0 ? "" : '-',  "Inf", sizeof(x)*8)
    else
        show(io, BigFloat(x, precision=precision(x)))
    end

end # module