module Manifests

import ..base_path, ..base_path_default, ..manifest_subdir_default, ..data_subdir_default
import ..speed_of_light_mps

export get_data_filenames, get_data_filenames_and_bsrv
export get_base_path, set_base_path!, get_manifest_path, set_manifest_path!, get_data_path, set_data_path!

using CSV
using FITSIO
using DataFrames
using Dates
using Printf

# Structure to define file format (works for HPF, see if need to generalize for EXPRES/HARPSN)
#=
struct ManifestFormatEntry <: AbstractManifestFormatEntry
    name::Symbol
    type::Type
    cols::UnitRange{Int64}
end
=#

# Specify paths to manifest and data files
manifest_path_default = joinpath(base_path_default,manifest_subdir_default)
data_path_default = joinpath(base_path_default,data_subdir_default)

get_manifest_path() = manifest_path_default
set_manifest_path!(path::String) = manifest_path_default = path

get_data_path() = data_path_default
set_data_path!(path::String) = data_path_default = path

get_base_path() = base_path
function set_base_path!(path::String)
   base_path = path
   set_manifest_path!( joinpath(base_path,manifest_subdir_default) )
   set_data_path!( joinpath(base_path,data_subdir_default) )
end

# Initialize base_path
set_base_path!(base_path_default)

# Read in manifest files
function read(filename::String; dvtol=0.0001, qthresh=0.99, date_start::Date, date_stop::Date = date_start, verbose = false)
   df = CSV.read(filename)
   df = df[.!ismissing.(df[:JD]),:]  # clean missing values
   vcorr = df[:RVfinal] - df[:RVhel]
   cond1 = abs.(vcorr) .< dvtol
   cond2 = df[:qualflag] .> qthresh
   date_fmt = DateFormat("d/m/y")
   df[:Date] = Date.(df[:Date],date_fmt)
   cond3 = date_start .<= df[:Date] .<= date_stop
   df = df[cond1 .& cond2 .& cond3,:]
   num_spectra = size(df,1)
   if verbose
      println("# Found ", num_spectra, " HARPS-N spectra matching requirements in manifest.")
   end
   #bsrv = df[:RVhel] .- df[:RVbary]
   return df
end
read_manifest = read
export read_manifest

"""Get data filenames from a manifest DataFrame.  Optioanlly specify path to data."""
function get_data_filenames(df::DataFrame; path::String = get_data_path(), suffix::String = "_e2ds_A.fits", verbose=false )
   #num_files = size(df,1)
   #files = Vector{String}()
   #df[:File] = map(df->joinpath(path,,@sprintf("%4d-%02d",year(df[1]),month(df[1])), @sprintf("%4d-%02d-%02d",year(df[1]),month(df[1]),day(df[1])), df[2] * "_e2ds_A.fits"),zip(df[:,:Date],df[:,:File]))
   df[:File] = map(df->joinpath(path,df[2] * suffix), zip(df[:,:Date],df[:,:File]) )
   return df[:,:File]
   #=
    for i in 1:num_files
       obsnum = Printf.@sprintf("%04d",df[i,:ObsNum])
       subdir = joinpath(df[i,:UTDate],obsnum)
       dirname = joinpath(path,subdir)
       file_pattern = Regex("(" * df[i,:Frame][1:end-5] * ".*fits)")
       local files_here
       try
          files_here = readdir(dirname)
       catch
          if verbose
             println("# Skipping " * string(file_pattern) * " not found in " * subdir )
          end
          continue
       end
       files_matched = map(f->match(file_pattern,f),files_here)
       idx_keep = findall(x->x!=nothing,files_matched)
       append!(files,map(m->joinpath(dirname,m.match),files_matched[idx_keep]))
    end
    =#
end

"""Get data filenames and corresponding barycentric solar radial velocities from searching all manifest entries.  Optioanlly specify path to data, constraints on which data files to read."""
function get_data_filenames_and_bsrv(; manifest_path::String = get_manifest_path(), manifest_filename::String = "Sun_harpsn_index.csv",
      data_path::String = get_data_path(), verbose=false,
      dvtol=0.0001, qthresh=0.99, date_start=Date(2015,01,01), date_stop::Date = date_start)
   df = read(joinpath(manifest_path,manifest_filename), dvtol=dvtol, qthresh=qthresh, date_start=date_start, date_stop=date_stop)
   bsrv = (df[:RVhel] .- df[:RVbary])  ./ speed_of_light_mps
   filenames = get_data_filenames(df, path=data_path, verbose=verbose)
   found_on_disk = isfile.(filenames)
   return (filenames=filenames[found_on_disk], z_bsrv=bsrv[found_on_disk])
end

"""Get data filenames from searching all manifest entries.  Optioanlly specify path to data, constraints on which data files to read."""
function get_data_filenames(; manifest_path::String = get_manifest_path(), manifest_filename::String = "Sun_harpsn_index.csv",
      data_path::String = get_data_path(), verbose=false,
      dvtol=0.0001, qthresh=0.99, date_start=Date(2015,01,01), date_stop::Date = date_start)
   output = get_data_filenames_and_bsrv(manifest_path=manifest_path, manifest_filename=manifest_filename,data_path=data_path,verbose=verbose,
               dvtol=dvtol, qthresh=qthresh, date_start=date_start, date_stop=date_stop)
   return output.filenames
end

end


using .Instantiation
using .ModelElaboration

using Unitful
using .StructuralTransform
using .Synchronous: sample, Clock, previous, hold, positive, positiveChange, positiveEdge

"""
Shortcut for `Variable`
"""
Var(; args...) = Variable(; args...)

"""
Create a floating-point `Variable`
"""
Float(value=nothing; info="", size=nothing, unit=NoUnits, displayUnit=NoUnits,
    min=nothing, max=nothing, start=nothing, fixed::Bool=false, nominal=nothing,
    variability=continuous,
    flow::Bool=false, state::Bool=true) = Variable(variability, Float64, size, value,
    unit, displayUnit, min, max, start, fixed, nominal, info, flow, state, general)

"""
Create a boolean `Variable`
"""
Boolean(value=nothing; info="", size=nothing, unit=NoUnits, displayUnit=NoUnits,
    min=nothing, max=nothing, start=nothing, fixed::Bool=false, nominal=nothing,
    variability=continuous,
    flow::Bool=false, state::Bool=true) = Variable(variability, Bool, size, value,
    unit, displayUnit, min, max, start, fixed, nominal, info, flow, state, general)

"""
Create an integer `Variable`
"""
Integ(value=nothing; info="", size=nothing, unit=NoUnits, displayUnit=NoUnits,
    min=nothing, max=nothing, start=nothing, fixed::Bool=false, nominal=nothing,
    variability=continuous,
    flow::Bool=false, state::Bool=true) = Variable(variability, Int, size, value,
    unit, displayUnit, min, max, start, fixed, nominal, info, flow, state, general)

"""
Create a string `Variable`
"""
Str(value=nothing; info="", size=nothing, unit=NoUnits, displayUnit=NoUnits,
    min=nothing, max=nothing, start=nothing, fixed::Bool=false, nominal=nothing,
    variability=continuous,
    flow::Bool=false, state::Bool=true) = Variable(variability, String, size, value,
    unit, displayUnit, min, max, start, fixed, nominal, info, flow, state, general)

#=
Float(; args...) = Var(T=Float64; args...)
Float0(; args...) = Float(size=(); args...)

Boolean(; args...) = Var(T=Bool; args...)
Boolean0(; args...) = Boolean(size=(); args...)

Integ(; args...) = Var(T=Int; args...)
Integ0(; args...) = Integ(size=(); args...)

Str(; args...) = Var(T=String; args...)
Str0(; args...) = Str(size=(); args...)
=#

"""
Create a `Variable` with `parameter` variability, meaning it
is an input variable that is constant with time
"""
Parameter(; args...) = Variable(variability=parameter; args...)
Parameter(value; args...) = Variable(variability=parameter, value=value; args...)

"""
Shortcut for `Parameter`
"""
Par(; args...) = Variable(variability=parameter; args...)
Par(value; args...) = Variable(variability=parameter, value=value; args...)

"""
A value meant to be filled in later
"""
const undefined = nothing


using Test, AdvancedHMC, Random
using Statistics: mean
using LinearAlgebra: dot
include("common.jl")

ϵ = 0.01
lf = Leapfrog(ϵ)

θ_init = randn(D)
h = Hamiltonian(UnitEuclideanMetric(D), ℓπ, ∂ℓπ∂θ)
τ = Trajectory{MultinomialTS}(Leapfrog(find_good_stepsize(h, θ_init)), GeneralisedNoUTurn())
r_init = AdvancedHMC.rand(h.metric)

@testset "Passing random number generator" begin
    τ_with_jittered_lf = Trajectory{MultinomialTS}(JitteredLeapfrog(find_good_stepsize(h, θ_init), 1.0), GeneralisedNoUTurn())
    for τ_test in [τ, τ_with_jittered_lf],
        seed in [1234, 5678, 90]
        rng = MersenneTwister(seed)
        z = AdvancedHMC.phasepoint(h, θ_init, r_init)
        z1′ = AdvancedHMC.transition(rng, τ_test, h, z).z

        rng = MersenneTwister(seed)
        z = AdvancedHMC.phasepoint(h, θ_init, r_init)
        z2′ = AdvancedHMC.transition(rng, τ_test, h, z).z

        @test z1′.θ == z2′.θ
        @test z1′.r == z2′.r
    end
end

@testset "TreeSampler" begin
    n_samples = 10_000
    z1 = AdvancedHMC.phasepoint(h, zeros(D), r_init)
    z2 = AdvancedHMC.phasepoint(h, ones(D), r_init)

    rng = MersenneTwister(1234)

    ℓu = rand()
    n1 = 2
    s1 = AdvancedHMC.SliceTS(z1, ℓu, n1)
    n2 = 1
    s2 = AdvancedHMC.SliceTS(z2, ℓu, n2)
    s3 = AdvancedHMC.combine(rng, s1, s2)
    @test s3.ℓu == ℓu
    @test s3.n == n1 + n2

    s3_θ = Vector(undef, n_samples)
    for i = 1:n_samples
        s3_θ[i] = AdvancedHMC.combine(rng, s1, s2).zcand.θ
    end
    @test mean(s3_θ) ≈ ones(D) * n2 / (n1 + n2) rtol=0.01

    w1 = 100
    s1 = AdvancedHMC.MultinomialTS(z1, log(w1))
    w2 = 150
    s2 = AdvancedHMC.MultinomialTS(z2, log(w2))
    s3 = AdvancedHMC.combine(rng, s1, s2)
    @test s3.ℓw ≈ log(w1 + w2)

    s3_θ = Vector(undef, n_samples)
    for i = 1:n_samples
        s3_θ[i] = AdvancedHMC.combine(rng, s1, s2).zcand.θ
    end
    @test mean(s3_θ) ≈ ones(D) * w2 / (w1 + w2) rtol=0.01
end

@testset "TerminationCriterion" begin
    tc = AdvancedHMC.ClassicNoUTurn()
    z1 = AdvancedHMC.phasepoint(h, θ_init, randn(D))
    z2 = AdvancedHMC.phasepoint(h, θ_init, randn(D))
    ts1 = AdvancedHMC.TurnStatistic(tc, z1)
    ts2 = AdvancedHMC.TurnStatistic(tc, z2)
    ts3 = AdvancedHMC.combine(ts1, ts2)
    @test ts1 == ts2 == ts3

    tc = AdvancedHMC.GeneralisedNoUTurn()
    r1 = randn(D)
    z1 = AdvancedHMC.phasepoint(h, θ_init, r1)
    r2 = randn(D)
    z2 = AdvancedHMC.phasepoint(h, θ_init, r2)
    ts1 = AdvancedHMC.TurnStatistic(tc, z1)
    ts2 = AdvancedHMC.TurnStatistic(tc, z2)
    ts3 = AdvancedHMC.combine(ts1, ts2)
    @test ts3.rho == r1 + r2
end

@testset "Termination" begin
    t00 = AdvancedHMC.Termination(false, false)
    t01 = AdvancedHMC.Termination(false, true)
    t10 = AdvancedHMC.Termination(true, false)
    t11 = AdvancedHMC.Termination(true, true)

    @test AdvancedHMC.isterminated(t00) == false
    @test AdvancedHMC.isterminated(t01) == true
    @test AdvancedHMC.isterminated(t10) == true
    @test AdvancedHMC.isterminated(t11) == true

    @test AdvancedHMC.isterminated(t00 * t00) == false
    @test AdvancedHMC.isterminated(t00 * t01) == true
    @test AdvancedHMC.isterminated(t00 * t10) == true
    @test AdvancedHMC.isterminated(t00 * t11) == true

    @test AdvancedHMC.isterminated(t01 * t00) == true
    @test AdvancedHMC.isterminated(t01 * t01) == true
    @test AdvancedHMC.isterminated(t01 * t10) == true
    @test AdvancedHMC.isterminated(t01 * t11) == true

    @test AdvancedHMC.isterminated(t10 * t00) == true
    @test AdvancedHMC.isterminated(t10 * t01) == true
    @test AdvancedHMC.isterminated(t10 * t10) == true
    @test AdvancedHMC.isterminated(t10 * t11) == true

    @test AdvancedHMC.isterminated(t11 * t00) == true
    @test AdvancedHMC.isterminated(t11 * t01) == true
    @test AdvancedHMC.isterminated(t11 * t10) == true
    @test AdvancedHMC.isterminated(t11 * t11) == true
end

@testset "BinaryTree" begin
    z = AdvancedHMC.phasepoint(h, θ_init, randn(D))

    t1 = AdvancedHMC.BinaryTree(z, z, AdvancedHMC.TurnStatistic(), 0.1, 1, -2.0)
    t2 = AdvancedHMC.BinaryTree(z, z, AdvancedHMC.TurnStatistic(), 1.1, 2, 1.0)
    t3 = AdvancedHMC.combine(t1, t2)

    @test t3.sum_α ≈ 1.2 atol=1e-9
    @test t3.nα == 3
    @test t3.ΔH_max == -2.0

    t4 = AdvancedHMC.BinaryTree(z, z, AdvancedHMC.TurnStatistic(), 1.1, 2, 3.0)
    t5 = AdvancedHMC.combine(t1, t4)

    @test t5.ΔH_max == 3.0
end

### Test ClassicNoUTurn and GeneralisedNoUTurn

function makeplot(
    plt,
    traj_θ,
    ts_list...
)
    function plotturn!(traj_θ, ts)
        s = 9.0
        idcs_nodiv = ts .== false
        idcs_div = ts .== true
        idcs_nodiv[1] = idcs_div[1] = false # avoid plotting the first point
        plt.scatter(traj_θ[1,idcs_nodiv], traj_θ[2,idcs_nodiv], s=s, c="black",  label="¬div")
        plt.scatter(traj_θ[1,idcs_div], traj_θ[2,idcs_div],  s=s, c="red",    label="div")
        plt.scatter(traj_θ[1,1], traj_θ[2,1], s=s, c="yellow", label="init")
    end

    fig = plt.figure(figsize=(16, 3))

    for (i, ts, title) in zip(
        1:length(ts_list),
        ts_list,
        [
            "Hand original (v = 1)",
            "AHMC original (v = 1)",
            "Hand generalised (v = 1)",
            "AHMC generalised (v = 1)"
        ]
    )
        plt.subplot(1, 4, i)
        plotturn!(traj_θ, ts)
        plt.gca().set_title(title)
        plt.legend()
    end

    return fig
end

function gettraj(rng, ϵ=0.1, n_steps=50)
    lf = Leapfrog(ϵ)

    q_init = randn(rng, D)
    p_init = AdvancedHMC.rand(rng, h.metric)
    z = AdvancedHMC.phasepoint(h, q_init, p_init)

    traj_z = Vector(undef, n_steps)
    traj_z[1] = z
    for i = 2:n_steps
        traj_z[i] = AdvancedHMC.step(lf, h, traj_z[i-1])
    end

    return traj_z
end

function hand_isturn(z0, z1, rho, v=1)
    θ0minusθ1 = z0.θ - z1.θ
    s = (dot(-θ0minusθ1, -z0.r) >= 0) || (dot(θ0minusθ1, z1.r) >= 0)
    return s
end

ahmc_isturn(z0, z1, rho, v=1) =
    AdvancedHMC.isterminated(ClassicNoUTurn(), h, AdvancedHMC.BinaryTree(z0, z1, AdvancedHMC.TurnStatistic(), 0, 0, 0.0)).dynamic

function hand_isturn_generalised(z0, z1, rho, v=1)
    s = (dot(rho, -z0.r) >= 0) || (dot(-rho, z1.r) >= 0)
    return s
end

ahmc_isturn_generalised(z0, z1, rho, v=1) =
    AdvancedHMC.isterminated(GeneralisedNoUTurn(), h, AdvancedHMC.BinaryTree(z0, z1, AdvancedHMC.TurnStatistic(rho), 0, 0, 0.0)).dynamic

function ahmc_isturn_strictgeneralised(z0, z1, rho, v=1)
    t = AdvancedHMC.isterminated(
        StrictGeneralisedNoUTurn(),
        h,
        AdvancedHMC.BinaryTree(z0, z1, AdvancedHMC.TurnStatistic(rho), 0, 0, 0.0),
        AdvancedHMC.BinaryTree(z0, z0, AdvancedHMC.TurnStatistic(rho - z1.r), 0, 0, 0.0),
        AdvancedHMC.BinaryTree(z1, z1, AdvancedHMC.TurnStatistic(rho - z0.r), 0, 0, 0.0)
    )
    return t.dynamic
end

"""
Check whether the subtree checks adequately detect U-turns.
"""
function check_subtree_u_turns(z0, z1, rho)
    t = AdvancedHMC.BinaryTree(z0, z1, AdvancedHMC.TurnStatistic(rho), 0, 0, 0.0)
    # The left and right subtree are created in such a way that the
    # check_left_subtree and check_right_subtree checks should be equivalent
    # to the general no U-turn check.
    tleft = AdvancedHMC.BinaryTree(z0, z0, AdvancedHMC.TurnStatistic(rho - z1.r), 0, 0, 0.0)
    tright = AdvancedHMC.BinaryTree(z1, z1, AdvancedHMC.TurnStatistic(rho - z0.r), 0, 0, 0.0)

    s1 = AdvancedHMC.isterminated(GeneralisedNoUTurn(), h, t)
    s2 = AdvancedHMC.check_left_subtree(h, t, tleft, tright)
    s3 = AdvancedHMC.check_right_subtree(h, t, tleft, tright)
    @test s1 == s2 == s3
end

@testset "ClassicNoUTurn" begin
    n_tests = 4
    for _ = 1:n_tests
        seed = abs(rand(Int8) + 128)
        rng = MersenneTwister(seed)
        @testset "seed = $seed" begin
            traj_z = gettraj(rng)
            traj_θ = hcat(map(z -> z.θ, traj_z)...)
            traj_r = hcat(map(z -> z.r, traj_z)...)
            rho = cumsum(traj_r, dims=2)

            ts_hand_isturn_fwd = hand_isturn.(Ref(traj_z[1]), traj_z, [rho[:,i] for i = 1:length(traj_z)], Ref(1))
            ts_ahmc_isturn_fwd = ahmc_isturn.(Ref(traj_z[1]), traj_z, [rho[:,i] for i = 1:length(traj_z)], Ref(1))

            ts_hand_isturn_generalised_fwd = hand_isturn_generalised.(Ref(traj_z[1]), traj_z, [rho[:,i] for i = 1:length(traj_z)], Ref(1))
            ts_ahmc_isturn_generalised_fwd = ahmc_isturn_generalised.(Ref(traj_z[1]), traj_z, [rho[:,i] for i = 1:length(traj_z)], Ref(1))

            ts_ahmc_isturn_strictgeneralised_fwd = ahmc_isturn_strictgeneralised.(Ref(traj_z[1]), traj_z, [rho[:,i] for i = 1:length(traj_z)], Ref(1))

            check_subtree_u_turns.(Ref(traj_z[1]), traj_z, [rho[:,i] for i = 1:length(traj_z)])

            @test ts_hand_isturn_fwd[2:end] ==
                ts_ahmc_isturn_fwd[2:end] ==
                ts_hand_isturn_generalised_fwd[2:end] ==
                ts_ahmc_isturn_generalised_fwd[2:end] ==
                ts_ahmc_isturn_strictgeneralised_fwd[2:end]

            if length(ARGS) > 0 && ARGS[1] == "--plot"
                import PyPlot
                fig = makeplot(
                    PyPlot,
                    traj_θ,
                    ts_hand_isturn_fwd,
                    ts_ahmc_isturn_fwd,
                    ts_hand_isturn_generalised_fwd,
                    ts_ahmc_isturn_generalised_fwd
                )
                fig.savefig("seed=$seed.png")
            end
        end
    end
end


insts = OrderedDict()
insts["minimal"] = [
    ((2, false, true, true, true),),
    ((2, false, false, true, true),),
    ((2, true, true, true, true),),
    ((2, true, false, true, true),),
    ((2, false, true, true, false),),
    ((2, false, false, true, false),),
    ((2, true, true, true, false),),
    ((2, true, false, true, false),),
    ((2, false, false, false, false),),
    ((2, true, false, false, false),),
    ]
insts["fast"] = [
    ((5, false, true, true, true),),
    ((5, false, false, true, false),),
    ((5, true, true, true, false),),
    ((5, true, false, true, true),),
    ((5, false, true, false, false),),
    ((5, false, false, false, false),),
    ((5, true, true, false, false),),
    ((5, true, false, false, false),),
    ((20, false, true, true, false),),
    ((20, false, false, true, true),),
    ((20, true, true, true, true),),
    ((20, true, false, true, false),),
    ((20, false, true, false, false),),
    ((20, false, false, false, false),),
    ((20, true, true, false, false),),
    ((20, true, false, false, false),),
    ((100, false, true, false, false),),
    ((100, false, false, false, false),),
    ]
insts["various"] = [
    ((50, false, true, true, true),),
    ((50, false, false, true, true),),
    ((50, true, true, true, false),),
    ((50, true, false, true, false),),
    ((50, false, true, false, false),),
    ((50, false, false, false, false),),
    ((50, true, true, false, false),),
    ((50, true, false, false, false),),
    ((100, false, true, true, true),),
    ((100, false, false, true, false),),
    ((100, true, true, true, false),),
    ((100, true, false, true, true),),
    ((100, false, true, false, false),),
    ((100, false, false, false, false),),
    ((100, true, true, false, false),),
    ((100, true, false, false, false),),
    ((200, false, true, false, false),),
    ((200, false, false, false, false),),
    ((200, false, true, true, false),),
    ((200, false, false, true, false),),
    ((200, true, true, true, true),),
    ((200, true, false, true, true),),
    ((400, false, true, false, false),),
    ((400, false, false, false, false),),
    ((400, false, true, true, false),),
    ((400, false, false, true, true),),
    ((400, true, true, true, true),),
    ((400, true, false, true, false),),
    ]
return (NearestPSDJuMP, insts)


mutable struct MCMC{S <: DiseaseStateSequence, M <: ILM}
  event_observations::EventObservations{S, M}
  event_extents::EventExtents{S}
  population::Population
  starting_states::DiseaseStates
  risk_functions::RiskFunctions{S}
  risk_priors::RiskPriors{S}
  substitution_model::Union{Nothing, Type{<:NucleicAcidSubstitutionModel}}
  substitution_model_prior::Union{Nothing, Vector{UnivariateDistribution}}
  transmission_network_prior::Union{Nothing, TNPrior}
  markov_chains::Vector{MarkovChain{S, M}}

  function MCMC(
    obs::EventObservations{S, M},
    ee::EventExtents{S},
    pop::Population,
    states::DiseaseStates,
    rf::RiskFunctions{S},
    rp::RiskPriors{S};
    tnprior::Union{Nothing, TNPrior}=nothing) where {
    S <: DiseaseStateSequence,
    M <: TNILM}
    return new{S, M}(obs, ee, pop, states, rf, rp, nothing, nothing, tnprior, MarkovChain{S, M}[])
  end

  function MCMC(
    obs::EventObservations{S, M},
    ee::EventExtents{S},
    pop::Population,
    states::DiseaseStates,
    rf::RiskFunctions{S},
    rp::RiskPriors{S},
    sm::Type{N},
    smp::Vector{UnivariateDistribution};
    tnprior::Union{Nothing, TNPrior}=nothing) where {
    S <: DiseaseStateSequence,
    M <: PhyloILM,
    N <: NucleicAcidSubstitutionModel}
    return new{S, M}(obs, ee, pop, states, rf, rp, sm, smp, tnprior, MarkovChain{S, M}[])
  end
end

function MCMC(
  obs::EventObservations{S, M},
  ee::EventExtents{S},
  pop::Population,
  rf::RiskFunctions{S},
  rp::RiskPriors{S};
  tnprior::Union{Nothing, TNPrior}=nothing) where {
  S <: DiseaseStateSequence,
  M <: TNILM}
  return MCMC(obs, ee, pop, fill(State_S, individuals(pop)), rf, rp, tnprior=tnprior)
end

function MCMC(
  obs::EventObservations{S, M},
  ee::EventExtents{S},
  pop::Population,
  rf::RiskFunctions{S},
  rp::RiskPriors{S},
  sm::Type{N},
  smp::Vector{UnivariateDistribution};
  tnprior::Union{Nothing, TNPrior}=nothing) where {
  S <: DiseaseStateSequence,
  M <: PhyloILM,
  N <: NucleicAcidSubstitutionModel}
  return MCMC(obs, ee, pop, fill(State_S, individuals(pop)), rf, rp, sm, smp, tnprior=tnprior)
end

function Base.show(io::IO, x::MCMC{S, M}) where {
                   S <: DiseaseStateSequence,
                   M <: ILM}
  return print(io, "$S $M MCMC with $(length(x.markov_chains)) Markov chains")
end

function TNDistribution(x::MCMC; burnin::Int64=0, thin::Int64=1)
  return TNDistribution([TNDistribution(y, burnin=burnin, thin=thin) for y in x.markov_chains])
end


function numericRoot(residFnc::Function, measurement, parameters, x0::Vector{Float64})
  return (nlsolve(   (res, X) -> residFnc(res, measurement, parameters, X), x0 )).zero
end

function freshSamples!(ccwl::CommonConvWrapper, N::Int=1)
  ccwl.measurement = getSample(ccwl.usrfnc!, N)
  nothing
end

function shuffleXAltD(X::Vector{Float64}, Alt::Vector{Float64}, d::Int, p::Vector{Int})
  # n = length(X)
  Y = deepcopy(Alt)
  for i in 1:d
    Y[p[i]] = X[i]
  end
  return Y
end

# Shuffle incoming X into random positions in fr.Y
# shuffled fr.Y will be placed back into fr.X[:,fr.gwp.particleidx] upon fr.gwp.usrfnc(x, res)
function shuffleXAltD!(ccwl::CommonConvWrapper, X::Vector{Float64})
  # populate defaults from existing values
  for i in 1:ccwl.xDim
    ccwl.cpt[Threads.threadid()].Y[i] = ccwl.cpt[Threads.threadid()].X[i, ccwl.cpt[Threads.threadid()].particleidx]
  end
  # populate as many measurment dimensions randomly for calculation
  for i in 1:ccwl.zDim
    ccwl.cpt[Threads.threadid()].Y[ccwl.cpt[Threads.threadid()].p[i]] = X[i]
  end
  nothing
end

function (ccw::CommonConvWrapper)(res::Vector{Float64}, x::Vector{Float64})
  shuffleXAltD!(ccw, x)
  ccw.params[ccw.varidx][:, ccw.cpt[Threads.threadid()].particleidx] = ccw.cpt[Threads.threadid()].Y
  ret = ccw.usrfnc!(res, ccw.cpt[Threads.threadid()].factormetadata, ccw.cpt[Threads.threadid()].particleidx, ccw.measurement, ccw.params[ccw.cpt[Threads.threadid()].activehypo]...) # optmize the view here, re-use the same memory
  return ret
end

function (ccw::CommonConvWrapper)(x::Vector{Float64})
  ccw.params[ccw.varidx][:, ccw.cpt[Threads.threadid()].particleidx] = x #ccw.Y
  ccw.usrfnc!(ccw.cpt[Threads.threadid()].res, ccw.cpt[Threads.threadid()].factormetadata, ccw.cpt[Threads.threadid()].particleidx, ccw.measurement, ccw.params[ccw.cpt[Threads.threadid()].activehypo]...)
end

function numericRootGenericRandomizedFnc!(
            ccwl::CommonConvWrapper{T};
            perturb::Float64=1e-10,
            testshuffle::Bool=false ) where {T <: FunctorPairwiseMinimize}
  #
  fill!(ccwl.cpt[Threads.threadid()].res, 0.0) # 1:frl.xDim
  r = optimize( ccwl, ccwl.cpt[Threads.threadid()].X[:, ccwl.cpt[Threads.threadid()].particleidx] ) # ccw.gg
  # TODO -- clearly lots of optmization to be done here
  ccwl.cpt[Threads.threadid()].Y[:] = r.minimizer
  ccwl.cpt[Threads.threadid()].X[:,ccwl.cpt[Threads.threadid()].particleidx] = ccwl.cpt[Threads.threadid()].Y
  nothing
end

## TODO desperately needs cleaning up and refactoring
# Solve free variable x by root finding residual function fgr.usrfnc(x, res)
# randomly shuffle x dimensions if underconstrained by measurement z dimensions
# small random perturbation used to prevent trivial solver cases, div by 0 etc.
# result stored in fgr.Y
# fr.X must be set to memory ref the param[varidx] being solved, at creation of fr
function numericRootGenericRandomizedFnc!(
            ccwl::CommonConvWrapper{T};
            perturb::Float64=1e-10,
            testshuffle::Bool=false ) where {T <: FunctorPairwise}
  #
  # ststr = "thrid=$(Threads.threadid()), zDim=$(ccwl.zDim), xDim=$(ccwl.xDim)\n"
  # ccall(:jl_, Void, (Any,), ststr)
  if ccwl.zDim < ccwl.xDim && !ccwl.partial || testshuffle
    # less measurement dimensions than variable dimensions -- i.e. shuffle
    shuffle!(ccwl.cpt[Threads.threadid()].p)
    for i in 1:ccwl.xDim
      ccwl.cpt[Threads.threadid()].perturb[1:ccwl.zDim] = perturb*randn(ccwl.zDim)
      ccwl.cpt[Threads.threadid()].X[ccwl.cpt[Threads.threadid()].p[1:ccwl.zDim], ccwl.cpt[Threads.threadid()].particleidx] += ccwl.cpt[Threads.threadid()].perturb
      r = nlsolve(  ccwl,
                    ccwl.cpt[Threads.threadid()].X[ccwl.cpt[Threads.threadid()].p[1:ccwl.zDim], ccwl.cpt[Threads.threadid()].particleidx] # this is x0
                 )
      if r.f_converged
        shuffleXAltD!( ccwl, r.zero )
        break;
      else
        # TODO -- report on this bottleneck, useful for optimization of code
        # @show i, ccwl.p, ccwl.xDim, ccwl.zDim
        temp = ccwl.cpt[Threads.threadid()].p[end]
        ccwl.cpt[Threads.threadid()].p[2:end] = ccwl.cpt[Threads.threadid()].p[1:(end-1)]
        ccwl.cpt[Threads.threadid()].p[1] = temp
        if i == ccwl.xDim
          error("numericRootGenericRandomizedFnc could not converge, i=$(i), ccwl.usrfnc!=$(typeof(ccwl.usrfnc!))")
        end
      end
    end
    #shuffleXAltD!( ccwl, r.zero ) # moved up
  elseif ccwl.zDim >= ccwl.xDim && !ccwl.partial
    # equal or more measurement dimensions than variable dimensions -- i.e. don't shuffle
    ccwl.cpt[Threads.threadid()].perturb[1:ccwl.xDim] = perturb*randn(ccwl.xDim)
    ccwl.cpt[Threads.threadid()].X[1:ccwl.xDim, ccwl.cpt[Threads.threadid()].particleidx] += ccwl.cpt[Threads.threadid()].perturb[1:ccwl.xDim] # moved up

    # str = "nlsolve, thrid_=$(Threads.threadid()), partidx=$(ccwl.cpt[Threads.threadid()].particleidx), X=$(ccwl.cpt[Threads.threadid()].X[1:ccwl.xDim,ccwl.cpt[Threads.threadid()].particleidx])"
    # ccall(:jl_, Void, (Any,), str)
    r = nlsolve( ccwl, ccwl.cpt[Threads.threadid()].X[1:ccwl.xDim,ccwl.cpt[Threads.threadid()].particleidx] )
    # ccall(:jl_, Void, (Any,), "nlsolve.zero=$(r.zero)")
    if sum(isnan.(( r ).zero)) == 0
      ccwl.cpt[Threads.threadid()].Y[1:ccwl.xDim] = ( r ).zero
    else
      # TODO print this output as needed
      # warn("got NaN, ccwl.cpt[Threads.threadid()].particleidx = $(ccwl.cpt[Threads.threadid()].particleidx), r=$(r)")
      str = "ccw.thrid_=$(Threads.threadid()), got NaN, ccwl.cpt[Threads.threadid()].particleidx = $(ccwl.cpt[Threads.threadid()].particleidx), r=$(r)\n"
      ccall(:jl_, Void, (Any,), str)
      ccall(:jl_, Void, (Any,), ccwl.usrfnc!)
      for thatlen in 1:length(ccwl.params)
        str = "thatlen=$thatlen, ccwl.params[thatlen][:, ccwl.cpt[Threads.threadid()].particleidx]=$(ccwl.params[thatlen][:, ccwl.cpt[Threads.threadid()].particleidx])\n"
        ccall(:jl_, Void, (Any,), str)
      end
    end
  elseif ccwl.partial
    # improve memory management in this function
    ccwl.cpt[Threads.threadid()].p[1:length(ccwl.usrfnc!.partial)] = Int[ccwl.usrfnc!.partial...] # TODO -- move this line up and out of inner loop
    r = nlsolve(  ccwl,
                  ccwl.cpt[Threads.threadid()].X[ccwl.cpt[Threads.threadid()].p[1:ccwl.zDim], ccwl.cpt[Threads.threadid()].particleidx] # this is x0
               )
    shuffleXAltD!( ccwl, r.zero )
  else
    ccall(:jl_, Void, (Any,), "ERROR: Unresolved numeric solve case")
    error("Unresolved numeric solve case")
  end
  ccwl.cpt[Threads.threadid()].X[:,ccwl.cpt[Threads.threadid()].particleidx] = ccwl.cpt[Threads.threadid()].Y
  nothing
end

"""
    $(SIGNATURES)

Perform multimodal incremental smoothing and mapping (mm-iSAM) computations over given factor graph `fgl::FactorGraph` on the local computer.  A pdf of the Bayes (Junction) tree will be generated in the working folder with `drawpdf=true`
"""
function batchSolve!(fgl::FactorGraph; drawpdf::Bool=false)
  tree = wipeBuildNewTree!(fgl, drawpdf=drawpdf)
  inferOverTree!(fgl, tree)
  tree
end

#


function _process_reaction_phrase(phrase_array::Array{String,1})

    tmp_string = ""
    for phrase in phrase_array
        tmp_string*="$(phrase) + "
    end

    # cutoff -
    tmp_string = tmp_string |> rstrip

    # cutoff the trailing +
    return (tmp_string[1:end-1] |> rstrip)
end

function build_stoichiometric_matrix(model::Dict{Symbol,Any})

    try

        # get the reaction table -
        reaction_table = model[:reactions]
        reaction_id_array = reaction_table[!, :reaction_number]
        compound_id_array = model[:compounds][!,:compound_id]

        # now - let's build the stm -
        number_of_species = length(compound_id_array)
        number_of_reactions = length(reaction_id_array)
        stoichiometric_matrix = zeros(number_of_species, number_of_reactions)

        # build the array -
        for reaction_index = 1:number_of_reactions

            # what is my reaction id?
            reaction_id = reaction_id_array[reaction_index]

            # get row from the reaction table -
            df_reaction = filter(:reaction_number => x -> (x == reaction_id), reaction_table)

            # grab the stm dictionary -
            stm_dictionary = df_reaction[1, :stoichiometric_dictionary]

            # ok, lets see if we have these species -
            for species_index = 1:number_of_species

                # species code -
                species_symbol = compound_id_array[species_index]
                if (haskey(stm_dictionary, species_symbol) == true)
                    stm_coeff_value = stm_dictionary[species_symbol]
                    stoichiometric_matrix[species_index, reaction_index] = stm_coeff_value
                end
            end
        end

        # return -
        return (compound_id_array, reaction_id_array, stoichiometric_matrix)
    catch error

        # what is our error policy? => for now, just print the message
        error_message = sprint(showerror, error, catch_backtrace())
        println(error_message)

    end
end

function find_compound_index(model::Dict{Symbol,Any},
    search::Pair{Symbol,String})

    # get the compounds table -
    compounds_table = model[:compounds]

    # get list of compound names -
    tmp_array = compounds_table[!,search.first] |> collect

    # do we have this name? (if yes, then return index)
    return findfirst(x->x==search.second,tmp_array)
end

function find_reaction_index(model::Dict{Symbol,Any},
    search::Pair{Symbol,String})

    # get the compounds table -
    reaction_table = model[:reactions]

    # get list of compound names -
    tmp_array = reaction_table[!,search.first] |> collect

    # do we have this name? (if yes, then return index)
    return findfirst(x->x==search.second,tmp_array)
end

function update_flux_bounds_directionality(MODEL,default_flux_bounds)

    # get the ΔG table -> for now, just use the mean value -
    ΔG_table = MODEL[:ΔG]

    # how many reactions do we have ΔG data for?
    (number_of_reactions,_) = size(ΔG_table)
    for reaction_index = 1:number_of_reactions

        # get ΔG value -
        μ_ΔG_value = ΔG_table[reaction_index,:μ_ΔG]

        # get the corresponding reaction number -
        reaction_id = ΔG_table[reaction_index, :reaction_number]

        # what index is this?
        index_in_bounds_array = find_reaction_index(MODEL,:reaction_number=>reaction_id)
        if (isempty(index_in_bounds_array) == false)

            # backward?
            is_reversible = -1*(sign(μ_ΔG_value)) <= 0.0 ? true : false
            if (is_reversible == false)
                default_flux_bounds[index_in_bounds_array,1] = 0.0
            end
        end
    end

    return default_flux_bounds
end

function translation_reaction_string_to_human(model::Dict{Symbol,Any})

    # get the compound table -
    compound_table = model[:compounds]
    reaction_table = model[:reactions]

    # build a translation table -
    (number_of_compounds, _) = size(compound_table)
    compound_translation_table = Dict{String,String}()
    for compound_index = 1:number_of_compounds
        kegg_name = compound_table[compound_index, :compound_id]
        human_name = compound_table[compound_index, :compound_name]
        compound_translation_table[kegg_name] = human_name
    end

    # hack: -
    compound_translation_table["C00138"] = "Reduced ferredoxin"
    compound_translation_table["C00139"] = "Oxidized ferredoxin"

    # number of _reactions -
    (number_of_reactions, _) = size(reaction_table)

    # initialze space -
    tmp_reaction_string_array = Array{String,1}(undef,number_of_reactions)

    # just so we are safe - copy the old strings into the tmp array in case something doesn't work
    for reaction_index = 1:number_of_reactions
        old_string = reaction_table[reaction_index, :reaction_markup]
        tmp_reaction_string_array[reaction_index] = old_string
    end

    # init some tmp storage -
    tmp_reactant_phrase_array = Array{String,1}()
    tmp_product_phrase_array = Array{String,1}()

    # main loop - here we go .... let's do this ...
    for reaction_index = 1:number_of_reactions

        # grab the st dictionary for this reaction -
        stoichiometric_dictionary = reaction_table[reaction_index, :stoichiometric_dictionary]
        for (key,value) in stoichiometric_dictionary

            # look up this key in the translation table -
            human_key = get(compound_translation_table, key, "?")

            if (abs(value) != 1)
                tmp_phrase = "$(abs(value)) $(human_key)"
            else
                tmp_phrase = "$(human_key)"
            end

            if (value<0)
                push!(tmp_reactant_phrase_array, tmp_phrase)
            else
                push!(tmp_product_phrase_array, tmp_phrase)
            end
        end

        # build new reaction string ...
        reactant_string = _process_reaction_phrase(tmp_reactant_phrase_array)
        product_string = _process_reaction_phrase(tmp_product_phrase_array)
        full_reaction_string = "$(reactant_string) <=> $(product_string)"
        tmp_reaction_string_array[reaction_index] = full_reaction_string

        # empty -
        empty!(tmp_reactant_phrase_array)
        empty!(tmp_product_phrase_array)
    end

    return tmp_reaction_string_array
end


function dh_simulation(LP::AbstractLProblem{T}, p::Float64, nTrials::Int64, alg::OrdinaryDiffEqAlgorithm, logger::Logger{T1,T2} = Logger{:no, :cmd}(), interval::Int64 = 1, offset1::Int64 = 0; links = (:all,:all), axes = [:x, :y, :z], kwargs...) where {T<:PureClassicalApprox,T1,T2}
    assert(nTrials > 0)
    RNGarray = initRNG(nprocs())
    path_part = @sprintf("Dis Tm%.2f dt%.3f del%d Obs%s Pr%.3f/", LP.Tmax, LP.tstep, LP.delimiter, get_string(LP.rules.str_vec), p)
    filenames = Vector{String}()
    A = LP.A
    l = LP.cb.affect!.save_func.len
    tspan = (0.0, LP.Tmax*LP.delimiter)
    t_means = collect(0.0:LP.tstep:(LP.Tmax*LP.delimiter))
    saved_values = SavedValues(t_means, [zeros(l) for i in eachindex(t_means)])
    for thread = (offset1+1):(offset1+nprocs())
        push!(filenames, get_string(nTrials, thread, alg))
    end
    log_o, save_os = set_Logging(LP, nTrials, logger, A, path_part, filenames)
    cf_vals0 = LP.cf_vals
    @sync for i in workers()
        @spawnat i begin
                       global cf_vals = cf_vals0
                       global RNG = RNGarray[i]
                   end
    end
    iter = get_status(log_o)
    offset = get_offset(log_o)
    if iter > 0
        load_data!(cf_vals0, save_os[1])
        for i in workers()
            @spawnat i load_data!(cf_vals, save_os[i])
        end
    end
    t0 = time()-offset
    while iter<nTrials
        t1 = time()
        iter += 1
        ARHS = build_RHS_function(A)
        rules, OSET = set_CorrFuncs(A, links, axes)
        @sync begin
            for i in workers()
                @spawnat i dh_run(iter, A, ARHS, OSET, rules, t_means, saved_values, tspan, p, cf_vals, alg, save_os[i], RNG; kwargs...)
            end
            @spawnat 1 dh_run(iter, A, ARHS, OSET, rules, t_means, saved_values, tspan, p, cf_vals0, alg, save_os[1], RNGarray[1]; kwargs...)
        end
        t2 = time()
        output(log_o, @sprintf("%10d %20.5f %20.5f %10d", iter, t2-t1, (t2-t0)/iter, nTrials - iter))
    end
end

function dh_simulation(LP::AbstractLProblem{T}, p::Float64, nTrials::Int64, alg::OrdinaryDiffEqAlgorithm, logger::Logger{T1,T2} = Logger{:no, :cmd}(), interval::Int64 = 1, offset1::Int64 = 0; links = (:all,:allq), axes = [:x, :y, :z], kwargs...) where {T<:HybridApprox,T1,T2}
    assert(nTrials > 0)
    RNGarray = initRNG(nprocs()+1)
    path_part = @sprintf("Dis Tm%.2f dt%.3f del%d Obs%s Pr%.3f/", LP.Tmax, LP.tstep, LP.delimiter, get_string(LP.rules.str_vec), p)
    filenames = Vector{String}()
    A = HybridApprox(LP.A.L, copy(LP.A.q_spins), LP.A.name)
    l = LP.cb.affect!.save_func.len
    tspan = (0.0, LP.Tmax*LP.delimiter)
    t_means = collect(0.0:LP.tstep:(LP.Tmax*LP.delimiter))
    saved_values = SavedValues(t_means, [zeros(l) for i in eachindex(t_means)])
    for thread = (offset1+1):(offset1+nprocs())
        push!(filenames, get_string(nTrials, thread, alg))
    end
    log_o, save_os = set_Logging(LP, nTrials, logger, LP.A, path_part, filenames)
    cf_vals0 = LP.cf_vals
    @sync for i in workers()
        @spawnat i begin
                       global cf_vals = cf_vals0
                       global RNG = RNGarray[i]
                   end
    end
    iter = get_status(log_o)
    offset = get_offset(log_o)
    if iter > 0
        load_data!(cf_vals0, save_os[1])
        for i in workers()
            @spawnat i load_data!(cf_vals, save_os[i])
        end
    end
    t0 = time()-offset
    while iter<nTrials
        t1 = time()
        iter += 1
        q_spins = randsubseq(RNGarray[end], LP.A.q_spins, 1-p)
        A.q_spins = q_spins
        A.q_num = length(q_spins)
        ARHS = build_RHS_function(A)
        rules, OSET = set_CorrFuncs(A, links, axes)
        @sync begin
            for i in workers()
                @spawnat i dh_run(iter, A, ARHS, OSET, rules, t_means, saved_values, tspan, p, cf_vals, alg, save_os[i], RNG; kwargs...)
            end
            @spawnat 1 dh_run(iter, A, ARHS, OSET, rules, t_means, saved_values, tspan, p, cf_vals0, alg, save_os[1], RNGarray[1]; kwargs...)
        end
        t2 = time()
        output(log_o, @sprintf("%10d %20.5f %20.5f %10d", iter, t2-t1, (t2-t0)/iter, nTrials - iter))
    end
end

function dh_simulation(LP::AbstractLProblem{Approx}, p::Float64, nTrials::Int64, alg::OrdinaryDiffEqAlgorithm, logger::Logger{T1,T2} = Logger{:no, :cmd}(), interval::Int64 = 1, offset1::Int64 = 0; links = (:all,:allq), axes = [:x, :y, :z], kwargs...) where {Approx <: CompositeApproximation{<:HybridApprox, <:PureClassicalApprox}, T1,T2}
    A = LP.A
    Tmax = LP.Tmax
    delimiter = LP.delimiter
    tstep = LP.tstep
    A_temp = CompositeApproximation(HybridApprox(A.A1.L, copy(A.A1.q_spins), A.A1.name), A.A2, A.gamma)
    #rules, OSET = set_CorrFuncs(A, links, axes)
    l = LP.cb.affect!.save_func.len
    tspan = (0.0, Tmax*delimiter)
    t_means = collect(0.0:tstep:(Tmax*delimiter))
    t_cors = collect(0.0:tstep:Tmax)
    saved_values = SavedValues(t_means, [zeros(l) for i in eachindex(t_means)])
    ##############################
    assert(nTrials > 0)
    RNGarray = initRNG(nprocs()+1)
    path_part = @sprintf("Dis Tm%.2f dt%.3f del%d Obs%s Pr%.3f/", Tmax, tstep, delimiter, get_string(LP.rules.str_vec), p)
    filenames = Vector{String}()
    for thread = (offset1+1):(offset1+nprocs())
        push!(filenames, get_string(nTrials, thread, alg))
    end
    log_o, save_os = set_Logging(LP, nTrials, logger, A, path_part, filenames)
    cf_vals0 = LP.cf_vals
    @sync for i in workers()
        @spawnat i begin
                       global cf_vals = cf_vals0
                       global RNG = RNGarray[i]
                   end
    end
    iter = get_status(log_o)
    offset = get_offset(log_o)
    if iter > 0
        load_data!(cf_vals0, save_os[1])
        for i in workers()
            @spawnat i load_data!(cf_vals, save_os[i])
        end
    end
    t0 = time()-offset
    ##############################
    while iter<nTrials
        t1 = time()
        iter += 1
        q_spins = randsubseq(RNGarray[end], A.A1.q_spins, 1-p)
        A_temp.A1.q_spins = q_spins
        A_temp.A1.q_num = length(q_spins)
        print(length(q_spins), "\n")
        ARHS = build_RHS_function(A_temp)
        rules, OSET = set_CorrFuncs(A_temp.A1, links, axes)
        @sync begin
            for i in workers()
                @spawnat i dh_run(iter, A_temp, ARHS, OSET, rules, t_means, saved_values, tspan, p, cf_vals, alg, save_os[i], RNG; kwargs...)
            end
            @spawnat 1 dh_run(iter, A_temp, ARHS, OSET, rules, t_means, saved_values, tspan, p, cf_vals0, alg, save_os[1], RNGarray[1]; kwargs...)
        end
        t2 = time()
        output(log_o, @sprintf("%10d %20.5f %20.5f %10d", iter, t2-t1, (t2-t0)/iter, nTrials - iter))
    end
end

function dh_run(iter::Int64, A::abstractApproximation, ARHS::AbstractRHSFunction, OSET::ObservablesSet, rules::ConvRules, t_means::Vector{Float64}, saved_values::SavedValues, tspan::NTuple{2, Float64}, p::Float64, cf_vals::T, alg::OrdinaryDiffEqAlgorithm, save_o::Saver, RNG::AbstractRNG; abstol_i = 1e-14, reltol_i = 1e-6, kwargs...) where T <: Union{CFVals, CFVals0}
    u0 = randomDState(A, p, RNG)
    #print(u0, "\n")
    prob = ODEProblem(ARHS, u0, tspan)
    cb = SavingCallback(OSET, saved_values, u0; saveat = t_means)
    solve(prob, alg; save_everystep = false, callback = cb, abstol = abstol_i, reltol = reltol_i, kwargs...)
    calculateCorrFunc!(A, saved_values, rules, cf_vals, iter)
    save_data(save_o, cf_vals, iter)
    #print(cf_vals, "\n")
end

function randomDState(A::PureClassicalApprox, p::Float64, RNG::AbstractRNG = Base.GLOBAL_RNG)
    cl_state = randomState(A, RNG)
    discarded = randsubseq(RNG, collect(1:A.L.tot_spin_num), p)
    for i in discarded
        cl_state[i,:] .= 0.0
    end
    return cl_state
end

function randomDState(A::HybridApprox, p::Float64, RNG::AbstractRNG = Base.GLOBAL_RNG)
    Dh = 2^A.q_num
    cl_state = random_cl_state(A.cl_num, RNG)
    discarded = randsubseq(RNG, collect(1:A.cl_num), p)
    for i in discarded
        cl_state[i,:] .= 0.0
    end
    return ArrayPartition(random_q_state(Dh, RNG), cl_state)
end

function randomDState(A::CompositeApproximation{T1,T2}, p::Float64, RNG::AbstractRNG = Base.GLOBAL_RNG) where {T1<:HybridApprox, T2<:PureClassicalApprox}
    Dh = 2^A.A1.q_num
    cl_state = random_cl_state(A.A1.cl_num, RNG)
    discarded = randsubseq(RNG, collect(1:A.A1.cl_num), p)
    for i in discarded
        cl_state[i,:] .= 0.0
    end
    ArrayPartition(random_q_state(Dh, RNG), cl_state, randomState(A.A2, RNG))
end

function getrsubseq(A::AbstractArray{T}, p::Float64, RNG::AbstractRNG = Base.GLOBAL_RNG) where {T}
    B = Vector{T}()
    for i in eachindex(A)
        if rand(RNG) < p
            push!(B, A[i])
        end
    end
    return B
end

export dh_simulation


seed!(42)
L = 3
D = 10
N = 20
nSamples = 10
b = 5
x = rand(N, D)
y = rand(N)

@testset "Gibbs Sampling" begin
    i = GibbsSampling(nBurnin = 0)
    @test repr(i) == "Gibbs Sampler"
    i = AGP.tuple_inference(i, L, D, N, b, [], [])

    @test AGP.getρ(i) == 1
    @test AGP.isStochastic(i) == false

end


# Structured optimizer interface
# ========================== #
"""
    supports_structure(optimizer::StochasticProgramOptimizerType, structure::AbstractStochasticStructure)

Return a `Bool` indicating whether `optimizer` supports the stochastic `structure`. That is, `load_structure!(optimizer, structure)` will not throw `UnsupportedStructure`
"""
function supports_structure(optimizer::StochasticProgramOptimizerType, structure::AbstractStochasticStructure)
    return false
end
"""
    check_loadable(optimizer::AbstractStructuredOptimizer, structure::AbstractStochasticStructure)

Throws an `UnloadableStructure` exception if `structure` is not loadable by `optimizer`.

    See also: [`load_structure!`](@ref)
"""
function check_loadable end
"""
    load_structure!(optimizer::AbstractStructuredOptimizer, structure::AbstractStochasticStructure, x₀::AbstractVector)

Instantiate the `optimizer` with the stochastic program represented in memory by the given `structure` and initial decision `x₀`.

See also: [`optimize!`](@ref)
"""
function load_structure! end
"""
    restore_structure!(optimizer::AbstractStructuredOptimizer)

Restore the stochastic program to the state it was in before a call to `optimize!`

See also: [`load_structure!`](@ref)
"""
function restore_structure! end
"""
    optimize!(optimizer::AbstractStructuredOptimizer)

Start the solution procedure for `optimizer` after a call to [`load_structure!`](@ref).

See also: [`load_structure!`](@ref)
"""
function optimize! end
"""
    optimizer_name(optimizer::AbstractStructuredOptimizer)

Optionally, return a string identifier of `AbstractStructuredOptimizer`.
"""
function optimizer_name(::AbstractStructuredOptimizer)
    return "SolverName() attribute not implemented by the optimizer."
end
"""
    master_optimizer(optimizer::AbstractStructuredOptimizer)

Return a MOI optimizer constructor
"""
function master_optimizer end
"""
   subproblem_optimizer(optimizer::AbstractStructuredOptimizer)

Return a MOI optimizer constructor for solving subproblems
"""
function subproblem_optimizer end
# Sample-based solver interface
# ========================== #
"""
    load_model!(optimizer::AbstractSampledOptimizer, model::StochasticModel, x₀::AbstractVector)

Instantiate the `optimizer` with the stochastic model and initial decision `x₀`.

See also: [`optimize!`](@ref)
"""
function load_model! end
"""
    optimizer_name(optimizer::AbstractSampledOptimizer)

Optionally, return a string identifier of `AbstractSampledOptimizer`.
"""
function optimizer_name(::AbstractSampledOptimizer)
    return "SolverName() attribute not implemented by the optimizer."
end
"""
    optimal_instance(optimizer::AbstractSampledOptimizer)

Return a stochastic programming instance of the stochastic model after a call to [`optimize!`](@ref).
"""
function optimal_instance end


## Integrator Dispatches

# Can get rid of an allocation here with a function
# get_tmp_arr(integrator.cache) which gives a pointer to some
# cache array which can be modified.

@inline function _searchsortedfirst(v::AbstractVector, x, lo::Integer, forward::Bool)
    u = oftype(lo, 1)
    lo = lo - u
    hi = length(v) + u
    @inbounds while lo < hi - u
        m = (lo + hi) >>> 1
        @inbounds if (forward && v[m] < x) || (!forward && v[m] > x)
            lo = m
        else
            hi = m
        end
    end
    return hi
end

@inline function _searchsortedlast(v::AbstractVector, x, lo::Integer, forward::Bool)
  u = oftype(lo, 1)
  lo = lo - u
  hi = length(v) + u
  @inbounds while lo < hi - u
      m = (lo + hi) >>> 1
      @inbounds if (forward && v[m] > x) || (!forward && v[m] < x)
          hi = m
      else
          lo = m
      end
  end
  return lo
end

@inline function _ode_addsteps!(integrator,f=integrator.f,always_calc_begin = false,allow_calc_end = true,force_calc_end = false)
  if !(typeof(integrator.cache) <: CompositeCache)
    DiffEqBase.addsteps!(integrator.k,integrator.tprev,integrator.uprev,integrator.u,
                  integrator.dt,f,integrator.p,integrator.cache,
                  always_calc_begin,allow_calc_end,force_calc_end)
  else
    DiffEqBase.addsteps!(integrator.k,integrator.tprev,integrator.uprev,integrator.u,
                  integrator.dt,f,integrator.p,
                  @inbounds(integrator.cache.caches[integrator.cache.current]),
                  always_calc_begin,allow_calc_end,force_calc_end)
  end
end
@inline DiffEqBase.addsteps!(integrator::ODEIntegrator,args...) = _ode_addsteps!(integrator,args...)

@inline function ode_interpolant(Θ,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  DiffEqBase.addsteps!(integrator)
  if !(typeof(integrator.cache) <: CompositeCache)
    val = ode_interpolant(Θ,integrator.dt,integrator.uprev,integrator.u,integrator.k,integrator.cache,idxs,deriv)
  else
    val = composite_ode_interpolant(Θ,integrator,integrator.cache.caches,integrator.cache.current,idxs,deriv)
  end
  val
end

@generated function composite_ode_interpolant(Θ, integrator, caches::T, current, idxs, deriv) where {T <: Tuple}
  expr = Expr(:block)
  for i in 1:length(T.types)
    push!(expr.args, quote
      if $i == current
        return ode_interpolant(Θ,integrator.dt,integrator.uprev,integrator.u,integrator.k,caches[$i],idxs,deriv)
      end
    end)
  end
  push!(expr.args, quote
    throw("Cache $current is not available. There are only $(length(caches)) caches.")
  end)
  return expr
end

@inline function ode_interpolant!(val,Θ,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  DiffEqBase.addsteps!(integrator)
  if !(typeof(integrator.cache) <: CompositeCache)
    ode_interpolant!(val,Θ,integrator.dt,integrator.uprev,integrator.u,integrator.k,integrator.cache,idxs,deriv)
  else
    ode_interpolant!(val,Θ,integrator.dt,integrator.uprev,integrator.u,integrator.k,integrator.cache.caches[integrator.cache.current],idxs,deriv)
  end
end

@generated function composite_ode_interpolant!(val, Θ, integrator, caches::T, current, idxs, deriv) where {T <: Tuple}
  expr = Expr(:block)
  for i in 1:length(T.types)
    push!(expr.args, quote
      if $i == current
        return ode_interpolant!(val,Θ,integrator.dt,integrator.uprev,integrator.u,integrator.k,caches[$i],idxs,deriv)
      end
    end)
  end
  push!(expr.args, quote
    throw("Cache $current is not available. There are only $(length(caches)) caches.")
  end)
  return expr
end

@inline function current_interpolant(t::Number,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  Θ = (t-integrator.tprev)/integrator.dt
  ode_interpolant(Θ,integrator,idxs,deriv)
end

@inline function current_interpolant(t,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  Θ = (t.-integrator.tprev)./integrator.dt
  [ode_interpolant(ϕ,integrator,idxs,deriv) for ϕ in Θ]
end

@inline function current_interpolant!(val,t::Number,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  Θ = (t-integrator.tprev)/integrator.dt
  ode_interpolant!(val,Θ,integrator,idxs,deriv)
end

@inline function current_interpolant!(val,t,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  Θ = (t.-integrator.tprev)./integrator.dt
  [ode_interpolant!(val,ϕ,integrator,idxs,deriv) for ϕ in Θ]
end

@inline function current_extrapolant(t::Number,integrator::DiffEqBase.DEIntegrator,idxs=nothing,deriv=Val{0})
  Θ = (t-integrator.tprev)/(integrator.t-integrator.tprev)
  ode_extrapolant(Θ,integrator,idxs,deriv)
end

@inline function current_extrapolant!(val,t::Number,integrator::DiffEqBase.DEIntegrator,idxs=nothing,deriv=Val{0})
  Θ = (t-integrator.tprev)/(integrator.t-integrator.tprev)
  ode_extrapolant!(val,Θ,integrator,idxs,deriv)
end

@inline function current_extrapolant(t::AbstractArray,integrator::DiffEqBase.DEIntegrator,idxs=nothing,deriv=Val{0})
  Θ = (t.-integrator.tprev)./(integrator.t-integrator.tprev)
  [ode_extrapolant(ϕ,integrator,idxs,deriv) for ϕ in Θ]
end

@inline function current_extrapolant!(val,t,integrator::DiffEqBase.DEIntegrator,idxs=nothing,deriv=Val{0})
  Θ = (t.-integrator.tprev)./(integrator.t-integrator.tprev)
  [ode_extrapolant!(val,ϕ,integrator,idxs,deriv) for ϕ in Θ]
end

@inline function ode_extrapolant!(val,Θ,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  DiffEqBase.addsteps!(integrator)
  if !(typeof(integrator.cache) <: CompositeCache)
    ode_interpolant!(val,Θ,integrator.t-integrator.tprev,integrator.uprev2,integrator.uprev,integrator.k,integrator.cache,idxs,deriv)
  else
    composite_ode_extrapolant!(val,Θ,integrator,integrator.cache.caches,integrator.cache.current,idxs,deriv)
  end
end

@generated function composite_ode_extrapolant!(val, Θ, integrator, caches::T, current, idxs, deriv) where {T <: Tuple}
  expr = Expr(:block)
  for i in 1:length(T.types)
    push!(expr.args, quote
      if $i == current
        return ode_interpolant!(val,Θ,integrator.t-integrator.tprev,integrator.uprev2,integrator.uprev,integrator.k,caches[$i],idxs,deriv)
      end
    end)
  end
  push!(expr.args, quote
    throw("Cache $current is not available. There are only $(length(caches)) caches.")
  end)
  return expr
end

@inline function ode_extrapolant(Θ,integrator::DiffEqBase.DEIntegrator,idxs,deriv)
  DiffEqBase.addsteps!(integrator)
  if !(typeof(integrator.cache) <: CompositeCache)
    ode_interpolant(Θ,integrator.t-integrator.tprev,integrator.uprev2,integrator.uprev,integrator.k,integrator.cache,idxs,deriv)
  else
    composite_ode_extrapolant(Θ,integrator,integrator.cache.caches,integrator.cache.current,idxs,deriv)
  end
end

@generated function composite_ode_extrapolant(Θ, integrator, caches::T, current, idxs, deriv) where {T <: Tuple}
  expr = Expr(:block)
  for i in 1:length(T.types)
    push!(expr.args, quote
      if $i == current
        return ode_interpolant(Θ,integrator.t-integrator.tprev,integrator.uprev2,integrator.uprev,integrator.k,caches[$i],idxs,deriv)
      end
    end)
  end
  push!(expr.args, quote
    throw("Cache $current is not available. There are only $(length(caches)) caches.")
  end)
  return expr
end

"""
ode_interpolation(tvals,ts,timeseries,ks)

Get the value at tvals where the solution is known at the
times ts (sorted), with values timeseries and derivatives ks
"""
function ode_interpolation(tvals,id,idxs,deriv,p,continuity::Symbol=:left)
  @unpack ts,timeseries,ks,f,cache = id
  @inbounds tdir = sign(ts[end]-ts[1])
  idx = sortperm(tvals,rev=tdir<0)

  # start the search thinking it's ts[1]-ts[2]
  i₋ = 1
  i₊ = 2
  vals = map(idx) do j
    t = tvals[j]

    if continuity === :left
      # we have i₋ = i₊ = 1 if t = ts[1], i₊ = i₋ + 1 = lastindex(ts) if t > ts[end],
      # and otherwise i₋ and i₊ satisfy ts[i₋] < t ≤ ts[i₊]
      i₊ = min(lastindex(ts), _searchsortedfirst(ts,t,i₊,tdir > 0))
      i₋ = i₊ > 1 ? i₊ - 1 : i₊
    else
      # we have i₋ = i₊ - 1 = 1 if t < ts[1], i₊ = i₋ = lastindex(ts) if t = ts[end],
      # and otherwise i₋ and i₊ satisfy ts[i₋] ≤ t < ts[i₊]
      i₋ = max(1, _searchsortedlast(ts,t,i₋,tdir > 0))
      i₊ = i₋ < lastindex(ts) ? i₋ + 1 : i₋
    end

    dt = ts[i₊] - ts[i₋]
    Θ = iszero(dt) ? oneunit(t) / oneunit(dt) : (t-ts[i₋]) / dt

    if typeof(cache) <: (FunctionMapCache) || typeof(cache) <: FunctionMapConstantCache
      return ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],0,cache,idxs,deriv)
    elseif !id.dense
      return linear_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],idxs,deriv)
    elseif typeof(cache) <: CompositeCache
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache.caches[id.alg_choice[i₊]]) # update the kcurrent
      return ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache.caches[id.alg_choice[i₊]],idxs,deriv)
    else
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache) # update the kcurrent
      return ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache,idxs,deriv)
    end
  end
  invpermute!(vals, idx)
  DiffEqArray(vals, tvals)
end

"""
ode_interpolation(tvals,ts,timeseries,ks)

Get the value at tvals where the solution is known at the
times ts (sorted), with values timeseries and derivatives ks
"""
function ode_interpolation!(vals,tvals,id,idxs,deriv,p,continuity::Symbol=:left)
  @unpack ts,timeseries,ks,f,cache = id
  @inbounds tdir = sign(ts[end]-ts[1])
  idx = sortperm(tvals,rev=tdir<0)

  # start the search thinking it's in ts[1]-ts[2]
  i₋ = 1
  i₊ = 2
  @inbounds for j in idx
    t = tvals[j]

    if continuity === :left
      # we have i₋ = i₊ = 1 if t = ts[1], i₊ = i₋ + 1 = lastindex(ts) if t > ts[end],
      # and otherwise i₋ and i₊ satisfy ts[i₋] < t ≤ ts[i₊]
      i₊ = min(lastindex(ts), _searchsortedfirst(ts,t,i₊,tdir > 0))
      i₋ = i₊ > 1 ? i₊ - 1 : i₊
    else
      # we have i₋ = i₊ - 1 = 1 if t < ts[1], i₊ = i₋ = lastindex(ts) if t = ts[end],
      # and otherwise i₋ and i₊ satisfy ts[i₋] ≤ t < ts[i₊]
      i₋ = max(1, _searchsortedlast(ts,t,i₋,tdir > 0))
      i₊ = i₋ < lastindex(ts) ? i₋ + 1 : i₋
    end

    dt = ts[i₊] - ts[i₋]
    Θ = iszero(dt) ? oneunit(t) / oneunit(dt) : (t-ts[i₋]) / dt

    if typeof(cache) <: (FunctionMapCache) || typeof(cache) <: FunctionMapConstantCache
      if eltype(timeseries) <: AbstractArray
        ode_interpolant!(vals[j],Θ,dt,timeseries[i₋],timeseries[i₊],0,cache,idxs,deriv)
      else
        vals[j] = ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],0,cache,idxs,deriv)
      end
    elseif !id.dense
      if eltype(timeseries) <: AbstractArray
        linear_interpolant!(vals[j],Θ,dt,timeseries[i₋],timeseries[i₊],idxs,deriv)
      else
        vals[j] = linear_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],idxs,deriv)
      end
    elseif typeof(cache) <: CompositeCache
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache.caches[id.alg_choice[i₊]]) # update the kcurrent
      if eltype(timeseries) <: AbstractArray
        ode_interpolant!(vals[j],Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache.caches[id.alg_choice[i₊]],idxs,deriv)
      else
        vals[j] = ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache.caches[id.alg_choice[i₊]],idxs,deriv)
      end
    else
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache) # update the kcurrent
      if eltype(vals[j]) <: AbstractArray
        ode_interpolant!(vals[j],Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache,idxs,deriv)
      else
        vals[j] = ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache,idxs,deriv)
      end
    end
  end

  vals
end

"""
ode_interpolation(tval::Number,ts,timeseries,ks)

Get the value at tval where the solution is known at the
times ts (sorted), with values timeseries and derivatives ks
"""
function ode_interpolation(tval::Number,id,idxs,deriv,p,continuity::Symbol=:left)
  @unpack ts,timeseries,ks,f,cache = id
  @inbounds tdir = sign(ts[end]-ts[1])

  if continuity === :left
    # we have i₋ = i₊ = 1 if tval = ts[1], i₊ = i₋ + 1 = lastindex(ts) if tval > ts[end],
    # and otherwise i₋ and i₊ satisfy ts[i₋] < tval ≤ ts[i₊]
    i₊ = min(lastindex(ts), _searchsortedfirst(ts,tval,2,tdir > 0))
    i₋ = i₊ > 1 ? i₊ - 1 : i₊
  else
    # we have i₋ = i₊ - 1 = 1 if tval < ts[1], i₊ = i₋ = lastindex(ts) if tval = ts[end],
    # and otherwise i₋ and i₊ satisfy ts[i₋] ≤ tval < ts[i₊]
    i₋ = max(1, _searchsortedlast(ts,tval,1,tdir > 0))
    i₊ = i₋ < lastindex(ts) ? i₋ + 1 : i₋
  end

  @inbounds begin
    dt = ts[i₊] - ts[i₋]
    Θ = iszero(dt) ? oneunit(tval) / oneunit(dt) : (tval-ts[i₋]) / dt

    if typeof(cache) <: (FunctionMapCache) || typeof(cache) <: FunctionMapConstantCache
      val = ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],0,cache,idxs,deriv)
    elseif !id.dense
      val = linear_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],idxs,deriv)
    elseif typeof(cache) <: CompositeCache
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache.caches[id.alg_choice[i₊]]) # update the kcurrent
      val = ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache.caches[id.alg_choice[i₊]],idxs,deriv)
    else
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache) # update the kcurrent
      val = ode_interpolant(Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache,idxs,deriv)
    end
  end

  val
end

"""
ode_interpolation!(out,tval::Number,ts,timeseries,ks)

Get the value at tval where the solution is known at the
times ts (sorted), with values timeseries and derivatives ks
"""
function ode_interpolation!(out,tval::Number,id,idxs,deriv,p,continuity::Symbol=:left)
  @unpack ts,timeseries,ks,f,cache = id
  @inbounds tdir = sign(ts[end]-ts[1])

  if continuity === :left
    # we have i₋ = i₊ = 1 if tval = ts[1], i₊ = i₋ + 1 = lastindex(ts) if tval > ts[end],
    # and otherwise i₋ and i₊ satisfy ts[i₋] < tval ≤ ts[i₊]
    i₊ = min(lastindex(ts), _searchsortedfirst(ts,tval,2,tdir > 0))
    i₋ = i₊ > 1 ? i₊ - 1 : i₊
  else
    # we have i₋ = i₊ - 1 = 1 if tval < ts[1], i₊ = i₋ = lastindex(ts) if tval = ts[end],
    # and otherwise i₋ and i₊ satisfy ts[i₋] ≤ tval < ts[i₊]
    i₋ = max(1, _searchsortedlast(ts,tval,1,tdir > 0))
    i₊ = i₋ < lastindex(ts) ? i₋ + 1 : i₋
  end

  @inbounds begin
    dt = ts[i₊] - ts[i₋]
    Θ = iszero(dt) ? oneunit(tval) / oneunit(dt) : (tval-ts[i₋]) / dt

    if typeof(cache) <: (FunctionMapCache) || typeof(cache) <: FunctionMapConstantCache
      ode_interpolant!(out,Θ,dt,timeseries[i₋],timeseries[i₊],0,cache,idxs,deriv)
    elseif !id.dense
      linear_interpolant!(out,Θ,dt,timeseries[i₋],timeseries[i₊],idxs,deriv)
    elseif typeof(cache) <: CompositeCache
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache.caches[id.alg_choice[i₊]]) # update the kcurrent
      ode_interpolant!(out,Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache.caches[id.alg_choice[i₊]],idxs,deriv)
    else
      DiffEqBase.addsteps!(ks[i₊],ts[i₋],timeseries[i₋],timeseries[i₊],dt,f,p,cache) # update the kcurrent
      ode_interpolant!(out,Θ,dt,timeseries[i₋],timeseries[i₊],ks[i₊],cache,idxs,deriv)
    end
  end

  out
end

"""
By default, Hermite interpolant so update the derivative at the two ends
"""
function DiffEqBase.addsteps!(k,t,uprev,u,dt,f,p,cache,always_calc_begin = false,allow_calc_end = true,force_calc_end = false)
  if length(k)<2 || always_calc_begin
    if typeof(cache) <: OrdinaryDiffEqMutableCache
      rtmp = similar(u, eltype(eltype(k)))
      f(rtmp,uprev,p,t)
      copyat_or_push!(k,1,rtmp)
      f(rtmp,u,p,t+dt)
      copyat_or_push!(k,2,rtmp)
    else
      copyat_or_push!(k,1,f(uprev,p,t))
      copyat_or_push!(k,2,f(u,p,t+dt))
    end
  end
  nothing
end

"""
ode_interpolant and ode_interpolant! dispatch
"""
function ode_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{TI}}) where TI
  _ode_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T)
end

function ode_interpolant(Θ,dt,y₀,y₁,k,cache::OrdinaryDiffEqMutableCache,idxs,T::Type{Val{TI}}) where TI
  if typeof(idxs) <: Number || typeof(y₀) <: Union{Number,SArray}
    # typeof(y₀) can be these if saveidxs gives a single value
    _ode_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T)
  elseif typeof(idxs) <: Nothing
    out = oneunit(Θ) .* y₁
    _ode_interpolant!(out,Θ,dt,y₀,y₁,k,cache,idxs,T)
  else
    out = oneunit(Θ) .* y₁[idxs]
    _ode_interpolant!(out,Θ,dt,y₀,y₁,k,cache,idxs,T)
  end
end

function ode_interpolant!(out,Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{TI}}) where TI
  _ode_interpolant!(out,Θ,dt,y₀,y₁,k,cache,idxs,T)
end

##################### Hermite Interpolants

# If no dispatch found, assume Hermite
function _ode_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{TI}}) where TI
  hermite_interpolant(Θ,dt,y₀,y₁,k,Val{typeof(cache) <: OrdinaryDiffEqMutableCache},idxs,T)
end

function _ode_interpolant!(out,Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{TI}}) where TI
  hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs,T)
end

"""
Hairer Norsett Wanner Solving Ordinary Differential Euations I - Nonstiff Problems Page 190

Herimte Interpolation, chosen if no other dispatch for ode_interpolant
"""
@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{false}},idxs::Nothing,T::Type{Val{0}}) # Default interpolant is Hermite
  #@.. (1-Θ)*y₀+Θ*y₁+Θ*(Θ-1)*((1-2Θ)*(y₁-y₀)+(Θ-1)*dt*k[1] + Θ*dt*k[2])
  @inbounds (1-Θ)*y₀+Θ*y₁+Θ*(Θ-1)*((1-2Θ)*(y₁-y₀)+(Θ-1)*dt*k[1] + Θ*dt*k[2])
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{true}},idxs::Nothing,T::Type{Val{0}}) # Default interpolant is Hermite
  #@.. (1-Θ)*y₀+Θ*y₁+Θ*(Θ-1)*((1-2Θ)*(y₁-y₀)+(Θ-1)*dt*k[1] + Θ*dt*k[2])
  @inbounds @.. (1-Θ)*y₀+Θ*y₁+Θ*(Θ-1)*((1-2Θ)*(y₁-y₀)+(Θ-1)*dt*k[1] + Θ*dt*k[2])
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{0}}) # Default interpolant is Hermite
  # return @.. (1-Θ)*y₀[idxs]+Θ*y₁[idxs]+Θ*(Θ-1)*((1-2Θ)*(y₁[idxs]-y₀[idxs])+(Θ-1)*dt*k[1][idxs] + Θ*dt*k[2][idxs])
  return (1-Θ)*y₀[idxs]+Θ*y₁[idxs]+Θ*(Θ-1)*((1-2Θ)*(y₁[idxs]-y₀[idxs])+(Θ-1)*dt*k[1][idxs] + Θ*dt*k[2][idxs])
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs::Nothing,T::Type{Val{0}}) # Default interpolant is Hermite
  @inbounds @.. out = (1-Θ)*y₀+Θ*y₁+Θ*(Θ-1)*((1-2Θ)*(y₁-y₀)+(Θ-1)*dt*k[1] + Θ*dt*k[2])
  #@inbounds for i in eachindex(out)
  #  out[i] = (1-Θ)*y₀[i]+Θ*y₁[i]+Θ*(Θ-1)*((1-2Θ)*(y₁[i]-y₀[i])+(Θ-1)*dt*k[1][i] + Θ*dt*k[2][i])
  #end
  #out
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs,T::Type{Val{0}}) # Default interpolant is Hermite
  @views @.. out = (1-Θ)*y₀[idxs]+Θ*y₁[idxs]+Θ*(Θ-1)*((1-2Θ)*(y₁[idxs]-y₀[idxs])+(Θ-1)*dt*k[1][idxs] + Θ*dt*k[2][idxs])
  #@inbounds for (j,i) in enumerate(idxs)
  #  out[j] = (1-Θ)*y₀[i]+Θ*y₁[i]+Θ*(Θ-1)*((1-2Θ)*(y₁[i]-y₀[i])+(Θ-1)*dt*k[1][i] + Θ*dt*k[2][i])
  #end
  #out
end

"""
Herimte Interpolation, chosen if no other dispatch for ode_interpolant
"""
@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{false}},idxs::Nothing,T::Type{Val{1}}) # Default interpolant is Hermite
  #@.. k[1] + Θ*(-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(3*dt*k[1] + 3*dt*k[2] + 6*y₀ - 6*y₁) + 6*y₁)/dt
  @inbounds k[1] + Θ*(-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(3*dt*k[1] + 3*dt*k[2] + 6*y₀ - 6*y₁) + 6*y₁)/dt
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{true}},idxs::Nothing,T::Type{Val{1}}) # Default interpolant is Hermite
  @inbounds @.. k[1] + Θ*(-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(3*dt*k[1] + 3*dt*k[2] + 6*y₀ - 6*y₁) + 6*y₁)/dt
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{1}}) # Default interpolant is Hermite
  # return @.. k[1][idxs] + Θ*(-4*dt*k[1][idxs] - 2*dt*k[2][idxs] - 6*y₀[idxs] + Θ*(3*dt*k[1][idxs] + 3*dt*k[2][idxs] + 6*y₀[idxs] - 6*y₁[idxs]) + 6*y₁[idxs])/dt
  return k[1][idxs] + Θ*(-4*dt*k[1][idxs] - 2*dt*k[2][idxs] - 6*y₀[idxs] + Θ*(3*dt*k[1][idxs] + 3*dt*k[2][idxs] + 6*y₀[idxs] - 6*y₁[idxs]) + 6*y₁[idxs])/dt
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs::Nothing,T::Type{Val{1}}) # Default interpolant is Hermite
  @inbounds @.. out = k[1] + Θ*(-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(3*dt*k[1] + 3*dt*k[2] + 6*y₀ - 6*y₁) + 6*y₁)/dt
  #@inbounds for i in eachindex(out)
  #  out[i] = k[1][i] + Θ*(-4*dt*k[1][i] - 2*dt*k[2][i] - 6*y₀[i] + Θ*(3*dt*k[1][i] + 3*dt*k[2][i] + 6*y₀[i] - 6*y₁[i]) + 6*y₁[i])/dt
  #end
  #out
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs,T::Type{Val{1}}) # Default interpolant is Hermite
  @views @.. out = k[1][idxs] + Θ*(-4*dt*k[1][idxs] - 2*dt*k[2][idxs] - 6*y₀[idxs] + Θ*(3*dt*k[1][idxs] + 3*dt*k[2][idxs] + 6*y₀[idxs] - 6*y₁[idxs]) + 6*y₁[idxs])/dt
  #@inbounds for (j,i) in enumerate(idxs)
  #  out[j] = k[1][i] + Θ*(-4*dt*k[1][i] - 2*dt*k[2][i] - 6*y₀[i] + Θ*(3*dt*k[1][i] + 3*dt*k[2][i] + 6*y₀[i] - 6*y₁[i]) + 6*y₁[i])/dt
  #end
  #out
end

"""
Herimte Interpolation, chosen if no other dispatch for ode_interpolant
"""
@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{false}},idxs::Nothing,T::Type{Val{2}}) # Default interpolant is Hermite
  #@.. (-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁) + 6*y₁)/(dt*dt)
  @inbounds (-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁) + 6*y₁)/(dt*dt)
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{true}},idxs::Nothing,T::Type{Val{2}}) # Default interpolant is Hermite
  #@.. (-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁) + 6*y₁)/(dt*dt)
  @inbounds @.. (-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁) + 6*y₁)/(dt*dt)
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{2}}) # Default interpolant is Hermite
  #out = similar(y₀,axes(idxs))
  #@views @.. out = (-4*dt*k[1][idxs] - 2*dt*k[2][idxs] - 6*y₀[idxs] + Θ*(6*dt*k[1][idxs] + 6*dt*k[2][idxs] + 12*y₀[idxs] - 12*y₁[idxs]) + 6*y₁[idxs])/(dt*dt)
  @views out = (-4*dt*k[1][idxs] - 2*dt*k[2][idxs] - 6*y₀[idxs] + Θ*(6*dt*k[1][idxs] + 6*dt*k[2][idxs] + 12*y₀[idxs] - 12*y₁[idxs]) + 6*y₁[idxs])/(dt*dt)
  out
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs::Nothing,T::Type{Val{2}}) # Default interpolant is Hermite
  @inbounds @.. out = (-4*dt*k[1] - 2*dt*k[2] - 6*y₀ + Θ*(6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁) + 6*y₁)/(dt*dt)
  #@inbounds for i in eachindex(out)
  #  out[i] = (-4*dt*k[1][i] - 2*dt*k[2][i] - 6*y₀[i] + Θ*(6*dt*k[1][i] + 6*dt*k[2][i] + 12*y₀[i] - 12*y₁[i]) + 6*y₁[i])/(dt*dt)
  #end
  #out
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs,T::Type{Val{2}}) # Default interpolant is Hermite
  @views @.. out = (-4*dt*k[1][idxs] - 2*dt*k[2][idxs] - 6*y₀[idxs] + Θ*(6*dt*k[1][idxs] + 6*dt*k[2][idxs] + 12*y₀[idxs] - 12*y₁[idxs]) + 6*y₁[idxs])/(dt*dt)
  #@inbounds for (j,i) in enumerate(idxs)
  #  out[j] = (-4*dt*k[1][i] - 2*dt*k[2][i] - 6*y₀[i] + Θ*(6*dt*k[1][i] + 6*dt*k[2][i] + 12*y₀[i] - 12*y₁[i]) + 6*y₁[i])/(dt*dt)
  #end
  #out
end

"""
Herimte Interpolation, chosen if no other dispatch for ode_interpolant
"""
@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{false}},idxs::Nothing,T::Type{Val{3}}) # Default interpolant is Hermite
  #@.. (6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁)/(dt*dt*dt)
  @inbounds (6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁)/(dt*dt*dt)
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,::Type{Val{true}},idxs::Nothing,T::Type{Val{3}}) # Default interpolant is Hermite
  #@.. (6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁)/(dt*dt*dt)
  @inbounds @.. (6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁)/(dt*dt*dt)
end

@muladd function hermite_interpolant(Θ,dt,y₀,y₁,k,cache,idxs,T::Type{Val{3}}) # Default interpolant is Hermite
  #out = similar(y₀,axes(idxs))
  #@views @.. out = (6*dt*k[1][idxs] + 6*dt*k[2][idxs] + 12*y₀[idxs] - 12*y₁[idxs])/(dt*dt*dt)
  @views out = (6*dt*k[1][idxs] + 6*dt*k[2][idxs] + 12*y₀[idxs] - 12*y₁[idxs])/(dt*dt*dt)
  out
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs::Nothing,T::Type{Val{3}}) # Default interpolant is Hermite
  @inbounds @.. out = (6*dt*k[1] + 6*dt*k[2] + 12*y₀ - 12*y₁)/(dt*dt*dt)
  #for i in eachindex(out)
  #  out[i] = (6*dt*k[1][i] + 6*dt*k[2][i] + 12*y₀[i] - 12*y₁[i])/(dt*dt*dt)
  #end
  #out
end

@muladd function hermite_interpolant!(out,Θ,dt,y₀,y₁,k,idxs,T::Type{Val{3}}) # Default interpolant is Hermite
  @views @.. out = (6*dt*k[1][idxs] + 6*dt*k[2][idxs] + 12*y₀[idxs] - 12*y₁[idxs])/(dt*dt*dt)
  #for (j,i) in enumerate(idxs)
  #  out[j] = (6*dt*k[1][i] + 6*dt*k[2][i] + 12*y₀[i] - 12*y₁[i])/(dt*dt*dt)
  #end
  #out
end

######################## Linear Interpolants

@muladd @inline function linear_interpolant(Θ,dt,y₀,y₁,idxs::Nothing,T::Type{Val{0}})
  Θm1 = (1-Θ)
  @.. Θm1*y₀ + Θ*y₁
end

@muladd @inline function linear_interpolant(Θ,dt,y₀,y₁,idxs,T::Type{Val{0}})
  Θm1 = (1-Θ)
  @.. Θm1*y₀[idxs] + Θ*y₁[idxs]
end

@muladd @inline function linear_interpolant!(out,Θ,dt,y₀,y₁,idxs::Nothing,T::Type{Val{0}})
  Θm1 = (1-Θ)
  @.. out = Θm1*y₀ + Θ*y₁
  out
end

@muladd @inline function linear_interpolant!(out,Θ,dt,y₀,y₁,idxs,T::Type{Val{0}})
  Θm1 = (1-Θ)
  @views @.. out = Θm1*y₀[idxs] + Θ*y₁[idxs]
  out
end

"""
Linear Interpolation
"""
@inline function linear_interpolant(Θ,dt,y₀,y₁,idxs::Nothing,T::Type{Val{1}})
  (y₁ - y₀)/dt
end

@inline function linear_interpolant(Θ,dt,y₀,y₁,idxs,T::Type{Val{1}})
  @.. (y₁[idxs] - y₀[idxs])/dt
end

@inline function linear_interpolant!(out,Θ,dt,y₀,y₁,idxs::Nothing,T::Type{Val{1}})
  @.. out = (y₁ - y₀)/dt
  out
end

@inline function linear_interpolant!(out,Θ,dt,y₀,y₁,idxs,T::Type{Val{1}})
  @views @.. out = (y₁[idxs] - y₀[idxs])/dt
  out
end


struct IndexedCounts{M<:AbstractMer,C<:CountMode}
    mer_index::Vector{M}            # Ordered list of kmer that contain counts
    count_names::Vector{Symbol}     # Names of the count vectors
    counts::Vector{Vector{UInt16}}  # Count vectors, each contains an entry per mer in the index
    name::Symbol                    # The name of this MerCounts
end

# Construction of empty MerCounts with a name
@inline function IndexedCounts{M,C}(name::Symbol) where {M<:AbstractMer,C<:CountMode}
    return IndexedCounts{M,C}(Vector{M}(), Vector{Symbol}(), Vector{Vector{UInt16}}(), name)
end

@inline function IndexedCounts(::Type{M}, name::Symbol, mode::C = CANONICAL) where {M<:AbstractMer,C<:CountMode}
    return IndexedCounts{M,C}(name)
end

# Construction from a sequence distance graph
function IndexedCounts(name::Symbol, sequences, mode::C = CANONICAL) where {M<:AbstractMer,C<:CountMode}
    @info string("Creating an empty kmer count datastore called ", name)
    mc = IndexedCounts{M,C}(name, mode)
    index!(mc, sequences)
    return mc
end

function _count_and_collapse!(mers::Vector{M}, counts::Vector{UInt16}) where {M<:AbstractMer}
    wi = 1
    ri = 1
    stop = lastindex(mers) + 1
    @inbounds while ri < stop
        mers[wi] = mers[ri]
        ci = one(UInt16)
        while (ri += 1) < stop && mers[ri] == mers[wi]
            ci = ci + one(UInt16)
        end
        push!(counts, ci)
        wi = wi + 1
    end
    resize!(mers, wi - 1)
    return nothing
end

function index!(mc::IndexedCounts{M,C}, ref_sequences) where {M<:AbstractMer,C<:CountMode}
    @assert eltype(ref_sequences) <: LongSequence
    @info "Indexing kmer counts of reference sequences"
    @info "Populating index with reference sequence mers"
    # Add all Kmers from reference sequences.
    # t = expected number of kmers.
    t = sum(length(x) for x in ref_sequences) - BioSequences.ksize(M) + 1
    mcindex = mc.mer_index
    resize!(mcindex, t)
    vec_i = 1
    f = C()
    for seq in ref_sequences
        for mer in each(M, seq)
            mcindex[vec_i] = f(mer)
            vec_i = vec_i + 1
        end
    end
    resize!(mcindex, vec_i)
    @info "Sorting the index"
    # Sort the index
    sort!(mcindex)
    @info "Counting and collapsing mers in index"
    # Create a first count;
    # Collapse the kmer index, saving the coverage to the first count.
    first_count = Vector{UInt16}()
    sizehint!(first_count, length(mcindex))
    push!(mc.counts, first_count)
    push!(mc.count_names, :sdg)
    _count_and_collapse!(mcindex, first_count)
    return mc
end

function Base.summary(io::IO, mc::IndexedCounts{M}) where {M}
    print(io, BioSequences.ksize(M), "-mer Indexed Counts Datastore '",  mc.name, "': ", length(mc.counts), " stored ", BioSequences.ksize(M), "-mer counts")
end

function Base.show(io::IO, mc::IndexedCounts{M}) where {M}
    summary(io, mc)
end

"""
    add_count!(mc::IndexedCounts{M}, name::Symbol, reads::ReadDatastore) where {M}

Count the k-mers in a ReadDatastore and add them to the IndexedCounts{M}.

Currently IndexedCounts use a serial and in-memory counting method, as

!!! note
    Only the k-mers in the IndexedCounts' index shall be counted from the reads.
"""
function add_count!(mc::IndexedCounts{M,C}, name::Symbol, reads::ReadDatastore) where {M,C}
    mode = C()
    # TODO: Check if name already exists.
    push!(mc.count_names, name)
    c = zeros(UInt16, length(mc.mer_index))
    push!(mc.counts, c)
    # Populate lookup map.
    kmer_map = Dict{M, Int}()
    sizehint!(kmer_map, length(mc.mer_index))
    for (i, j) in enumerate(mc.mer_index)
        kmer_map[j] = i
    end
    for read in reads
        for mer in each(M, read)
            findk = get(kmer_map, mode(mer), 0)
            @inbounds if findk > 0
                old = c[findk]
                inc = old + one(UInt16)
                c[findk] = ifelse(old > inc, typemax(UInt16), inc)
            end
        end
    end
end

function project_count(mc::IndexedCounts{M,C}, name::Symbol, sequence::BioSequence) where {M,C}
    countidx = findfirst(isequal(name), mc.count_names)
    if isnothing(countidx)
        throw(ArgumentError(string("No count dataset called", string(name), "exists")))
    end
    counts = mc.counts[countidx]
    querymers = collect_mers(M, C(), sequence)
    projection = zeros(UInt16, length(querymers))
    indexsize = length(mc.mer_index)
    index = mc.mer_index
    for (i, mer) in enumerate(querymers)
        kidx = min(searchsortedfirst(index, mer), indexsize)
        if index[kidx] === mer
            projection[i] = counts[kidx]
        end
    end
    return projection
end


#some useful variables
msb1    = UInt64(0x8000_0000_0000_0000)
lsb1    = UInt64(0x0000_0000_0000_0001)
msb8    = UInt64(0xFF00_0000_0000_0000)
msb6    = UInt64(0xFC00_0000_0000_0000)
lsb6    = UInt64(0x0000_0000_0000_003F)
allbits = UInt64(0xFFFF_FFFF_FFFF_FFFF)
nobits  = UInt64(0x0000_0000_0000_0000)

@test_throws ArgumentError Unums.__check_ArrayNum(0, [z64])
@test_throws ArgumentError Unums.__check_ArrayNum(6, [z64])
@test_throws ArgumentError Unums.__check_ArrayNum(7, [z64])
@test_throws ArgumentError Unums.__check_ArrayNum(8, [z64, z64])

#test developer safety being disabled on ArrayNum type

@devmode_on begin
  #ArrayNums of size < 7 are disallowed.
  @test_throws ArgumentError Unums.ArrayNum{0}([z64])
  @test_throws ArgumentError Unums.ArrayNum{1}([z64])
  @test_throws ArgumentError Unums.ArrayNum{2}([z64])
  @test_throws ArgumentError Unums.ArrayNum{3}([z64])
  @test_throws ArgumentError Unums.ArrayNum{4}([z64])
  @test_throws ArgumentError Unums.ArrayNum{5}([z64])
  @test_throws ArgumentError Unums.ArrayNum{6}([z64])

  #ArrayNums must have the minimum number of elements.
  @test_throws ArgumentError Unums.ArrayNum{7}([z64])
  @test_throws ArgumentError Unums.ArrayNum{8}([z64, z64])
end

@devmode_off begin

  #these are OK now.
  Unums.ArrayNum{0}([z64])
  Unums.ArrayNum{6}([z64])
  Unums.ArrayNum{7}([z64])
end

#but it's ok if we pass an ArrayNum an array with more elements than necessary.
bigger_array = Unums.ArrayNum{7}([z64, z64, z64, z64])
@test bigger_array.a == [z64, z64, z64, z64]
################################################################################
## SUPERINT CONSTANT GENERATION

@test zero(Unums.ArrayNum{7}).a == [nobits, nobits]
@test one(Unums.ArrayNum{7}).a  == [nobits, lsb1]
@test top(Unums.ArrayNum{7}).a  == [msb1, nobits]
@test zero(Unums.ArrayNum{8}).a == [nobits, nobits, nobits, nobits]
@test one(Unums.ArrayNum{8}).a  == [nobits, nobits, nobits, lsb1]
@test top(Unums.ArrayNum{8}).a  == [msb1, nobits, nobits, nobits]

#bit_from_top
#=
@test Unums.__bit_from_top(0,1)    == msb1
@test Unums.__bit_from_top(1,1)    == 0x4000_0000_0000_0000
@test Unums.__bit_from_top(63,1)   == lsb1
@test Unums.__bit_from_top(0, 2)   == [msb1, nobits]
@test Unums.__bit_from_top(63, 2)  == [lsb1, nobits]
@test Unums.__bit_from_top(64, 2)  == [nobits, msb1]
@test Unums.__bit_from_top(127, 2) == [nobits, lsb1]
=#

################################################################################
## BIT PATTERN DETECTION

#is_all_zero
@test  Unums.is_all_zero(nobits)
@test  Unums.is_all_zero(Unums.ArrayNum{7}([nobits, nobits]))
@test  Unums.is_all_zero(Unums.ArrayNum{8}([nobits, nobits, nobits, nobits]))
@test !Unums.is_all_zero(lsb1)
@test !Unums.is_all_zero(Unums.ArrayNum{7}([lsb1, nobits]))
@test !Unums.is_all_zero(Unums.ArrayNum{7}([nobits, msb1]))
@test !Unums.is_all_zero(Unums.ArrayNum{7}([lsb1, msb1]))
@test !Unums.is_all_zero(Unums.ArrayNum{8}([nobits, nobits, lsb1, nobits]))

#is_not_zero
@test !Unums.is_not_zero(nobits)
@test !Unums.is_not_zero(Unums.ArrayNum{7}([nobits, nobits]))
@test !Unums.is_not_zero(Unums.ArrayNum{8}([nobits, nobits, nobits, nobits]))
@test  Unums.is_not_zero(lsb1)
@test  Unums.is_not_zero(Unums.ArrayNum{7}([lsb1, nobits]))
@test  Unums.is_not_zero(Unums.ArrayNum{7}([nobits, msb1]))
@test  Unums.is_not_zero(Unums.ArrayNum{7}([lsb1, msb1]))
@test  Unums.is_not_zero(Unums.ArrayNum{8}([nobits, nobits, lsb1, nobits]))

#is_top
@test Unums.is_top(msb1)
@test Unums.is_top(Unums.ArrayNum{7}([msb1, nobits]))
@test Unums.is_top(Unums.ArrayNum{8}([msb1, nobits, nobits, nobits]))
@test !Unums.is_top(nobits)
@test !Unums.is_top(msb1 & lsb1)
@test !Unums.is_top(Unums.ArrayNum{7}([msb1, lsb1]))
@test !Unums.is_top(Unums.ArrayNum{7}([nobits, msb1]))
@test !Unums.is_top(Unums.ArrayNum{8}([msb1, nobits, nobits, msb1]))

#is_not_top
@test !Unums.is_not_top(msb1)
@test !Unums.is_not_top(Unums.ArrayNum{7}([msb1, nobits]))
@test !Unums.is_not_top(Unums.ArrayNum{8}([msb1, nobits, nobits, nobits]))
@test Unums.is_not_top(nobits)
@test Unums.is_not_top(msb1 & lsb1)
@test Unums.is_not_top(Unums.ArrayNum{7}([msb1, lsb1]))
@test Unums.is_not_top(Unums.ArrayNum{7}([nobits, msb1]))
@test Unums.is_not_top(Unums.ArrayNum{8}([msb1, nobits, nobits, msb1]))

#inward_ubit_crosses_zero
@test !Unums.inward_ubit_crosses_zero(t64, z16)                  #doesn't happen.
@test  Unums.inward_ubit_crosses_zero(0x4000_0000_0000_0000, z16)
@test !Unums.inward_ubit_crosses_zero(0x4000_0000_0000_0000, o16)
@test  Unums.inward_ubit_crosses_zero(0x0000_0000_0000_0001, o16)
@test !Unums.inward_ubit_crosses_zero(0x0000_0000_0000_0001, 0x003F)
@test  Unums.inward_ubit_crosses_zero(0x0000_0000_0000_0000, z16)
@test  Unums.inward_ubit_crosses_zero(0x0000_0000_0000_0000, o16)
@test  Unums.inward_ubit_crosses_zero(0x0000_0000_0000_0000, 0x003F)
@test  Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([o64,z64]), 0x0000)
@test !Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([o64,z64]), 0x003F)
@test  Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,t64]), 0x003F)
@test !Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,t64]), 0x0040)
@test  Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,o64]), 0x003F)
@test  Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,o64]), 0x0040)
@test !Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,o64]), 0x007F)
@test  Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,z64]), 0x003F)
@test  Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,z64]), 0x0040)
@test  Unums.inward_ubit_crosses_zero(Unums.ArrayNum{7}([z64,z64]), 0x007F)

################################################################################
## Int64 ubit contraction tests.

@test Unums.contract_outer_fsize(0xFFFF_0000_0000_0000, 0x000F) == 0x0000
@test Unums.contract_outer_fsize(0xFFFF_0000_0000_0000, 0x0010) == 0x0000
@test Unums.contract_outer_fsize(0xFFFF_0000_0000_0000, 0x0011) == 0x0011

################################################################################
## CLZ AND CTZ

#leading_zeros
@test clz(allbits) == 0
@test clz(nobits) == 64
@test clz(msb8) == 0
@test clz(lsb6) == 58
@test clz(UInt64(0b0001111000)) == 57
@test clz(Unums.ArrayNum{7}([nobits, 0x00FF_0000_0000_0000])) == 72
@test clz(Unums.ArrayNum{7}([0x0000_0000_0000_F00F, nobits])) == 48
@test clz(Unums.ArrayNum{7}([nobits, nobits])) == 128
#test trailing_zeros
@test ctz(allbits) == 0
@test ctz(nobits) == 64
@test ctz(msb8) == 56
@test ctz(lsb6) == 0
@test ctz(UInt64(0b0001111000)) == 3
@test ctz(Unums.ArrayNum{7}([nobits, 0x00FF_0000_0000_0000])) == 48
@test ctz(Unums.ArrayNum{7}([0x0000_0000_0000_F00F, nobits])) == 64
@test ctz(Unums.ArrayNum{7}([nobits, nobits])) == 128

################################################################################
## MASKS

#test mask generation
@test Unums.mask_top(UInt16(0)) == msb1
@test Unums.mask_bot(UInt16(62)) == lsb1
@test Unums.mask_top(UInt16(7)) == msb8
@test Unums.mask_bot(UInt16(57)) == lsb6
@test Unums.mask_top(UInt16(63)) == allbits

#test two cells.
twocellint = zero(Unums.ArrayNum{7})

Unums.mask_top!(twocellint, UInt16(0))
@test twocellint.a == [msb1, nobits]
Unums.mask_bot!(twocellint, UInt16(126))
@test twocellint.a == [nobits, lsb1]
Unums.mask_top!(twocellint, UInt16(63))
@test twocellint.a == [allbits, nobits]
Unums.mask_bot!(twocellint, UInt16(63))
@test twocellint.a == [nobits, allbits]
Unums.mask_top!(twocellint, UInt16(64))
@test twocellint.a == [allbits, msb1]
Unums.mask_bot!(twocellint, UInt16(62))
@test twocellint.a == [lsb1, allbits]
Unums.mask_bot!(twocellint, UInt16(127))
@test twocellint.a == [nobits, nobits]
Unums.mask_top!(twocellint, UInt16(127))
@test twocellint.a == [allbits, allbits]

#test four cells
fourcellint = zero(Unums.ArrayNum{8})
Unums.mask_top!(fourcellint, UInt16(64))
@test fourcellint.a == [allbits, msb1, nobits, nobits]
Unums.mask_bot!(fourcellint, UInt16(190))
@test fourcellint.a == [nobits, nobits, lsb1, allbits]
Unums.mask_top!(fourcellint, UInt16(255))
@test fourcellint.a == [allbits, allbits, allbits, allbits]
Unums.mask_bot!(fourcellint, UInt16(255))
@test fourcellint.a == [nobits, nobits, nobits, nobits]

#mask filling.
test_texture = [0x1234_5678_9ABC_DEF1, 0x2345_6789_ABCD_EF12]
test_mask    = [0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_0000_0000_0000]
test_masktex = [0x1234_5678_9ABC_DEF1, 0x2345_0000_0000_0000]
#test the and-ing process
txt = Unums.ArrayNum{7}(copy(test_texture))
msk = Unums.ArrayNum{7}(copy(test_mask))
Unums.fill_mask!(msk, txt)
@test msk.a == test_masktex

## bottom bits
@test Unums.bottom_bit(UInt16(0))  == 0x8000_0000_0000_0000
@test Unums.bottom_bit(UInt16(1))  == 0x4000_0000_0000_0000
@test Unums.bottom_bit(UInt16(2))  == 0x2000_0000_0000_0000
@test Unums.bottom_bit(UInt16(3))  == 0x1000_0000_0000_0000
@test Unums.bottom_bit(UInt16(13)) == 0x0004_0000_0000_0000
@test Unums.bottom_bit(UInt16(14)) == 0x0002_0000_0000_0000
@test Unums.bottom_bit(UInt16(15)) == 0x0001_0000_0000_0000
@test Unums.bottom_bit(UInt16(63)) == 0x0000_0000_0000_0001

twocellint = zero(Unums.ArrayNum{7})
@test Unums.bottom_bit!(twocellint, UInt16(63)).a == [o64, z64]
@test Unums.bottom_bit!(twocellint, UInt16(64)).a == [z64, t64]
@test Unums.bottom_bit!(twocellint).a == [z64, o64]
fourcellint = zero(Unums.ArrayNum{8})
@test Unums.bottom_bit!(fourcellint).a == [z64, z64, z64, o64]

################################################################################
## SHIFTS

#test leftshifts and rightshifts on multi-arrays
@test Unums.lsh(allbits, 4) == 0xFFFF_FFFF_FFFF_FFF0
@test Unums.rsh(allbits, 4) == 0x0FFF_FFFF_FFFF_FFFF
twocellint = Unums.ArrayNum{7}([allbits, allbits])
Unums.lsh!(twocellint, 4)
@test twocellint.a == [allbits, 0xFFFF_FFFF_FFFF_FFF0]

twocellint.a = [allbits, allbits]
Unums.rsh!(twocellint, 4)
@test twocellint.a == [0x0FFF_FFFF_FFFF_FFFF, allbits]

#test really long distance leftshifts and rightshifts
const allfour = [allbits, allbits, allbits, allbits]
const texture = [0x1234_5678_9ABC_DEF1, 0x2345_6789_ABCD_EF12, 0x3456_789A_BCDE_F123, 0x4567_89AB_CDEF_1234]

fourcellint = Unums.ArrayNum{8}(copy(allfour))
Unums.lsh!(fourcellint, 68)
@test fourcellint.a == [allbits, allbits, 0xFFFF_FFFF_FFFF_FFF0, nobits]

fourcellint.a = copy(allfour)
Unums.rsh!(fourcellint, 68)
@test fourcellint.a == [nobits, 0x0FFF_FFFF_FFFF_FFFF, allbits, allbits]

#textured shifts to make sure things are actually ok.
fourcellint.a = copy(texture)
Unums.lsh!(fourcellint, 4)
@test fourcellint.a ==  [0x2345_6789_ABCD_EF12, 0x3456_789A_BCDE_F123, 0x4567_89AB_CDEF_1234, 0x5678_9ABC_DEF1_2340]

fourcellint.a = copy(texture)
Unums.lsh!(fourcellint, 68)
@test fourcellint.a == [0x3456_789A_BCDE_F123, 0x4567_89AB_CDEF_1234, 0x5678_9ABC_DEF1_2340, nobits]

fourcellint.a = copy(texture)
Unums.rsh!(fourcellint, 4)
@test fourcellint.a == [0x0123_4567_89AB_CDEF, 0x1234_5678_9ABC_DEF1, 0x2345_6789_ABCD_EF12, 0x3456_789A_BCDE_F123]

fourcellint.a = copy(texture)
Unums.rsh!(fourcellint, 68)
@test fourcellint.a == [nobits, 0x0123_4567_89AB_CDEF, 0x1234_5678_9ABC_DEF1, 0x2345_6789_ABCD_EF12]

################################################################################
## UTILITIES

#test __minimum_data_width
@test Unums.__minimum_data_width(allbits) == 63
@test Unums.__minimum_data_width(nobits) == 0
@test Unums.__minimum_data_width(msb1) == 0
@test Unums.__minimum_data_width(lsb1) == 63
@test Unums.__minimum_data_width(msb8) == 7
@test Unums.__minimum_data_width(Unums.ArrayNum{7}([nobits, nobits])) == 0
@test Unums.__minimum_data_width(Unums.ArrayNum{7}([msb1, nobits])) == 0
@test Unums.__minimum_data_width(Unums.ArrayNum{7}([nobits, msb1])) == 64
@test Unums.__minimum_data_width(Unums.ArrayNum{7}([allbits, allbits])) == 127
@test Unums.__minimum_data_width(Unums.ArrayNum{7}([nobits, lsb1])) == 127
@test Unums.__minimum_data_width(Unums.ArrayNum{8}([nobits, nobits, nobits, lsb1])) == 255

#=
#test __allones_for_length
@test Unums.__allones_for_length(0x8000_0000_0000_0000,                  UInt16(0))
@test !Unums.__allones_for_length(z64,                                   UInt16(0))
@test Unums.__allones_for_length(0xF000_0000_0000_0000,                  UInt16(3))
@test !Unums.__allones_for_length(0xD000_0000_0000_0000,                 UInt16(3))
@test Unums.__allones_for_length(0xFFFF_0000_0000_0000,                  UInt16(15))
@test Unums.__allones_for_length(f64,                                    UInt16(63))
@test Unums.__allones_for_length(ArrayNum{7}([0x8000_0000_0000_0000, z64]),           UInt16(0))
@test Unums.__allones_for_length(ArrayNum{7}([f64, z64]),                             UInt16(63))
@test Unums.__allones_for_length(ArrayNum{7}([f64, t64]),                             UInt16(64))
@test Unums.__allones_for_length(ArrayNum{7}([f64, 0xC000_0000_0000_0000]),           UInt16(65))
@test Unums.__allones_for_length(ArrayNum{8}([f64, f64, 0xC000_0000_0000_0000, z64]), UInt16(129))
=#