escapename(n::String) =
    replace(n,r"(^#)" => "_#")

fmtcof(c::Float64) =
    if     c < 0.0
        if     c < -1.0 || c > -1.0
             "- $(-c)"
        else
            "-"
        end
    elseif c > 0.0
        if c < 1.0 || c > 1.0
            "+ $c"
        else
            "+"
        end
    else
        "0"
    end

#Base.showall(f::IO, t::Task) = showlimited(f,t,0)
Base.show(f::IO, t::Task) = showlimited(f,t,20)

Base.show(io::IOContext, t::Task) =
    if get(io, :short, false)
        showlimited(io,t,20)
    else
        showlimited(io,t,0)
    end

function showlimited(f::IO, t::Mosek.Task, limit :: Int)
    numvar    = getnumvar(t)
    numbarvar = getnumbarvar(t)
    numcon    = getnumcon(t)
    numcone   = getnumcone(t)
    numanz    = getnumanz(t)
    numqobjnz = getnumqobjnz(t)
    numqconnz = if numcon > 0 sum(i -> getnumqconknz(t,i),1:numcon) else 0 end

    limitnumcon = if limit > 0 min(limit,numcon) else numcon end
    limitnumvar = if limit > 0 min(limit,numvar) else numvar end
    limitnumcone = if limit > 0 min(limit,numcone) else numcone end
    limitnumbarvar = if limit > 0 min(limit,numbarvar) else numbarvar end

    termslimit = numvar+numbarvar+numqobjnz+numqconnz
    if limit > 0 termslimit = min(limit,termslimit) end

    if numvar == 0 && numcon == 0 && numbarvar == 0
        println(f,"Empty Task")
    else
        varnames = String[ escapename(getvarname(t,i)) for i in 1:numvar]
        barvarnames = String[ escapename(getbarvarname(t,i)) for i in 1:numbarvar]
        connames = String[ escapename(getconname(t,i)) for i in 1:numcon]
        conenames = String[ escapename(getconename(t,i)) for i in 1:numcone]

        for i in 1:numvar
            if length(varnames[i]) == 0
                varnames[i] = "#x$i"
            end
        end

        for i in 1:numcon
            if length(connames[i]) == 0
                connames[i] = "#c$i"
            end
        end

        for i in 1:numcone
            if length(conenames[i]) == 0
                conenames[i] = "#k$i"
            end
        end

        for i in 1:numbarvar
            if length(barvarnames[i]) == 0
                barvarnames[i] = "#X̄$i"
            end
        end

        maxvarnamelen = if numvar > 0 max(map(n -> length(n),varnames)...) else 0 end
        maxconnamelen = if numcon > 0 max(map(n -> length(n),connames)...) else 0 end

        taskname = gettaskname(t)

        println(f,"Task '$(escape_string(taskname))'")

        _,barcidx = getbarcsparsity(t)

        _,baraidx = getbarasparsity(t)
        baraptrb = zeros(Int,numcon+1)
        for idx in baraidx
            (barai,baraj) = getbaraidxij(t,idx)
            baraptrb[barai+1] += 1
        end
        for i in 1:numcon
            baraptrb[i+1] += baraptrb[i]
        end
        baraptrb .+= 1

        baraptre = copy(baraptrb)
        baraidx_idx = Vector{Int}(undef,length(baraidx))
        for idx in baraidx
            (barai,baraj) = getbaraidxij(t,idx)
            baraidx_idx[baraptre[barai]] = idx
            baraptre[barai] += 1
        end

        # Objective
        objsense = getobjsense(t)
        objname  = escapename(getobjname(t))
        if objname == ""
            objname = "#obj"
        end

        if objsense == MSK_OBJECTIVE_SENSE_MAXIMIZE
            println(f,"    Maximize")
        else
            println(f,"    Minimize")
        end
        c = getc(t)
        print(f,"        $objname: ")
        terms = 0
        omittedterms = 0
        for j in 1:numvar
            if c[j] < 0.0 || c[j] > 0.0
               if terms < termslimit
                   print(f,"$(fmtcof(c[j])) $(varnames[j]) ")
                   terms += 1
               else
                   omittedterms += 1
               end
            end
        end

        for idx in barcidx
            if terms < termslimit
                (barcj,num,sub,w) = getbarcidx(t,idx)
                if num == 1
                    print(f,"$(fmtcof(w[1])) #MX$(sub[1]) ⋅ $(barvarnames[barcj]) ")
                elseif num > 1
                    print(f,"(")
                    printf(f,"$(fmtcof(w[1])) #MX$(sub[1])")
                    print(f,") ⋅ $(barvarnames[barcj]) ")
                end
                terms += 1
            else
                omittedterms += 1
            end
        end

        numqobjnz = getnumqobjnz(t)
        if numqobjnz > 0
            (nnz,qcsubi,qcsubj,qcval) = getqobj(t)
            for k in 1:nnz
                if terms < limitterms
                    if qcsubi[k] == qcsubj[k]
                        print(f,"$(Mosek.fmtcof(qcval[k])) $(varnames[qcsubi[k]]) ^ 2")
                    else
                        ni = varnames[qcsubi[k]]
                        nj = varnames[qcsubj[k]]
                        print(f,"$(Mosek.fmtcof(qcval[k])) $ni ⋅ $nj")
                    end
                    terms += 1
                else
                    omittedterms += 1
                end
            end
        end

        if omittedterms > 0
            print(f," ... ($omittedterms terms omitted)")
        end

        cfix = getcfix(t)
        if cfix < 0 || cfix > 0
            print(f,fmtcof(cfix))
        end
        println(f)

        # Constraints
        if numcon+numcone > 0
            println(f,"    Subject to")

            for i in 1:limitnumcon
                (nzi,subi,vali) = getarow(t,Int32(i))
                (bk,bl,bu) = getconbound(t,Int32(i))

                print(f,"        $(connames[i]): ")
                if bk == MSK_BK_LO || bk == MSK_BK_RA
                    print(f,"$bl < ")
                end
                terms = 0
                omittedterms = 0
                for k in 1:nzi
                    if terms < termslimit
                        print(f,"$(fmtcof(vali[k])) $(varnames[subi[k]]) ")
                        terms += 1
                    else
                        omittedterms += 1
                    end
                end

                for k in baraptrb[i]:baraptrb[i+1]-1
                    if terms < termslimit
                        idx = baraidx_idx[k]
                        (barai,baraj,num,sub,w) = getbaraidx(t,idx)
                        if num == 1
                            print(f,"$(fmtcof(w[1])) #MX$(sub[1]) ⋅ $(barvarnames[baraj]) ")
                        elseif num > 1
                            print(f,"(")
                            printf(f,"$(fmtcof(w[1])) #MX$(sub[1])")
                            print(f,") ⋅ $(barvarnames[baraj]) ")
                        end
                        terms += 1
                    else
                        omittedterms += 1
                    end
                end

                numqconnz = getnumqconknz(t,i)
                if numqconnz > 0
                    (nnz,qcsubi,qcsubj,qcval) = getqconk(t,i)
                    for k in 1:nnz
                        if terms < limitterms
                            if qcsubi[k] == qcsubj[k]
                                print(f,"$(Mosek.fmtcof(qcval[k])) $(varnames[qcsubi[k]]) ^ 2")
                            else
                                ni = varnames[qcsubi[k]]
                                nj = varnames[qcsubj[k]]
                                print(f,"$(Mosek.fmtcof(qcval[k])) $ni ⋅ $nj")
                            end
                            terms += 1
                        else
                            omittedterms += 1
                        end
                    end
                end

                if omittedterms > 0
                    print(f," ... ($omittedterms terms omitted)")
                end

                if bk == MSK_BK_UP || bk == MSK_BK_RA
                    println(f,"< $bu")
                elseif bk == MSK_BK_FX
                    println(f,"= $bu")
                else
                    println(f,"")
                end
            end

            numacc = getnumacc(t)
            for i in 1:numacc
                domidx = getaccdomain(t,i)
                sz = getdomainn(t,i)
                name = getaccname(t,i)
                afeidxs = getaccafeidxlist(t,i)
                domtp = getdomaintype(t,domidx)

                domname = ( if     domtp == MSK_DOMAIN_R "Unbounded"
                            elseif domtp == MSK_DOMAIN_RZERO "Zero"
                            elseif domtp == MSK_DOMAIN_RPLUS "Nonnegative"
                            elseif domtp == MSK_DOMAIN_RMINUS "Nonpositive"
                            elseif domtp == MSK_DOMAIN_INF_NORM_CONE "K_inf"
                            elseif domtp == MSK_DOMAIN_ONE_NORM_CONE "K_1"
                            elseif domtp == MSK_DOMAIN_QUADRATIC_CONE  "𝒞_q"
                            elseif domtp == MSK_DOMAIN_RQUADRATIC_CONE  "𝒞_qr"
                            elseif domtp == MSK_DOMAIN_PRIMAL_GEO_MEAN_CONE "GeoMean"
                            elseif domtp == MSK_DOMAIN_PRIMAL_POWER_CONE "𝒞_pow"
                            elseif domtp == MSK_DOMAIN_DUAL_POWER_CONE "𝒞*_pow"
                            elseif domtp == MSK_DOMAIN_PRIMAL_EXP_CONE "𝒞_exp"
                            elseif domtp == MSK_DOMAIN_DUAL_EXP_CONE "𝒞*_exp"
                            elseif domtp == MSK_DOMAIN_PSD_CONE "PSD"
                            else   "<?>"
                            end )
                println(f,"        \"$name\" $domname")
                for j in 1:sz
                    g = getafeg(t,afeidxs[j])
                    (nzi,subj,cof) = getafefrow(t,afeidxs[j])
                    (barj,termptr,midx,w) = getafebarfrow(t,afeidxs[j])
                    print(f,"            ")

                    terms = 0
                    for k in 1:nzi
                        if terms < termslimit
                            print(f,"$(fmtcof(cof[k])) $(varnames[subj[k]]) ")
                            terms += 1
                        else
                            omittedterms += 1
                        end
                    end

                    # for idx in barj
                    #     if terms < termslimit
                    #         (barcj,num,sub,w) = getbarcidx(t,idx)
                    #         if num == 1
                    #             print(f,"$(fmtcof(w[1])) #MX$(sub[1]) ⋅ $(barvarnames[barcj]) ")
                    #         elseif num > 1
                    #             print(f,"(")
                    #             printf(f,"$(fmtcof(w[1])) #MX$(sub[1])")
                    #             print(f,") ⋅ $(barvarnames[barcj]) ")
                    #         end
                    #         terms += 1
                    #     else
                    #         omittedterms += 1
                    #     end
                    # end
                    print(f,"\n")
                end
            end

            if limitnumcon < numcon
                println(f,"        ... ($(numcon-limitnumcon) constraints omitted)")
            end

            if numcone > 0
                for k in 1:limitnumcone
                    (ct,conepar,nummem,submem) = getcone(t,k)
                    dom =
                        if     ct == MSK_CT_ZERO "𝒞_0"
                        elseif ct == MSK_CT_QUAD "𝒞_q"
                        elseif ct == MSK_CT_RQUAD "𝒞_qr"
                        elseif ct == MSK_CT_PPOW "𝒞_pow{$conepar}"
                        elseif ct == MSK_CT_PEXP "𝒞_exp"
                        else "?{$ct}"
                        end

                    print(f,"        $(conenames[k]): (")

                    if nummem > 0
                        print(f,"$(varnames[submem[1]])")
                        for i in 2:nummem
                            print(f,",$(varnames[submem[i]])")
                        end
                    end
                    println(f,") ∈ $dom($nummem)")
                end

                if limitnumcone < numcone
                    println(f,"        ... ($(numcone-limitnumcone) cones omitted)")
                end
            end
        end

        # Variable bounds
        if numvar+numbarvar > 0
            println(f,"    Variables")
            if numvar > 0
                for j in 1:limitnumvar
                    bk,bl,bu = getvarbound(t,j)

                    if bk == MSK_BK_FX
                        println(f,"        $(varnames[j]) = $bl")
                    else
                        blstr =
                            if     bk == MSK_BK_LO || bk == MSK_BK_RA "[$bl"
                            else   "]-Inf"
                            end
                        bustr =
                            if     bk == MSK_BK_UP || bk == MSK_BK_RA "$bu]"
                            else   "+Inf["
                            end

                        println(f,"        $(varnames[j]) ∈ $blstr;$bustr")
                    end
                end
            end
            if limitnumvar < numvar
                println(f,"        ... ($(numvar-limitnumvar) variable bounds omitted)")
            end

            for j in 1:limitnumbarvar
                dim = getdimbarvarj(t,j)
                println(f,"        $(barvarnames[j]) ∈ S($dim)")
            end
            if limitnumbarvar < numbarvar
                println(f,"        ... ($(numbarvar-limitnumbarvar) semidefinite variables omitted)")
            end
        end

        # Matsto
        numsymmat = getnumsymmat(t)
        limitnumsymmat = if limit > 0 min(limit,numsymmat) else numsymmat end
        if numsymmat > 0
            println(f,"    Symmetric matrixes")
            for i in 1:limitnumsymmat
                dim,nz,tp = getsymmatinfo(t,i)
                (msubi,msubj,mvalij) = getsparsesymmat(t,i)
                print(f,"        #MX$i ($dim): [ " )
                if nz > 0
                    limitnz = if limit > 0 min(limit,nz) else nz end
                    print(f,"($(msubi[1]),$(msubj[1]),$(mvalij[1]))")
                    for i in 2:limitnz
                        print(f,",($(msubi[i]),$(msubj[i]),$(mvalij[i]))")
                    end
                    if limitnz < nz
                        print(f,"... ($(nz-limitnz) nonzeros omitted)")
                    end
                end
                println(f," ]")
            end
            if limitnumsymmat < numsymmat
                println(f,"        ... ($(numsymmat-limitnumsymmat) matrixes omitted)")
            end

        end
    end
end


mutable struct LockedPriorityQueue{K,V}
    # TODO: find a better queue
    items::Vector{Pair{K,V}}
    nitems::Int
    @atomic minkey::K
    @atomic locked::Bool
end

struct MultiQueue{K,V}
    queues::Vector{LockedPriorityQueue{K,V}}
end

function LockedPriorityQueue{K,V}(buffersize::Integer) where {K,V}
    items = Vector{Pair{K,V}}(undef, buffersize)
    return LockedPriorityQueue{K,V}(items, 0, typemax(K), false)
end

function MultiQueue{K,V}() where {K,V}
    nqueues = 2 * Threads.nthreads()
    queues = [LockedPriorityQueue{K,V}(128) for _ in 1:nqueues]
    return MultiQueue{K,V}(queues)
end

function Base.trylock(pq::LockedPriorityQueue)
    if @atomic :monotonic pq.locked
        return false
    end
    _, ok = @atomicreplace pq.locked false => true
    return ok
end

function Base.unlock(pq::LockedPriorityQueue)
    @_assert pq.locked
    @atomic pq.locked = false
    return
end

function _insert!(xs, n::Integer, i::Integer, x)
    @_assert n < length(xs)
    for j in n:-1:i
        @inbounds xs[j+1] = xs[j]
    end
    xs[i] = x
end

function trypush!(pq::LockedPriorityQueue{K,V}, x::Pair{K,V}) where {K,V}
    @_assert pq.locked
    @_assert first(x) < typemax(K)
    if length(pq.items) == pq.nitems
        # Main.@tlc lpq_full
        return false
    end
    i = searchsortedlast(view(pq.items, 1:pq.nitems), x; by = first, rev = true) + 1
    _insert!(pq.items, pq.nitems, i, x)
    pq.nitems += 1
    if i == 1
        @atomic :monotonic pq.minkey = first(x)
    end
    # Main.@tlc lpq_inserted
    return true
end

function maybepopmin!(pq::LockedPriorityQueue{K,V}) where {K,V}
    pq.minkey < typemax(K) || return nothing
    @_assert pq.locked
    @_assert pq.nitems > 0
    x = pq.items[pq.nitems]
    n = pq.nitems -= 1
    k = n == 0 ? typemax(K) : first(pq.items[n])
    @atomic :monotonic pq.minkey = k
    return Some{V}(last(x))
end

function trypush!(mq::MultiQueue{K,V}, x::Pair{K,V}) where {K,V}
    (; queues) = mq
    nfails = 0
    while true
        # TODO: cheaper RNG?
        a = rand(queues)
        if trylock(a)
            try
                trypush!(a, x) && return true
            finally
                unlock(a)
            end
        end
        nfails += 1
        if nfails > length(queues)  # TODO: what's the good threshold?
            # contention or full; not a good sign anyway
            return false
        end
        GC.safepoint()
    end
end

"""
    maybepopmin!(mq::MultiQueue{K,V}) -> Some(v::V) or nothing

Return approximately minimum element from the multi-queue `mq`; return `nothing`
if empty.
"""
function maybepopmin!(mq::MultiQueue{K,V}) where {K,V}
    (; queues) = mq
    while true
        # TODO: cheaper RNG?
        a = rand(queues)
        b = rand(queues)
        if a.minkey > b.minkey
            a = b
        end
        if a.minkey == typemax(K)
            found = false
            for outer a in queues
                if a.minkey < typemax(K)
                    found = true
                    break
                end
            end
            found || return nothing
        end
        if trylock(a)
            try
                y = maybepopmin!(a)
                y === nothing || return y
            finally
                unlock(a)
            end
        end
        GC.safepoint()
    end
end


struct GradientLocalOptimizer <: LocalOptimizer
    algo::Symbol
    options::Dict{Symbol,Any}
end

need_gradient(::GradientLocalOptimizer) = true

function LBFGSOptimizer(; options...)
    GradientLocalOptimizer(:LD_LBFGS, Dict{Symbol,Any}(options))
end

function minimize(o::GradientLocalOptimizer, f, lb, ub, x0)
    opt = create_nlopt(o.algo, lb, ub; min_objective = f, o.options...)
    fx, x, ret = NLopt.optimize(opt, x0)
    ret == :FORCED_STOP && throw(InterruptException())
    return fx, x, ret
end

function maximize(o::GradientLocalOptimizer, f, lb, ub, x0)
    opt = create_nlopt(o.algo, lb, ub; max_objective = f, o.options...)
    fx, x, ret = NLopt.optimize(opt, x0)
    ret == :FORCED_STOP && throw(InterruptException())
    return fx, x, ret
end


export RockPaperScissorsEnv

"""
    RockPaperScissorsEnv()

[Rock Paper Scissors](https://en.wikipedia.org/wiki/Rock_paper_scissors) is a
simultaneous, zero sum game.
"""
Base.@kwdef mutable struct RockPaperScissorsEnv <: AbstractEnv
    reward::Tuple{Int,Int} = (0, 0)
    is_done::Bool = false
end

RLBase.players(::RockPaperScissorsEnv) = (1, 2)

"""
Note that although this is a two player game. The current player is always a
dummy simultaneous player.
"""
RLBase.current_player(::RockPaperScissorsEnv) = SIMULTANEOUS_PLAYER

# Defining the `action_space` of each independent player can help to transform
# this SIMULTANEOUS environment into a SEQUENTIAL environment with
# [`simultaneous2sequential`](@ref).
RLBase.action_space(::RockPaperScissorsEnv, ::Int) = Space(('💎', '📃', '✂'))

RLBase.action_space(::RockPaperScissorsEnv, ::SimultaneousPlayer) =
    Space(Tuple((i, j) for i in ('💎', '📃', '✂') for j in ('💎', '📃', '✂')))

RLBase.action_space(env::RockPaperScissorsEnv) = action_space(env, SIMULTANEOUS_PLAYER)

RLBase.legal_action_space(env::RockPaperScissorsEnv, p) =
    is_terminated(env) ? () : action_space(env, p)

"Since it's a one-shot game, the state space doesn't have much meaning."
RLBase.state_space(::RockPaperScissorsEnv, ::Observation, p) = Space(OneTo(1))

"""
For multi-agent environments, we usually implement the most detailed one.
"""
RLBase.state(::RockPaperScissorsEnv, ::Observation, p) = 1

RLBase.reward(env::RockPaperScissorsEnv) = env.is_done ? env.reward : (0, 0)
RLBase.reward(env::RockPaperScissorsEnv, p::Int) = reward(env)[p]

RLBase.is_terminated(env::RockPaperScissorsEnv) = env.is_done
RLBase.reset!(env::RockPaperScissorsEnv) = env.is_done = false

function (env::RockPaperScissorsEnv)((x, y))
    if x == y
        env.reward = (0, 0)
    elseif x == '💎' && y == '✂' || x == '✂' && y == '📃' || x == '📃' && y == '💎'
        env.reward = (1, -1)
    else
        env.reward = (-1, 1)
    end
    env.is_done = true
end

RLBase.NumAgentStyle(::RockPaperScissorsEnv) = MultiAgent(2)
RLBase.DynamicStyle(::RockPaperScissorsEnv) = SIMULTANEOUS
RLBase.ActionStyle(::RockPaperScissorsEnv) = MINIMAL_ACTION_SET
RLBase.InformationStyle(::RockPaperScissorsEnv) = IMPERFECT_INFORMATION
RLBase.StateStyle(::RockPaperScissorsEnv) = Observation{Int}()
RLBase.RewardStyle(::RockPaperScissorsEnv) = TERMINAL_REWARD
RLBase.UtilityStyle(::RockPaperScissorsEnv) = ZERO_SUM
RLBase.ChanceStyle(::RockPaperScissorsEnv) = DETERMINISTIC


function verifymeasure(;
    measure::DataFrame,
    measureinfo::NamedTuple,
    lpfilterperiods::Vector{<:Union{Day, Nothing}} = PARAM[:analysislpfilterperiods],
    benchmarkfunds::Bool = PARAM[:analysisbenchmarkfunds],
    benchmarkcomomentum::Bool=PARAM[:analysisbenchmarkcomomentum],
    comomcols::Union{Vector{Symbol}, Nothing} = PARAM[:analysiscomomcols],
    hfrname::String = PARAM[:hfrname],
    mfname::String = PARAM[:mfname],
    plname::String = PARAM[:plname],
    weeklyaggregatename = PARAM[:analysisweeklyaggregatename]
  )

  @unpack inputpath, outputpath, resultid, returntype, datefrequency = measureinfo

  hf = CSV.File("$inputpath\\$(returntype)_$(hfrname).csv") |> DataFrame
  mf = CSV.File("$inputpath\\$(returntype)_$(mfname).csv") |> DataFrame

    #compare the measure to particular benchmarks
    if datefrequency === :month
      validatedates(measure.date, frequency=:month, validateexactfrequency=true)

      if benchmarkfunds
        hfresults = comparetofundassets(measure, measureinfo, hf;
            fundslabel=:hf,
            datefrequency=:month,
            measurecols = findcols(measure, :loggrowth),
            benchmarkcols = findcols(hf, :loggrowth),
            usefocalcolheuristics=true,
            matchonlpfilter=true)

        mfresults = comparetofundassets(measure, measureinfo, mf;
            fundslabel=:mf,
            datefrequency=:month,
            measurecols = findcols(measure, :loggrowth),
            benchmarkcols = findcols(mf, :loggrowth),
            usefocalcolheuristics=true,
            matchonlpfilter=true)
        fundresults = [hfresults.results; mfresults.results;]
        sort!(fundresults, :cor)
        hfresults.data |> CSV.write("$outputpath\\$(resultid)_$(hfrname)_series.csv")
        mfresults.data |> CSV.write("$outputpath\\$(resultid)_$(mfname)_series.csv")
        fundresults |> CSV.write("$outputpath\\$(resultid)_fund_corresults.csv")

    elseif datefrequency === :week
      validatedates(measure.date, frequency=:week, validateexactfrequency=true)
      benchmarkfunds && throw("Only monthly data is supported when comparetofunds=true")
    end

    else

      @info "Note: fund benchmark analysis not run as analysisbenchmarkfunds=false"

    end

    if benchmarkcomomentum
      throw("Momentum not set up!!! Need to re-run everything before disabling this error.
        To set up, re-run/refresh momentum soup-2-nuts, then disable this error
        (can also disable the load warning in the analyze file)")

      comom = CSV.File("$inputpath\\$(returntype)_$(plname).csv") |> DataFrame

      benchmarkcols = isnothing(comomcols) ? setdiff(propertynames(comom), [:date]) : comomcols
      measurecols = [findcols(measure, :loggrowth); findcols(measure, :loggrowthpressure);
        findcols(measure, :assetmarketcap); findcols(measure, :loggrowthvolumepressure);
        findcols(measure, :growthvolumepressure)]

      println("benchmarkcols: $benchmarkcols")
      measurecols = [measurecols; measurecols .|> differencecol]
      comomresults = comparetofundassets(measure, measureinfo, comom,
        fundslabel = :pl,
        matchonlpfilter=false,
        usefocalcolheuristics=false;
        datefrequency, benchmarkcols, measurecols)
      sort!(comomresults.results, :cor)
      comomresults.data |> CSV.write("$outputpath\\$(resultid)_$(plname)_series.csv")
      comomresults.results |> CSV.write("$outputpath\\$(resultid)_comom_corresults.csv")
    else
      @info "Note: comomentum not run as analysisbenchmarkcomomentum=false"
    end

end

#useful function to translate monthyl dates between dataframes
function conformmonthlydates(old::AbstractVector{Date}, valid::AbstractVector{Date};
  datejoin = leftjoin)

  validatedates(old, frequency=:month, validateexactfrequency=true)
  validatedates(valid, frequency=:month, validateexactfrequency=true)

  #form a lookup table out of the available dates
  olddf = DataFrame(idx=1:length(old) |> collect, old=old)
  olddf.year = year.(old)
  olddf.month = month.(old)

  validdf = DataFrame(valid=valid)
  validdf.year = year.(valid)
  validdf.month = month.(valid)
  @assert sum(nonunique(validdf, [:year, :month])) == 0

  joint = datejoin(olddf, validdf, on=[:year, :month], validate=(false, true))
  @assert sum(nonunique(joint, [:year, :month])) == 0
  sort!(joint, :idx)

  return joint.valid
end

#useful function to translate weekly dates between dataframes
function conformweeklydates(old::AbstractVector{Date}, valid::AbstractVector{Date};
  datejoin = leftjoin)

  validatedates(old, frequency=:week, validateexactfrequency=true)
  validatedates(valid, frequency=:week, validateexactfrequency=true)

  olddf = DataFrame(idx=1:length(old) |> collect, old=old, lastdow = lastdayofweek.(old))
  validdf = DataFrame(valid=valid, lastdow = lastdayofweek.(valid))
  @assert sum(nonunique(validdf, [:lastdow])) == 0

  joint = datejoin(olddf, validdf, on=:lastdow, validate=(false, true))
  @assert sum(nonunique(joint, :lastdow)) == 0
  sort!(joint, :idx)

  return joint.valid
end

#compute correlations and statistics
function comparetofundassets(measure, measureinfo, funds;
    fundslabel::Symbol = throw("results label is required"),
    datefrequency::Symbol = throw("datefrequency is required"),
    measurecols = setdiff(propertynames(measure), [:date]),
    matchonlpfilter=true,
    benchmarkcols = setdiff(propertynames(funds), [:date]), #benchmarkcols
    extrabetaweights = PARAM[:fundextrabetaweights],
    extrabetaweightsidx = PARAM[:fundextrabetaweightsidx],
    minσ = eps(Float64)^0.5,
    usefocalcolheuristics = true
    )

  function bsstatfunc(sim)
    b,m = sim[1], sim[2]

    ρ = cor(b,m)
    return Dict(
      :cor  => ρ,
      :kend =>corkendall(b,m),
      :spear =>corspearman(b,m),
      :betabm => ρ * std(b)/std(m),
      :betamb => ρ * std(m)/std(b))
  end

  nullar1statfunc(b,m) = bsstatfunc((b,m))

  @unpack (neweylags, nwΣ,
    bsaggfunc, bsmeasurecolsregex, bsbenchmarkcolsregex, bssamples, bswidth,
      nullar1aggfunc, nullar1simulations) = measureinfo

  if datefrequency ≡ :month
    funds.measuredate = conformmonthlydates(funds.date, measure.date)
  elseif datefrequency ≡ :week
    funds.measuredate = conformweeklydates(funds.date, measure.date)
  else
    throw("unrecognized date frequency")
  end

  funds = funds[completecases(funds, :measuredate),:]
  select!(funds, Not(:date))
  joint = innerjoin(measure, funds,
    on=:date=>:measuredate, validate=(true,true))
  sort!(joint, :date)

  #compute the statistics pairwise in a semi-long format
  results = DataFrame(
    label=Symbol[], #label in case we concatenate this df with others
    measure=Symbol[], #measure field that we are benchmarking
    benchmark=Symbol[], #benchmark field

    cor=MFloat64[],
    cort=MFloat64[],
    corhac=MFloat64[],
    cormbbz = MFloat64[],
    cormbbp = MFloat64[],
    corar1h0z = MFloat64[],
    corar1h0p = MFloat64[],

    spear=MFloat64[],
    speart=MFloat64[],
    spearmbbz=MFloat64[],
    spearmbbp=MFloat64[],
    spearar1h0z=MFloat64[],
    spearar1h0p=MFloat64[],
    #spearz=MFloat64[],
    kend=MFloat64[],
    kendz=MFloat64[],
    kendmbbz=MFloat64[],
    kendmbbp=MFloat64[],
    kendar1h0z=MFloat64[],
    kendar1h0p=MFloat64[],

    betabm=MFloat64[],
    betabmmw=MFloat64[],
    betabmhac=MFloat64[],
    betabmmbbp=MFloat64[],
    betabmmbbz=MFloat64[],
    betabmar1h0z=MFloat64[],
    betabmar1h0p=MFloat64[],

    betamb=MFloat64[],
    betambmw=MFloat64[],
    betambhac=MFloat64[],
    betambmbbp=MFloat64[],
    betambmbbz=MFloat64[],
    betambar1h0z=MFloat64[],
    betambar1h0p=MFloat64[],
    N=MInt[])

  resultsbythread = [deepcopy(results) for t ∈ 1:Threads.nthreads()]

  #this subfunction determines which measure cols will be tested against a bcol
  function formfocalcols(bcol)
    matchedcols = matchonlpfilter ? matchcolsonlpsuffix(bcol, measurecols) : measurecols
    (!usefocalcolheuristics) && return measurecols

    focalmeasurecols = Symbol[]

    #=if (strippedbcol |> Symbol) ∉ extrabetaweights
      #along with maybe the s
      strippedbcol = stripcolprefix(bcol)=#

    for mcol ∈ matchedcols
      local isfocal::Bool
      strippedmcol = stripcolprefix(mcol) |> string

      if occursin(strippedmcol, bcol |> string)
        isfocal=true

      #see if we explicitly want to include it as a focal col
      else
        isfocal = false
        for k ∈ extrabetaweights
          #in this case, the specified extra column is embeded in the name
          # of the bcol
          if occursin(k|> string, bcol |> string) && (
            any((m->
              occursin(m|>string, strippedmcol)).(extrabetaweightsidx[k])))

            isfocal=true
          end
        end
      end

      if isfocal
        push!(focalmeasurecols, mcol)
      end
    end

    return focalmeasurecols
  end
  for bcol ∈ benchmarkcols
    tresults = resultsbythread[Threads.threadid()]

    focalmeasurecols = formfocalcols(bcol)
    #focalmeasurecols = matchonlpfilter ? matchcolsonlpsuffix(bcol, measurecols) : measurecols
    #we want to match the smoothing/lp filter level between the measure and the benchmark

    for mcol ∈ focalmeasurecols

      jointnomissing = view(joint, completecases(joint, [mcol,bcol]), [:date, mcol, bcol])
      m::Vector{Float64} = jointnomissing[:,mcol]
      b::Vector{Float64} = jointnomissing[:,bcol]
      N=nrow(jointnomissing)

      #try
        #stats all from wikipedia, so probably want better sources if used
        pear = cor(m,b) #pearson correlation
        peart = pear*((N-2)/(1-pear^2))^(0.5)
        spear = StatsBase.corspearman(m, b) #spearman correlation
        #spearz = atanh(spear)*((N-3)/1.06)^0.5
        speart = spear*((N-2)/(1-spear^2))^(0.5)
        kend = StatsBase.corkendall(m, b) #kendall's tau
        kendz = kend/((2*(2*N+5))/(9*N*(N-1)))^0.5 #not sure about this- investigate if using

        #=Xb = hcat(b, ones(N))
        lm = FMLM(Xb, m)
        beta=lm.β[1]
        @assert lm.β[1] ≈ pear * std(m)/std(b)=#
        Xm = hcat(m, ones(N))
        lmbm = FMLM(Xm, b)
        betabm=lmbm.β[1]
        @assert (lmbm.β[1] ≈ pear * std(b)/std(m)) || (lmbm.β[1] ≡ pear * std(b)/std(m)) "
          lmbm.β[1]= $(lmbm.β[1]) while pear * std(b)/std(m) = $(pear * std(b)/std(m))!!"
        betabmmw = betabm/modifiedwhiteΣ!(lmbm)[1,1]^0.5
        betabmhac = betabm/nwΣ(lmbm)[1,1]^0.5

        Xb = hcat(b, ones(N))
        lmmb = FMLM(Xb, m)
        betamb=lmmb.β[1]
        @assert ((lmmb.β[1] ≈ pear * std(m)/std(b)) || (lmmb.β[1] ≡ pear * std(m)/std(b)))
        betambmw = betamb/modifiedwhiteΣ!(lmmb)[1,1]^0.5
        betambhac = betamb/nwΣ(lmmb)[1,1]^0.5

        #compute the pearson correlation as a regression
        #this is kinda hacky- might want to replace with a block bootstrap
        #=Xbcor = hcat(b .* std(m) ./ std(b), ones(N))
        lmcor = FMLM(Xbcor, m)=#
        Xmcor = hcat(m .* std(b) ./ std(m), ones(N))
        lmcor = FMLM(Xmcor, b)
        @assert ((lmcor.β[1] ≈ pear) || (lmcor.β[1] ≡ pear))
        pearhacz = pear/nwΣ(lmcor)[1,1]^0.5

        #perform moving block bootstrap if desired
        if (occursin(bsmeasurecolsregex, string(mcol)) &&
            occursin(bsbenchmarkcolsregex, string(bcol)) &&
            (std(b) > minσ) && (std(m) > minσ))
          #@info "bcol:$bcol, mcol:$mcol"
          bs = mbbootstrap([b,m], bsstatfunc, bsaggfunc, B=bssamples, w=bswidth)
          cormbbz = pear/bs.cor.se
          spearmbbz = spear/bs.spear.se
          kendmbbz =  kend/bs.kend.se
          betabmmbbz =  betabm/bs.betabm.se
          betambmbbz =  betamb/bs.betamb.se

          cormbbp = bs.cor.p
          spearmbbp = bs.spear.p
          kendmbbp =  bs.kend.p
          betabmmbbp =  bs.betabm.p
          betambmbbp =  bs.betamb.p

          ar1h0teststats = Dict(:cor=>pear, :kend=>kend, :spear=>spear,
            :betabm => betabm, :betamb => betamb)
          ar1h0 = comparear1undernull(b,m, nullar1statfunc,
            (v,s)->nullar1aggfunc(v,s,ar1h0teststats),
            Nsimulations=nullar1simulations)
          #these are the alternate hypotheses to test

          corar1h0z =  pear/ar1h0.cor.se
          spearar1h0z = spear/ar1h0.spear.se
          kendar1h0z =  kend/ar1h0.kend.se
          betabmar1h0z =  betabm/ar1h0.betabm.se
          betambar1h0z =  betamb/ar1h0.betamb.se

          corar1h0p = ar1h0.cor.p
          spearar1h0p = ar1h0.spear.p
          kendar1h0p =  ar1h0.kend.p
          betabmar1h0p =  ar1h0.betabm.p
          betambar1h0p =  ar1h0.betamb.p
        else
          cormbbp=missing; spearmbbp=missing; kendmbbp=missing; betabmmbbp=missing; betambmbbp=missing;
          cormbbz=missing; spearmbbz=missing; kendmbbz=missing; betabmmbbz=missing; betambmbbz=missing;

          corar1h0p=missing; spearar1h0p=missing; kendar1h0p=missing; betabmar1h0p=missing; betambar1h0p=missing;
          corar1h0z=missing; spearar1h0z=missing; kendar1h0z=missing; betabmar1h0z=missing; betambar1h0z=missing;
        end

        push!(tresults,(;
          label=fundslabel,
          measure=mcol,
          benchmark=bcol,
          cor=pear,
          cort=peart,
          corhac=pearhacz,
          cormbbz, cormbbp, corar1h0z, corar1h0p,
          spear, speart, spearmbbz, spearmbbp, spearar1h0z, spearar1h0p,
          kend, kendz, kendmbbp, kendmbbz, kendar1h0p, kendar1h0z,
          betabm, betabmmw, betabmhac, betabmmbbz,  betabmmbbp, betabmar1h0z,  betabmar1h0p,
          betamb, betambmw, betambhac, betambmbbz,  betambmbbp, betambar1h0z,  betambar1h0p,
          N))

    end
  end

  joint |> CSV.write(
    "$(PARAM[:testpath])\\$(PARAM[:analysismeasurefilename])_$(fundslabel)_jointcor.csv")

  #collect the results in a single df
  append!(results, reduce(append!, resultsbythread))
  sort!(results, [:benchmark, :measure])

  return (;results, data=joint)
end

#normalizedassetcols(measure) = propertynames(measure[!, r"^NA_"])


using ..Mesh.Grids: _x1, _x2, _x3
"""
    InitStateBC

Set the value at the boundary to match the `init_state_prognostic!` function. This is
mainly useful for cases where the problem has an explicit solution.

# TODO: This should be fixed later once BCs are figured out (likely want
# different things here?)
"""
struct InitStateBC end
function boundary_state!(
    ::Union{NumericalFluxFirstOrder, NumericalFluxGradient},
    bc::InitStateBC,
    m::AtmosModel,
    state⁺::Vars,
    aux⁺::Vars,
    n⁻,
    state⁻::Vars,
    aux⁻::Vars,
    t,
    _...,
)
    # Put cood in a NamedTuple to mimmic LocalGeometry
    init_state_prognostic!(m, state⁺, aux⁺, (coord = aux⁺.coord,), t)
end

function boundary_state!(
    ::NumericalFluxSecondOrder,
    bc::InitStateBC,
    m::AtmosModel,
    state⁺::Vars,
    diff⁺::Vars,
    hyperdiff⁺::Vars,
    aux⁺::Vars,
    n⁻,
    state⁻::Vars,
    diff⁻::Vars,
    hyperdiff⁻::Vars,
    aux⁻::Vars,
    t,
    args...,
)
    # Put coord in a NamedTuple to mimmic LocalGeometry
    init_state_prognostic!(m, state⁺, aux⁺, (coord = aux⁺.coord,), t)
end


# Tabix
# =====
#
# Generic index for tab-delimited files.
#
# Li, Heng. "Tabix: fast retrieval of sequence features from generic
# TAB-delimited files." Bioinformatics 27.5 (2011): 718-719.
# Specification: http://samtools.github.io/hts-specs/tabix.pdf

include("index.jl")
include("reader.jl")

"""
    overlapchunks(index::Tabix, seqname::String, interval::UnitRange)
    overlapchunks(index::Tabix, seqid::Integer, interval::UnitRange)

Return chunks possibly overlapping the range specified by `seqname` (or `seqid`)
and `interval`; `seqid` and `interval` must be 1-based index and inclusive.

NOTE: Records within the returned chunks are not guaranteed to overlap the query
region. The user need to check the condition as:
    line = readline(stream)
    values = split(chomp(line), '\t')
    seqname = value[index.columns[1]]
    start = parse(Int, index.columns[2])
    stop = parse(Int, index.columns[3]) - 1  # needs -1 for the BED format
    if seqname == target_seqname && intersect(start:stop, target_interval)
        # the line overlaps the query region
    end
"""
function overlapchunks(index::Union{BAI,Tabix}, seqid::Integer, interval::UnitRange)
    if !(1 ≤ seqid ≤ endof(index.indexes))
        throw(ArgumentError("sequence id $(seqid) is out of range"))
    end

    if isempty(interval)
        return Chunk[]
    end

    binindex, linindex, pbin = index.indexes[seqid]
    bins = reg2bins(first(interval), last(interval))
    offset = linindex[cld(first(interval), LinearWindowSize)]
    ret = Chunk[]
    for bin in bins
        if haskey(binindex, bin)
            for chunk in binindex[bin]
                if chunk.stop > offset
                    push!(ret, chunk)
                end
            end
        end
    end

    sort!(ret)
    reduce!(ret)

    return ret
end

function overlapchunks(index::Tabix, seqname::AbstractString, interval::UnitRange)
    seqid = findfirst(index.names, seqname)
    if seqid == 0
        throw(ArgumentError("sequence name $(seqname) is not included in the index"))
    end
    return overlapchunks(index, seqid, interval)
end

# Merge chunks to minimize the number of seek operations.
function reduce!(chunks)
    @assert issorted(chunks)
    i = 1
    while i < endof(chunks)
        chunk = chunks[i]
        next = chunks[i+1]
        if chunk.stop < next.start
            # neither overlapping nor adjacent
            i += 1
            continue
        end
        chunks[i] = Chunk(chunk.start, max(chunk.stop, next.stop))
        deleteat!(chunks, i + 1)
    end
    return chunks
end


export lsh

import ..LSHSearch

using mlpack._Internal.io

import mlpack_jll
const lshLibrary = mlpack_jll.libmlpack_julia_lsh

# Call the C binding of the mlpack lsh binding.
function lsh_mlpackMain()
  success = ccall((:lsh, lshLibrary), Bool, ())
  if !success
    # Throw an exception---false means there was a C++ exception.
    throw(ErrorException("mlpack binding error; see output"))
  end
end

" Internal module to hold utility functions. "
module lsh_internal
  import ..lshLibrary

import ...LSHSearch

# Get the value of a model pointer parameter of type LSHSearch.
function IOGetParamLSHSearch(paramName::String)::LSHSearch
  LSHSearch(ccall((:IO_GetParamLSHSearchPtr, lshLibrary), Ptr{Nothing}, (Cstring,), paramName))
end

# Set the value of a model pointer parameter of type LSHSearch.
function IOSetParamLSHSearch(paramName::String, model::LSHSearch)
  ccall((:IO_SetParamLSHSearchPtr, lshLibrary), Nothing, (Cstring, Ptr{Nothing}), paramName, model.ptr)
end

# Serialize a model to the given stream.
function serializeLSHSearch(stream::IO, model::LSHSearch)
  buf_len = UInt[0]
  buf_ptr = ccall((:SerializeLSHSearchPtr, lshLibrary), Ptr{UInt8}, (Ptr{Nothing}, Ptr{UInt}), model.ptr, Base.pointer(buf_len))
  buf = Base.unsafe_wrap(Vector{UInt8}, buf_ptr, buf_len[1]; own=true)
  write(stream, buf)
end
# Deserialize a model from the given stream.
function deserializeLSHSearch(stream::IO)::LSHSearch
  buffer = read(stream)
  LSHSearch(ccall((:DeserializeLSHSearchPtr, lshLibrary), Ptr{Nothing}, (Ptr{UInt8}, UInt), Base.pointer(buffer), length(buffer)))
end
end # module

"""
    lsh(; [bucket_size, hash_width, input_model, k, num_probes, projections, query, reference, second_hash_size, seed, tables, true_neighbors, verbose])

This program will calculate the k approximate-nearest-neighbors of a set of
points using locality-sensitive hashing. You may specify a separate set of
reference points and query points, or just a reference set which will be used as
both the reference and query set.

For example, the following will return 5 neighbors from the data for each point
in `input` and store the distances in `distances` and the neighbors in
`neighbors`:

```julia
julia> using CSV
julia> input = CSV.read("input.csv")
julia> distances, neighbors, _ = lsh(k=5, reference=input)
```

The output is organized such that row i and column j in the neighbors output
corresponds to the index of the point in the reference set which is the j'th
nearest neighbor from the point in the query set with index i.  Row j and column
i in the distances output file corresponds to the distance between those two
points.

Because this is approximate-nearest-neighbors search, results may be different
from run to run.  Thus, the `seed` parameter can be specified to set the random
seed.

This program also has many other parameters to control its functionality; see
the parameter-specific documentation for more information.

# Arguments

 - `bucket_size::Int`: The size of a bucket in the second level hash.
      Default value `500`.

 - `hash_width::Float64`: The hash width for the first-level hashing in
      the LSH preprocessing. By default, the LSH class automatically estimates a
      hash width for its use.  Default value `0`.

 - `input_model::LSHSearch`: Input LSH model.
 - `k::Int`: Number of nearest neighbors to find.  Default value `0`.

 - `num_probes::Int`: Number of additional probes for multiprobe LSH; if
      0, traditional LSH is used.  Default value `0`.

 - `projections::Int`: The number of hash functions for each table
      Default value `10`.

 - `query::Array{Float64, 2}`: Matrix containing query points (optional).
 - `reference::Array{Float64, 2}`: Matrix containing the reference
      dataset.
 - `second_hash_size::Int`: The size of the second level hash table.
      Default value `99901`.

 - `seed::Int`: Random seed.  If 0, 'std::time(NULL)' is used.  Default
      value `0`.

 - `tables::Int`: The number of hash tables to be used.  Default value
      `30`.

 - `true_neighbors::Array{Int, 2}`: Matrix of true neighbors to compute
      recall with (the recall is printed when -v is specified).
 - `verbose::Bool`: Display informational messages and the full list of
      parameters and timers at the end of execution.  Default value `false`.

# Return values

 - `distances::Array{Float64, 2}`: Matrix to output distances into.
 - `neighbors::Array{Int, 2}`: Matrix to output neighbors into.
 - `output_model::LSHSearch`: Output for trained LSH model.

"""
function lsh(;
             bucket_size::Union{Int, Missing} = missing,
             hash_width::Union{Float64, Missing} = missing,
             input_model::Union{LSHSearch, Missing} = missing,
             k::Union{Int, Missing} = missing,
             num_probes::Union{Int, Missing} = missing,
             projections::Union{Int, Missing} = missing,
             query = missing,
             reference = missing,
             second_hash_size::Union{Int, Missing} = missing,
             seed::Union{Int, Missing} = missing,
             tables::Union{Int, Missing} = missing,
             true_neighbors = missing,
             verbose::Union{Bool, Missing} = missing,
             points_are_rows::Bool = true)
  # Force the symbols to load.
  ccall((:loadSymbols, lshLibrary), Nothing, ());

  IORestoreSettings("K-Approximate-Nearest-Neighbor Search with LSH")

  # Process each input argument before calling mlpackMain().
  if !ismissing(bucket_size)
    IOSetParam("bucket_size", convert(Int, bucket_size))
  end
  if !ismissing(hash_width)
    IOSetParam("hash_width", convert(Float64, hash_width))
  end
  if !ismissing(input_model)
    lsh_internal.IOSetParamLSHSearch("input_model", convert(LSHSearch, input_model))
  end
  if !ismissing(k)
    IOSetParam("k", convert(Int, k))
  end
  if !ismissing(num_probes)
    IOSetParam("num_probes", convert(Int, num_probes))
  end
  if !ismissing(projections)
    IOSetParam("projections", convert(Int, projections))
  end
  if !ismissing(query)
    IOSetParamMat("query", query, points_are_rows)
  end
  if !ismissing(reference)
    IOSetParamMat("reference", reference, points_are_rows)
  end
  if !ismissing(second_hash_size)
    IOSetParam("second_hash_size", convert(Int, second_hash_size))
  end
  if !ismissing(seed)
    IOSetParam("seed", convert(Int, seed))
  end
  if !ismissing(tables)
    IOSetParam("tables", convert(Int, tables))
  end
  if !ismissing(true_neighbors)
    IOSetParamUMat("true_neighbors", true_neighbors, points_are_rows)
  end
  if verbose !== nothing && verbose === true
    IOEnableVerbose()
  else
    IODisableVerbose()
  end

  IOSetPassed("distances")
  IOSetPassed("neighbors")
  IOSetPassed("output_model")
  # Call the program.
  lsh_mlpackMain()

  return IOGetParamMat("distances", points_are_rows),
         IOGetParamUMat("neighbors", points_are_rows),
         lsh_internal.IOGetParamLSHSearch("output_model")
end


export
    ParticleFilter,
    SyncPF,
    AsyncPF,
    CoherentPF

function ParticleFilter(window_creator::WindowCreator, num_particles::Int, resampling_size::Int = num_particles)
    function pp!(run)
        particles = get_state(run, :particles)
        newparticles = resample(particles, resampling_size)
        set_state!(run, :particles, newparticles)
    end
    return WindowFilter(window_creator, LW(num_particles), pp!)
end

"""
    SyncPF(num_particles::Int, resampling_size::Int = num_particles)

A particle filter that uses a synchronous window with the given number of particles and resampling buffer size.
"""
SyncPF(num_particles::Int, resampling_size::Int = num_particles) = ParticleFilter(SyncWindow(), num_particles, resampling_size)

"""
    AsyncPF(num_particles::Int, resampling_size::Int = num_particles, T = Float64)

A particle filter that uses an asynchronous window with the given number of particles and resampling buffer size.
`T` represents the time type and must be the same as used in creation of the runtime.
"""
AsyncPF(num_particles::Int, resampling_size::Int = num_particles, T = Float64) = ParticleFilter(AsyncWindow{T}(), num_particles, resampling_size)

"""
    CoherentPF(num_particles::Int, resampling_size::Int = num_particles, T = Float64)

A particle filter that uses a coherent window with the given number of particles and resampling buffer size.
`T` represents the time type and must be the same as used in creation of the runtime.
"""
CoherentPF(num_particles::Int, resampling_size::Int = num_particles, T = Float64) = ParticleFilter(CoherentWindow{T}(), num_particles, resampling_size)

#=
"""
General type of particle filters. Implementations must have fields runtime, num_particles, and resampling_size.
Some implementations will also provide APIs that organize the filtering in a specific way.
"""
abstract type ParticleFilter end

"""
current_particles(pf::ParticleFilter)

Returns a Particles data structure consisting of the current view of the most recent samples
of all variables in the network, along with the current log weights.
"""
# Samples are stored in a distributed manner across variables, possibly at different time points.
# Coherence of global samples is maintained by consistent indexing into arrays of variable samples.
function current_particles(pf::ParticleFilter)
    vars = get_variables(get_network(pf.runtime))
    # Need to make sure samples is initialized with distinct empty dictionaries
    samples = Sample[]
    for i in 1:pf.num_particles
        push!(samples, Sample())
    end
    for var in vars
        inst = current_instance(pf.runtime, var)
        varsample = get_value(pf.runtime, inst, :samples)
        for i in 1:pf.num_particles
            samples[i][var.name] = varsample[i]
        end
    end
    lws = get_state(pf.runtime, :log_weights)
    return Particles(samples, lws)
end

"""
store_particles!(pf::ParticleFilter, ps::Particles)

Stores the information in ps in a form pf can use.
"""
function store_particles!(pf::ParticleFilter, ps::Particles)
    vars = get_variables(get_network(pf.runtime))
    # Reset the samples and log weights in the runtime
    for var in vars
        inst = current_instance(pf.runtime, var)
        varsample = Vector{output_type(get_sfunc(inst))}()
        for i in 1:pf.num_particles
            # correction for 1-indexing
            push!(varsample, ps.samples[(i-1) % length(ps.samples) + 1][var.name])
        end
        set_value!(pf.runtime, inst, :samples, varsample)
    end
    set_state!(pf.runtime, :log_weights, ps.log_weights)
end

function resample!(pf::ParticleFilter)
    current_ps = current_particles(pf)
    new_ps = resample(current_ps, pf.resampling_size)
    store_particles!(pf, new_ps)
end

function init_filter(pf::ParticleFilter)
    ensure_all!(pf.runtime, 0)
    likelihood_weighting(pf.runtime, pf.num_particles)
    ps = get_state(pf.runtime, :particles)
    store_particles!(pf, ps)
end

# TODO: Handle evidence at different points in time
function filter_step(pf::ParticleFilter, variables_to_sample::Vector{<:Variable}, time::Int, evidence::Dict{Symbol, Score})
    @assert time > current_time(pf.runtime)
    set_time!(pf.runtime, time)

    # The log weights of the current evidence are added to the existing log weights for each sample.
    scores = get_state(pf.runtime, :log_weights)

    for var in variables_to_sample
        varsamples = Vector{output_type(var)}()
        inst = instantiate!(pf.runtime, var, time)
        sf = get_sfunc(inst)
        varev = var.name in keys(evidence) ? evidence[var.name] : nothing

        # Get the sample sets of each of the parents of the current variable in the transition model.
        # These parents may have been previously instantiated at any time.
        # instantiate! will make sure that the sfunc correctly takes into account the time lags,
        # as long as the model of the variable is defined appropriately.
        parents = get_transition_parents(get_network(pf.runtime), var)
        n = length(parents)
        parsamples = map(parents) do p
            if p == var # self edge from previous
                parinst = previous_instance(pf.runtime, var)
            else
                parinst = current_instance(pf.runtime, p)
            end
            get_value(pf.runtime, parinst, :samples)
        end

        for i in 1:pf.num_particles
            parvals = tuple([parsamples[j][i] for j in 1:n]...)
            if isa(varev, HardScore)
                # Since the value is known, force it and score it instead of sampling, LW style
                evval = varev.value
                push!(varsamples, evval)
                scores[i] += logcpdf(sf, parvals, evval)
            else
                # Otherwise, we sample it, but if there is evidence we still score it
                sampval = sample(sf, parvals)
                push!(varsamples, sampval)
                if !isnothing(varev)
                    scores[i] += get_log_score(varev, sampval)
                end
            end
        end

        set_value!(pf.runtime, inst, :samples, varsamples)
    end

    set_state!(pf.runtime, :log_weights, scores)
end

=#


# include("time_parse.jl")
# include("multifrequency-opf.jl")
# include("utilities.jl")

# function to run multifrequency_opf on one case within the given time range.
#
# parameters:
# month_st, day_st, period_st, month_en, day_en, period_en: the parameters specifying the period to plot
# folder: the directory containing all subnetwork data, subnetworks.csv, interfaces.csv
# subne_folder: the number of subnet that is to be optimized (which directs to the sub-folder containing its data)
# output_folder: the folder specified to save the output files
# plot_results: choice of plot the optimization result (optimal value vs time points) directly after optimization, default false Below are the similar arguments as multifrequency-opf:
# obj::String: the objective function to use, from the following:
# "mincost": minimize generation cost
# "areagen": minimize generation in the areas specified in gen_areas
# "minredispatch": minimize the change in generator dispatch from the initial values defined in the network data
# gen_areas: integer array of all areas in which generation should be minimized if obj=="mincost"
# area_transfer: two areas with power transfer between them that should be saved and plotted. Results for P, Q, S, and loss are saved for power transfer between the two areas. Must have exactly two elements.
# print_results: if true, print the optimization results
###########################################################################################
# month_st, day_st, period_st, month_en, day_en, period_en: the
# folder::String:: the directory containing all subnetwork data, subnetworks.csv, interfaces.csv
# subne_folder: the number of subnet that is to be optimized (which directs to the sub-folder containing its data)
# output_folder: the folder specified to save the output files
# plot_results::Bool = true: choice of plot the optimization result (optimal value vs time points) directly after optimization, default false Below are the similar arguments as multifrequency-opf:
# obj::String: the objective function to use, from the following:
#     "mincost": minimize generation cost
#     "areagen": minimize generation in the areas specified in gen_areas
#     "minredispatch": minimize the change in generator dispatch from the initial values defined in the network data
# gen_areas: integer array of all areas in which generation should be minimized if obj=="mincost"
# area_transfer: two areas with power transfer between them that should be saved and plotted. Results for P, Q, S, and loss are saved for power transfer between the two areas. Must have exactly two elements.
# print_results::Bool: if true, print the optimization results
# function to run multifrequency-opf on the specified case with the specified range of time
function hour_opf(month_st, day_st, period_st, month_en, day_en, period_en,
      folder:: String,
      subnet_folder:: String,
      output_folder::String,
      obj::String,
      gen_areas = [],
      area_transfer = [],
      plot_results::Bool = true,
      print_results::Bool = false)
      # override_param::Dict{Any}=Dict(),
      # fix_f_override::Bool=false,
      # direct_pq::Bool=true,
      # master_subnet::Int64=1,
      # suffix::String=""
      dir = "$(folder)/"
      #file_name = "$(dir)/RTS_GMLC.m"

      # hour_data: store the data
      hour_data = Dict{Int64, Any}

      # time range
      start_t = DateTime(2020,month_st,day_st,period_st)
      end_t = DateTime(2020,month_en,day_en,period_en)
      range_t = start_t:Hour(1):end_t

      # input the corresponding data of the specific subnetwork
      sub_folder = "$(folder)/enum_net/$(subnet_folder)"
      println("reading the file $(sub_folder)")
      hour_data = hour_parse(month_st, day_st, period_st, month_en, day_en, period_en, sub_folder)

      # s = JSON.json(hour_data)
      # open("test_hour_cost.json", "w") do f
      # write(f, s)
      # end

      # dictionaries store the output of the optimization for each period.
      opt_net = Dict()
      res_summary_dict = Dict()
      solution_pm_dict = Dict()
      hour_mn_data = Dict()
      objective_val = Dict()

      # output binding constraints
      binding_constraints = Dict()

      # manually used enumerate_branch to make data of subnetworks from RTS_GMLC.m before
      # running the make_mn_data function
      for i in 1:size(range_t, 1)
            #for j in 1:119
                  #subnetwork_file = "$(dir)/enum_nets/br$(j)/subnetworks.csv"
                  #interface_file = "$(dir)/enum_nets/br$(j)/interfaces.csv"
                  subnetwork_file = "$(dir)/enum_net/$(subnet_folder)/subnetworks.csv"
                  interface_file = "$(dir)/enum_net/$(subnet_folder)/interfaces.csv"
                  subnetworks = DataFrame(index=Int64[], file=String[], variable_f=Bool[],
                                          f_base=Float64[], f_min=Float64[], f_max=Float64[])
                  interfaces = DataFrame(index=Int64[], subnet_index=Int64[], bus=Int64[],
                                         s_max=Float64[], p_max=Float64[], q_max=Float64[],
                                         loss_param=Float64[])

                  # files ready for input to make_mn_data
                  subnetworks = CSV.read(subnetwork_file, DataFrame)
                  interfaces = CSV.read(interface_file, DataFrame)
                  hour_mn_data[i] = VariableFrequencyOPF.make_mn_data(subnetworks, interfaces, hour_data[i])

                  (opt_net[i], res_summary_dict[i], solution_pm_dict[i]) =
                  VariableFrequencyOPF.multifrequency_opf(hour_mn_data[i], output_folder, obj, gen_areas, area_transfer, print_results)
                  # open("test_mn_cost_$(i).json", "w") do f
                  #  write(f, s1)
                  # end

                  # write the binding constriants to a new file (save binding constraints of each case individually)
                  binding_constraints_part = Dict()
                  open("$(output_folder)/binding_constraints.json", "r") do f
                        global binding_constraints_part
                        #bind = read(f, String)
                        binding_constraints_part = JSON.parse(f)
                        binding_constraints[i] = binding_constraints_part
                  end
                  #binding_constraints_json = JSON.json(binding_constraints)

            #end
      end

      # save files of outputs
      binding_constraints_json = JSON.json(binding_constraints)
      open("$(output_folder)/binding_constraints$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f1
            write(f1, binding_constraints_json)
      end

      opt_net_string = JSON.json(opt_net)
      open("$(output_folder)/opt_net_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
        write(f, opt_net_string)
      end

      res_summary_string = JSON.json(res_summary_dict)
      open("$(output_folder)/res_summary_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
        write(f, res_summary_string)
      end

      # solution_pm_string = JSON.json(solution_pm_dict)
      # open("$(output_folder)/solution_pm_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
      #   write(f, solution_pm_string)
      # end

      # plot of the results (optional, now we mostly use the function of multi_plot)
      # if plot_results == true
      #       objval = Array{Float64, 1}()
      #       objval_n = Array{Float64, 1}()
      #       size_obj = Array{DateTime, 1}()
      #       size_obj_n = Array{DateTime, 1}()
      #       for i in 1:size(range_t, 1)
      #             if res_summary_dict[i].status[1] in [LOCALLY_SOLVED, ALMOST_LOCALLY_SOLVED]
      #                   push!(objval, res_summary_dict[i]["objective"][1])
      #                   push!(size_obj, range_t[i])
      #             else
      #                   push!(objval_n, res_summary_dict[i]["objective"][1])
      #                   push!(size_obj_n, range_t[i])
      #             end
      #       end
      #       #size_obj_n = size(range_t, 1) - size_obj
      #       # below are the parts of plotting
      #       println(range_t)
      #       println(objval)
      #       gr()
      #       p = plot(size_obj, objval,  color = :blue, seriestype = :line, title = "optimal result plot",
      #                xlabel = "time/h", ylabel = "optimal $(obj)", right_margin = 10px)
      #       plot!(size_obj_n, objval_n, color = :red, seriestype = :scatter)
      #       # seriestype = :histogram
      #       #savefig(fn)
      #       #savefig(p, )
      #
      #       # save output plot file
      #       savefig(p, "$(output_folder)/opt_res_plot_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).pdf")
      # end

      return opt_net, res_summary_dict, solution_pm_dict
end

function hour_opf_regularized(month_st, day_st, period_st, month_en, day_en, period_en,
      folder:: String,
      subnet_folder:: String,
      output_folder::String,
      obj::String;
      gen_areas = [],
      area_transfer = [],
      gen_zones = [],
      zone_transfer = [],
      plot_results::Bool = true,
      print_results::Bool = false,
      regularize_f::Float64=0.0,
      ipopt_max_iter::Int64=20000
      )
      # override_param::Dict{Any}=Dict(),
      # fix_f_override::Bool=false,
      # direct_pq::Bool=true,
      # master_subnet::Int64=1,
      # suffix::String=""
      dir = "$(folder)/"
      #file_name = "$(dir)/RTS_GMLC.m"

      # hour_data: store the data
      hour_data = Dict{Int64, Any}

      if !isdir(output_folder)
            mkpath(output_folder)
      end
      # time range
      start_t = DateTime(2020,month_st,day_st,period_st)
      end_t = DateTime(2020,month_en,day_en,period_en)
      range_t = start_t:Hour(1):end_t

      # input the corresponding data of the specific subnetwork
      # sub_folder = "$(folder)/enum_net_new/$(subnet_folder)"
      sub_folder = subnet_folder
      println("reading the file $(sub_folder)")
      hour_data = hour_parse(month_st, day_st, period_st, month_en, day_en, period_en, sub_folder)

      # s = JSON.json(hour_data)
      # open("test_hour_cost.json", "w") do f
      # write(f, s)
      # end

      # dictionaries store the output of the optimization for each period.
      opt_net = Dict()
      res_summary_dict = Dict()
      solution_pm_dict = Dict()
      hour_mn_data = Dict()
      objective_val = Dict()

      # output binding constraints
      binding_constraints = Dict()

      # manually used enumerate_branch to make data of subnetworks from RTS_GMLC.m before
      # running the make_mn_data function
      for i in 1:size(range_t, 1)
            #for j in 1:119
                  #subnetwork_file = "$(dir)/enum_nets/br$(j)/subnetworks.csv"
                  #interface_file = "$(dir)/enum_nets/br$(j)/interfaces.csv"
                  # subnetwork_file = "$(dir)/enum_net_new/$(subnet_folder)/subnetworks.csv"
                  # interface_file = "$(dir)/enum_net_new/$(subnet_folder)/interfaces.csv"
                  subnetwork_file = "$(subnet_folder)/subnetworks.csv"
                  interface_file = "$(subnet_folder)/interfaces.csv"
                  subnetworks = DataFrame(index=Int64[], file=String[], variable_f=Bool[],
                                          f_base=Float64[], f_min=Float64[], f_max=Float64[])
                  interfaces = DataFrame(index=Int64[], subnet_index=Int64[], bus=Int64[],
                                         s_max=Float64[], p_max=Float64[], q_max=Float64[],
                                         loss_param=Float64[])

                  # files ready for input to make_mn_data
                  subnetworks = CSV.read(subnetwork_file, DataFrame)
                  interfaces = CSV.read(interface_file, DataFrame)
                  hour_mn_data[i] = VariableFrequencyOPF.make_mn_data(subnetworks, interfaces, hour_data[i])

                  override_param=Dict()
                  fix_f_override=false
                  direct_pq=true
                  master_subnet=1
                  start_vals=Dict{String, Dict}("sn"=>Dict())
                  uniform_gen_scaling=false
                  unbounded_pg=false
                  output_to_files=true
                  # regularize_f=regularize_f
                  # ipopt_max_iter = ipopt_max_iter

                  (opt_net[i], res_summary_dict[i], solution_pm_dict[i]) =
                  VariableFrequencyOPF.multifrequency_opf(
                        hour_mn_data[i],
                        output_folder,
                        obj,
                        gen_areas,
                        area_transfer,
                        gen_zones,
                        zone_transfer,
                        print_results,
                        override_param,
                        fix_f_override,
                        direct_pq,
                        master_subnet,
                        start_vals,
                        uniform_gen_scaling,
                        unbounded_pg,
                        output_to_files,
                        regularize_f = regularize_f,
                        ipopt_max_iter = ipopt_max_iter
                        )
                  # open("test_mn_cost_$(i).json", "w") do f
                  #  write(f, s1)
                  # end

                  # write the binding constriants to a new file (save binding constraints of each case individually)
                  binding_constraints_part = Dict()
                  open("$(output_folder)/binding_constraints.json", "r") do f
                        global binding_constraints_part
                        #bind = read(f, String)
                        binding_constraints_part = JSON.parse(f)
                        binding_constraints[i] = binding_constraints_part
                  end
                  #binding_constraints_json = JSON.json(binding_constraints)

            #end
      end

      # save files of outputs
      if !isdir("$(output_folder)/binding_constraints/$(subnet_folder)")
            mkpath("$(output_folder)/binding_constraints/$(subnet_folder)")
      end
      binding_constraints_json = JSON.json(binding_constraints)
      open("$(output_folder)/binding_constraints/$(subnet_folder)/($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f1
            write(f1, binding_constraints_json)
      end
      if !isdir("$(output_folder)/opt_net/$(subnet_folder)")
            mkpath("$(output_folder)/opt_net/$(subnet_folder)")
      end
      opt_net_string = JSON.json(opt_net)
      open("$(output_folder)/opt_net/$(subnet_folder)/($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
        write(f, opt_net_string)
      end
      if !isdir("$(output_folder)/res_summary/$(subnet_folder)")
            mkpath("$(output_folder)/res_summary/$(subnet_folder)")
      end
      res_summary_string = JSON.json(res_summary_dict)
      open("$(output_folder)/res_summary/$(subnet_folder)/($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
        write(f, res_summary_string)
      end

      return opt_net, res_summary_dict, solution_pm_dict
end

function hour_opf_modified(month_st, day_st, period_st, month_en, day_en, period_en,
      folder:: String,
      subnet_folder:: String,
      output_folder::String,
      obj::String,
      gen_areas = [],
      area_transfer = [],
      plot_results::Bool = true,
      print_results::Bool = false)
      # override_param::Dict{Any}=Dict(),
      # fix_f_override::Bool=false,
      # direct_pq::Bool=true,
      # master_subnet::Int64=1,
      # suffix::String=""
      dir = "$(folder)/"
      #file_name = "$(dir)/RTS_GMLC.m"

      # hour_data: store the data
      hour_data = Dict{Int64, Any}

      # time range
      start_t = DateTime(2020,month_st,day_st,period_st)
      end_t = DateTime(2020,month_en,day_en,period_en)
      range_t = start_t:Hour(1):end_t

      # input the corresponding data of the specific subnetwork
      sub_folder = "$(folder)/enum_net_new/$(subnet_folder)"
      println("reading the file $(sub_folder)")
      hour_data = hour_parse(month_st, day_st, period_st, month_en, day_en, period_en, sub_folder)

      # s = JSON.json(hour_data)
      # open("test_hour_cost.json", "w") do f
      # write(f, s)
      # end

      # dictionaries store the output of the optimization for each period.
      opt_net = Dict()
      res_summary_dict = Dict()
      solution_pm_dict = Dict()
      hour_mn_data = Dict()
      objective_val = Dict()

      # output binding constraints
      binding_constraints = Dict()

      # manually used enumerate_branch to make data of subnetworks from RTS_GMLC.m before
      # running the make_mn_data function
      for i in 1:size(range_t, 1)
            #for j in 1:119
                  #subnetwork_file = "$(dir)/enum_nets/br$(j)/subnetworks.csv"
                  #interface_file = "$(dir)/enum_nets/br$(j)/interfaces.csv"
                  subnetwork_file = "$(dir)/enum_net_new/$(subnet_folder)/subnetworks.csv"
                  interface_file = "$(dir)/enum_net_new/$(subnet_folder)/interfaces.csv"
                  subnetworks = DataFrame(index=Int64[], file=String[], variable_f=Bool[],
                                          f_base=Float64[], f_min=Float64[], f_max=Float64[])
                  interfaces = DataFrame(index=Int64[], subnet_index=Int64[], bus=Int64[],
                                         s_max=Float64[], p_max=Float64[], q_max=Float64[],
                                         loss_param=Float64[])

                  # files ready for input to make_mn_data
                  subnetworks = CSV.read(subnetwork_file, DataFrame)
                  interfaces = CSV.read(interface_file, DataFrame)
                  hour_mn_data[i] = VariableFrequencyOPF.make_mn_data(subnetworks, interfaces, hour_data[i])

                  (opt_net[i], res_summary_dict[i], solution_pm_dict[i]) =
                  VariableFrequencyOPF.multifrequency_opf(hour_mn_data[i], output_folder, obj, gen_areas, area_transfer, print_results)
                  # open("test_mn_cost_$(i).json", "w") do f
                  #  write(f, s1)
                  # end

                  # write the binding constriants to a new file (save binding constraints of each case individually)
                  binding_constraints_part = Dict()
                  open("$(output_folder)/binding_constraints.json", "r") do f
                        global binding_constraints_part
                        #bind = read(f, String)
                        binding_constraints_part = JSON.parse(f)
                        binding_constraints[i] = binding_constraints_part
                  end
                  #binding_constraints_json = JSON.json(binding_constraints)

            #end
      end

      # save files of outputs
      binding_constraints_json = JSON.json(binding_constraints)
      open("$(output_folder)/binding_constraints$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f1
            write(f1, binding_constraints_json)
      end

      opt_net_string = JSON.json(opt_net)
      open("$(output_folder)/opt_net_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
        write(f, opt_net_string)
      end

      res_summary_string = JSON.json(res_summary_dict)
      open("$(output_folder)/res_summary_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
        write(f, res_summary_string)
      end

      # solution_pm_string = JSON.json(solution_pm_dict)
      # open("$(output_folder)/solution_pm_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).json", "w") do f
      #   write(f, solution_pm_string)
      # end

      # plot of the results (optional, now we mostly use the function of multi_plot)
      # if plot_results == true
      #       objval = Array{Float64, 1}()
      #       objval_n = Array{Float64, 1}()
      #       size_obj = Array{DateTime, 1}()
      #       size_obj_n = Array{DateTime, 1}()
      #       for i in 1:size(range_t, 1)
      #             if res_summary_dict[i].status[1] in [LOCALLY_SOLVED, ALMOST_LOCALLY_SOLVED]
      #                   push!(objval, res_summary_dict[i]["objective"][1])
      #                   push!(size_obj, range_t[i])
      #             else
      #                   push!(objval_n, res_summary_dict[i]["objective"][1])
      #                   push!(size_obj_n, range_t[i])
      #             end
      #       end
      #       #size_obj_n = size(range_t, 1) - size_obj
      #       # below are the parts of plotting
      #       println(range_t)
      #       println(objval)
      #       gr()
      #       p = plot(size_obj, objval,  color = :blue, seriestype = :line, title = "optimal result plot",
      #                xlabel = "time/h", ylabel = "optimal $(obj)", right_margin = 10px)
      #       plot!(size_obj_n, objval_n, color = :red, seriestype = :scatter)
      #       # seriestype = :histogram
      #       #savefig(fn)
      #       #savefig(p, )
      #
      #       # save output plot file
      #       savefig(p, "$(output_folder)/opt_res_plot_$(subnet_folder)_($(month_st).$(day_st).$(period_st)__$(month_en).$(day_en).$(period_en)).pdf")
      # end

      return opt_net, res_summary_dict, solution_pm_dict
end


# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/AbaqusReader.jl/blob/master/LICENSE

"""
    element_mapping

This mapping table contains information what node ids locally match
each side of element.
"""
const element_mapping = Dict(
    :Tet4 => Dict(
        :S1 => (:Tri3, [1, 3, 2]),
        :S2 => (:Tri3, [1, 2, 4]),
        :S3 => (:Tri3, [2, 3, 4]),
        :S4 => (:Tri3, [1, 4, 3])),
    :Tet10 => Dict(
        :S1 => (:Tri6, [1, 3, 2, 7, 6, 5]),
        :S2 => (:Tri6, [1, 2, 4, 5, 9, 8]),
        :S3 => (:Tri6, [2, 3, 4, 6, 10, 9]),
        :S4 => (:Tri6, [1, 4, 3, 8, 10, 7])),
    :Hex8 => Dict(
        :S1 => (:Quad4, [1, 2, 3, 4]),
        :S2 => (:Quad4, [5, 8, 7, 6]),
        :S3 => (:Quad4, [1, 5, 6, 2]),
        :S4 => (:Quad4, [2, 6, 7, 3]),
        :S5 => (:Quad4, [3, 7, 8, 4]),
        :S6 => (:Quad4, [4, 8, 5, 1])))

""" Given element code, element side and global connectivity, determine boundary
element. E.g. for Tet4 we have 4 sides S1..S4 and boundary element is of type Tri3.
"""
function create_surface_element(element_type::Symbol, element_side::Symbol,
    element_connectivity::Vector{Int64})

    if !haskey(element_mapping, element_type)
        error("Unable to find surface element for element of type ",
              "$element_type for side $element_side, update element ",
              "mapping table.")
    end

    if !haskey(element_mapping[element_type], element_side)
        error("Unable to find child element side mapping for element ",
              "of type $element_type for side $element_side, update ",
              "element mapping table.")
    end

    surfel, surfel_lconn = element_mapping[element_type][element_side]
    surfel_gconn = element_connectivity[surfel_lconn]
    return surfel, surfel_gconn
end

"""
    create_surface_elements(mesh, surface_name)

Create surface elements for `surface` using mesh `mesh`.
Mesh can be obtained by using `abaqus_read_mesh`.
"""
function create_surface_elements(mesh::Dict, surface_name::String)
    surface = mesh["surface_sets"][surface_name]
    elements = mesh["elements"]
    eltypes = mesh["element_types"]
    result = Tuple{Symbol, Vector{Integer}}[]
    for (elid, side) in surface
        surface_element = create_surface_element(eltypes[elid], side, elements[elid])
        push!(result, surface_element)
    end
    return result
end


using DiffEqJump: AbstractAggregatorAlgorithm, NullAggregator

PDMPProblem(prob,jumps::ConstantRateJump;kwargs...) = PDMPProblem(prob,JumpSet(jumps);kwargs...)
PDMPProblem(prob,jumps::VariableRateJump;kwargs...) = PDMPProblem(prob,JumpSet(jumps);kwargs...)
PDMPProblem(prob,jumps::RegularJump;kwargs...) = PDMPProblem(prob,JumpSet(jumps);kwargs...)
PDMPProblem(prob,jumps::MassActionJump;kwargs...) = PDMPProblem(prob,JumpSet(jumps);kwargs...)
PDMPProblem(prob,jumps::DiffEqJump.AbstractJump...;kwargs...) = PDMPProblem(prob,JumpSet(jumps...);kwargs...)

PDMPProblem(prob,aggregator::AbstractAggregatorAlgorithm,jumps::ConstantRateJump;kwargs...) = PDMPProblem(prob,aggregator,JumpSet(jumps);kwargs...)
PDMPProblem(prob,aggregator::AbstractAggregatorAlgorithm,jumps::VariableRateJump;kwargs...) = PDMPProblem(prob,aggregator,JumpSet(jumps);kwargs...)
PDMPProblem(prob,aggregator::AbstractAggregatorAlgorithm,jumps::RegularJump;kwargs...) = PDMPProblem(prob,aggregator,JumpSet(jumps);kwargs...)
PDMPProblem(prob,aggregator::AbstractAggregatorAlgorithm,jumps::MassActionJump;kwargs...) = PDMPProblem(prob,aggregator,JumpSet(jumps);kwargs...)
PDMPProblem(prob,aggregator::AbstractAggregatorAlgorithm,jumps::DiffEqJump.AbstractJump...;kwargs...) = PDMPProblem(prob,aggregator,JumpSet(jumps...);kwargs...)
PDMPProblem(prob,jumps::JumpSet;kwargs...) = PDMPProblem(prob,NullAggregator(),jumps;kwargs...)

struct DiffeqJumpWrapper
	diffeqpb
	jumps
	u
end

# encode the vector field
function (wrap::DiffeqJumpWrapper)(ẋ, xc, xd, p, t::Float64)
	wrap.diffeqpb.f(ẋ, xc, p, t)
	nothing
end

# encode the rate function
function (wrap::DiffeqJumpWrapper)(rate, xc, xd, p, t::Float64, issum::Bool)
	for ii in eachindex(rate)
		rate[ii] = wrap.jumps.variable_jumps[ii].rate(xc, p, t)
	end
	return sum(rate)
end

# encode the jump function
function (wrap::DiffeqJumpWrapper)(xc, xd, p, t::Float64, ind_reaction::Int64)
	# this is a hack to be able to call affect! from DiffEqJump which requires an integrator as an argument. But if the type of affect! is not enforced, it should work
	@inbounds for ii=1:length(wrap.u)
		wrap.u[ii] = xc[ii]
	end
	wrap.jumps.variable_jumps[ind_reaction].affect!(wrap)
	@inbounds for ii=1:length(wrap.u)
		xc[ii] = wrap.u[ii]
	end
	nothing
end

function PDMPProblem(prob, aggregator::AbstractAggregatorAlgorithm, jumps::JumpSet;
	save_positions = typeof(prob) <: DiffEqBase.AbstractDiscreteProblem ? (false,true) : (true, true), kwargs...)

	@assert isinplace(prob) "The current interface requires the ODE to be written inplace"
	@assert jumps.regular_jump == nothing
	@assert jumps.massaction_jump == nothing

	pb_wrapper = DiffeqJumpWrapper(prob, jumps, copy(prob.u0))

	# get PDMP characteristics
	F = (xdot,xc,xd,p,t::Float64) -> pb_wrapper(xdot,xc,xd,p,t)
	R = (rate,xc,xd,p,t::Float64,issum::Bool) -> pb_wrapper(rate,xc,xd,p,t,issum)
	Delta = (xc,xd,p,t::Float64,ind_reaction::Int) -> pb_wrapper(xc,xd,p,t,ind_reaction)

	xc0 = copy(prob.u0)

	xd0 = [0]
	tspan = prob.tspan
	p = prob.p

	# determine the number of reactions
	nb_reactions = length(jumps.variable_jumps)

	return PDMPProblem(F,R,Delta,nb_reactions,xc0,xd0,p,tspan)
end


#=
read_rdb_hdr.jl
based on
https://gitlab.com/fMRI/toppe/blob/master/+toppe/+utils/read_rdb_hdr.m
=#

export read_rdb_hdr

include("rdb-26_002.jl")

"""
`s = read_rdb_hdr(fid::IOStream)`

Read GE raw (RDB) header for MRI scans

Returns NamedTuple `s` with header values accessible by `s.key`

Matlab version is:
Copyright (c) 2012 by General Electric Company. All rights reserved.

2019-05-22 Julia version by Jeff Fessler
"""
function read_rdb_hdr(fid::IOStream)
	seek(fid, 0)
	rdbm_rev = read(fid, Float32) # raw header (RDBM) revision number

	if rdbm_rev == Float32(26.002)
		s = read_rdb_hdr_26_002(fid)
	else
		throw("unknown RDBM rev $rdbm_rev")
	end

	if (s.rdbm_rev != rdbm_rev)
		throw("rev mismatch: $(s.rdbm_rev) != $rdbm_rev")
	end

	return s
end

"""
`s = read_rdb_hdr(file::String)`
read from `file`
"""
function read_rdb_hdr(file::String)
	return open(read_rdb_hdr, file)
end

"""
`read_rdb_hdr(:test)`
self test
"""
function read_rdb_hdr(test::Symbol)
	file = "/n/ir71/d3/fessler/fmri-data-michelle-L+S/P97792.7"
	if isfile(file)
		return read_rdb_hdr(file).dab[2] == 31
	end
	true
end


struct TriangleMesh_2D <: LinearUnstructuredMesh_2D
    name::String
    points::Vector{Point_2D}
    edges::Vector{SVector{2, UInt32}}
    materialized_edges::Vector{LineSegment_2D}
    faces::Vector{SVector{3, UInt32}}
    materialized_faces::Vector{Triangle_2D}
    edge_face_connectivity::Vector{SVector{2, UInt32}}
    face_edge_connectivity::Vector{SVector{3, UInt32}}
    boundary_edges::Vector{Vector{UInt32}}
    face_sets::Dict{String, Set{UInt32}}
end

function TriangleMesh_2D(;
        name::String = "DefaultMeshName",
        points::Vector{Point_2D} = Point_2D[],
        edges::Vector{SVector{2, UInt32}} = SVector{2, UInt32}[],
        materialized_edges::Vector{LineSegment_2D} = LineSegment_2D[],
        faces::Vector{SVector{3, UInt32}} = SVector{3, U}[],
        materialized_faces::Vector{Triangle_2D} = Triangle_2D[],
        edge_face_connectivity::Vector{SVector{2, UInt32}} = SVector{2, UInt32}[],
        face_edge_connectivity ::Vector{SVector{3, UInt32}} = SVector{3, UInt32}[],
        boundary_edges::Vector{Vector{UInt32}} = Vector{UInt32}[],
        face_sets::Dict{String, Set{UInt32}} = Dict{String, Set{UInt32}}()
    )
        return TriangleMesh_2D(name,
                               points,
                               edges,
                               materialized_edges,
                               faces,
                               materialized_faces,
                               edge_face_connectivity,
                               face_edge_connectivity,
                               boundary_edges,
                               face_sets,
                              )
end

Base.broadcastable(mesh::TriangleMesh_2D) = Ref(mesh)

struct QuadrilateralMesh_2D <: LinearUnstructuredMesh_2D
    name::String
    points::Vector{Point_2D}
    edges::Vector{SVector{2, UInt32}}
    materialized_edges::Vector{LineSegment_2D}
    faces::Vector{SVector{4, UInt32}}
    materialized_faces::Vector{Quadrilateral_2D}
    edge_face_connectivity::Vector{SVector{2, UInt32}}
    face_edge_connectivity::Vector{SVector{4, UInt32}}
    boundary_edges::Vector{Vector{UInt32}}
    face_sets::Dict{String, Set{UInt32}}
end

function QuadrilateralMesh_2D(;
        name::String = "DefaultMeshName",
        points::Vector{Point_2D} = Point_2D[],
        edges::Vector{SVector{2, UInt32}} = SVector{2, UInt32}[],
        materialized_edges::Vector{LineSegment_2D} = LineSegment_2D[],
        faces::Vector{SVector{4, UInt32}} = SVector{4, U}[],
        materialized_faces::Vector{Quadrilateral_2D} = Quadrilateral_2D[],
        edge_face_connectivity::Vector{SVector{2, UInt32}} = SVector{2, UInt32}[],
        face_edge_connectivity ::Vector{SVector{4, UInt32}} = SVector{4, UInt32}[],
        boundary_edges::Vector{Vector{UInt32}} = Vector{UInt32}[],
        face_sets::Dict{String, Set{UInt32}} = Dict{String, Set{UInt32}}()
    )
        return QuadrilateralMesh_2D(name,
                                    points,
                                    edges,
                                    materialized_edges,
                                    faces,
                                    materialized_faces,
                                    edge_face_connectivity,
                                    face_edge_connectivity,
                                    boundary_edges,
                                    face_sets,
                                   )
end

Base.broadcastable(mesh::QuadrilateralMesh_2D) = Ref(mesh)

# Axis-aligned bounding box, in 2d a rectangle.
function boundingbox(mesh::LinearUnstructuredMesh_2D; boundary_shape::String="Unknown")
    # If the mesh does not have any quadratic faces, the bounding_box may be determined
    # entirely from the points.
    nsides = length(mesh.boundary_edges)
    if nsides !== 0
        boundary_edge_IDs = reduce(vcat, mesh.boundary_edges)
        point_IDs = reduce(vcat, mesh.edges[boundary_edge_IDs])
        return boundingbox(mesh.points[point_IDs])
    else
        return boundingbox(mesh.points)
    end
end

# A vector of SVectors, denoting the edge ID each face is connected to.
function face_edge_connectivity(mesh::QuadrilateralMesh_2D)
    if length(mesh.edges) === 0
        @error "Mesh does not have edges!"
    end
    # A vector of MVectors of zeros for each face
    # Each MVector is the length of the number of edges
    face_edge = [MVector{4, UInt32}(0, 0, 0, 0) for _ in eachindex(mesh.faces)]
    # For each face in the mesh, generate the edges.
    # Search for the index of the edge in the mesh.edges vector
    # Insert the index of the edge into the face_edge connectivity vector
    for i in eachindex(mesh.faces)
        for (j, edge) in enumerate(edges(mesh.faces[i]))
            face_edge[i][j] = searchsortedfirst(mesh.edges, SVector(edge.data))
        end
        if any(x->x === 0x00000000, face_edge[i])
            @error "Could not determine the face/edge connectivity of face $i"
        end
    end
    return [SVector(sort(conn).data) for conn in face_edge]
end

# A vector of SVectors, denoting the edge ID each face is connected to.
function face_edge_connectivity(mesh::TriangleMesh_2D)
    if length(mesh.edges) === 0
        @error "Mesh does not have edges!"
    end
    # A vector of MVectors of zeros for each face
    # Each MVector is the length of the number of edges
    face_edge = [MVector{3, UInt32}(0, 0, 0) for _ in eachindex(mesh.faces)]
    # For each face in the mesh, generate the edges.
    # Search for the index of the edge in the mesh.edges vector
    # Insert the index of the edge into the face_edge connectivity vector
    for i in eachindex(mesh.faces)
        for (j, edge) in enumerate(edges(mesh.faces[i]))
            face_edge[i][j] = searchsortedfirst(mesh.edges, SVector(edge.data))
        end
        if any(x->x === 0x00000000, face_edge[i])
            @error "Could not determine the face/edge connectivity of face $i"
        end
    end
    return [SVector(sort(conn).data) for conn in face_edge]
end


"""
    module AtomicMod

Simple module for atom type.
"""
module AtomicMod

using ..Utility:str_w_spaces
using ..BandTools:band_energy
using ..BandTools:smearing_energy

"""
    struct atom

Hold basic atomic information

- `name::String`
- `Z::Int64` Atomic number
- `row::Float64` Periodic table row
- `col::Float64` Periodic table col
- `mass::Float64` Mass in amu
- `nval::Float64` Number of valence electrons in TB calculation.
- `nsemicore::Int64` Number of semicore electrons. Depends on pseudopotential choice
- `nwan::Int64` Number of TB orbitals
- `orbitals::Array{Symbol,1}` Names of the orbitals, like `[:s, :p]`
- `total_energy::Float64` DFT total energy, depends on pseudopotentials.
- `eigs::Dict` orbital eigenvalues of isolated non-spin-polarized nuetral atom.
- `energy_offset::Float64` energy to add to TB calculation to make isolated atoms have zero energy.
- `U::Float64` U value for Ewald correction

"""
struct atom
    name::String
    Z::Int64
    row::Float64
    col::Float64
    mass::Float64
    nval::Float64
    nsemicore::Int64
    nwan::Int64
    orbitals::Array{Symbol,1}
    total_energy::Float64
    eigs::Dict
    energy_offset::Float64
    U::Float64
end

#printing
Base.show(io::IO, a::atom) = begin

    println(io,str_w_spaces([a.name, "Z=", a.Z, "Mass=",a.mass, "RC=",a.row, a.col]))
    println(io,str_w_spaces(["Nval=", a.nval, "Nsemicore=", a.nsemicore, "Orbitals=",a.orbitals]))
    println(io,"U = ",a.U)
    println(io,"total energy=", a.total_energy)
    for i in a.orbitals
        println(i, "  " , a.eigs[i])
    end
end

"""
    function makeatom(name, Z, row, col, mass, nval, nsemicore, orbitals, etot, eigs, vac_potential=0.0, U=0.0)

Constructor for atom.
"""
function makeatom(name, Z, row, col, mass, nval, nsemicore, orbitals, etot, eigs, vac_potential=0.0, U=0.0)
    #vac potential in ryd

    orb2 = map(x->convert(Symbol, x), orbitals)

    nwan = 0

    for o in orb2
        if o == :s
            nwan += 2
        elseif o == :p
            nwan += 6
        elseif o == :d
            nwan += 10
        elseif o == :f
            nwan += 14
        else
            exit("oribtal entry incorrect, good values are s p d f: ", orbitals, orb2)
        end
    end

    convert_ev_ryd = 1.0/13.605693122
    EIGS = zeros(Int64(nwan/2), 1)
    c=0
    for (i,o) in enumerate(orb2)
        if o == :s
            c+=1
            EIGS[c,1] = eigs[i]*convert_ev_ryd
        elseif o == :p
            c += 3
            EIGS[c-2,1] = eigs[i]*convert_ev_ryd
            EIGS[c-1,1] = eigs[i]*convert_ev_ryd
            EIGS[c  ,1] = eigs[i]*convert_ev_ryd
        elseif o == :d
            c += 5
#            EIGS[c-5,1] = eigs[i]*convert_ev_ryd
            EIGS[c-4,1] = eigs[i]*convert_ev_ryd
            EIGS[c-3,1] = eigs[i]*convert_ev_ryd
            EIGS[c-2,1] = eigs[i]*convert_ev_ryd
            EIGS[c-1,1] = eigs[i]*convert_ev_ryd
            EIGS[c  ,1] = eigs[i]*convert_ev_ryd
        elseif o == :f
            c += 7
#            EIGS[c-7,1] = eigs[i]*convert_ev_ryd
            EIGS[c-6,1] = eigs[i]*convert_ev_ryd
            EIGS[c-5,1] = eigs[i]*convert_ev_ryd
            EIGS[c-4,1] = eigs[i]*convert_ev_ryd
            EIGS[c-3,1] = eigs[i]*convert_ev_ryd
            EIGS[c-2,1] = eigs[i]*convert_ev_ryd
            EIGS[c-1,1] = eigs[i]*convert_ev_ryd
            EIGS[c  ,1] = eigs[i]*convert_ev_ryd
        end
    end

    EIGS = EIGS .- vac_potential

#    println("EIGS after subtraction")
#    println(EIGS)

    band_energy0, efermi = band_energy(EIGS, [1.0], nval, returnef=true)
    smear_energy = smearing_energy(EIGS, [1.0], efermi)

#    if name=="Ta"
#        println("name $name")
#        println("EIGS before subtraction")
#        println(EIGS)
#        println("band_energy0 ", band_energy0, " ", efermi)
#    end

#    shift = (0.0 - band_energy0)/nval

    energy_offset = -(band_energy0+smear_energy)  #this is the atomic energy, no spin, after vac correction

    if name == "X" || name == "Xa"
        energy_offset = 0.0
    end

#    println("$name energy_offset $energy_offset    $band_energy0 $smear_energy  efermi $efermi")

    d = Dict()

    for (o, i) in zip(orb2, eigs)
#        d[o] = (i * convert_ev_ryd + shift)
        d[o] = (i * convert_ev_ryd - vac_potential)

#        println("d ", d[o], " ", o)
    end

#    band_energy_new = band_energy(EIGS .- band_energy0 / nval , [1.0], nval)#
#
#    println("Loading $name, new band energy $band_energy_new, ", d[:s])

    return atom(name, Z, row, col, mass, nval, nsemicore, nwan, orb2, etot, d, energy_offset, U)

end

end #end module