using CPLEX, Base.Test, MathOptInterface

const MOI  = MathOptInterface
const MOIT = MathOptInterface.Test
const MOIB = MathOptInterface.Bridges

@testset "Unit Tests" begin
    config = MOIT.TestConfig()
    solver = CPLEX.Optimizer()

    MOIT.unittest(solver, config, [
        "solve_affine_interval",     # not implemented
        "solve_qp_edge_cases",       # not implemented
        "solve_qcp_edge_cases",      # not implemented
        "solve_objbound_edge_cases", # Requires MOI.ObjectiveBound()
    ])
    @testset "solve_affine_interval" begin
        MOIT.solve_affine_interval(
            MOIB.SplitInterval{Float64}(CPLEX.Optimizer()),
            config
        )
    end
    MOIT.modificationtest(solver, config, [
        "solve_func_scalaraffine_lessthan"
    ])
end
@testset "Linear tests" begin
    linconfig = MOIT.TestConfig()
    @testset "Default Solver"  begin
        solver = CPLEX.Optimizer()
        MOIT.contlineartest(solver, linconfig, ["linear10","linear11",
                "linear12", "linear8a","linear8b","linear8c"])
    end
    @testset "linear10" begin
        MOIT.linear10test(
            MOIB.SplitInterval{Float64}(CPLEX.Optimizer()),
            MOIT.TestConfig()
        )
    end
    @testset "No certificate" begin
        MOIT.linear12test(
            CPLEX.Optimizer(),
            MOIT.TestConfig(infeas_certificates=false)
        )
    end
end
@testset "Integer Linear tests" begin
    intconfig = MOIT.TestConfig()
    solver = CPLEX.Optimizer()
    MOIT.intlineartest(solver, intconfig, ["int1", # requires objbound impl
                                           "int2", "int3"])
    # 3 is ranged, 2 has sos
    @testset "int3" begin
        MOIT.int3test(
            MOIB.SplitInterval{Float64}(CPLEX.Optimizer()),
            MOIT.TestConfig()
        )
    end
end
@testset "ModelLike tests" begin
    solver = CPLEX.Optimizer()
    @testset "nametest" begin
        MOIT.nametest(solver)
    end
    @testset "validtest" begin
        MOIT.validtest(solver)
    end
    @testset "emptytest" begin
        MOIT.emptytest(solver)
    end
    @testset "orderedindicestest" begin
        MOIT.orderedindicestest(solver)
    end
    @testset "copytest" begin
        MOIT.copytest(solver, CPLEX.Optimizer())
    end
end


@testset "Career progression report" begin

@testset "function generateCareerProgression" begin
    report = generateCareerProgression( mpSim, "Test", "Sim1", "Sim100",
        "Sim500", "Sim1000" )
    @test all( haskey.( Ref( report ), ["Sim1", "Sim100", "Sim500"] ) ) &&
        !any( haskey.( Ref( report ), ["Test", "Sim1000"] ) )
    @test all( [report["Sim1"][1] == 240, report["Sim100"][1] == 240,
        report["Sim500"][1] == 240] )
    @test all( [
        all( report["Sim1"][2][:, "timeIndex"] .== [0, 24] ),
        all( report["Sim100"][2][:, "timeIndex"] .== [36, 60, 156] ),
        all( report["Sim500"][2][:, "timeIndex"] .==
            [192, 216, 252] )] )
    @test all( [
        all( report["Sim1"][2][:, "transition"] .== ["EW", "B-"] ),
        all( report["Sim100"][2][:, "transition"] .==
            ["EW", "Promotion", "PE"] ),
        all( report["Sim500"][2][:, "transition"] .==
            ["EW", "Promotion", "Promotion"] )] )
end  # @testset "function generateCareerProgression"

end  # @testset "Career progression report"


function forward(backend::GPUBackend, state::ConcatLayerState, inputs::Vector{Blob})
  output = state.blobs[1]
  shift = 0
  for i = 1:length(inputs)
    copy_to_shifted!(backend, output, inputs[i], (state.layer.dim, shift))
    shift += size(inputs[i], state.layer.dim)
  end
end

function backward(backend::GPUBackend, state::ConcatLayerState, inputs::Vector{Blob}, diffs::Vector{Blob})
  grad = state.blobs_diff[1]
  shift = 0
  for i = 1:length(diffs)
    if !isa(diffs[i], NullBlob)
      copy_from_shifted!(backend, diffs[i], grad, (state.layer.dim, shift))
    end
    shift += size(inputs[i], state.layer.dim)
  end
end


printstyled("  ungap!\n", color=:light_green)
printstyled("    subsample negative time gaps (Issue 29)\n", color=:light_green)

gf1 = path * "/SampleFiles/SEED/CIRIO__BHE___2017101.mseed"
opf = path * "/SampleFiles/SEED/obspy.dat"

# read merge/ungap target file
S = read_data(gf1)
C1 = deepcopy(S[1])
i1 = C1.t[2,1]
merge!(S)
ungap!(S)
C2 = S[1]
i2 = C2.t[2,1]

# read ObsPy merge output to compare
io = open(opf, "r")
X2 = Array{Int32,1}(undef, length(C2.x))
read!(io, X2)
X = map(Float32, X2)

# these should be approximately equal
@test isapprox(C2.x, X)


using Knet
using AutoGrad
import AutoGrad: gcheck
using Test
using CUDA

include("layers.jl")


getindex(a::Number, ::Block{0}) = a

axes(A::AbstractTriangular{<:Any,<:AbstractBlockMatrix}) = axes(parent(A))
axes(A::HermOrSym{<:Any,<:AbstractBlockMatrix}) = axes(parent(A))

Base.print_matrix_row(io::IO,
        X::Union{AbstractTriangular{<:Any,<:AbstractBlockMatrix},
                 Symmetric{<:Any,<:AbstractBlockMatrix},
                 Hermitian{<:Any,<:AbstractBlockMatrix},
                 Adjoint{<:Any,<:AbstractBlockMatrix},
                 Transpose{<:Any,<:AbstractBlockMatrix}}, A::Vector,
        i::Integer, cols::AbstractVector, sep::AbstractString) =
        _blockarray_print_matrix_row(io, X, A, i, cols, sep)


using .Plots
function corner(
    table,
    labels = map(string, Tables.columnnames(table)),
    units = ["" for _ in labels],
    ;
    title="",
    plotcontours=true,
    plotscatter=true,
    plotpercentiles=[15,50,84],
    histfunc=prepare_hist,
    filterscatter=true,
    hist_kwargs=(;),
    hist2d_kwargs=(;),
    contour_kwargs=(;),
    scatter_kwargs=(;),
    percentiles_kwargs=(;),
    appearance=(;),
    lens=nothing,
    lens_kwargs=(;),
    bonusplot=nothing,

    # Format of 1D histogram titles
    titlefmt="\$\\mathrm{%s = %.2f^{+%.2f}_{-%.2f} \\; %s}\$",

    # By how much should we oversize the limits? Use this option with caution.
    lim_factor = 1.00,

    # Plot layout options (subject to change)
    # Overall plot scale factor
    sf = 1,
    pad_top = 20sf,
    pad_left = nothing,
    w=40sf,
    h=40sf,
    p=nothing, # 1sf noramlly, 10sf if 3D
    pad_right = 0sf,
    pad_bottom = 10sf,
    pad_bonus = nothing,
    # Remaining kwargs are for the overall plot group
    kwargs...
)

if !Tables.istable(table)
    error("You must supply data in a format consistent with the definition in Tables.jl, e.g. a Named Tuple of vectors.")
end

if any(!isascii, labels) ||  any(!isascii, units)
    @warn "Non-ascii labels or units detected. Some plotting backends require passing these using LaTeX escapes, e.g. \\alpha instead of α"
end

columns = Tables.columnnames(table)
n = length(columns)

# filter out empty (0 variance) variables
ii_empty = findall(==(0),  var.(collect(Tables.columns(table))))
if length(ii_empty) > 0
    @warn "Skipping entries with zero variance" skip=columns[ii_empty]
end

if isnothing(pad_left)
    pad_left = 20sf + 1sf*length(columns)
end

# Merge keywords for different subplots & series
hist_kwargs=merge((; nbins=20, color=:black, yticks=[], minorgrid=false, minorticks=false, ylims=(0,NaN)), hist_kwargs)
hist2d_kwargs=merge((; nbins=32, color=:Greys, colorbar=:none), hist2d_kwargs)
contour_kwargs=merge((; color=:black, linewidth=1.5), contour_kwargs)
scatter_kwargs=merge((; color=:black, alpha=0.5, markersize=0.6, markerstrokewidth=0), scatter_kwargs)
percentiles_kwargs=merge((;linestyle=:dash, color=:black), percentiles_kwargs)
appearance = merge((;
    framestyle=:box,
    xrotation = 45,
    yrotation = 45,
    # margin=-6Measures.mm,
    grid=:none,
    xformatter=:plain,
    yformatter=:plain,
    zformatter=t->"",
    titlefontsize=10,
    tick_direction=:out,
    label="",
), appearance)

threeD = get(hist2d_kwargs, :seriestype, nothing) == :wireframe
# Increase padding for 3D plots
if isnothing(p)
    p = threeD ? 10sf : 1sf
end

if isnothing(pad_bonus)
    pad_bonus = iseven(n) ? 20sf : 5sf
end

# Calculate a default reasonable size unless overriden
# s = 250*n
# kwargs = merge((;size = (s,s)), kwargs)

# Rather than messing with linking axes, let's just take over setting the plot limits ourselves
ex = extrema.(collect(Tables.columns(table)))

mids = [(ex[1]+ex[2])/2 for ex in ex]
widths = [(ex[2]-ex[1])/2 for ex in ex]
mins = [mid-width*lim_factor for (mid,width) in zip(mids,widths)]
maxs = [mid+width*lim_factor for (mid,width) in zip(mids,widths)]
# mins = [ex[1] for ex in ex]
# maxs = [ex[2] for ex in ex]

full_width = (pad_left+(w+p)*n+pad_right)
full_height = (pad_top+(h+p)*n+pad_bottom)
parent = RecipesBase.plot(
    # size=((w+p)*n, ((w+p)*n+pad_top)),
    size=(full_width*4,full_height*4),
    framestyle=:none,
    title=title;
    # left_margin=-10Measures.mm,
    background_color_inside=:transparent,
    kwargs...
)

fmt(label, unit) = "\$\\mathrm{$label \\; $unit}\$"

# Build grid of nxn plots
k = 1
for row in 1:n, col in 1:n

    if row < col
        # push!(subplots, emptyplot())
        continue
    end
    k+=1

    # fmt(label) = "\$\\mathrm{$label}\$"
    kw = (;
        title = row == col ? labels[row] : "",
        xguide = threeD || row == n ? fmt(labels[col], units[col]) : "",
        yguide = col == 1  && row > 1 ? fmt(labels[row], units[row]) : "",
        xformatter = threeD || row == n ? appearance.xformatter : t->"",
        yformatter = threeD || col == 1 && row > 1 ? appearance.yformatter : t->"",

        # For crudlely tweaking aspect ratios of the plots to account for
        # top_margin = row == 1 ? 10Measures.mm : :match,
        # left_margin = col == 1 ? 8Measures.mm : :match,
        # right_margin = col == n ? 6Measures.mm : :match,

        xlims=(mins[col], maxs[col]),
        label="",
        subplot=k,
    )
    x=(col-1)*(w+p)+pad_left
    y=(row-1)*(h+p)+pad_top
    inset = bbox(x*Measures.mm, y*Measures.mm, w*Measures.mm, h*Measures.mm, :left)

    if row != col
        kw = (;kw..., ylims=(mins[row], maxs[row]))
    end

    table_keys = Tables.columnnames(table)
    if row == col
        # 1D histogram
        unit = units[row]
        hist(Tables.getcolumn(table, table_keys[row]), histfunc, merge(appearance, kw, hist_kwargs, (;inset)), plotpercentiles, merge(kw, percentiles_kwargs), titlefmt, unit)
    else row > col
        # 2D histogram
        hist(Tables.getcolumn(table, table_keys[row]), Tables.getcolumn(table, table_keys[col]), histfunc, merge(appearance, kw, hist2d_kwargs, (;inset)), merge(kw,contour_kwargs), merge(kw,scatter_kwargs), plotcontours, plotscatter, filterscatter)
    end
    # push!(subplots, subplot)
end

if !isnothing(lens)
    k += 1
    nspan = floor(mean(1:n))
    x=nspan*(w+p)+pad_bonus+pad_left
    y=pad_top
    bonuswidth = full_width-x
    bonusheight = full_height-pad_bottom-pad_top-nspan*(h+p) - pad_bonus
    inset = bbox(x*Measures.mm, y*Measures.mm, bonuswidth*Measures.mm, bonusheight*Measures.mm, :left)
    if lens ∈ columns
        kw = (;label="", subplot=k, title=labels[findfirst(==(lens), columns)])
        hist(Tables.getcolumn(table, lens), histfunc, merge(appearance, kw, hist_kwargs, (;inset), get(lens_kwargs,:hist_kwargs,(;))), plotpercentiles, merge(kw, percentiles_kwargs, get(lens_kwargs,:hist_kwargs,(;))))
    elseif lens[1] ∈ columns && lens[2] ∈ columns
        kw = (;label="", subplot=k, yguide=fmt(labels[findfirst(==(lens[1]), columns)]), xguide=fmt(labels[findfirst(==(lens[2]), columns)]))
        hist(Tables.getcolumn(table, lens[1]), Tables.getcolumn(table, lens[2]), histfunc, merge(appearance, kw, hist2d_kwargs, (;inset), lens_kwargs, get(lens_kwargs,:hist2d_kwargs,(;))), merge(kw,contour_kwargs,get(lens_kwargs,:contour_kwargs,(;))), merge(kw,scatter_kwargs,get(lens_kwargs,:scatter_kwargs,(;))), get(lens_kwargs,:plotcontours,plotcontours), get(lens_kwargs,:plotscatter,plotscatter), get(lens_kwargs,:filterscatter,filterscatter))
    end
elseif !isnothing(bonusplot)
    k += 1
    nspan = floor(mean(1:n))
    x=nspan*(w+p)+pad_bonus+pad_left
    y=pad_top
    bonuswidth = full_width-x-pad_bonus
    bonusheight = full_height-pad_bottom-pad_top-nspan*(h+p) - pad_bonus
    inset = bbox(x*Measures.mm, y*Measures.mm, bonuswidth*Measures.mm, bonusheight*Measures.mm, :left)
    kw = (;inset,subplot=k)
    bonusplot(kw)
end

return parent
end


function optimize(diag::Union{Diagram,Tuple,AbstractVector}, optlevel = 1; verbose = 0, kwargs...)
    single = false
    diag = deepcopy(diag)
    if diag isa Diagram
        single = true
        diag = [diag,]
    end
    removeOneChildParent!(diag, verbose = verbose)
    removeDuplicatedLeaves!(diag, verbose = verbose)
    return single ? diag[1] : diag
end

"""
    removeOneChildParent!(diags::AbstractVector; verbose = 0)

    remove duplicated nodes such as:  ---> ver4 ---> InteractionId. Leaf will not be touched!
"""
function removeOneChildParent!(diags::AbstractVector; verbose = 0)
    verbose > 0 && println("remove nodes with only one child.")
    for diag in diags
        #deep first search, remove one-child parent from the leaf level first
        removeOneChildParent!(diag.subdiagram)
        #then remove the one-child subdiagram of the current diagram
        for (si, subdiag) in enumerate(diag.subdiagram)
            if length(subdiag.subdiagram) == 1
                subdiag.subdiagram[1].factor *= subdiag.factor
                diag.subdiagram[si] = subdiag.subdiagram[1]
            end
        end
    end
    return diags
end

"""
    removeDuplicatedLeaves!(diags::AbstractVector; verbose = 0)

    remove duplicated nodes such as:  ---> ver4 ---> InteractionId. Leaf will not be touched!
"""
function removeDuplicatedLeaves!(diags::AbstractVector; verbose = 0)
    verbose > 0 && println("remove duplicated leaves.")
    leaves = []
    for diag in diags
        #leaves must be the propagators
        append!(leaves, collect(Leaves(diag)))
    end
    # println([d.hash for d in leaves])
    sort!(leaves, by = x -> x.hash) #sort the hash of the leaves in an asscend order
    unique!(x -> x.hash, leaves) #filter out the leaves with the same hash number

    for l in leaves
        #make sure all leaves are either Green's functions or interactions
        @assert l.id isa PropagatorId
    end

    function uniqueLeaves(_diags::AbstractVector)
        ############### find the unique Leaves #####################
        uniqueDiag = []
        mapping = Dict{Int,Any}()
        for diag in _diags
            flag = true
            for (ei, e) in enumerate(uniqueDiag)
                if e.factor ≈ diag.factor && e.id == diag.id
                    mapping[diag.hash] = e
                    flag = false
                    break
                end
            end
            if flag
                push!(uniqueDiag, diag)
                # push!(mapping, length(uniqueDiag))
                mapping[diag.hash] = diag
            end
        end
        return uniqueDiag, mapping
    end

    # println(leaves)
    green = [l for l in leaves if l.id isa BareGreenId]
    interaction = [l for l in leaves if l.id isa BareInteractionId]
    greenN = [l for l in leaves if l.id isa BareGreenNId]
    hopping = [l for l in leaves if l.id isa BareHoppingId]
    # println(green)
    # println(interaction)

    uniqueGreen, greenMap = uniqueLeaves(green)
    uniqueGreenN, greenNMap = uniqueLeaves(greenN)
    uniqueInteraction, interactionMap = uniqueLeaves(interaction)
    uniqueHopping, hoppingMap = uniqueLeaves(hopping)
    # println(uniqueInteraction)
    # display(greenMap)

    verbose > 0 && length(green) > 0 && println("Number of independent Greens $(length(green)) → $(length(uniqueGreen))")
    verbose > 0 && length(greenN) > 0 && println("Number of independent GreenNs $(length(greenN)) → $(length(uniqueGreenN))")
    verbose > 0 && length(interaction) > 0 && println("Number of independent Interactions $(length(interaction)) → $(length(uniqueInteraction))")
    verbose > 0 && length(hopping) > 0 && println("Number of independent Hopping $(length(hopping)) → $(length(uniqueHopping))")

    for diag in diags
        for n in PreOrderDFS(diag)
            for (si, subdiag) in enumerate(n.subdiagram)
                @assert (n.id isa PropagatorId) == false "the diagram $n with subdiagrams cannot be a proapgator!"

                if subdiag.id isa PropagatorId
                    if subdiag.id isa BareGreenId
                        n.subdiagram[si] = greenMap[subdiag.hash]
                    elseif subdiag.id isa BareInteractionId
                        n.subdiagram[si] = interactionMap[subdiag.hash]
                    elseif subdiag.id isa BareGreenNId
                        n.subdiagram[si] = greenNMap[subdiag.hash]
                    elseif subdiag.id isa BareHoppingId
                        n.subdiagram[si] = hoppingMap[subdiag.hash]
                    else
                        error("not implemented!")
                    end
                end
            end
        end
    end

    return uniqueGreen, uniqueInteraction
    # return diags
end


# This file is a part of Julia. License is MIT: https://julialang.org/license

## type join (closest common ancestor, or least upper bound) ##

"""
    typejoin(T, S)

Return the closest common ancestor of `T` and `S`, i.e. the narrowest type from which
they both inherit.
"""
typejoin() = (@_pure_meta; Bottom)
typejoin(@nospecialize(t)) = (@_pure_meta; t)
typejoin(@nospecialize(t), ts...) = (@_pure_meta; typejoin(t, typejoin(ts...)))
typejoin(@nospecialize(a), @nospecialize(b)) = (@_pure_meta; join_types(a, b, typejoin))

function join_types(@nospecialize(a), @nospecialize(b), f::Function)
    if a <: b
        return b
    elseif b <: a
        return a
    elseif isa(a,UnionAll)
        return UnionAll(a.var, join_types(a.body, b, f))
    elseif isa(b,UnionAll)
        return UnionAll(b.var, join_types(a, b.body, f))
    elseif isa(a,TypeVar)
        return f(a.ub, b)
    elseif isa(b,TypeVar)
        return f(a, b.ub)
    elseif isa(a,Union)
        a′ = f(a.a,a.b)
        return a′ === a ? typejoin(a, b) : f(a′, b)
    elseif isa(b,Union)
        b′ = f(b.a,b.b)
        return b′ === b ? typejoin(a, b) : f(a, b′)
    elseif a <: Tuple
        if !(b <: Tuple)
            return Any
        end
        ap, bp = a.parameters, b.parameters
        lar = length(ap)::Int; lbr = length(bp)::Int
        if lar == 0
            return Tuple{Vararg{tailjoin(bp,1,f)}}
        end
        if lbr == 0
            return Tuple{Vararg{tailjoin(ap,1,f)}}
        end
        laf, afixed = full_va_len(ap)
        lbf, bfixed = full_va_len(bp)
        if laf < lbf
            if isvarargtype(ap[lar]) && !afixed
                c = Vector{Any}(uninitialized, laf)
                c[laf] = Vararg{f(unwrapva(ap[lar]), tailjoin(bp,laf,f))}
                n = laf-1
            else
                c = Vector{Any}(uninitialized, laf+1)
                c[laf+1] = Vararg{tailjoin(bp,laf+1,f)}
                n = laf
            end
        elseif lbf < laf
            if isvarargtype(bp[lbr]) && !bfixed
                c = Vector{Any}(uninitialized, lbf)
                c[lbf] = Vararg{f(unwrapva(bp[lbr]), tailjoin(ap,lbf,f))}
                n = lbf-1
            else
                c = Vector{Any}(uninitialized, lbf+1)
                c[lbf+1] = Vararg{tailjoin(ap,lbf+1,f)}
                n = lbf
            end
        else
            c = Vector{Any}(uninitialized, laf)
            n = laf
        end
        for i = 1:n
            ai = ap[min(i,lar)]; bi = bp[min(i,lbr)]
            ci = f(unwrapva(ai),unwrapva(bi))
            c[i] = i == length(c) && (isvarargtype(ai) || isvarargtype(bi)) ? Vararg{ci} : ci
        end
        return Tuple{c...}
    elseif b <: Tuple
        return Any
    end
    while b !== Any
        if a <: b.name.wrapper
            while a.name !== b.name
                a = supertype(a)
            end
            aprimary = unwrap_unionall(a.name.wrapper)
            # join on parameters
            n = length(a.parameters)
            if n == 0
                return aprimary
            end
            p = Vector{Any}(uninitialized, n)
            for i = 1:n
                ai, bi = a.parameters[i], b.parameters[i]
                if ai === bi || (isa(ai,Type) && isa(bi,Type) && typeseq(ai,bi))
                    p[i] = ai
                else
                    p[i] = aprimary.parameters[i]
                end
            end
            return rewrap_unionall(a.name.wrapper{p...}, a.name.wrapper)
        end
        b = supertype(b)
    end
    return Any
end

"""
    promote_typejoin(T, S)

Compute a type that contains both `T` and `S`, which could be
either a parent of both types, or a `Union` if appropriate.
Falls back to [`typejoin`](@ref).
"""
promote_typejoin(@nospecialize(a), @nospecialize(b)) =
    (@_pure_meta; join_types(a, b, promote_typejoin))
promote_typejoin(::Type{Nothing}, ::Type{T}) where {T} =
    isconcretetype(T) ? Union{T, Nothing} : Any
promote_typejoin(::Type{T}, ::Type{Nothing}) where {T} =
    isconcretetype(T) ? Union{T, Nothing} : Any
promote_typejoin(::Type{Missing}, ::Type{T}) where {T} =
    isconcretetype(T) ? Union{T, Missing} : Any
promote_typejoin(::Type{T}, ::Type{Missing}) where {T} =
    isconcretetype(T) ? Union{T, Missing} : Any
promote_typejoin(::Type{Nothing}, ::Type{Missing}) = Union{Nothing, Missing}
promote_typejoin(::Type{Missing}, ::Type{Nothing}) = Union{Nothing, Missing}
promote_typejoin(::Type{Nothing}, ::Type{Nothing}) = Nothing
promote_typejoin(::Type{Missing}, ::Type{Missing}) = Missing

# Returns length, isfixed
function full_va_len(p)
    isempty(p) && return 0, true
    last = p[end]
    if isvarargtype(last)
        N = unwrap_unionall(last).parameters[2]
        if isa(N, Integer)
            return (length(p) + N - 1)::Int, true
        end
        return length(p)::Int, false
    end
    return length(p)::Int, true
end

# reduce join_types over A[i:end]
function tailjoin(A, i, f::Function)
    if i > length(A)
        return unwrapva(A[end])
    end
    t = Bottom
    for j = i:length(A)
        t = f(t, unwrapva(A[j]))
    end
    return t
end

## promotion mechanism ##

"""
    promote_type(type1, type2)

Promotion refers to converting values of mixed types to a single common type.
`promote_type` represents the default promotion behavior in Julia when
operators (usually mathematical) are given arguments of differing types.
`promote_type` generally tries to return a type which can at least approximate
most values of either input type without excessively widening.  Some loss is
tolerated; for example, `promote_type(Int64, Float64)` returns
[`Float64`](@ref) even though strictly, not all [`Int64`](@ref) values can be
represented exactly as `Float64` values.

```jldoctest
julia> promote_type(Int64, Float64)
Float64

julia> promote_type(Int32, Int64)
Int64

julia> promote_type(Float32, BigInt)
BigFloat

julia> promote_type(Int16, Float16)
Float16

julia> promote_type(Int64, Float16)
Float16

julia> promote_type(Int8, UInt16)
UInt16
```
"""
function promote_type end

promote_type()  = Bottom
promote_type(T) = T
promote_type(T, S, U, V...) = (@_inline_meta; promote_type(T, promote_type(S, U, V...)))

promote_type(::Type{Bottom}, ::Type{Bottom}) = Bottom
promote_type(::Type{T}, ::Type{T}) where {T} = T
promote_type(::Type{T}, ::Type{Bottom}) where {T} = T
promote_type(::Type{Bottom}, ::Type{T}) where {T} = T

function promote_type(::Type{T}, ::Type{S}) where {T,S}
    @_inline_meta
    # Try promote_rule in both orders. Typically only one is defined,
    # and there is a fallback returning Bottom below, so the common case is
    #   promote_type(T, S) =>
    #   promote_result(T, S, result, Bottom) =>
    #   typejoin(result, Bottom) => result
    promote_result(T, S, promote_rule(T,S), promote_rule(S,T))
end

"""
    promote_rule(type1, type2)

Specifies what type should be used by [`promote`](@ref) when given values of types `type1` and
`type2`. This function should not be called directly, but should have definitions added to
it for new types as appropriate.
"""
function promote_rule end

promote_rule(::Type{<:Any}, ::Type{<:Any}) = Bottom
# To fix ambiguities
promote_rule(::Type{Any}, ::Type{<:Any}) = Any
promote_rule(::Type{<:Any}, ::Type{Any}) = Any
promote_rule(::Type{Any}, ::Type{Any}) = Any

promote_result(::Type{<:Any},::Type{<:Any},::Type{T},::Type{S}) where {T,S} = (@_inline_meta; promote_type(T,S))
# If no promote_rule is defined, both directions give Bottom. In that
# case use typejoin on the original types instead.
promote_result(::Type{T},::Type{S},::Type{Bottom},::Type{Bottom}) where {T,S} = (@_inline_meta; typejoin(T, S))

"""
    promote(xs...)

Convert all arguments to a common type, and return them all (as a tuple).
If no arguments can be converted, an error is raised.

# Examples
```jldoctest
julia> promote(Int8(1), Float16(4.5), Float32(4.1))
(1.0f0, 4.5f0, 4.1f0)
```
"""
function promote end

function _promote(x::T, y::S) where {T,S}
    @_inline_meta
    R = promote_type(T, S)
    return (convert(R, x), convert(R, y))
end
promote_typeof(x) = typeof(x)
promote_typeof(x, xs...) = (@_inline_meta; promote_type(typeof(x), promote_typeof(xs...)))
function _promote(x, y, z)
    @_inline_meta
    R = promote_typeof(x, y, z)
    return (convert(R, x), convert(R, y), convert(R, z))
end
function _promote(x, y, zs...)
    @_inline_meta
    R = promote_typeof(x, y, zs...)
    return (convert(R, x), convert(R, y), convert(Tuple{Vararg{R}}, zs)...)
end
# TODO: promote(x::T, ys::T...) where {T} here to catch all circularities?

## promotions in arithmetic, etc. ##

# Because of the promoting fallback definitions for Number, we need
# a special case for undefined promote_rule on numeric types.
# Otherwise, typejoin(T,S) is called (returning Number) so no conversion
# happens, and +(promote(x,y)...) is called again, causing a stack
# overflow.
function promote_result(::Type{T},::Type{S},::Type{Bottom},::Type{Bottom}) where {T<:Number,S<:Number}
    @_inline_meta
    promote_to_supertype(T, S, typejoin(T,S))
end

# promote numeric types T and S to typejoin(T,S) if T<:S or S<:T
# for example this makes promote_type(Integer,Real) == Real without
# promoting arbitrary pairs of numeric types to Number.
promote_to_supertype(::Type{T}, ::Type{T}, ::Type{T}) where {T<:Number}           = (@_inline_meta; T)
promote_to_supertype(::Type{T}, ::Type{S}, ::Type{T}) where {T<:Number,S<:Number} = (@_inline_meta; T)
promote_to_supertype(::Type{T}, ::Type{S}, ::Type{S}) where {T<:Number,S<:Number} = (@_inline_meta; S)
promote_to_supertype(::Type{T}, ::Type{S}, ::Type) where {T<:Number,S<:Number} =
    error("no promotion exists for ", T, " and ", S)

promote() = ()
promote(x) = (x,)

function promote(x, y)
    @_inline_meta
    px, py = _promote(x, y)
    not_sametype((x,y), (px,py))
    px, py
end
function promote(x, y, z)
    @_inline_meta
    px, py, pz = _promote(x, y, z)
    not_sametype((x,y,z), (px,py,pz))
    px, py, pz
end
function promote(x, y, z, a...)
    p = _promote(x, y, z, a...)
    not_sametype((x, y, z, a...), p)
    p
end

promote(x::T, y::T, zs::T...) where {T} = (x, y, zs...)

not_sametype(x::T, y::T) where {T} = sametype_error(x)

not_sametype(x, y) = nothing

function sametype_error(input)
    @_noinline_meta
    error("promotion of types ",
          join(map(x->string(typeof(x)), input), ", ", " and "),
          " failed to change any arguments")
end

+(x::Number, y::Number) = +(promote(x,y)...)
*(x::Number, y::Number) = *(promote(x,y)...)
-(x::Number, y::Number) = -(promote(x,y)...)
/(x::Number, y::Number) = /(promote(x,y)...)

"""
    ^(x, y)

Exponentiation operator. If `x` is a matrix, computes matrix exponentiation.

If `y` is an `Int` literal (e.g. `2` in `x^2` or `-3` in `x^-3`), the Julia code
`x^y` is transformed by the compiler to `Base.literal_pow(^, x, Val(y))`, to
enable compile-time specialization on the value of the exponent.
(As a default fallback we have `Base.literal_pow(^, x, Val(y)) = ^(x,y)`,
where usually `^ == Base.^` unless `^` has been defined in the calling
namespace.)

```jldoctest
julia> 3^5
243

julia> A = [1 2; 3 4]
2×2 Array{Int64,2}:
 1  2
 3  4

julia> A^3
2×2 Array{Int64,2}:
 37   54
 81  118
```
"""
^(x::Number, y::Number) = ^(promote(x,y)...)

fma(x::Number, y::Number, z::Number) = fma(promote(x,y,z)...)
muladd(x::Number, y::Number, z::Number) = muladd(promote(x,y,z)...)

==(x::Number, y::Number) = (==)(promote(x,y)...)
<( x::Real, y::Real)     = (< )(promote(x,y)...)
<=(x::Real, y::Real)     = (<=)(promote(x,y)...)

div(x::Real, y::Real) = div(promote(x,y)...)
fld(x::Real, y::Real) = fld(promote(x,y)...)
cld(x::Real, y::Real) = cld(promote(x,y)...)
rem(x::Real, y::Real) = rem(promote(x,y)...)
mod(x::Real, y::Real) = mod(promote(x,y)...)

mod1(x::Real, y::Real) = mod1(promote(x,y)...)
fld1(x::Real, y::Real) = fld1(promote(x,y)...)

max(x::Real, y::Real) = max(promote(x,y)...)
min(x::Real, y::Real) = min(promote(x,y)...)
minmax(x::Real, y::Real) = minmax(promote(x, y)...)

# "Promotion" that takes a function into account and tries to preserve
# non-concrete types. These are meant to be used mainly by elementwise
# operations, so it is advised against overriding them
_default_type(T::Type) = (@_inline_meta; T)

if isdefined(Core, :Compiler)
    const _return_type = Core.Compiler.return_type
else
    _return_type(@nospecialize(f), @nospecialize(t)) = Any
end

"""
    promote_op(f, argtypes...)

Guess what an appropriate container eltype would be for storing results of
`f(::argtypes...)`. The guess is in part based on type inference, so can change any time.

!!! warning
    In pathological cases, the type returned by `promote_op(f, argtypes...)` may not even
    be a supertype of the return value of `f(::argtypes...)`. Therefore, `promote_op`
    should _not_ be used e.g. in the preallocation of an output array.

!!! warning
    Due to its fragility, use of `promote_op` should be avoided. It is preferable to base
    the container eltype on the type of the actual elements. Only in the absence of any
    elements (for an empty result container), it may be unavoidable to call `promote_op`.
"""
promote_op(::Any...) = (@_inline_meta; Any)
function promote_op(f, ::Type{S}) where S
    @_inline_meta
    TT = Tuple{_default_type(S)}
    T = _return_type(f, TT)
    isdispatchtuple(Tuple{S}) && return isdispatchtuple(Tuple{T}) ? T : Any
    return typejoin(S, T)
end
function promote_op(f, ::Type{R}, ::Type{S}) where {R,S}
    @_inline_meta
    TT = Tuple{_default_type(R), _default_type(S)}
    T = _return_type(f, TT)
    isdispatchtuple(Tuple{R}) && isdispatchtuple(Tuple{S}) && return isdispatchtuple(Tuple{T}) ? T : Any
    return typejoin(R, S, T)
end

## catch-alls to prevent infinite recursion when definitions are missing ##

no_op_err(name, T) = error(name," not defined for ",T)
(+)(x::T, y::T) where {T<:Number} = no_op_err("+", T)
(*)(x::T, y::T) where {T<:Number} = no_op_err("*", T)
(-)(x::T, y::T) where {T<:Number} = no_op_err("-", T)
(/)(x::T, y::T) where {T<:Number} = no_op_err("/", T)
(^)(x::T, y::T) where {T<:Number} = no_op_err("^", T)

fma(x::T, y::T, z::T) where {T<:Number} = no_op_err("fma", T)
fma(x::Integer, y::Integer, z::Integer) = x*y+z
muladd(x::T, y::T, z::T) where {T<:Number} = x*y+z

(&)(x::T, y::T) where {T<:Integer} = no_op_err("&", T)
(|)(x::T, y::T) where {T<:Integer} = no_op_err("|", T)
xor(x::T, y::T) where {T<:Integer} = no_op_err("xor", T)

(==)(x::T, y::T) where {T<:Number} = x === y
(< )(x::T, y::T) where {T<:Real} = no_op_err("<" , T)
(<=)(x::T, y::T) where {T<:Real} = no_op_err("<=", T)

rem(x::T, y::T) where {T<:Real} = no_op_err("rem", T)
mod(x::T, y::T) where {T<:Real} = no_op_err("mod", T)

min(x::Real) = x
max(x::Real) = x
minmax(x::Real) = (x, x)

max(x::T, y::T) where {T<:Real} = select_value(y < x, x, y)
min(x::T, y::T) where {T<:Real} = select_value(y < x, y, x)
minmax(x::T, y::T) where {T<:Real} = y < x ? (y, x) : (x, y)

flipsign(x::T, y::T) where {T<:Signed} = no_op_err("flipsign", T)


"""
    read data, sample ID;s and observation ID's to DataFrame.
"""
function readCooccurrence(source::String, sort_on=nothing; abs_cutoff=nothing, rel_cutoff=nothing, uncommon_observation=-1)::DataFrame
    #= parse parameters =#
    if sort_on === nothing || startswith(lowercase(sort_on), "sample")
        x = "sample"
        y = "observation"
    elseif startswith(lowercase(sort_on), "observation")
        x = "observation"
        y = "sample"
    else
        throw(ArgumentError("Can't sort on $(sort_on). Choose sample [default] or observation"))
    end
    if abs_cutoff !== nothing && rel_cutoff !== nothing
        throw(ArgumentError("Can't have both a relative and absolute cutoff."))
    end
    if rel_cutoff !== nothing && (0 > rel_cutoff || rel_cutoff > 1)
        throw(ArgumentError("rel_cutoff is $(rel_cutoff), but should `nothing` or be between 0 and 1."))
    end
    #= read HDF5 file =#
    h5file = HDF5.h5open(source, "r") do h5
        read(h5)
    end
    #= set preprocess parameters =#
    n_obs = length(h5file["sample"]["matrix"]["indptr"])-1
    if abs_cutoff !== nothing && (0 > abs_cutoff || abs_cutoff > n_obs)
        throw(ArgumentError("abs_cutoff is $(abs_cutoff), but should be `nothing` or between 0 and $(n_obs)."))
    end
    if abs_cutoff !== nothing
        preprocess = true
        min_obs = abs_cutoff
    elseif rel_cutoff !== nothing
        preprocess = true
        min_obs = n_obs*rel_cutoff
    else
        preprocess = false
    end
    #= write data to dataframe =#
    indptr = h5file[x]["matrix"]["indptr"]
    # x_col  = [parse(Int, h5file[y]["ids"][i+1]) for i in h5file[x]["matrix"]["indices"]]
    x_col  = [i for i in h5file[x]["matrix"]["indices"]]
    y_col  = [Int32(i) for a  in 1:length(indptr)-1 for i in fill(a, indptr[a+1]-indptr[a])]
    data   = h5file[x]["matrix"]["data"]
    if x == "sample"
        df = DataFrame(data = data, observation = x_col, sample = y_col)
    else
        df = DataFrame(data = data, observation = y_col, sample = x_col)
    end

    # df.observation = parse.(Int, df.observation)
    #= preprocess rare/uncommon observations =#
    c = countmap(df.observation)
    if preprocess
        for i in 1:length(df.observation)
            if c[df.observation[i]] < min_obs
                df.observation[i] = uncommon_observation
            end
        end
    end
    return df
end

function readBIOM(path::String)::Dict{String, Any}
    dict = h5open(path, "r") do h5
        read(h5)
    end
end

function isBIOM(path::String)::Bool
    di = readBIOM(path)
    isBIOM(di)
end

function isBIOM(di::Dict{String, Any})::Bool
    k₁ = keys(di)
    if  length(k₁) != 2 && ["sample", "observation"] ⊆ k₁
        return false
    else
        k₂ₒ = keys(di["observation"])
        k₂ₛ = keys(di["sample"])
        if  length(k₂ₒ) != 4 && ["ids", "matrix", "metadata", "group-metadata"] ⊆ k₂ₒ &&
            length(k₂ₛ) != 4 && ["ids", "matrix", "metadata", "group-metadata"] ⊆ k₂ₛ
            return false
        else
            k₃ₒ = keys(di["observation"]["matrix"])
            k₃ₛ = keys(di["sample"]["matrix"])
            if  length(k₃ₒ) != 3 && ["data", "indptr", "indices"] ⊆ k₃ₒ &&
                length(k₃ₛ) != 3 && ["data", "indptr", "indices"] ⊆ k₃ₛ
                return false
            else
                return true
            end
        end
    end

end

function writeBIOM(path::String, df::DataFrame;
                    sample_meta::Union{Nothing,Dict{String, <:Any}}       = nothing,
                    sample_group_meta::Union{Nothing,Dict{String, <:Any}} = nothing,
                    obs_meta::Union{Nothing,Dict{String, <:Any}}          = nothing,
                    obs_group_meta::Union{Nothing,Dict{String, <:Any}}    = nothing)
    if  ["sample", "observation", "data"] ⊈ names(df)
        Throw(ArgumentError("Something bad happend."))
    end
    dfₒ   = sort(df, :observation)
    dfₛ   = sort(df, :sample)
	dataₒ = Array{Float64}(dfₒ.data)
	samₒ  = Array{Int32}(dfₒ.sample)
	obsₒ  = Array{Int32}([[findfirst(==(i), dfₒ.observation)-1 for i in unique(dfₒ.observation)]... , length(dfₒ.observation)])
	idₒ   = Array{String}(string.(sort(unique(dfₒ.observation))))
    dataₛ = Array{Float64}(dfₛ.data)
	obsₛ  = Array{Int32}(dfₛ.observation)
	samₛ  = Array{Int32}([[findfirst(==(i), dfₛ.sample)-1 for i in unique(dfₛ.sample)]... , length(dfₛ.sample)])
	idₛ   = Array{String}(string.(sort(unique(dfₛ.sample))))
    HDF5.h5write(path, "sample/ids", idₛ)
    HDF5.h5write(path, "sample/matrix/data", dataₛ)
    HDF5.h5write(path, "sample/matrix/indptr", samₛ)
    HDF5.h5write(path, "sample/matrix/indices",obsₛ)
    HDF5.h5write(path, "observation/ids", idₒ)
    HDF5.h5write(path, "observation/matrix/data", dataₒ)
    HDF5.h5write(path, "observation/matrix/indptr", obsₒ)
    HDF5.h5write(path, "observation/matrix/indices", samₒ)
    if sample_meta !== nothing
        for k in keys(sample_meta)
            HDF5.h5write(path, "sample/metadata/$(k)", sample_meta[k])
        end
    end
    if sample_group_meta !== nothing
        for k in keys(sample_group_meta)
            HDF5.h5write(path, "sample/group-metadata/$(k)", sample_group_meta[k])
        end
    end
    if obs_meta !== nothing
        for k in keys(obs_meta)
            HDF5.h5write(path, "observation/metadata/$(k)", obs_meta[k])
        end
    end
    if obs_group_meta !== nothing
        for k in keys(obs_group_meta)
            HDF5.h5write(path, "sample/group-metadata/$(k)", obs_group_meta[k])
        end
    end
end

function collapseBIOM(biomin::String, biomout::String, field::String; on::Int)
    if isfile(biomout)
        throw(ArgumentError("$(biomout) already exists. Can't overwrite files."))
    end
    if biomin == biomout
        throw(ArgumentError("Can't change/override/append to biom file."))
    end
    if field != "taxonomy"
        throw(ErrorException("Collapsing on $(field) is not implemented (yet)."))
    end
    indict = readBIOM(biomin)
    if !haskey(indict["observation"]["metadata"], field)
        throw(KeyError(field))
    end
    if size(indict["observation"]["metadata"][field], 1) < on
        throw(BoundsError())
    end
    df = readCooccurrence(biomin)
    tax = indict["observation"]["metadata"]["taxonomy"]
    dftax =  transform!(df, :observation => (x -> tax[on, x]) => :taxonomy)
    remove = Array{Int}([])
    sort!(dftax, [:taxonomy, :observation])
    for i in 2:size(dftax,1)
        if length(dftax[i, :taxonomy]) > 3 && dftax[i,:taxonomy] == dftax[i-1, :taxonomy]
            dftax[i,:observation] = dftax[i-1,:observation]
        end
    end
    sort!(dftax, [:sample, :observation])
    for i in 2:size(dftax,1)
        if dftax[i,:observation] == dftax[i-1,:observation] &&
            dftax[i, :sample] == dftax[i-1, :sample] &&
            length(dftax[i, :taxonomy]) > 3
            dftax[i,:data] += dftax[i-1,:data]
            push!(remove, i-1)
        end
    end
    delete!(dftax, remove)

    sort!(dftax, :observation)
    col_tax = [dftax[1, :taxonomy]]
    for i in 2:size(dftax,1)
        if dftax[i,:observation] != dftax[i-1,:observation]
            push!(col_tax, dftax[i,:taxonomy])
        end
    end
    outdict = writeBIOM(biomout, select(dftax, :sample, :data, :observation), sample_meta=Dict("taxonomy" => col_tax))
end

# function preprocessBIOM!(inpath::String, outpath::String, rel_cutoff::Float64; uncommon_observation=-1)
#     @warn "This function is not very well tested yet. Please don't overwrite files that shouldn't be potentially corrupted"
#     if rel_cutoff !== nothing && (0 > rel_cutoff || rel_cutoff > 1)
#         throw(ArgumentError("rel_cutoff is $(rel_cutoff), but should `nothing` or be between 0 and 1."))
#     end
#     #= read HDF5 file =#
#     h5in = HDF5.h5open(inpath, "r") do h5
#         read(h5)
#     end
#     #= set preprocess parameters =#
#     n_obs = length(h5in["sample"]["matrix"]["indptr"])-1
#     min_obs = n_obs*rel_cutoff
#     #= add synthetic observation =#
#     push!(hdin["observation"]["ids"], "synthetic")
#     syn_id = length(hdin["observation"]["ids"])
#     #= change observation under cut off to synthetic =#
#     observations = hdin["sample"]["matrix"]["indices"]
#     obs_count = countmap(observations)
#     removed_ids = Set()
#     if preprocess
#         for i in 1:length(observations)
#             if obs_count[i] < min_obs
#                 push!(observations[i])
#                 observations[i] = syn_id
#             end
#         end
#     end
#     #= adjust ids index =#
#     obs_indptr = hdin["observation"]["matrix"]["indptr"]
#     updated_obs_indptr = [i for i in obs_indptr if !contains(removed_ids, i)]

#     #= sort data for hdf5 =#
#     sample_indptr = h5in["sample"]["matrix"]["indptr"]
#     df = DataFrame(observation = observations,
#                    data        = h5in["sample"]["matrix"]["data"],
#                    sample      = [i-1 for a  in 1:length(sample_indptr)-1 for i in fill(a, sample_indptr[a+1]-sample_indptr[a])] )
#     sort!(df, :observation)
#     h5in["observation"]["matrix"]["data"] = df.data
#     h5in["observation"]["matrix"]["indices"] = df.sample

# end


using JLD, PyPlot

for m in 25
    for sgn in 0
        for numChanges in [1, 3, 5]
            file = jldopen("res/power_$(m)_$(sgn)_$(numChanges).jld", "r")
            power = read(file, "power")
            close(file)

            fig = figure(figsize=(3, 3), dpi=300)

            scatter(0.0:0.05:0.4, power[1:9, 1], s=5, marker="o")
            scatter(0.0:0.05:0.4, power[1:9, 2], s=5, marker="v")

            minorticks_on()
            grid()
            grid(which="minor", ls="dotted", lw=".5")

            xlabel("δ", size="xx-small")
            ylabel("proportion of rejections", size="xx-small")

            ylim(0, 1.05)

            tick_params("both", labelsize="xx-small")

            tight_layout()

            savefig("res/power_$(m)_$(sgn)_$(numChanges).png")

            close(fig)
        end
    end
end

for m in 25
    for sgn in 0
        for numChanges in [1, 3, 5]
            file = jldopen("res/power_$(m)_$(sgn)_$(numChanges).jld", "r")
            power = read(file, "power")
            close(file)

            fig = figure(figsize=(3, 3), dpi=300)

            scatter(0.0:0.05:0.4, power[1:9, 3], s=5, marker="o")
            scatter(0.0:0.05:0.4, power[1:9, 4], s=5, marker="v")

            minorticks_on()
            grid()
            grid(which="minor", ls="dotted", lw=".5")

            xlabel("δ", size="xx-small")
            ylabel("proportion of rejections", size="xx-small")

            ylim(0, 1.05)

            tick_params("both", labelsize="xx-small")

            tight_layout()

            savefig("res/power_studentized_$(m)_$(sgn)_$(numChanges).png")

            close(fig)
        end
    end
end

for m in 25
    for sgn in 0
        fig = figure(figsize=(27, 9), dpi=300)

        k = 0
        for numChanges in [1, 3, 5]
            for lbInd = 1:9
                k += 1

                ax = subplot(3, 9, k)

                file = jldopen("res/coverage_$(m)_$(sgn)_$(numChanges)_$(lbInd).jld", "r")
                coverage = read(file, "coverage")
                close(file)

                plot(0.0:0.05:1.0, 0.0:0.05:1.0, color="grey", linestyle=":", linewidth=.25)

                plot(0.05:0.05:0.95, coverage[1:19, 1], linewidth=.25, marker="o", markersize=2)
                plot(0.05:0.05:0.95, coverage[1:19, 2], linewidth=.25, marker="v", markersize=2)

                ax.set_xlim(0.0, 1.0)
                ax.set_ylim(0.0, 1.0)

                ax[:tick_params]("both", labelsize=2)
            end
        end

        tight_layout()

        savefig("res/coverage_$(m)_$(sgn).png")

        close(fig)
    end
end

for m in 25
    for sgn in 0
        fig = figure(figsize=(27, 9), dpi=300)

        k = 0
        for numChanges in [1, 3, 5]
            for lbInd = 1:9
                k += 1

                ax = subplot(3, 9, k)

                file = jldopen("res/coverage_$(m)_$(sgn)_$(numChanges)_$(lbInd).jld", "r")
                coverage = read(file, "coverage")
                close(file)

                plot(0.0:0.05:1.0, 0.0:0.05:1.0, color="grey", linestyle=":", linewidth=.25)

                plot(0.05:0.05:0.95, coverage[1:19, 3], linewidth=.25, marker="o", markersize=2)
                plot(0.05:0.05:0.95, coverage[1:19, 4], linewidth=.25, marker="v", markersize=2)

                ax.set_xlim(0.0, 1.0)
                ax.set_ylim(0.0, 1.0)

                ax[:tick_params]("both", labelsize=2)
            end
        end

        tight_layout()

        savefig("res/coverage_studentized_$(m)_$(sgn).png")

        close(fig)
    end
end


using Printf
using Oceananigans.Grids: size_summary, scalar_summary

location_str(::Type{Face})    = "Face"
location_str(::Type{Center})  = "Center"
location_str(::Type{Nothing}) = "⋅"
show_location(LX, LY, LZ) = "($(location_str(LX)), $(location_str(LY)), $(location_str(LZ)))"
show_location(field::AbstractField) = show_location(location(field)...)

function Base.summary(field::Field)
    LX, LY, LZ = location(field)
    prefix = string(size_summary(size(field)), " Field{$LX, $LY, $LZ}")

    grid_name = typeof(field.grid).name.wrapper
    reduced_dims = reduced_dimensions(field)

    suffix = reduced_dims === () ?
        string(" on ", grid_name, " on ", summary(architecture(field))) :
        string(" reduced over dims = ", reduced_dims,
               " on ", grid_name, " on ", summary(architecture(field)))

    return string(prefix, suffix)
end

data_summary(field) = string("max=", scalar_summary(maximum(field)), ", ",
                             "min=", scalar_summary(minimum(field)), ", ",
                             "mean=", scalar_summary(mean(field)))

function Base.show(io::IO, field::Field)

    prefix =
        string("$(summary(field))\n",
               "├── grid: ", summary(field.grid), '\n',
               "├── boundary conditions: ", summary(field.boundary_conditions), '\n')

    middle = isnothing(field.operand) ? "" :
        string("├── operand: ", summary(field.operand), '\n',
               "├── status: ", summary(field.status), '\n')

    suffix = string("└── data: ", summary(field.data), '\n',
                    "    └── ", data_summary(field))

    print(io, prefix, middle, suffix)
end

Base.summary(status::FieldStatus) = "time=$(status.time)"

Base.summary(::ZeroField{N}) where N = "ZeroField{$N}"
Base.show(io::IO, z::ZeroField) = print(io, summary(z))

Base.show(io::IO, ::MIME"text/plain", f::AbstractField) = show(io, f)


# ```@setup covmat
# # the example command line input for this script
# ARGS = ["example.toml", "143", "143", "143", "143", "T", "T", "T", "T", "1", "2", "1", "2", "--plot"]
# ```
#
# # [Covariance Matrix Estimation (covmat.jl)](@id covmat)
# This part of the pipeline dominates the total computational cost. Many of the steps like
# spectra estimation could have been run on a laptop, but estimating the covariance for the
# full Planck data vector involves computing ``\sim 10^3`` covariance matrix blocks. The
# covariance matrix involves four fields, and those fields each have a frequency, channel,
# and map split. We thus
# need every combination of frequencies ``f_1``, ``f_2``, ``f_3``, ``f_4`` for
# ``f_i \in \{ 100, 143, 217 \}`` with channels ``A``, ``B``, ``C``, ``D``
# for ``X_i \in \{T, E\}``, and splits
# ``s_1``, ``s_2``, ``s_3``, ``s_4`` for ``s_i \in \{ \mathrm{hm1}, \mathrm{hm2} \}``.
#
# These are the inputs for the command line interface for this script, `covmat.jl`.
#
# ```@raw html
# <pre class="shell">
# <code class="language-shell hljs">$ julia covmat.jl global.toml f1 f2 f3 f3 X1 X2 X3 X4 s1 s2 s3 s4 --plot</code></pre>
# ```
# As usual, the `--plot` argument is optional.
#
#

# ## Setup
# As usual, we need to load up our dependencies and then unpack
# the command line arguments into variables. We'll keep the
# `freqs`, `specs`, and `splits` as just arrays of strings for now.

freqs = [ARGS[2], ARGS[3], ARGS[4], ARGS[5]]
specs = [ARGS[6], ARGS[7], ARGS[8], ARGS[9]]
splits = [ARGS[10], ARGS[11], ARGS[12], ARGS[13]]

using TOML
using PowerSpectra
using Healpix
using JLD2
using Plots
include("util.jl")

# As usual, we get some general information from the configuration file
# specified as the first command line argument.

configfile = ARGS[1]
config = TOML.parsefile(configfile)
nside = config["general"]["nside"]
lmax = nside2lmax(nside)
lmax_planck = min(2508, lmax)
run_name = config["general"]["name"]

# ## Change to canonical ordering
# The covariance matrix is over fields ``F_1``, ``F_2``, ``F_3``, ``F_4`` and represents
# the covariance between the ``\langle F_1, F_2 \rangle`` and
# ``\langle F_3 \times F_4 \rangle`` cross-spectra.
# However, the covariance matrix is invariant when swapping ``F_1 \leftrightarrow F_2``
# as well as ``F_3 \leftrightarrow F_4``. Under the transformation that swaps the
# first pair for the second pair (i.e. ``F_1 \leftrightarrow F_3`` and
# ``F_2 \leftrightarrow F_4`` simultaneously), the covariance matrix is transposed.
#
# Given these convenient properties, we define a canonical ordering of the fields
# in a covariance matrix. We insist that no temperature field can ever be after a
# polarization field in an ordering. Thus, we don't need to implement an expression
# for a ``TETT`` covariance, because it can always be transformed into the canonical
# ``TTTE`` (this example requires a transposition pf the matrix at the end).
# We'll write a few utility functions to enable these transformations here.
#
## This utility tests if it's not a temperature channel
ispol(spec_str) = (spec_str == "T") ? 0 : 1

function swap!(specs, freqs, splits, i, j)
    specs[i], specs[j] = specs[j], specs[i]
    freqs[i], freqs[j] = freqs[j], freqs[i]
    splits[i], splits[j] = splits[j], splits[i]
end

## canonical form: T as far left as possible

"""
    canonical!(specs, freqs, splits)

Given four fields, described by length-4 arrays of specs (T, E), freqs, and splits,
modify these input arrays so they are in the canonical ordering, i.e. with
temperature fields before polarization.

### Arguments:
- `specs`: array of four strings, containing "T" and "E" elements
- `freqs`: array of four frequencies
- `splits`: array of four choices of data split

### Returns:
- `Boolean`: some canonical orderings are a matrix transpose of the inputs. If this
    function returns true, then one needs to perform this transposition.
"""
function canonical!(specs, freqs, splits)
    transposed = false

    if sum(ispol.(specs[1:2])) > sum(ispol.(specs[3:4]))
        swap!(specs, freqs, splits, 1, 3)
        swap!(specs, freqs, splits, 2, 4)
        transposed = true
    end
    if specs[1] == "E" && specs[2] == "T"
        swap!(specs, freqs, splits, 1, 2)
    end
    if specs[3] == "E" && specs[4] == "T"
        swap!(specs, freqs, splits, 3, 4)
    end

    return transposed
end

# We re-arrange the `specs`, `freqs`, and `splits` spectra so they are in the
# canonical ordering. We save whether we need to transpose the result.

transposed = canonical!(specs, freqs, splits)
@show specs freqs splits transposed

## convenience names since we looking for covariance between AB and CD spectra
specAB = Symbol(specs[1]*specs[2])
specCD = Symbol(specs[3]*specs[4])
@show specAB specCD

# Next, we need to create dictionaries that contain the fruits of our labors from
# earlier parts of the pipeline. These include the theory spectra, the ratio of the
# noise power spectrum to the white noise power spectrum.

## store info needed for covariance calculations
spectra = Dict{SpectrumName, SpectralVector{Float64, Vector{Float64}}}()
noiseratios = Dict{SpectrumName, SpectralVector{Float64, Vector{Float64}}}()
identity_spectrum = SpectralVector(ones(lmax+1));

# The point source holes in the Planck likelihood maps are not sufficiently apodized,
# which causes problems in the covariance matrix estimation. Recall that we ran
# signal-only simulations in `signalsim.jl` so that we could correct for them here.
# We read in the correction curve estimated in `corrections.jl` for those simuulations,
# for use in the covariance matrix.

function extend_signal(s::Vector{T}, nside) where T
    lmax = nside2lmax(nside)
    es = SpectralVector(zeros(T, lmax+1))
    for ℓ ∈ 0:min(length(s) - 1, lmax)
        es[ℓ] = s[ℓ+1]
    end
    return es
end

function get_correction(freq1, freq2, spec)
    spec_str = string(spec)
    if parse(Float64, freq1) > parse(Float64, freq2)
        freq1, freq2 = freq2, freq1
        spec_str = reverse(spec_str)
    end
    correction_path = joinpath(config["scratch"], "point_source_corrections")
    correction_file = joinpath(correction_path, "$(freq1)_$(freq2)_$(spec_str)_corr.dat")
    correction_gp = readdlm(correction_file)[:,1]
    correction_gp[1:3] .= 0.0
    return extend_signal(correction_gp, nside)
end

# We want to supply the ratios of the noise power spectrum and the white noise power spectrum.
# To do that, we need to read in the estimates of the white noise power spectrum that
# were obtained from the maps.
function whitenoiselevel(config, freq::String, split::String)
    whitenoisefile = joinpath(config["scratch"], "whitenoise.dat")
    df = CSV.read(whitenoisefile, DataFrame)
    for row in 1:nrow(df)
        if (string(df[row,:freq]) == freq) && (string(df[row,:split]) == split)
            return df[row, :noiseT], df[row, :noiseP]
        end
    end
    throw(ArgumentError("freq and split combo missing from white noise file"))
end

# We save the noise power spectrum models in terms of parameters of the smooth
# model fit. We use the expression from CamSpec.
@. camspec_model(ℓ, α) =  α[1] * (100. / ℓ)^α[2] + α[3] * (ℓ / 1000.)^α[4] / ( 1 + (ℓ / α[5])^α[6] )^α[7]

# Now to put it all together. We need to put all of this information into our
# various dictionaries.

ells = collect(0:(lmax))
coefficientpath = joinpath(config["scratch"], "noise_model_coeffs")
beampath = joinpath(config["scratch"], "beams")

## loop over combinations and put stuff into the dictionaries
for (f1, s1) in zip(freqs, splits)
    for (f2, s2) in zip(freqs, splits)
        f1_name = "P$(f1)hm$(s1)"
        f2_name = "P$(f2)hm$(s2)"
        signal, theory_dict = signal_and_theory(f1, f2, config)

        WlTT = util_planck_beam_Wl(f1, "hm"*s1, f2, "hm"*s2, :TT, :TT;
            lmax=lmax, beamdir=beampath)
        WlTE = util_planck_beam_Wl(f1, "hm"*s1, f2, "hm"*s2, :TE, :TE;
            lmax=lmax, beamdir=beampath)
        WlEE = util_planck_beam_Wl(f1, "hm"*s1, f2, "hm"*s2, :EE, :EE;
            lmax=lmax, beamdir=beampath)

        spectra[(:TT, f1_name, f2_name)] = extend_signal(signal["TT"], nside) .* WlTT .*
            sqrt.(1 .+ get_correction(f1, f2, "TT"))
        spectra[(:TE, f1_name, f2_name)] = extend_signal(signal["TE"], nside) .* WlTE .*
            sqrt.(1 .+ get_correction(f1, f2, "TE"))
        spectra[(:EE, f1_name, f2_name)] = extend_signal(signal["EE"], nside) .* WlEE .*
            sqrt.(1 .+ get_correction(f1, f2, "EE"))

        if f1_name == f2_name
            whitenoiseT, whitenoiseP = whitenoiselevel(config, f1, s1)
            coeffTT = readdlm(joinpath(coefficientpath, "$(run_name)_$(f1)_TT_hm$(s1).dat"))[:,1]
            coeffEE = readdlm(joinpath(coefficientpath, "$(run_name)_$(f1)_EE_hm$(s1).dat"))[:,1]
            nlTT = camspec_model(ells, coeffTT)
            nlEE = camspec_model(ells, coeffEE)
            ratioTT = SpectralVector(nlTT ./ whitenoiseT)
            ratioEE = SpectralVector(nlEE ./ whitenoiseP)
            ratioTT[0:1] .= 1.0
            ratioEE[0:1] .= 1.0
            noiseratios[(:TT, f1_name, f2_name)] = sqrt.(ratioTT)
            noiseratios[(:EE, f1_name, f2_name)] = sqrt.(ratioEE)
        else
            noiseratios[(:TT, f1_name, f2_name)] = identity_spectrum
            noiseratios[(:EE, f1_name, f2_name)] = identity_spectrum
        end
    end
end

fnames = ["P$(freq)hm$(split)" for (freq, split) in zip(freqs, splits)]

if "--plot" ∈ ARGS
    plot(noiseratios[(:TT, fnames[1], fnames[1])].parent, label="$(freqs[1]) hm1 TT")
    plot!(noiseratios[(:EE, fnames[1], fnames[1])].parent, label="$(freqs[1]) hm1 EE",
        ylim=(0.0,2), xlim=(0,200), size=(600,300))
end

# Each of the four fields involved with the covariance matrix involves a polarized map,
# with associated mask and variance map. These are packed toegether in the `CovField` struct
# in `PowerSpectra.jl`.

## pack all the information that the covmat needs into a structure
function loadcovfield(freq, split)
    ## read maskT, maskP, covariances
    mapid = "P$(freq)hm$(split)"
    maskfileT = joinpath(config["scratch"], "masks", "$(run_name)_$(mapid)_maskT.fits")
    maskfileP = joinpath(config["scratch"], "masks", "$(run_name)_$(mapid)_maskP.fits")
    mapfile = joinpath(config["scratch"], "maps", config["map"][mapid])
    maskT = readMapFromFITS(maskfileT, 1, Float64)
    maskP = readMapFromFITS(maskfileP, 1, Float64)
    covII = nest2ring(readMapFromFITS(mapfile, 5, Float64)) * 1e12
    covQQ = nest2ring(readMapFromFITS(mapfile, 8, Float64)) * 1e12
    covUU = nest2ring(readMapFromFITS(mapfile, 10, Float64)) * 1e12
    return CovField(mapid, maskT, maskP, PolarizedHealpixMap(covII, covQQ, covUU))
end

fields = [loadcovfield(freqs[i], splits[i]) for i in 1:4];

# Next, we need to compute the mode-coupling matrices. These are multiplied with
# the covariance matrices in order to correct for the effects of the mask, just
# like in the case of the spectra.
## compute mode-coupling matrices
mcm_type_AB = (specAB == :EE) ? :M⁺⁺ : specAB
mcm_type_CD = (specCD == :EE) ? :M⁺⁺ : specCD

@time M₁₂ = mcm(mcm_type_AB, fields[1], fields[2])
@time M₃₄ = mcm(mcm_type_CD, fields[3], fields[4]);

## compute mode-coupled covariance matrix
workspace = CovarianceWorkspace(fields...);
@time C = coupledcov(Symbol(specs[1]*specs[2]), Symbol(specs[3]*specs[4]),
    workspace, spectra, noiseratios)

## this is the product of A and B pixel windows
function spec2pixwin(XY::Symbol, nside)
    pixwinT, pixwinP = pixwin(nside; pol=true)
    pixwinP[1:2] .= 1.0
    pX = (first(string(XY)) == 'T') ? pixwinT : pixwinP
    pY =  (last(string(XY)) == 'T') ? pixwinT : pixwinP
    return SpectralVector(pX .* pY)
end

pixwinAB = spec2pixwin(specAB, nside)
pixwinCD = spec2pixwin(specCD, nside)

# Next, we need decouple the modes from the effect of the mask, correct for the
# instrumental beam and pixel window function, and then finally bin the covariance matrix.
#

## load binnings
binfile = joinpath(@__DIR__, "../", "input", "binused.dat")
P, lb = util_planck_binning(binfile; lmax=lmax)
beampath = joinpath(config["scratch"], "beams")

C₀ = decouple_covmat(C, M₁₂, M₃₄)
WlAB = util_planck_beam_Wl(freqs[1], "hm"*splits[1], freqs[2], "hm"*splits[2], specAB, specAB;
    lmax=lmax, beamdir=beampath)
WlCD = util_planck_beam_Wl(freqs[3], "hm"*splits[3], freqs[4], "hm"*splits[4], specCD, specCD;
    lmax=lmax, beamdir=beampath)

for l1 in 0:lmax
    for l2 in 0:lmax
        C₀[l1, l2] /= WlAB[l1] * WlCD[l2] * pixwinAB[l1] * pixwinCD[l2]
    end
end

Cbb = P * parent(C₀) * (P')

## transpose if needed, from turning into the canonical form
if transposed
    Cbb = Array(transpose(Cbb))
end

spectrapath = joinpath(config["scratch"], "rawspectra")
covpath = joinpath(config["scratch"], "covmatblocks")
mkpath(covpath)

covmat_filename = joinpath(covpath, join(specs) * "_" *
    join(freqs) * "_" * join(splits) * ".jld2")
@save covmat_filename Cbb


module OceanBoundaryLayerParameterizations

pushfirst!(LOAD_PATH, joinpath(@__DIR__, "..", ".."))

using OceanTurbulenceParameterEstimation
using OceanTurbulenceParameterEstimation.Models: set!
using CairoMakie, LaTeXStrings, OrderedCollections
using Oceananigans, FileIO

export
    # lesbrary_paths.jl
    TwoDaySuite, FourDaySuite, SixDaySuite, GeneralStrat,

    # one_dimensional_ensemble_model.jl
    OneDimensionalEnsembleModel,

    # EKI_hyperparameter_search.jl
    plot_stds_within_bounds, plot_prior_variance, plot_num_ensemble_members,
    plot_observation_noise_level, plot_prior_variance_and_obs_noise_level,

    # visualize_profile_predictions.jl
    visualize!,
    visualize_and_save!

include("utils/lesbrary_paths.jl")
include("utils/one_dimensional_ensemble_model.jl")
include("EKI_hyperparameter_search.jl")
include("visualize_profile_predictions_utils.jl")
include("utils/visualize_profile_predictions.jl")

end # module


using Test, SpeedDate, BioSequences

@testset "Mutation Counting" begin
    @testset "Catching every mutation type" begin
        function generate_possibilities_tester(::Type{A}) where {A <: NucleicAcidAlphabet}
            sym = symbols(A())
            arra = Vector{eltype(A)}()
            arrb = Vector{eltype(A)}()
            for i in 1:length(sym), j in i:length(sym)
                push!(arra, sym[i])
                push!(arrb, sym[j])
            end
            return LongSequence{A}(arra), LongSequence{A}(arrb)
        end

        @testset "2 - Bit" begin
            dnaA, dnaB = generate_possibilities_tester(DNAAlphabet{2})
            @test SpeedDate.count_n_mutations(dnaA, dnaB) == SpeedDate.count_n_mutations(dnaB, dnaA) == 6
        end

        @testset "4 - Bit" begin
            dnaA, dnaB = generate_possibilities_tester(DNAAlphabet{4})
            @test SpeedDate.count_n_mutations(dnaA, dnaB) == SpeedDate.count_n_mutations(dnaB, dnaA) == (6, 10)
        end
    end

    @testset "Randomized tests" begin

    end
end


"""
    AbstractTreeNode

Abstract type of tree nodes
"""

abstract type AbstractTreeNode end

"""
    TreeNode

Tree node stores information of stage problem

    - `id`: index of node
    - `stage_builder`: adds variables and constraints to model with input (tree::Tree, subtree::SubTree, node::SubTreeNode)
    - `parent`: index of parent node
    - `children`: indices of child nodes
    - `stage`: current stage
    - `ξ`: current scenario
    - `p`: probability of node
"""

mutable struct TreeNode <: AbstractTreeNode
    id::Int
    stage_builder::Union{Nothing,Function}
    parent::Int
    children::Vector{Tuple{Int, Float64}}
    stage::Int
    ξ::Dict{Symbol, Union{Float64,<:AbstractArray{Float64}}}
    p::Float64

    function TreeNode(id::Int, parent::Int, stage::Int, ξ::Dict{Symbol, Union{Float64,<:AbstractArray{Float64}}}, p::Float64)
        tn = new()
        tn.id = id
        tn.stage_builder = nothing
        tn.parent = parent
        tn.children = Vector{Tuple{Int, Float64}}()
        tn.stage = stage
        tn.ξ = ξ
        tn.p = p
        return tn
    end
end

function TreeNode(ξ::Dict{Symbol, Union{Float64,<:AbstractArray{Float64}}})
    return TreeNode(1, 0, 1, ξ, 1.0)
end

get_id(node::TreeNode) = node.id
get_parent(node::TreeNode) = node.parent
get_children(node::TreeNode) = node.children
get_stage(node::TreeNode) = node.stage
get_scenario(node::TreeNode) = node.ξ
get_probability(node::TreeNode) = node.p
function set_stage_builder!(node::TreeNode, func::Function)
    node.stage_builder = func
end

"""
    AbstractTree

Abstract type of tree
"""

abstract type AbstractTree end

"""
    Tree

Tree keeps information of tree nodes.

    - `nodes`: list of nodes
"""

mutable struct Tree <: AbstractTree
    nodes::Dict{Int,TreeNode}
end

Tree() = Tree(Dict{Int,TreeNode}())

Tree(ξ::Dict{Symbol, Union{Float64,<:AbstractArray{Float64}}}) = Tree(Dict{Int,TreeNode}(1 => TreeNode(ξ)))

"""
    add_node!

adds abstract tree node to abstract tree

# Arguments
    - `tree`: Abstract tree
    - `node`: Abstract tree node
"""

function add_node!(tree::AbstractTree, node::AbstractTreeNode)
    @assert !haskey(tree.nodes, get_id(node))
    id = get_id(node)
    tree.nodes[id] = node
end

"""
    add_child!

creates child nodee and adds to tree

# Arguments
    - `tree`: Tree
    - `pt`: Parent ID
    - 'ξ': Dictionary of random variable name as symbols and scenarios
    - `prob`: probability of transitioning from the parent to this node
"""

function add_child!(tree::Tree, pt::Int, ξ::Dict{Symbol, Union{Float64,<:AbstractArray{Float64}}}, prob::Float64)::Int
    #   adds child node to tree.nodes[pt]
    @assert haskey(tree.nodes, pt)                                  # check if pt is valid
    stage = get_stage(tree, pt) + 1                                 # get new stage value
    p = get_probability(tree, pt) * prob                            # get new node probability
    id = length(tree.nodes) + 1                                     # get node id
    add_node!(tree, TreeNode(id, pt, stage, ξ, p ))                 # create node and add to tree
    push!(get_children(tree, pt), (id, prob))                      # push id to parent node children
    return id
end

get_children(tree, id) = get_children(tree.nodes[id])
get_parent(tree, id) = get_parent(tree.nodes[id])
get_stage(tree, id) = get_stage(tree.nodes[id])
get_scenario(tree, id) = get_scenario(tree.nodes[id])
get_probability(tree, id) = get_probability(tree.nodes[id])
function set_stage_builder!(tree, id, func::Function)
    set_stage_builder!(tree.nodes[id], func)
end

"""
    SubTreeNode

Contains TreeNode and other information that are used for dual decomposition.
    - `treenode`: an instance of TreeNode
    - `weight`: multiplied to the objective
    - `in`: dictionary of incoming variables from the previous stage
    - `out`: dictionary of outgoing variablees to the subsequent stage
    - `obj`: objective expression
"""

mutable struct SubTreeNode <: AbstractTreeNode
    treenode::TreeNode
    weight::Float64
    in::Dict{Symbol, Union{JuMP.VariableRef, <:AbstractArray{JuMP.VariableRef}}}    # incoming variables
    out::Dict{Symbol, Union{JuMP.VariableRef, <:AbstractArray{JuMP.VariableRef}}}   # outgoing variables
    obj::Union{Float64, JuMP.AbstractJuMPScalar}
    function SubTreeNode(treenode::TreeNode, weight::Float64)
        stn = new()
        stn.treenode = treenode
        stn.weight = weight
        stn.in = Dict{Symbol, Union{JuMP.VariableRef, <:AbstractArray{JuMP.VariableRef}}}()
        stn.out = Dict{Symbol, Union{JuMP.VariableRef, <:AbstractArray{JuMP.VariableRef}}}()
        stn.obj = 0.0
        return stn
    end
end

get_id(node::SubTreeNode) = get_id(node.treenode)
get_parent(node::SubTreeNode) = get_parent(node.treenode)
get_children(node::SubTreeNode) = get_children(node.treenode)
get_stage(node::SubTreeNode) = get_stage(node.treenode)
get_scenario(node::SubTreeNode) = get_scenario(node.treenode)
get_probability(node::SubTreeNode) = get_probability(node.treenode)

function set_input_variable!(nd::SubTreeNode, symb::Symbol, var::Union{JuMP.VariableRef, <:AbstractArray{JuMP.VariableRef}})
    nd.in[symb] = var
end

function set_output_variable!(nd::SubTreeNode, symb::Symbol, var::Union{JuMP.VariableRef, <:AbstractArray{JuMP.VariableRef}})
    nd.out[symb] = var
end

function set_stage_objective(nd::SubTreeNode, obj::Union{Float64, JuMP.AbstractJuMPScalar})
    nd.obj = obj
end

"""
    SubTree

Used for creeating and keeeping subproblems

    - `block_id`: ID of the subtrees
    - `nodes`: dictionary of SubTreeNodes
    - `model`: JuMP model
    - `parent`: ID of parent if exists (default is 0)
    - `root`: ID of the root node of the subtree (default is 1)
"""

mutable struct SubTree <: AbstractTree
    block_id::Int
    nodes::Dict{Int,SubTreeNode}     # list of nodes
    model::JuMP.AbstractModel
    parent::Int
    root::Int
    function SubTree(block_id::Int)
        return new(block_id, Dict{Int,SubTreeNode}(), JuMP.Model(), 0, 1)
    end
end

"""
    create_subtree!

creates subtree from vector of nodes and adds coupling variables

# Arguments
    - `tree`: Tree
    - `block_id`: ID of subtree
    - 'coupling_variables': list of coupling variables to be modified
    - `nodes`: vector of nodes
"""

function create_subtree!(tree::Tree, block_id::Int, coupling_variables::Vector{CouplingVariableRef}, nodes::Vector{Tuple{TreeNode,Float64}})::SubTree
    subtree = SubTree(block_id)
    # add nodes to subtree
    for (node, weight) in nodes
        subnode = SubTreeNode(node, weight)
        add_node!(subtree, subnode)
    end

    obj = 0
    for (id, subnode) in subtree.nodes
        subnode.treenode.stage_builder(tree, subtree, subnode) # make macro for changing variable names and constraint names to include node id
        obj += subnode.weight * subnode.obj
    end
    set_objective!(subtree, MOI.MIN_SENSE, obj)

    #
    for (id, subnode) in subtree.nodes
        couple_common_variables!(coupling_variables, block_id, subnode)
        parent = get_parent(subnode)
        if parent!=0 && haskey(subtree.nodes, parent)
            add_links!(subtree, id, parent)
        elseif parent!=0 # assuming 1st stage node is 1
            subtree.parent = parent
            subtree.root = id
            couple_incoming_variables!(coupling_variables, block_id, subnode)
        end
    end
    return subtree
end

function set_objective!(tree::SubTree, sense::MOI.OptimizationSense, obj::Union{Float64, JuMP.AbstractJuMPScalar})
    JuMP.set_objective(tree.model, sense, obj)
end

"""
    add_links!

creates linking constraints for nodes within the subtree by connecting the incoming and outgoing variables

# Arguments
    - `tree`: SubTree
    - `id`: ID of node
    - 'pt': ID of parent node
"""

function add_links!(tree::SubTree, id::Int, pt::Int)
    node = tree.nodes[id]
    parent = tree.nodes[pt]
    for (symb, var1) in node.in
        var2 = parent.out[symb]
        @constraint(tree.model, var1 .== var2)
    end
end

"""
    couple_common_variables!

couple outgoing variables

# Arguments
    - `coupling_variables`: vector of coupling variables
    - `block_id`: ID of block
    - `node`: SubTreeNode
"""

function couple_common_variables!(coupling_variables::Vector{CouplingVariableRef}, block_id::Int, node::SubTreeNode)
    label = get_id(node)
    for (symb, var) in node.out
        couple_variables!(coupling_variables, block_id, label, symb, var)
    end
end

"""
    couple_incoming_variables!

couple incoming variables with the root node of the subtree

# Arguments
    - `coupling_variables`: vector of coupling variables
    - `block_id`: ID of block
    - `child`: SubTreeNode
"""

function couple_incoming_variables!(coupling_variables::Vector{CouplingVariableRef}, block_id::Int, child::SubTreeNode)
    label = get_parent(child)
    for (symb, var) in child.in
        couple_variables!(coupling_variables, block_id, label, symb, var)
    end
end

"""
    couple_variables!

adds variables to coupling_variables

# Arguments
    - `coupling_variables`: vector of coupling variables
    - `block_id`: ID of block
    - `label`: ID of node
    - `symb`: symbol for variable
    - `var`: variable or vector of variables
"""

function couple_variables!(coupling_variables::Vector{CouplingVariableRef}, block_id::Int, label::Int, symb::Symbol,
        var::JuMP.VariableRef)
    push!(coupling_variables, CouplingVariableRef(block_id, [label, symb], var))
end

function couple_variables!(coupling_variables::Vector{CouplingVariableRef}, block_id::Int, label::Int, symb::Symbol,
        var::Array{JuMP.VariableRef})
    for (index, value) in pairs(var)
        push!(coupling_variables, CouplingVariableRef(block_id, [label, symb, Tuple(index)], value))
    end
end

function couple_variables!(coupling_variables::Vector{CouplingVariableRef}, block_id::Int, label::Int, symb::Symbol,
        var::JuMP.Containers.DenseAxisArray{JuMP.VariableRef})
    for (index, value) in pairs(var.data)
        push!(coupling_variables, CouplingVariableRef(block_id, [label, symb, keys(var)[index].I], value))
    end
end

function couple_variables!(coupling_variables::Vector{CouplingVariableRef}, block_id::Int, label::Int, symb::Symbol,
        var::JuMP.Containers.SparseAxisArray{JuMP.VariableRef})
    for (index, value) in var.data
        push!(coupling_variables, CouplingVariableRef(block_id, [label, symb, index], value))
    end
end

"""
    decomposition_not

outputs the entire tree

# Arguments
    - `tree`: Tree
"""

function decomposition_not(tree::Tree)::Vector{Vector{Tuple{TreeNode,Float64}}}
    nodes = Vector{Tuple{TreeNode,Float64}}()
    for (id, node) in tree.nodes
        push!(nodes,(node, get_probability(node)))
    end
    return [nodes]
end

"""
    decomposition_scenario

outputs the scenario decomposition at each leaf nodes

# Arguments
    - `tree`: Tree
"""

function decomposition_scenario(tree::Tree)::Vector{Vector{Tuple{TreeNode,Float64}}}
    node_cluster = Vector{Vector{Tuple{TreeNode,Float64}}}()
    for (id, node) in tree.nodes
        if check_leaf(node)
            nodes = Vector{Tuple{TreeNode,Float64}}()
            prob = get_probability(node)
            current = node
            while true
                push!(nodes,(current, prob))
                pt = get_parent(current)
                current = tree.nodes[pt]
                if check_root(current)
                    push!(nodes,(current, prob))
                    break
                end
            end
            push!(node_cluster, nodes)
        end
    end
    return node_cluster
end

"""
    decomposition_temporal

outputs the temporal decomposition at each nodes

# Arguments
    - `tree`: Tree
"""
#need fix

function decomposition_temporal(tree::Tree)::Vector{Vector{Tuple{TreeNode,Float64}}}
    node_cluster = Vector{Vector{Tuple{TreeNode,Float64}}}()
    for (id, node) in tree.nodes
        push!(node_cluster,[(node, get_probability(node))])
    end
    return node_cluster
end

function check_leaf(node::TreeNode)::Bool
    if length(get_children(node)) == 0
        return true
    else
        return false
    end
end

function check_root(node::TreeNode)::Bool
    if get_parent(node) == 0
        return true
    else
        return false
    end
end


using Test
using Yota
using Yota: gradtape, gradcheck, update_chainrules_primitives!
using Yota: trace, compile, play!
using Yota: make_rrule, YotaRuleConfig
import ChainRulesCore: Tangent, ZeroTangent, rrule_via_ad

# test-only dependencies
using CUDA

include("test_helpers.jl")
include("test_chainrules.jl")
include("test_grad.jl")
include("test_update.jl")
include("test_examples.jl")


"""
    side_indices(c::GridContent)::RowCols{Int}

Indices of the rows / cols for each side
"""
function side_indices(c::GridContent)
    return RowCols(
        c.span.cols.start,
        c.span.cols.stop,
        c.span.rows.start,
        c.span.rows.stop,
    )
end

# These functions tell whether an object in a grid touches the left, top, etc. border
# of the grid. This means that it is relevant for the grid's own protrusion on that side.
ismostin(gc::GridContent, grid, ::Left) = gc.span.cols.start == 1
ismostin(gc::GridContent, grid, ::Right) = gc.span.cols.stop == grid.ncols
ismostin(gc::GridContent, grid, ::Bottom) = gc.span.rows.stop == grid.nrows
ismostin(gc::GridContent, grid, ::Top) = gc.span.rows.start == 1

function protrusion(x::T, side::Side) where T
    protrusions = protrusionsobservable(x)
    @match side begin
        si::Left => protrusions[].left
        si::Right => protrusions[].right
        si::Bottom => protrusions[].bottom
        si::Top => protrusions[].top
        si => error("Can't get a protrusion value for side $(typeof(si)), only
            Left, Right, Bottom, or Top.")
    end
end

function protrusion(gc::GridContent, side::Side)
    prot = @match gc.side begin
        gcside::Inner => protrusion(gc.content, side)
        # gcside::Outer => BBox(l - pl, r + pr, b - pb, t + pt)
        gcside::Union{Left, Right} => @match side begin
            si::typeof(gcside) => determinedirsize(gc.content, Col(), gc.side)
            si => 0.0
        end
        gcside::Union{Top, Bottom} => @match side begin
            si::typeof(gcside) => determinedirsize(gc.content, Row(), gc.side)
            si => 0.0
        end
        gcside::TopLeft => @match side begin
            si::Top => determinedirsize(gc.content, Row(), gc.side)
            si::Left => determinedirsize(gc.content, Col(), gc.side)
            si => 0.0
        end
        gcside::TopRight => @match side begin
            si::Top => determinedirsize(gc.content, Row(), gc.side)
            si::Right => determinedirsize(gc.content, Col(), gc.side)
            si => 0.0
        end
        gcside::BottomLeft => @match side begin
            si::Bottom => determinedirsize(gc.content, Row(), gc.side)
            si::Left => determinedirsize(gc.content, Col(), gc.side)
            si => 0.0
        end
        gcside::BottomRight => @match side begin
            si::Bottom => determinedirsize(gc.content, Row(), gc.side)
            si::Right => determinedirsize(gc.content, Col(), gc.side)
            si => 0.0
        end
        gcside => error("Invalid side $gcside")
    end
    ifnothing(prot, 0.0)
end

getside(m::Mixed, ::Left) = m.sides.left
getside(m::Mixed, ::Right) = m.sides.right
getside(m::Mixed, ::Top) = m.sides.top
getside(m::Mixed, ::Bottom) = m.sides.bottom

function inside_protrusion(gl::GridLayout, side::Side)
    prot = 0.0
    for elem in gl.content
        if ismostin(elem, gl, side)
            # take the max protrusion of all elements that are sticking
            # out at this side
            prot = max(protrusion(elem, side), prot)
        end
    end
    return prot
end

function protrusion(gl::GridLayout, side::Side)
    # when we align with the outside there is by definition no protrusion
    if gl.alignmode[] isa Outside
        return 0.0
    elseif gl.alignmode[] isa Inside
        inside_protrusion(gl, side)
    elseif gl.alignmode[] isa Mixed
        si = getside(gl.alignmode[], side)
        if isnothing(si)
            inside_protrusion(gl, side)
        elseif si isa Protrusion
            si.p
        else
            # Outside alignment
            0.0
        end
    else
        error("Unknown AlignMode of type $(typeof(gl.alignmode[]))")
    end
end

function bbox_for_solving_from_side(maxgrid::RowCols, bbox_cell::Rect2f, idx_rect::RowCols, side::Side)
    pl = maxgrid.lefts[idx_rect.lefts]
    pr = maxgrid.rights[idx_rect.rights]
    pt = maxgrid.tops[idx_rect.tops]
    pb = maxgrid.bottoms[idx_rect.bottoms]

    l = left(bbox_cell)
    r = right(bbox_cell)
    b = bottom(bbox_cell)
    t = top(bbox_cell)

    @match side begin
        s::Inner => bbox_cell
        s::Outer => BBox(l - pl, r + pr, b - pb, t + pt)
        s::Left => BBox(l - pl, l, b, t)
        s::Top => BBox(l, r, t, t + pt)
        s::Right => BBox(r, r + pr, b, t)
        s::Bottom => BBox(l, r, b - pb, b)
        s::TopLeft => BBox(l - pl, l, t, t + pt)
        s::TopRight => BBox(r, r + pr, t, t + pt)
        s::BottomRight => BBox(r, r + pr, b - pb, b)
        s::BottomLeft => BBox(l - pl, l, b - pb, b)
        s => error("Invalid side $s")
    end
end

startside(c::Col) = Left()
stopside(c::Col) = Right()
startside(r::Row) = Top()
stopside(r::Row) = Bottom()

getspan(gc::GridContent, dir::Col) = gc.span.cols
getspan(gc::GridContent, dir::Row) = gc.span.rows

"""
Determine the size of a protrusion layout along a dimension. This size is dependent
on the `Side` at which the layout is placed in its parent grid. An `Inside` side
means that the protrusion layout reports its width but not its protrusions. `Left`
means that the layout reports only its full width but not its height, because
an element placed in the left protrusion loses its ability to influence height.
"""
function determinedirsize(content, gdir::GridDir, side::Side)
    reportedsize = reportedsizeobservable(content)
    if gdir isa Row
        @match side begin
            # TODO: is reportedsize the correct thing to return? or plus protrusions depending on the side
            si::Union{Inner, Top, Bottom, TopLeft, TopRight, BottomLeft, BottomRight} =>
                ifnothing(reportedsize[][2], nothing)
            si::Union{Left, Right} => nothing
            si => error("$side not implemented")
        end
    else
        @match side begin
            si::Union{Inner, Left, Right, TopLeft, TopRight, BottomLeft, BottomRight} =>
                ifnothing(reportedsize[][1], nothing)
            si::Union{Top, Bottom} => nothing
            si => error("$side not implemented")
        end
    end
end

function to_ranges(g::GridLayout, rows::Indexables, cols::Indexables)
    if rows isa Int
        rows = rows:rows
    elseif rows isa Colon
        rows = 1:g.nrows
    end
    if cols isa Int
        cols = cols:cols
    elseif cols isa Colon
        cols = 1:g.ncols
    end
    rows, cols
end

function adjust_rows_cols!(g::GridLayout, rows, cols; update = true)
    rows, cols = to_ranges(g, rows, cols)

    if rows.start < 1
        n = 1 - rows.start
        prependrows!(g, n, update = update)
        # adjust rows for the newly prepended ones
        rows = rows .+ n
    end
    if rows.stop > g.nrows
        n = rows.stop - g.nrows
        appendrows!(g, n, update = update)
    end
    if cols.start < 1
        n = 1 - cols.start
        prependcols!(g, n, update = update)
        # adjust cols for the newly prepended ones
        cols = cols .+ n
    end
    if cols.stop > g.ncols
        n = cols.stop - g.ncols
        appendcols!(g, n, update = update)
    end

    rows, cols
end


module ChromNet

include("util.jl")
include("groupgm.jl")

end


# this script would install TestImages and ImageMagick packages if not installed,
# but better you install them in advance manually to avoid ZMQ timeout in LabVIEW
# during installation.

try
    using TestImages, ImageMagick
catch
    import Pkg
    Pkg.add("TestImages")
    Pkg.add("ImageMagick")
end

is2d_img(s) = ! any(occursin.(["_3d_", "series","stack"], lowercase(s)))

function get2d_images()
    rf = TestImages.remotefiles
    rf = filter(is2d_img, rf)
    return (;img_list=rf,)
end

function send_testimage(;img_name)
    img = testimage(img_name)
    img_type = string(typeof(img))
    return (;img_type, bigarrs=(;img))
end

fns = (;get2d_images, send_testimage, )


import Pkg
Pkg.add("Pkg")
#Pkg.add("TOML")
#using TOML
# tml = TOML.tryparsefile("../Project.toml")

#Pkg.add("PyPlot")
#Pkg.add(url = "https://github.com/russelljjarvis/SpikeSynchrony.jl")
#Pkg.add(url = "https://github.com/paulmthompson/SpikeSorting.jl.git")
Pkg.add("Conda")
Pkg.add("PyCall")
Pkg.add("SpikeSynchrony")
Pkg.add("SpikingNeuralNetworks")
Pkg.add("UnPack")
Pkg.add("LightGraphs")
Pkg.add("OrderedCollections")
#Pkg.add("Metaheuristics")
Pkg.add("Evolutionary")
Pkg.add("JLD")
Pkg.add("DataStructures")
Pkg.add("Distributions")
Pkg.add("Plots")
Pkg.add("Unitful")
Pkg.add("Evolutionary")
Pkg.add("DataStructures")
Pkg.add("JLD")


# ------------------------------------------------------------------
# Licensed under the ISC License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    AbstractData{T,N}

Spatial data in a `N`-dimensional space with coordinates of type `T`.
"""
abstract type AbstractData{T,N} <: AbstractSpatialObject{T,N} end

"""
    variables(sdata)

Return the variable names in spatial data `sdata` and their types.
"""
variables(sdata::AbstractData) =
  OrderedDict([var => eltype(array) for (var,array) in sdata.data])

#----------------
# DATAFRAME API
#----------------

"""
    sdata[inds,vars]

Return the value of `var` for the `ind`-th point in `sdata`.
"""
Base.getindex(sdata::AbstractData, ind::Int, var::Symbol) =
  sdata.data[var][ind]

Base.getindex(sdata::AbstractData, inds::AbstractVector{Int}, var::Symbol) =
  [getindex(sdata, ind, var) for ind in inds]

Base.getindex(sdata::AbstractData, ind::Int, vars::AbstractVector{Symbol}) =
  [getindex(sdata, ind, var) for var in vars]

Base.getindex(sdata::AbstractData, inds::AbstractVector{Int}, vars::AbstractVector{Symbol}) =
  [getindex(sdata, ind, var) for ind in inds, var in vars]

"""
    getindex(sdata, var)

Return the values of `var` for all points in `sdata` with the shape
of the underlying domain.
"""
Base.getindex(sdata::AbstractData, var::Symbol) =
  [getindex(sdata, ind, var) for ind in 1:npoints(sdata)]

Base.getindex(sdata::AbstractData, vars::AbstractVector{Symbol}) =
  [getindex(sdata, var) for var in vars]

"""
    setindex!(sdata, val, ind, var)

Set the value `val` of variable `var` for point `ind` in spatial
data `sdata`.
"""
Base.setindex!(sdata::AbstractData, val, ind::Int, var::Symbol) =
  sdata.data[var][ind] = val

"""
    setindex!(sdata, vals, var)

Set the values `vals` of variable `var` for all points in spatial
data `sdata`.
"""
Base.setindex!(sdata::AbstractData, vals::AbstractArray, var::Symbol) =
  sdata.data[var] = vals

#-----------
# VIEW API
#-----------

"""
    view(sdata, inds)
    view(sdata, vars)
    view(sdata, inds, vars)

Return a view of `sdata` with all points in `inds` and
all variables in `vars`.
"""
Base.view(sdata::AbstractData, inds::AbstractVector{Int}) =
  DataView(sdata, inds, collect(keys(variables(sdata))))

Base.view(sdata::AbstractData, vars::AbstractVector{Symbol}) =
  DataView(sdata, 1:npoints(sdata), vars)

Base.view(sdata::AbstractData, inds::AbstractVector{Int},
                               vars::AbstractVector{Symbol}) =
  DataView(sdata, inds, vars)

#---------------
# ITERATOR API
#---------------

"""
    iterate(sdata, state=1)

Iterate over samples in `sdata`.
"""
Base.iterate(sdata::AbstractData, state=1) =
  state > npoints(sdata) ? nothing : (sdata[state], state + 1)

"""
    length(sdata)

Return the number of samples in `sdata`.
"""
Base.length(sdata::AbstractData) = npoints(sdata)

#----------------
# INDEXABLE API
#----------------

"""
    getindex(sdata, ind)

Return `ind`-th sample in `sdata`.
"""
function Base.getindex(sdata::AbstractData, ind::Int)
  vars = [var for (var,V) in variables(sdata)]
  vals = [getindex(sdata, ind, var) for var in vars]
  NamedTuple{tuple(vars...)}(vals)
end

Base.getindex(sdata::AbstractData, inds::AbstractVector{Int}) =
  [getindex(sdata, ind) for ind in inds]

"""
    firstindex(sdata)

Return the first index of `sdata`.
"""
Base.firstindex(sdata::AbstractData) = 1

"""
    lastindex(sdata)

Return the last index of `sdata`.
"""
Base.lastindex(sdata::AbstractData) = npoints(sdata)

#-------------
# TABLES API
#-------------
Tables.istable(::Type{<:AbstractData}) = true

Tables.columnaccess(::Type{<:AbstractData}) = true

function Tables.columns(sdata::AbstractData)
  vars = keys(variables(sdata))
  vals = [getindex(sdata, 1:npoints(sdata), var) for var in vars]
  NamedTuple{tuple(vars...)}(vals)
end

#-----------------
# MISSING VALUES
#-----------------
"""
    isvalid(sdata, ind, var)

Return `true` if the `ind`-th point in `sdata` has a valid value for `var`.
"""
function Base.isvalid(sdata::AbstractData, ind::Int, var::Symbol)
  val = getindex(sdata, ind, var)
  !(ismissing(val) || (val isa Number && isnan(val)))
end

"""
    valid(sdata, var)

Return all points in `sdata` with a valid value for `var`. The output is
a tuple with the matrix of coordinates as the first item and the vector
of values as the second item.
"""
function valid(sdata::AbstractData{T,N}, var::Symbol) where {N,T}
  V = variables(sdata)[var]
  npts = npoints(sdata)

  # pre-allocate memory for result
  X = Matrix{T}(undef, N, npts)
  z = Vector{V}(undef, npts)

  nvalid = 0
  for ind in 1:npoints(sdata)
    if isvalid(sdata, ind, var)
      nvalid += 1
      coordinates!(view(X,:,nvalid), sdata, ind)
      z[nvalid] = getindex(sdata, ind, var)
    end
  end

  X[:,1:nvalid], z[1:nvalid]
end

# ------------
# IO methods
# ------------
function Base.show(io::IO, sdata::AbstractData{T,N}) where {N,T}
  npts = npoints(sdata)
  print(io, "$npts SpatialData{$T,$N}")
end

function Base.show(io::IO, ::MIME"text/plain", sdata::AbstractData{T,N}) where {N,T}
  println(io, sdata)
  println(io, "  variables")
  varlines = ["    └─$var ($V)" for (var,V) in variables(sdata)]
  print(io, join(sort(varlines), "\n"))
end

#------------------
# IMPLEMENTATIONS
#------------------
include("data/curve_data.jl")
include("data/geodataframe.jl")
include("data/point_set_data.jl")
include("data/regular_grid_data.jl")
include("data/structured_grid_data.jl")


import Benchmark

Benchmark.run_paper_benchmark([1_000, 5_000, 10_000, 50_000])


using GeometryBasics
export visualize

"""
    visualize(pcloud::PointCloud; kwargs...)

Visualize PointCloud `pcloud` at `index`.

Dimension of points in PointCloud `pcloud` must be 3.

### Optional Arguments:
- color (Symbol)       - Color of the marker, default `:blue`
- markersize (Number)  - Size of the marker, default `npoints(pcloud)/5000`

"""
function visualize(p::PointCloud, index::Number = 1; kwargs...)
    points = cpu(p[index])
    size(points, 1) == 3 || error("dimension of points in PointCloud must be 3.")

    kwargs = convert(Dict{Symbol,Any}, kwargs)
    get!(kwargs, :color, :lightgreen)
    get!(kwargs, :markersize, npoints(p) / 5000)

    meshscatter(points[1, :], points[2, :], points[3, :]; kwargs...)
end

"""
    visualize(m::TriMesh, index::Int=1; kwargs...)

Visualize mesh at `index` in TriMesh `m`.

### Optional Arguments:
- color (Symbol)       - Color of the marker, default `:red`

"""
function visualize(m::GeometryBasics.Mesh; kwargs...) where {T,R}
    kwargs = convert(Dict{Symbol,Any}, kwargs)
    get!(kwargs, :color, :orange)

    mesh(GeometryBasics.normal_mesh(m); kwargs...)
end

visualize(m::TriMesh, index::Int = 1; kwargs...) = visualize(GBMesh(m, index); kwargs...)

"""
    visualize(v::VoxelGrid, index::Int=1; kwargs...)

Visualize voxel at `index` in VoxelGrid `v`.

### Optional Arguments:
- color (Symbol)       - Color of the marker, default `:red`

"""
function visualize(
    v::VoxelGrid,
    index::Int = 1,
    thresh::Number = 0.49f0;
    algo = :Exact,
    kwargs...,
)
    algo in [:Exact, :MarchingCubes, :MarchingTetrahedra, :NaiveSurfaceNets] ||
        error("given algo: $(algo) is not supported. Accepted algo are
              {:Exact,:MarchingCubes, :MarchingTetrahedra, :NaiveSurfaceNets}.")
    kwargs = convert(Dict{Symbol,Any}, kwargs)
    get!(kwargs, :color, :violet)
    method = algo == :Exact ? _voxel_exact : _voxel_algo
    v, f = method(cpu(v[index]), Float32(thresh), algo)

    m = GBMesh(v, f)
    mesh(GeometryBasics.normal_mesh(m); kwargs...)
end

visualize(v::Dataset.AbstractDataPoint; kwargs...) = visualize(v.data; kwargs...)

visualize(v::AbstractCustomObject; kwargs...) =
    error("Define visualize function for custom type: $(typeof(v)).
            Use `import Flux3D.visualize` and define function
            `visualize(v::$(typeof(v)); kwargs...)`")


module SequentialMonteCarlo

import Statistics.mean
using RNGPool

include("structures.jl")
include("categorical.jl")
include("common.jl")
include("serial.jl")
include("parallel.jl")
include("interface.jl")

export smc!, csmc!, SMCModel, SMCIO

end # module


"""
GenX: An Configurable Capacity Expansion Model
Copyright (C) 2021,  Massachusetts Institute of Technology
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
A complete copy of the GNU General Public License v2 (GPLv2) is available
in LICENSE.txt.  Users uncompressing this from an archive may not have
received this license file.  If not, see <http://www.gnu.org/licenses/>.
"""

@doc raw"""
	load_data(setup::Dict, path::AbstractString, inputs_load::Dict)

Function for reading input parameters related to electricity load (demand)
"""
function load_load_data(setup::Dict, path::AbstractString, inputs_load::Dict)

	# Load related inputs
	#data_directory = chop(replace(path, pwd() => ""), head = 1, tail = 0)
	data_directory = joinpath(path, setup["TimeDomainReductionFolder"])
	if setup["TimeDomainReduction"] == 1  && isfile(joinpath(data_directory,"Load_data.csv")) && isfile(joinpath(data_directory,"Generators_variability.csv")) && isfile(joinpath(data_directory,"Fuels_data.csv")) # Use Time Domain Reduced data for GenX
		load_in = DataFrame(CSV.File(joinpath(data_directory,"Load_data.csv"), header=true), copycols=true)
	else # Run without Time Domain Reduction OR Getting original input data for Time Domain Reduction
		load_in = DataFrame(CSV.File(joinpath(path,"Load_data.csv"), header=true), copycols=true)
	end

	# Number of time steps (periods)
	inputs_load["T"] = size(collect(skipmissing(load_in[!,:Time_Index])),1)
	# Number of demand curtailment/lost load segments
	inputs_load["SEG"]=size(collect(skipmissing(load_in[!,:Demand_Segment])),1)

	## Set indices for internal use
	T = inputs_load["T"]   # Total number of time steps (hours)
	Z = inputs_load["Z"]   # Number of zones
	L = inputs_load["L"]   # Number of lines

	inputs_load["omega"] = zeros(Float64, T) # weights associated with operational sub-period in the model - sum of weight = 8760
	inputs_load["REP_PERIOD"] = 1   # Number of periods initialized
	inputs_load["H"] = 1   # Number of sub-periods within each period

	if setup["OperationWrapping"]==0 # Modeling full year chronologically at hourly resolution
		# Total number of subtime periods
		inputs_load["REP_PERIOD"] = 1
		# Simple scaling factor for number of subperiods
		inputs_load["omega"][:] .= 1 #changes all rows of inputs["omega"] from 0.0 to 1.0
	elseif setup["OperationWrapping"]==1
		# Weights for each period - assumed same weights for each sub-period within a period
		inputs_load["Weights"] = collect(skipmissing(load_in[!,:Sub_Weights])) # Weights each period

		# Total number of periods and subperiods
		inputs_load["REP_PERIOD"] = convert(Int16, collect(skipmissing(load_in[!,:Rep_Periods]))[1])
		inputs_load["H"] = convert(Int64, collect(skipmissing(load_in[!,:Timesteps_per_Rep_Period]))[1])

		# Creating sub-period weights from weekly weights
		for w in 1:inputs_load["REP_PERIOD"]
			for h in 1:inputs_load["H"]
				t = inputs_load["H"]*(w-1)+h
				inputs_load["omega"][t] = inputs_load["Weights"][w]/inputs_load["H"]
			end
		end
	end

	# Create time set steps indicies
	inputs_load["hours_per_subperiod"] = div.(T,inputs_load["REP_PERIOD"]) # total number of hours per subperiod
	hours_per_subperiod = inputs_load["hours_per_subperiod"] # set value for internal use

	inputs_load["START_SUBPERIODS"] = 1:hours_per_subperiod:T 	# set of indexes for all time periods that start a subperiod (e.g. sample day/week)
	inputs_load["INTERIOR_SUBPERIODS"] = setdiff(1:T,inputs_load["START_SUBPERIODS"]) # set of indexes for all time periods that do not start a subperiod

	# Demand in MW for each zone
	#println(names(load_in))
	start = findall(s -> s == "Load_MW_z1", names(load_in))[1] #gets the starting column number of all the columns, with header "Load_MW_z1"
	if setup["ParameterScale"] ==1  # Parameter scaling turned on
		# Max value of non-served energy
		inputs_load["Voll"] = collect(skipmissing(load_in[!,:Voll])) /ModelScalingFactor # convert from $/MWh $ million/GWh (assuming objective is divided by 1000)
		# Demand in MW
		inputs_load["pD"] =Matrix(load_in[1:inputs_load["T"],start:start-1+inputs_load["Z"]])/ModelScalingFactor  # convert to GW
	else # No scaling
		# Max value of non-served energy
		inputs_load["Voll"] = collect(skipmissing(load_in[!,:Voll]))
		# Demand in MW
		inputs_load["pD"] =Matrix(load_in[1:inputs_load["T"],start:start-1+inputs_load["Z"]]) #form a matrix with columns as the different zonal load MW values and rows as the hours
	end

	# Cost of non-served energy/demand curtailment (for each segment)
	SEG = inputs_load["SEG"]  # Number of demand segments
	inputs_load["pC_D_Curtail"] = zeros(SEG)
	inputs_load["pMax_D_Curtail"] = zeros(SEG)
	for s in 1:SEG
		# Cost of each segment reported as a fraction of value of non-served energy - scaled implicitly
		inputs_load["pC_D_Curtail"][s] = collect(skipmissing(load_in[!,:Cost_of_Demand_Curtailment_per_MW]))[s]*inputs_load["Voll"][1]
		# Maximum hourly demand curtailable as % of the max demand (for each segment)
		inputs_load["pMax_D_Curtail"][s] = collect(skipmissing(load_in[!,:Max_Demand_Curtailment]))[s]
	end

	println("Load_data.csv Successfully Read!")

	return inputs_load
end