function _plot_fspec_grid(fs::Dict{Symbol,Dict{Int64,JAXTAM.FFTData}},
        obs_row)
    example = _recursive_first(fs)

    plt_1 = plot(fs; norm=:rms,   save=false, save_csv=true)
    plt_2 = plot(fs; norm=:leahy, save=false)

    plt_3 = plot(fs, norm=:leahy, save=false, freq_lims=(0, 1),     rebin=(:linear, 1),
        logx=false, logy=false, title_append=" - 0 to 1 Hz")
    plt_4 = plot(fs, norm=:leahy, save=false, freq_lims=(1, :end),  rebin=(:linear, 1),
        logx=false, logy=false, title_append=" - 1 to :end Hz")
    plt_4 = plot(fs, norm=:leahy, save=false, freq_lims=(50, :end), rebin=(:linear, 1),
        logx=false, logy=false, title_append=" - 50 to :end Hz")

    Plots.plot(plt_1, plt_2, plt_3, plt_4, layout=grid(4,1), size=(1140,600*4))

    savefig(example, obs_row, example.e_range; plot_name="grid")
end

# Sort of works but doesn't
# macro error_wrapper(mission, obs_row, func)
#     local escaped_obs_row = esc(obs_row)

#     return quote
#         local _wrapped_result = nothing
#         try
#             _wrapped_result = $func
#         catch err
#             if typeof(err) == JAXTAMError
#                 _wrapped_result = err
#                 _log_add($mission, $escaped_obs_row, Dict{String,Any}("errors"=>Dict(err.step=>err)))
#                 @warn err
#             else
#                 rethrow(err)
#             end
#         end

#         _wrapped_result
#     end
# end

function _error_wrapper(func, args...; mission=nothing, obs_row=nothing, kwargs...)
    if mission == nothing
        # Assume first arg is mission
        @assert args[1] isa Mission
        mission = args[1]
    end

    if obs_row == nothing
        # Assume second arg is obs_row or obsid
        if args[2] isa String
            obs_row = master_query(mission, :obsid, args[2])
        elseif args[2] isa DataFrameRow
            obs_row = args[2]
        end
    end

    try
        return func(args...; kwargs...)
    catch err
        if typeof(err) == JAXTAMError
            _log_add(mission, obs_row, Dict{String,Any}("errors"=>Dict(err.step=>err)))
            @warn err
            return nothing
        else
            rethrow(err)
        end
    end
end

"""
    report(mission::Mission, obs_row::DataFrameRow; e_range::Tuple{Float64,Float64}=_mission_good_e_range(mission), overwrite::Bool, nuke::Bool=false, update_masterpage::Bool=true)

Generates a report for the default energy range

Creates plots for:

    * Lightcurve (+ grouped lightcurves)

    * Periodigram (+ grouped periodograms)

    * Power Spectra

        * :rms, full range, log-rebinned, log-log plot

        * :leahy, full range, log-rebinned, log-log plot

        * :leahy, 0 to 1 Hz, no rebin, linear-linear plot

        * :leahy, 1 to end Hz, no rebin, linear-linear plot

        * :leahy, 50 to end Hz, no regin, linear-linear plot

    * Spectrogram

    * Pulsation search plot

Produces HTML report page

Updates the homepage
"""
function report(mission, obs_row; e_range=_mission_good_e_range(mission), overwrite=false, nuke=false, update_masterpage=true)
    path_jaxtam = abspath(_obs_path_local(mission, obs_row; kind=:jaxtam), "JAXTAM")
    path_web    = abspath(_obs_path_local(mission, obs_row; kind=:web), "JAXTAM")

    if nuke
        GC.gc() # Required due to Feather.jl loading files lazily, meaning they can't be removed from disk
                # until garbage collection runs and un-lazily-loads them
        ispath(path_jaxtam) ? rm(path_jaxtam, recursive=true) : false
        ispath(path_web)    ? rm(path_web,    recursive=true) : false
    end

    if ismissing(_log_query(mission, obs_row, "meta", :downloaded; surpress_warn=true)) || !_log_query(mission, obs_row, "meta", :downloaded)
        _error_wrapper(download, mission, obs_row)
    end

    if !ismissing(JAXTAM._log_query(mission, obs_row, "errors", :read_cl; surpress_warn=true))
        @warn "Error logged at :read_cl stage, no files to be analysed, skipping report gen"
        return nothing
    end

    images = _log_query(mission, obs_row, "images", e_range)

    img_count_groupless = ismissing(images) ? 0 : size(filter(x->ismissing(x[:group]), images), 1)
    # Expect five 'groupless' plots: lightcurve, periodogram, powerspectra, spectrogram, pulsations
    if img_count_groupless < 5 || overwrite
        lc = _error_wrapper(JAXTAM.lcurve, mission, obs_row, 2.0^0; e_range=e_range)
        # If the initial 1s lightcurve failed, everything else will fail, so just exit now
        lc == nothing ? (return nothing) : ""

        _error_wrapper(JAXTAM.plot, lc; save=true, mission=mission, obs_row=obs_row)
        _error_wrapper(JAXTAM.plot_groups, lc; save=true, size_in=(1140,400/2), mission=mission, obs_row=obs_row)

        pg = _error_wrapper(JAXTAM.pgram, lc, mission=mission, obs_row=obs_row)
        if pg != nothing
            # Only try plotting the periodogram if there is one
            _error_wrapper(JAXTAM.plot, pg; save=true, mission=mission, obs_row=obs_row)
            pg = _error_wrapper(JAXTAM.pgram, lc; per_group=true, mission=mission, obs_row=obs_row)
            _error_wrapper(JAXTAM.plot_groups, pg; save=true, size_in=(1140,600/2), mission=mission, obs_row=obs_row)
        end

        lc = nothing; pg = nothing; GC.gc()

        lc = _error_wrapper(JAXTAM.lcurve, mission, obs_row, 2.0^-13; e_range=e_range)
        lc == nothing ? (return nothing) : ""

        gtis = _error_wrapper(JAXTAM.gtis, mission, obs_row, 2.0^-13; lcurve_data=lc, e_range=e_range)
        gtis == nothing ? (return nothing) : ""
        lc = nothing;

        fs = _error_wrapper(JAXTAM.fspec, mission, obs_row, 2.0^-13, 128; gtis_data=gtis, e_range=e_range)
        if fs != nothing # Only try plotting if fspec didn't fail
            @info "Plotting fspec grid"
            _error_wrapper(JAXTAM._plot_fspec_grid, fs, obs_row, mission=mission, obs_row=obs_row)
            @info "Plotting fspec groups"
            _error_wrapper(JAXTAM.plot_groups, fs; save=true, size_in=(1140,600/2), mission=mission, obs_row=obs_row)
            @info "Plotting sgram"
            _error_wrapper(JAXTAM.plot_sgram, fs;  save=true, size_in=(1140,600), mission=mission, obs_row=obs_row)
            @info "Plotting pulses"
            _error_wrapper(JAXTAM.plot_pulses_candle, fs; save=true, size_in=(1140,600/2), mission=mission, obs_row=obs_row)
            @info "Plotting pulses groups"
            _error_wrapper(JAXTAM.plot_pulses_candle_groups, fs; save=true, size_in=(1140,600/2), mission=mission, obs_row=obs_row)
            fs = 0; GC.gc()
        end
    end

    subpage_path = _error_wrapper(_webgen_subpage, mission, obs_row; e_range=e_range)

    if update_masterpage
        webgen_mission(mission)
    end

    return subpage_path
end

"""
    report(mission::Mission, obsid::String; kwargs...)

Multiple dispath to report(mission::Mission, obs_row::DataFrameRow; kwargs...)

Uses `obsid` to select the observation
"""
function report(mission::Mission, obsid::String; e_range=_mission_good_e_range(mission), overwrite=false, nuke=false, update_masterpage=true)
    obs_row = master_query(mission, :obsid, obsid)

    return report(mission, obs_row; e_range=e_range, overwrite=overwrite, nuke=nuke, update_masterpage=update_masterpage)
end

"""
    report_all(::Mission, ::DataFrameRow; e_ranges=[(0.2,12.0), (2.0,10.0), (0.2,2.0)], overwrite::Bool=false, nuke::Bool=false, update_masterpage::Bool=true)

Calls `report` with three default energy ranges
"""
function report_all(mission::Mission, obs_row::DataFrames.DataFrameRow; e_ranges=[(0.2,12.0), (2.0,10.0), (0.2,2.0)], overwrite=false, nuke=false, update_masterpage=true)
    if nuke
        path_jaxtam = abspath(_obs_path_local(mission, obs_row; kind=:jaxtam), "JAXTAM")
        path_web    = abspath(_obs_path_local(mission, obs_row; kind=:web), "JAXTAM")

        GC.gc() # Required due to Feather.jl loading files lazily, meaning they can't be removed from disk
                # until garbace collection runs and un-lazily-loads them
        ispath(path_jaxtam) ? rm(path_jaxtam, recursive=true) : false
        ispath(path_web)    ? rm(path_web,    recursive=true) : false
    end

    for e_range in e_ranges
        report(mission, obs_row; e_range=e_range, overwrite=overwrite, update_masterpage=update_masterpage)
    end

    # Run it again to re-generate report pages so that they all have each others links in them
    for e_range in e_ranges
        println("Report path:\n\t$(report(mission, obs_row; e_range=e_range, overwrite=false))")
    end
end

"""
    report_all(mission::Mission, obsid::String; kwargs...)

Multiple dispath to report_all(mission::Mission, obs_row::DataFrameRow; kwargs...)

Uses `obsid` to select the observation
"""
function report_all(mission::Mission, obsid::String; e_ranges=[(0.2,12.0), (2.0,10.0), (0.2,2.0)], overwrite=false, nuke=false, update_masterpage=true)
    obs_row = master_query(mission, :obsid, obsid)

    return report_all(mission, obs_row; e_ranges=e_ranges, overwrite=overwrite, nuke=nuke, update_masterpage=update_masterpage)
end


@doc """
    merge!(S::SeisData, U::SeisData[, prune_only=true])

Merge channels of two SeisData structures.

    merge!(S::SeisData[, prune_only=true])

"Flatten" a SeisData structure by merging channels with identical properties.

If `prune_only=true`, the only action taken is deletion of empty and duplicate
channels; `merge!(S, U, prune_only=true)` is identical to an in-place `S+U`.
""" merge!
function merge!(S::Y; v::Int64=KW.v, purge_only::Bool=false) where Y<:GphysData
  # Required preprocessing
  prune!(S)

  # Initialize variables
  ID    = getfield(S, :id)
  NAME  = getfield(S, :name)
  LOC   = getfield(S, :loc)
  FS    = getfield(S, :fs)
  GAIN  = getfield(S, :gain)
  RESP  = getfield(S, :resp)
  UNITS = getfield(S, :units)
  SRC   = getfield(S, :src)
  MISC  = getfield(S, :misc)
  NOTES = getfield(S, :notes)
  T     = getfield(S, :t)
  X     = getfield(S, :x)

  UID = unique(getfield(S, :id))
  cnt = 0
  N_warn = 0
  note_head = string(SeisIO.timestamp(), ": ")
  to_delete = Array{Int64,1}(undef, 0)
  while cnt < length(UID)
    cnt = cnt + 1
    id = getindex(UID, cnt)
    GRP = findall(S.id.==id)
    SUBGRPS = get_subgroups(LOC, FS, RESP, UNITS, GRP)

    for subgrp in SUBGRPS
      N = length(subgrp)
      if N > 1
        # Check for duplicates
        sort!(subgrp)
        u = trues(N)
        while N > 1
          t1 = getindex(T, getindex(subgrp, N))
          x1 = getindex(X, getindex(subgrp, N))

          # if a channel is already known to be a duplicate, skip it
          if getindex(u, N) == true
            j = N
            while j > 1
              j = j-1
              t2 = getindex(T, getindex(subgrp, j))
              x2 = getindex(X, getindex(subgrp, j))
              if t1 == t2 && x1 == x2
                setindex!(u, false, j)
              end
            end
          end

          # flag duplicates for deletion
          append!(to_delete, subgrp[u.==false])
          N = N-1
        end

        subgrp = subgrp[u.==true]
      end
      purge_only == true && continue

      N     = length(subgrp)
      i1    = getindex(subgrp, 1)
      fs    = getindex(FS, i1)
      if fs == 0.0
        Ω = merge_non_ts!(S, subgrp)
        append!(to_delete, subgrp[subgrp.!=Ω])
        continue
      end
      Δ     = round(Int64, sμ/fs)
      w_tmp = Array{Array{Int64,2},1}(undef, N)
      te    = Array{Int64,1}(undef, N)
      for i = 1:N
        m     = getindex(subgrp, i)
        w_m   = t_win(getindex(T, m), Δ)
        n_w   = size(w_m, 1)

        # Store: w_start, w_end, channel_number, window_number, x_start, x_end
        w = hcat(w_m, Array{Int64,2}(undef, n_w, 4))
        j  = 0
        ws = 0
        we = 0
        while j < n_w
          j       = j + 1
          we      = max(we, getindex(w_m, j, 2))
          setindex!(w, m, j, 3)                     # channel number
          setindex!(w, j, j, 4)                     # window number
          setindex!(w, ws+1, j, 5)                  # x_start
          ws      = ws + div(w[j,2]-w[j,1], Δ)+1    # x_end
          setindex!(w, ws, j, 6)
        end
        setindex!(te, we, i)
        setindex!(w_tmp, w, i)
      end
      W   = vcat(w_tmp...)
      ii  = sortperm(W[:,2], rev=true)
      W   = W[ii,:]

      # ====================================
      # Let Ω be the channel number in subgrp with the last end time
      Ω     = subgrp[argmax(te)]
      rest  = subgrp[subgrp.!=Ω]
      # ====================================

      # GAIN, MISC, NAME, NOTES, SRC ==========================================
      gain  = getindex(GAIN, Ω)
      notes = getindex(NOTES, Ω)
      misc  = getindex(MISC, Ω)
      for i in rest
        scalefac = gain / getindex(GAIN, i)
        if scalefac != 1.0
          rmul!(getindex(X, i), scalefac)
        end
        if getindex(SRC, i) != getindex(SRC, Ω)
          push!(notes, string(note_head, "+src: ", getindex(SRC, i)))
        end
        if getindex(NAME, i) != getindex(NAME, Ω)
          push!(notes, string(note_head, "alternate name: ", getindex(NAME, i)))
        end
        append!(notes, getindex(NOTES, i))
        merge!(misc, getindex(MISC, i))
      end
      sort!(notes)

      # EventTraceData extra fields ===========================================
      if Y == EventTraceData
        pha   = PhaseCat()
        az    = getindex(getfield(S, :az), Ω)
        baz   = getindex(getfield(S, :baz), Ω)
        dist  = getindex(getfield(S, :dist), Ω)
        for i in rest
          if az == 0.0
            θ = getindex(getfield(S, :az), i)
            (θ != 0.0) && (az = θ)
          end
          if baz == 0.0
            β = getindex(getfield(S, :baz), i)
            (β != 0.0) && (baz = β)
          end
          if dist == 0.0
            # Δ is already in use, so...
            d = getindex(getfield(S, :dist), i)
            (d != 0.0) && (dist = d)
          end
          merge!(pha, getindex(getfield(S, :pha), i))
        end
        # This guarantees that the phase catalog of Ω overwrites others
        merge!(pha, getindex(getfield(S, :pha), Ω))
        setindex!(getfield(S, :az),     az, Ω)
        setindex!(getfield(S, :baz),   baz, Ω)
        setindex!(getfield(S, :dist), dist, Ω)
        setindex!(getfield(S, :pha),   pha, Ω)
      end

      # T,X ===================================================================
      nW = size(W,1)
      (src, dest) = get_next_pair(W)
      while (src, dest) != (zeros(Int64, 7), zeros(Int64, 7))
        ts_i = src[1];  te_i = src[2];  p = src[3];  p_i = src[4];  os_p = src[5];  W_p = src[7]
        ts_j = dest[1]; te_j = dest[2]; q = dest[3]; q_i = dest[4]; os_q = dest[5]; W_q = dest[7]
        ts_max = max(ts_i, ts_j)
        te_min = min(te_i, te_j)
        nov = 1 + div(te_min - ts_max, Δ)
        Xq = getindex(X, q)

        # (1) determine the times and indices of overlap within each pair
        # a. determine sample times of overlap
        Ti = collect(ts_max:Δ:te_min)
        Tj = deepcopy(Ti)

        # b. get sample indices within each overlap window
        # i
        xsi_i = div(ts_max - ts_i, Δ) + os_p
        xei_i = xsi_i + nov - 1
        # j
        xsi_j = div(ts_max - ts_j, Δ) + os_q
        xei_j = xsi_j + nov - 1

        # (2) Extract sample windows
        Xi = getindex(getindex(X, p), xsi_i:xei_i)
        Xj = getindex(getindex(X, q), xsi_j:xei_j)
        lxp = length(getindex(X, p))
        lxq = length(getindex(X, q))

        # ================================================================
        # check for duplicate windows
        if (ts_i == ts_j) && (te_i == te_j) && (Xi == Xj)
          # delete time window
          W = W[setdiff(1:end, W_p), :]
        else
          # Check for misalignment:
          τ, χ, do_xtmerge, δj = check_alignment(Ti, Tj, Xi, Xj, Δ)
          if do_xtmerge
            xtmerge!(τ, χ, div(Δ,2))
          end
          if δj != 0
            xsi_i += δj
            xei_j -= δj
          end

          # (3) Merge X,T into S[q]
          deleteat!(Xq, xsi_j:xei_j)
          if xsi_j == 1
            prepend!(Xq, χ)
          else
            splice!(Xq, xsi_j:xsi_j-1, χ)
          end

          # (4) Adjust start, end indices of windows ≥ q_i in q
          # structure: w_start, w_end, channel_number, window_number, x_start, x_end
          nxq = length(Xq) - lxq
          i = 0
          while i < nW
            i += 1
            if W[i, 3] == q && W[i, 4] ≥ q_i
              W[i, 1] += nxq*Δ
              W[i, 2] += nxq*Δ
              W[i, 5] += nxq
              W[i, 6] += nxq
            end
          end

          #= if xsi_i ≤ os_p (which is always true, at this point in
          the control flow), we decrease W[W_p, 1:2] =#
          nxp = xei_i-xsi_i+1
          W[W_p, 1] -= δj*Δ
          W[W_p, 2] -= (nxp + δj)*Δ

          #= Control for when window P is emptied; the above two statements
          make this possible =#
          if (W[W_p, 2] < W[W_p, 1])
            W = W[setdiff(1:end, W_p), :]
          else
            W[W_p, 6] -= nxp
          end
        end
        # Sort by end time, to ensure we pick the window with latest end next
        k = sortperm(W[:,2], rev=true)
        W = W[k,:]
        nW = size(W,1)
        (src, dest) = get_next_pair(W)
      end
      # Repeat until no further merges are possible
      kk = sortperm(W[:,1])
      W = W[kk,:]

      #= At this point, we have nothing left that can be merged. So we're going
      to arrange T[subgrp] and X[subgrp] in windows using t_win =#

      n = size(W, 1)
      nx = broadcast(+, getindex(W, :, 6).-getindex(W, :,5), 1)
      X_Ω = Array{eltype(X[Ω]),1}(undef, sum(nx))
      xi = 1
      i = 0
      while i < n
        i   = i + 1
        p   = getindex(W, i, 3)
        lx  = getindex(nx, i)
        copyto!(X_Ω, xi, getindex(X, p), getindex(W, i, 5), lx)
        xi  = xi + lx
      end

      # Shrink W and eliminate windows with no actual gap between them
      m = trues(n)
      while n > 1
        if W[n-1, 2] + Δ == W[n,1]
          W[n-1, 2] = W[n,2]
          m[n] = false
        end
        n = n - 1
      end
      W = W[m, [1,2]]
      setindex!(T, w_time(W, Δ), Ω)
      setindex!(X, X_Ω, Ω)
      note!(S, Ω, string("merge!, combined channels ",
                          replace(repr(subgrp), ","=>""),
                          " (N = ", N, ") into :t, :x"))
      append!(to_delete, rest)
    end
    # Done with SUBGRPS
  end
  if N_warn > 0
    @info(string("Total warnings: ", N_warn))
  end
  deleteat!(S, to_delete)
  sort!(S)
  return nothing
end

# The following must be the same (or unset) to merge:
# :loc
# :resp
# :fs         cannot be unset; not a nullable array
# :units

# The following are easily dealt with:
# :gain       can be translated with no trouble
# :misc       can call merge! method for dictionaries
# :notes      append and sort
# :name       not important, can log extra names to :notes
# :src        use most recent, log extras to :notes


using SimpleTraits.BaseTraits, SimpleTraits

export Add

"""
    Add{N} <: CompositeBlock{N}
    Add{N}(iterable) -> Add
    Add(blocks::AbstractBlock{N}...) -> Add
"""
struct Add{N} <: CompositeBlock{N}
    list::Vector{AbstractBlock{N}}

    Add{N}(list::Vector{AbstractBlock{N}}) where {N} = new{N}(list)
    Add{N}() where {N} = new{N}(AbstractBlock{N}[])
end

#Add{N}() where N = Add{N}(AbstractBlock{N}[])
Add{N}(blocks) where {N} = Add{N}(collect(AbstractBlock{N}, blocks))
Add(blocks::AbstractBlock{N}...) where {N} = Add{N}(blocks)

function mat(::Type{T}, x::Add{N}) where {N,T}
    length(x.list) == 0 && return Diagonal(zeros(T, 1 << N))
    mapreduce(x -> mat(T, x), +, x.list)
end

chsubblocks(x::Add{N}, it) where {N} = Add{N}(it)

function _apply!(r::AbstractRegister, x::Add)
    isempty(x.list) && return r
    length(x.list) == 1 && return _apply!(r, x.list[])

    res = mapreduce(blk -> _apply!(copy(r), blk), regadd!, x.list[1:end-1])
    _apply!(r, x.list[end])
    regadd!(r, res)
    r
end

export Add

subblocks(x::Add) = x.list
cache_key(x::Add) = map(cache_key, x.list)
Base.copy(x::Add{N}) where {N} = Add{N}(copy(x.list))
Base.similar(c::Add{N}) where {N} = Add{N}(empty!(similar(c.list)))

function Base.:(==)(lhs::Add{N}, rhs::Add{N}) where {N}
    (length(lhs.list) == length(rhs.list)) && all(lhs.list .== rhs.list)
end

for FUNC in [:length, :iterate, :getindex, :eltype, :eachindex, :popfirst!, :lastindex]
    @eval Base.$FUNC(x::Add, args...) = $FUNC(subblocks(x), args...)
end

Base.getindex(c::Add{N}, index::Union{UnitRange,Vector}) where {N} = Add{N}(getindex(c.list, index))
Base.setindex!(c::Add{N}, val::AbstractBlock{N}, index::Integer) where {N} =
    (setindex!(c.list, val, index); c)
Base.insert!(c::Add{N}, index::Integer, val::AbstractBlock{N}) where {N} =
    (insert!(c.list, index, val); c)
Base.adjoint(blk::Add{N}) where {N} = Add{N}(map(adjoint, subblocks(blk)))

## Iterate contained blocks
occupied_locs(c::Add) = (unique(Iterators.flatten(occupied_locs(b) for b in subblocks(c)))...,)

# Additional Methods for Add
Base.push!(c::Add{N}, val::AbstractBlock{N}) where {N} = (push!(c.list, val); c)

function Base.push!(c::Add{N}, val::Function) where {N}
    push!(c, val(N))
end

function Base.append!(c::Add, list)
    for blk in list
        push!(c, blk)
    end
    c
end

function Base.prepend!(c::Add, list)
    for blk in list[end:-1:1]
        insert!(c, 1, blk)
    end
    c
end

YaoBase.ishermitian(ad::Add) = all(ishermitian, ad.list) || ishermitian(mat(ad))


function cumul_grasp_prob(arm::Arm, ref_time::Float64, ie::InteractionEvent)

    true_ref_time = (ref_time < ie.timestamps[START]) ? ie.timestamps[START] : ref_time
    grasp_time = convert(Int64, floor(ie.timestamps[FINISH] - arm.downtime/2.0 - true_ref_time))

    return cumprob(arm.grasp_model, grasp_time)
end

# Utility of a successful pickup is just 1
pickup_util(ie::InteractionEvent) = 1.0

"""
For conveyor belt, the CG is a directed chain, so we do not need any higher level coordination.
A later arm only needs to consider the previous arm's policy tree to identify objects
that are either inevitable or potentially likely to enter its own workspace.
"""
function identify_inevitable_adversarial_entries(tree::SearchTree, workspace_objs::Set{String}, decision_idx::Int64)

    if decision_idx == -1
        return workspace_objs, ""
    end

    # Now only explore a subtree if there IS a decision node that will be taken
    # N.B - This will be the chosen option in most cases
    @assert typeof(tree.nodes[decision_idx]) == SCoBASolver.DecisionNode

    # Attempted box may fail - adversarial entry
    adversarial_entry = tree.nodes[decision_idx].task_name

    possibly_considered = Set{String}()

    # Traverse the subtree and note which objects are possibly considered
    # Then add the remainder to entries
    subtree_fringe = Deque{Int64}()
    push!(subtree_fringe, decision_idx)

    while !isempty(subtree_fringe)

        stf_top = popfirst!(subtree_fringe)

        if typeof(tree.nodes[stf_top]) == SCoBASolver.DecisionNode &&
            tree.nodes[stf_top].attempt == true
            push!(possibly_considered, tree.nodes[stf_top].task_name)
        end

        if haskey(tree.child_ids, stf_top)
            for stfc in tree.child_ids[stf_top]
                push!(subtree_fringe, stfc)
            end
        end
    end

    # Get all the objs not considered during subtree search
    not_considered = setdiff(workspace_objs, possibly_considered)

    return not_considered, adversarial_entry
end

"""
Run SCoBA on the conveyor belt domain.
"""
function scoba_conveyor!(server::ArmAllocationServer, cbelt::ConveyorBelt)

    first_arm = server.agent_ordering[1]

    if server.agent_prop_set[first_arm].obj_grabbed == ""
        # Create wrapper for success probability

        @debug first_arm
        @debug server.agent_prop_set[first_arm].workspace_objs

        success_prob_fn_first(rt, ie) = cumul_grasp_prob(server.agent_set[first_arm], rt, ie)
        generate_search_tree!(server, first_arm, server.agent_prop_set[first_arm].workspace_objs,
                             success_prob_fn_first, pickup_util,server.agent_set[first_arm].downtime)

        tree = server.agent_prop_set[first_arm].tree
        entries = Set{String}()
        adv_entry = ""
        if ~(isempty(tree))
            dec_idx = get_next_attempt_idx(tree)

            if dec_idx != -1
                dec_node = tree.nodes[dec_idx]
                @debug dec_node
                grasp_coords = get_box_point(cbelt, dec_node.task_name, dec_node.timeval)
                @assert grasp_coords[1] != Inf

                server.agent_task_allocation[first_arm] = (name=dec_node.task_name, time=dec_node.timeval)
            end

            entries, adv_entry = identify_inevitable_adversarial_entries(tree,
                                    server.agent_prop_set[first_arm].workspace_objs, dec_idx)
        end
    else
        # Need to populate entries and adv_entry
        entries = server.agent_prop_set[first_arm].workspace_objs
        adv_entry = ""
    end

    for arm_name in server.agent_ordering[2:end]

        if server.agent_prop_set[arm_name].obj_grabbed != ""
            continue
        end

        @debug arm_name
        # Objs to consider is the union of inevitable entries and workspace objs
        objs_to_consider = union(entries, server.agent_prop_set[arm_name].workspace_objs)
        @debug objs_to_consider

        # If the above yields nothing, THEN just consider adversarial_entry
        if isempty(objs_to_consider)
            if adv_entry != ""
                objs_to_consider = Set{String}()
                push!(objs_to_consider, adv_entry)
            end
        end

        if !isempty(objs_to_consider)

            success_prob_fn_rest(rt, ie) = cumul_grasp_prob(server.agent_set[arm_name], rt, ie)
            generate_search_tree!(server, arm_name, objs_to_consider,
                                 success_prob_fn_rest, pickup_util,
                                 server.agent_set[arm_name].downtime)
            tree = server.agent_prop_set[arm_name].tree
            dec_idx = get_next_attempt_idx(tree)

            if dec_idx != -1
                dec_node = tree.nodes[dec_idx]
                @debug dec_node
                grasp_coords = get_box_point(cbelt, dec_node.task_name, dec_node.timeval)
                @assert grasp_coords[1] != Inf

                server.agent_task_allocation[arm_name] = (name=dec_node.task_name, time=dec_node.timeval)
            end

            entries, adv_entry = identify_inevitable_adversarial_entries(tree,
                            server.agent_prop_set[arm_name].workspace_objs, dec_idx)
        end

        # If empty, then all relevant things would anyway be empty
    end # end for arm_name

    @debug server.agent_task_allocation
end


module ClimateERA

# Main file for the ClimateERA module that downloads and processes ECMWF
# reanalysis data.

## Modules Used
using Crayons
using Dates
using DelimitedFiles
using GeoRegions
using Glob
using JLD2
using Logging
using NCDatasets
using Printf
using Statistics

## Exporting the following functions:
export
        erastartup, erainitialize, eradownload, eraanalysis, eracompile, eralvlsplit,
        erasubregion, erancread,
        erarawname, erarawread, erarawfolder, erarawsave,
        eraananame, eraanaread, eraanafolder,
        eracmpname, eracmpread,
        erafolder, eraregfolder, eravarfolder, eraimgfolder,
        erarawregion, erarawpoint, erarawgrid,
        eraparameterload, eraparametercopy, eraparameteradd, erapressureload,
        eraregionstep,
        eplotsubregion, eplotlsm, eplotlsmgrid,
        queryeaction, queryedataset, queryemod, queryepar, putinfo,
        hrstep, hrindy

## Including other files in the module

include("startup.jl")
include("initialize.jl")
include("download.jl")
include("analysis.jl")
include("compile.jl")
#include("calculate.jl")
include("eraplots.jl")

include("frontend.jl")
include("backend.jl")
include("query.jl")

end # module


function gamesize(g::AirDefenseGame)
    ncity, ndp, nap, ndpdc, ndpac, ndc, nac = g.ncity, g.ndp, g.nap, g.ndpdc, g.ndpac, g.ndc, g.nac
    nIc1 = 1
    nIa2 = binomial(ndp, ndpac)
    nIa5 = nIa2 * binomial(ndp, nac) * binomial(ndp, ndc)
    nIc3 = 1
    nId4 = binomial(ndp, ndpdc) * (2 ^ ndp)
    nIa = @sprintf("%.2E", nIa2 + nIa5)
    nId = @sprintf("%.2E", nId4)
    na1c = binomial(ndp, ndpdc)
    na2c = binomial(ndp, ndpac)
    na3a = binomial(ndpac, nac)
    na4c = 2 ^ nac
    na5d = binomial(ndpdc, ndc)
    na6a = binomial(ncity, nap)
    nas = [na1c, na2c, na3a, na4c, na5d, na6a]
    leafnodes = @sprintf("%.2E", prod(nas))
    nnodes = @sprintf("%.2E", sum(cumprod(nas)))
    (nIa, nId, leafnodes, nnodes)
end

# N : player indices (nature not included)
getplayers(g::AirDefenseGame) = 1:2

# Player index of current node (note: this is distinct from which player is currently updating CFR)
function getplayer(depth::Int64)
    (depth == 6 || depth == 3) ? 2 : (depth == 5) ? 1 : 3
    # 1 = defender, 2 = attacker, 3 = nature
end

actions(depth::Int64, An::Vector{UnitRange{Int64}}) = An[depth]
actions(depth::Int64, A::Vector{Vector{Vector{Int64}}}) = 1:length(A[depth])

function numactions(I::Int64, ni_stage, na_stage)
    if I in 1:ni_stage[3]
        return na_stage[3] # na3a
    elseif I in (ni_stage[3] + 1):(ni_stage[3] + ni_stage[5])
        return na_stage[5] # na5d
    elseif I in (ni_stage[3] + ni_stage[5] + 1):(ni_stage[3] + ni_stage[5] + ni_stage[6])
        return na_stage[6] # na5a
    else
        @warn "Infoset $I not found."
        return 0
    end
end

terminal(depth::Int64) = depth == 7

function details(h::SVector, A)
    len = findfirst(x -> x == 0, h) - 1
    (len == nothing) && (len = length(h))
    df = DataFrame(Stage = Int[], Domain = String[], Player = String[], ActionNumber = Int[], Action = Array[])
    for i = 1:len
        stage = i
        domain = (i == 6) ? "Physical" : "Cyber"
        p = getplayer(h[1:i])
        pstring = p == 2 ? "Attacker" : p == 1 ? "Defender" : "Chance"
        push!(df, (stage, domain, pstring, h[i], A[i][h[i]]))
    end
    df
end

function infoset(h::SVector, depth::Int64, g::AirDefenseGame, gs::GameSet) # btime: ~150 ns
    # findfirst benchmarks faster than numerically calculating infoset (see earlier commit)
    A, ni_stage, na_stage, sensors, iadsdefenses = gs.A, gs.ni_stage, gs.na_stage, gs.sensors, gs.iadsdefenses
    cyber_attacks, cyber_defenses = A[3], A[5]
    ndp, nac = g.ndp, g.nac
    len = depth - 1
    if len == 5
        defended_iads = getindex(gs.A[1][h[1]], gs.A[5][h[5]])
        idc = findfirst(x -> x == defended_iads, iadsdefenses)
        niadsdefenses = length(gs.iadsdefenses)
        return sum(ni_stage[1:5]) + (h[2] - 1) * na_stage[3] * niadsdefenses + (h[3] - 1) * niadsdefenses + idc
    elseif len == 4
        nsensor = length(sensors) # number of combinations of sensed cyber attacks
        sensed = getindex(gs.A[2][h[2]], getindex(A[3][h[3]], A[4][h[4]] .== 2)) # ndp id's of sensed attacks
        isensor = findfirst(x -> x == sensed, sensors) # infoset id of that combination
        return sum(ni_stage[1:4]) + (h[1] - 1) * nsensor + isensor
    elseif len == 2
        return h[2]
    else
        player = getplayer(h)
        @warn "No infoset found. Chance node? h = $h, player = $player."
        return 0
    end
end

#TODO: Make these probabilities more rational?
function chance(depth::Int64, a::Int64, g::AirDefenseGame, gs::GameSet)
    len = depth - 1
    if len == 3 # nature determines defense cyber SA
        nsensed = sum(gs.A[4][a] .== 2) # number of sensed attacks
        return (g.psc ^ nsensed) * (1 - g.psc)^(g.nac - nsensed)
    elseif len == 1 # nature deals offensive capes
        return 1.0 / gs.na_stage[2]
    elseif len == 0 # nature deals defensive capes
        return 1.0 / gs.na_stage[1]
    else
        @warn "Node at depth $depth not a chance node."
        return 0.0
    end
end

function leafutility(h::SVector, player::Int64, πi::Float64, πo::Float64, g::AirDefenseGame, gs::GameSet)
    c1, c2, ac, c4, dc, ap = h
    attacked_cities = @view gs.A[6][h[6]][:]
    defended_iads = getindex(g.iads, getindex(gs.A[1][h[1]], gs.A[5][h[5]]))
    attacked_iads = getindex(g.iads, getindex(gs.A[2][h[2]], gs.A[3][h[3]]))
    leaf_value = utility(g, gs.C, defended_iads, attacked_iads, attacked_cities)
    u = player == 1 ? leaf_value * πo : -1.0 * leaf_value * πo
    return u
end

# convert behavioral strategy to realization strategy (i.e. sequence form?)
function real_strat_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
    length(σ) != ni && @warn "Unexpected behavioral strategy length: $(length(σ))."
    σ1 = σ[(ni_stage[3] + 1):sum(ni_stage[1:5])]
    σ2 = vcat(σ[1:ni_stage[3]], σ[(sum(ni_stage[1:5]) + 1):end])
    rp1 = zeros(Float64, ns1)
    rp2 = zeros(Float64, ns2)
    rp1[1], rp2[1] = 1.0, 1.0
    for (i, s1) in enumerate(σ1)
        rp1[nextseqI1[i]] .= rp1[seqI1[i]] .* s1
    end
    for (i, s2) in enumerate(σ2)
        rp2[nextseqI2[i]] .= rp2[seqI2[i]] .* s2
    end
    return (rp1, rp2)
end

function real_strat_struct_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
    length(σ) != ni && @warn "Unexpected behavioral strategy length: $(length(σ))."
    σ1 = σ[(ni_stage[3] + 1):sum(ni_stage[1:5])]
    σ2 = vcat(σ[1:ni_stage[3]], σ[(sum(ni_stage[1:5]) + 1):end])
    rp1_struct, rp2_struct = similar(σ1), similar(σ2)
    rp1, rp2 = real_strat_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
    for (i, s1) in enumerate(σ1)
        rp1_struct[i] = rp1[nextseqI1[i]]
    end
    for (i, s2) in enumerate(σ2)
        rp2_struct[i] = rp2[nextseqI2[i]]
    end
    return (rp1_struct, rp2_struct)
end

function real_strat_struct_combined_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
    σ1 = σ[(ni_stage[3] + 1):sum(ni_stage[1:5])]
    σ2 = vcat(σ[1:ni_stage[3]], σ[(sum(ni_stage[1:5]) + 1):end])
    rp1_struct, rp2_struct = similar(σ1), similar(σ2)
    rp1, rp2 = real_strat_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
    for (i, s1) in enumerate(σ1)
        rp1_struct[i] = rp1[nextseqI1[i]]
    end
    for (i, s2) in enumerate(σ2)
        rp2_struct[i] = rp2[nextseqI2[i]]
    end
    rp_struct = similar(σ)
    rp_struct[1:ni_stage[3]] .= rp2_struct[1:ni_stage[3]]
    rp_struct[(ni_stage[3] + 1):sum(ni_stage[1:5])] .= rp1_struct
    rp_struct[(sum(ni_stage[1:5]) + 1):end] .= rp2_struct[(ni_stage[3] + 1):end]
    return rp_struct
end

function real_strat(σ::Vector{Vector{Float64}}, gs::GameSet, gos::GameOptSet)
    ni, ni_stage, ns1, ns2 = gs.ni, gs.ni_stage, gos.ns1, gos.ns2
    seqI1, seqI2, nextseqI1, nextseqI2 = gos.seqI1, gos.seqI2, gos.nextseqI1, gos.nextseqI2
    return real_strat_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
end

function real_strat(σ::Vector{Vector{Float64}}, g::AirDefenseGame)
    A, _, na_stage = build_action(g)
    ni, ni1, ni2, ni_stage = build_info(g)
    ns1, ns2, ns1_stage, ns2_stage = build_nseq(g, na_stage, ni_stage)
    _, _, (seqI1, seqI2), (nextseqI1, nextseqI2) = build_Iset(g, A, na_stage, ns1_stage, ns2_stage, ni1, ni2, ns1, ns2)
    return real_strat_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
end

function real_strat_struct(σ::Vector{Vector{Float64}}, g::AirDefenseGame)
    A, _, na_stage = build_action(g)
    ni, ni1, ni2, ni_stage = build_info(g)
    ns1, ns2, ns1_stage, ns2_stage = build_nseq(g, na_stage, ni_stage)
    _, _, (seqI1, seqI2), (nextseqI1, nextseqI2) = build_Iset(g, A, na_stage, ns1_stage, ns2_stage, ni1, ni2, ns1, ns2)
    return real_strat_struct_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
end

function real_strat_struct_combined(σ::Vector{Vector{Float64}}, g::AirDefenseGame)
    A, _, na_stage = build_action(g)
    ni, ni1, ni2, ni_stage = build_info(g)
    ns1, ns2, ns1_stage, ns2_stage = build_nseq(g, na_stage, ni_stage)
    _, _, (seqI1, seqI2), (nextseqI1, nextseqI2) = build_Iset(g, A, na_stage, ns1_stage, ns2_stage, ni1, ni2, ns1, ns2)
    return real_strat_struct_combined_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
end

function real_strat_struct_combined(σ::Vector{Vector{Float64}}, gs::GameSet, seqvals)
    ni, ni_stage = gs.ni, gs.ni_stage
    ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2 = seqvals
    return real_strat_struct_combined_calc(σ, ni, ni_stage, ns1, ns2, seqI1, seqI2, nextseqI1, nextseqI2)
end

# build consraint set on sequence-form strategy for CCFR
function build_ccfr_constraints(g, σfix, conf::Float64, sd::Float64)
    noise = Normal(0, sd)
    hw = quantile(noise, 1 - (1 - conf) / 2)
    σlb = [max.(σi .- hw, 0.0) for σi in σfix]
    σub = [min.(σi .+ hw, 1.0) for σi in σfix]
    rp2fix = real_strat_struct(σfix, g)[2]
    rp2lb = real_strat_struct(σlb, g)[2]
    rp2ub = real_strat_struct(σub, g)[2]
    !all(rp2lb .<= rp2fix .<= rp2ub) && @warn "Sequence-form strategy out of bounds."
    constraint_coeff = [[1, -1 .* rp2lb[i][j]] for i in 1:length(rp2lb), j in 1:length(rep2lb[i])]
    constraint_coeff = vcat(hcat(ones(length(rp2fix)), -1 .* rp2lb), hcat(-1 .* ones(length(rp2fix)), rp2ub))
    @show length(constraint_coeff), length(constraint_coeff[:,1])
    grad_constraint_coeff = hcat(constraint_coeff[:,1], zeros(length(constraint_coeff[:,1])))
    return constraint_coeff, grad_constraint_coeff
end


module BenchMixLookup
using BenchmarkTools
using SymSpellChecker
using SymSpellChecker: VerbosityALL, VerbosityCLOSEST

ASSETS_PATH = joinpath(@__DIR__, "..", "assets")

function load_noisy()
    map(readlines(joinpath(ASSETS_PATH, "noisy_query_en_1000.txt"))) do x
        String(strip(split(x)[1]))
    end
end

function mix(d, data, verbosity)
    sum(length(lookup(d, word, verbosity = verbosity)) for word in data)
end

dicts = [
    joinpath(ASSETS_PATH, "frequency_dictionary_en_30_000.txt"),
    joinpath(ASSETS_PATH, "frequency_dictionary_en_82_765.txt"),
    joinpath(ASSETS_PATH, "frequency_dictionary_en_500_000.txt"),
]

query1k = load_noisy()

suite = BenchmarkGroup()

for edit_distance in 1:3
    for prefix_length in 5:7
        d = SymSpell(dicts[2], max_dictionary_edit_distance = edit_distance, prefix_length = prefix_length)
        suite["mix_lookup_closest_$(edit_distance)_$(prefix_length)"] = @benchmarkable mix($d, $query1k, $VerbosityCLOSEST)
    end
end
end

BenchMixLookup.suite


macro judge_var(model, variable, class, aargs, aakws)
   lag=0
   span=1000
   initial=0.0
   ub=nothing
   lb=nothing
   penalty=nothing
   state_name=:nothing

   vartype=:Con

   if length(aargs)==0
      push!(aargs,:Con)
   end

   if length(aargs)!=1
      ex = quote
         error("@"*string($class)*" macro takes at most three positional arguments")
      end
      return ex
   elseif aargs[1] ∉ [:Con,:Bin,:Int]
      ex = quote
         error("Optional third positional argument must be \'Bin\' or \'Int\'")
      end
      return ex
   end

   for (a,b) in aakws
      if a==:lag
         lag=b
      elseif a==:duration
         span=b
      elseif a==:initial
         initial=b
      elseif a==:lb
         lb=b
      elseif a==:ub
         ub=b
      elseif a==:penalty
         penalty=b
      elseif a==:state_name
         state_name=b
      else
         ex = quote
            error("Invalid keyword argument for @"*string($class)*" macro")
         end
         return ex
      end
   end

   if class==:(:state)
      if lag!=0
         @warn("'lag' keyword has been ignored")
         lag=0
      end
      if span!=1000
         @warn("'duration' keyword has been ignored")
      end
      span=1
   elseif state_name!=:nothing
      @warn("'state_name' keyword has been ignored")
      state_name=:nothing
   end

   if !(class==:state || class==:enforced) && penalty!=nothing
      @warn("'penalty' keyword has been ignored")
      penalty=nothing
   end

   tmp=nothing

   ex = quote
      if !haskey($model.ext, :expansions)
         $model.ext[:expansions] = Dict{Symbol,Any}()
         $model.ext[:options] = Dict{Symbol,Tuple}()
      end
      if length($aargs)==1
         if $aargs[1]==:Con
            tmp=@variable($model, $variable)
         elseif $aargs[1]==:Bin
            tmp=@variable($model, $variable, Bin)
         else
            tmp=@variable($model, $variable, Int)
         end
      end

      if $(Meta.quot(state_name))==:nothing
         sym=[k for (k,v) in $model.obj_dict if v===tmp][1]
         $model.ext[:expansions][sym]=tmp
         $model.ext[:options][sym]=($class,$lag,$span,$aargs[1],$lb,$ub,$initial,$penalty)
      else
         $model.ext[:expansions][$(Meta.quot(state_name))]=tmp
         $model.ext[:options][$(Meta.quot(state_name))]=($class,$lag,$span,$aargs[1],$lb,$ub,$initial,$penalty)
         $state_name=tmp
      end

   end
   return esc(ex)
end

"""
	expansion(model, variable, args...)

Defines an expansion variable `variable` within a subproblem `model`. Note that all subproblems must have the same set of expansion variables.

### Required Arguments
`model` is the JuDGE subproblem that we are adding the expansion variable to

`variable` is the name of the variable being created, this will be continuous by default; follows JuMP syntax if defining a set of variables.

### Optional Arguments
This macro has a third, unnamed, argument which can be set to Con, Bin, or Int, similar to the `@variable` macro.

`lag` is the number of nodes in the scenario between an expansion being decided, and it becoming available.

`duration` is the number of consecutive nodes in the scenario over which an expansion is available.

`lb` is the lower bound for this variable in the master problem (typically omitted).

`ub` is the upper bound for this variable in the master problem (typically omitted).

### Examples
    @expansion(model, expand[1:5], Bin) #defines an array of 5 binary variables with no lag, and unlimited lifespan
    @expansion(model, expand[1:5,1:2]>=0, lag=1) #defines a matrix of 10 continuous variables with a lag of 1, and unlimited duration
    @expansion(model, 0<=expand<=10, Int, duration=2) #defines a single integer variable with a lag of 0, and a duration of 2
"""
macro expansion(model, variable, args...)
   aargs = []
   aakws = Pair{Symbol,Any}[]
   for el in args
      if Meta.isexpr(el, :(=))
         push!(aakws, Pair(el.args...))
      else
         push!(aargs, el)
      end
   end
   ex = quote
      JuDGE.@judge_var($model, $variable, :expansion, $aargs, $aakws)
   end

   return esc(ex)
end

"""
	shutdown(model, variable, args...)

Defines an shutdown variable `variable` within a subproblem `model`. Note that all subproblems must have the same set of shutdown variables.

### Required Arguments
`model` is the JuDGE subproblem that we are adding the shutdown variable to

`variable` is the name of the variable being created, this will be continuous by default; follows JuMP syntax if defining a set of variables.

### Optional Arguments
This macro has a third, unnamed, argument which can be set to Con, Bin, or Int, similar to the `@variable` macro.

`lag` is the number of nodes in the scenario between an shutdown being decided, and it becoming unavailable.

`duration` is the number of consecutive nodes in the scenario over which the shutdown will last.

`lb` is the lower bound for this variable in the master problem (typically omitted).

`ub` is the upper bound for this variable in the master problem (typically omitted).

### Examples
    @shutdown(model, shut[1:5], Bin) #defines an array of 5 binary variables with no lag, and unlimited lifespan
    @shutdown(model, shut[1:5,1:2]>=0, lag=1) #defines a matrix of 10 continuous variables with a lag of 1, and unlimited duration
    @shutdown(model, 0<=shut<=10, Int, duration=2) #defines a single integer variable with a lag of 0, and a duration of 2
"""
macro shutdown(model, variable, args...)
   aargs = []
   aakws = Pair{Symbol,Any}[]
   for el in args
      if Meta.isexpr(el, :(=))
         push!(aakws, Pair(el.args...))
      else
         push!(aargs, el)
      end
   end
   ex = quote
      JuDGE.@judge_var($model, $variable, :shutdown, $aargs, $aakws)
   end

   return esc(ex)
end

"""
	enforced(model, variable, args...)

Defines an enforced variable `variable` within a subproblem `model`. Note that all subproblems must have the same set of enforced variables. These
variables can be used as either expansion or shutdown variables, but since the constraint in the master problem is an equality, convergence is
more difficult since there is less flexibility when solving the master problem.

### Required Arguments
`model` is the JuDGE subproblem that we are adding the expansion variable to

`variable` is the name of the variable being created, this will be continuous by default; follows JuMP syntax if defining a set of variables.

### Optional Arguments
This macro has a third, unnamed, argument which can be set to Con, Bin, or Int, similar to the `@variable` macro.

`lag` is the number of nodes in the scenario between an expansion being decided, and it becoming available.

`duration` is the number of consecutive nodes in the scenario over which an expansion is available.

`lb` is the lower bound for this variable in the master problem (typically omitted).

`ub` is the upper bound for this variable in the master problem (typically omitted).

`penalty` is a placeholder for a future feature, which may allow the violation of master/subproblem equality constraint, at a cost.

### Examples
    @expansion(model, forced[1:5], Bin) #defines an array of 5 binary variables with no lag, and unlimited lifespan
    @expansion(model, forced[1:5,1:2]>=0, lag=1) #defines a matrix of 10 continuous variables with a lag of 1, and unlimited duration
    @expansion(model, 0<=forced<=10, Int, duration=2) #defines a single integer variable with a lag of 0, and a duration of 2
"""
macro enforced(model, variable, args...)
   aargs = []
   aakws = Pair{Symbol,Any}[]
   for el in args
      if Meta.isexpr(el, :(=))
         push!(aakws, Pair(el.args...))
      else
         push!(aargs, el)
      end
   end
   ex = quote
      JuDGE.@judge_var($model, $variable, :enforced, $aargs, $aakws)
   end

   return esc(ex)
end

"""
	state(model, variable, args...)

Defines a state variable `variable` within a subproblem `model`. Note that all subproblems must have the same set of state variables. These
variables can be used to model inventory that is carried forward between the subproblems.

### Required Arguments
`model` is the JuDGE subproblem that we are adding the expansion variable to

`variable` is the name of the variable being created in the subproblem, this will be continuous by default; follows JuMP syntax if defining a set of variables.
The subproblem variable corresponds to the change in the state.

### Optional Arguments
This macro has a third, unnamed, argument which can be set to Con, Bin, or Int, similar to the `@variable` macro.

`state_name` is the name for the state variable in the master problem. If omitted, the name of the master problem variable
will match the subproblem variable (but this may cause confusion, since only the master problem variable is the state).
See the `inventory.jl` example to see how this should be implemented.

`initial` is the initial value for the master problem's state variable at the root node.

`lb` is the lower bound for the variable in the master problem (typically omitted).

`ub` is the upper bound for the variable in the master problem (typically omitted).

### Examples
    @state(sp, -50<=Δstock<=50, state_name=stock, lb=0, ub=200, initial=0) #defines a state variable called stock in the master
                                                                           #(starting at 0, and able to take values 0 to 200),
                                                                           #and Δstock in the subproblem (able to change the stock level by ±50).
"""
macro state(model, variable, args...)
   aargs = []
   aakws = Pair{Symbol,Any}[]
   for el in args
      if Meta.isexpr(el, :(=))
         push!(aakws, Pair(el.args...))
      else
         push!(aargs, el)
      end
   end
   ex = quote
      JuDGE.@judge_var($model, $variable, :state, $aargs, $aakws)
   end

   return esc(ex)
end

"""
	capitalcosts(model, expr)

Defines a linear expression specifying the capital cost of expansions and shutdowns at the current node

### Required Arguments
`model` is the JuDGE subproblem corresponding to the node in the scenario tree that we are adding specifying the costs for

`expr` is an `AffExpr` which gives the total cost of choosing expansion and shutdown variables at the current node

### Example
    @capitalcosts(model, sum(expand[i]*cost[node][i] for i in 1:5))
"""
macro capitalcosts(model, expr)
   ex = quote
      #$model.ext[:capitalcosts] = @expression($model, $expr)
      $model.ext[:capitalcosts] = Dict()
      if typeof($expr)==AffExpr
         for (term,coef) in $expr.terms
            $model.ext[:capitalcosts][term]=coef
         end
         $model.ext[:capitalcosts][:constant]=$expr.constant
      elseif typeof($expr)==VariableRef
         $model.ext[:capitalcosts][$expr]=1.0
         $model.ext[:capitalcosts][:constant]=0.0
      elseif typeof($expr)==Float64
         $model.ext[:capitalcosts][:constant]=$expr
      end
   end
   return esc(ex)
end

"""
	ongoingcosts(model, expr)

Defines a linear expression specifying the ongoing costs of expansions and shutdowns available at the current node

### Required Arguments
`model` is the JuDGE subproblem corresponding to the node in the scenario tree that we are adding specifying the costs for

`expr` is an `AffExpr` which gives the ongoing cost of expansions and shutdowns available at the current node

### Example
    @ongoingcosts(model, sum(expand[i]*ongoingcosts[node][i] for i in 1:5))
"""
macro ongoingcosts(model, expr)
   ex = quote
#      $model.ext[:ongoingcosts] = @expression($model, $expr)
      $model.ext[:ongoingcosts] = Dict()
      if typeof($expr)==AffExpr
         for (term,coef) in $expr.terms
            $model.ext[:ongoingcosts][term]=coef
         end
         $model.ext[:ongoingcosts][:constant]=$expr.constant
      elseif typeof($expr)==VariableRef
         $model.ext[:ongoingcosts][$expr]=1.0
         $model.ext[:ongoingcosts][:constant]=0.0
      elseif typeof($expr)==Float64
         $model.ext[:ongoingcosts][:constant]=$expr
      end
   end
   return esc(ex)
end


"""

    oparse(pt::Type{T<:Parser}, c::Union{Corpus,Sentence},
           ndeps:Integer, feats=nothing; usepmap=false)

The oracle parser executes the moves that will lead to the best (least
cost) parse.  It returns the final parse(s) and, if `feats` is
specified, the input features and correct moves for training.

## Arguments:
* pt::ParserType: ArcHybridR1, ArcEagerR1, ArcHybrid13, ArcEager13
* s::Sentence or c::Corpus: input sentence(s), a single parser is returned for s, a vector of parsers for c
* ndeps::Integer: number of dependency types
* feats::Fvec: (optional) specification of features, a (p,x,y) tuple returned if specified, only p if not
* usepmap::Bool: (optional) performs parallel processing

"""
function oparse{T<:Parser}(pt::Type{T}, s::Sentence, feats=nothing; o...)
    p = pt(s)
    op_work(p, s, feats)
end

function oparse{T<:Parser}(pt::Type{T}, c::Corpus, feats=nothing; usepmap::Bool=false)
    if usepmap
        op_pmap(pt, c, feats)
    else
        # op_main returns x,y in vectors for pcat
        p = op_main(pt, c, feats)
        if isa(p[1],Vector); (p[1],p[2][1],p[3][1]); else; p; end
    end
end

function op_pmap{T<:Parser}(pt::Type{T}, c::Corpus, feats)
    d = distribute(c)           # TODO: test what happens to common s.vocab
    p = pmap(procs(d)) do x
        op_main(pt, localpart(d), feats)
    end
    return pcat(p)
end

function op_main{T<:Parser}(pt::Type{T}, c::Corpus, feats)
    pa = map(pt, c)
    op_work(pa, c, feats)
end

# With no feats specified, just parse the sentences

function op_work{T<:Parser}(pa::Vector{T}, c::Corpus, feats::Void=nothing)
    for i=1:length(c)
        op_work(pa[i], c[i], feats)
    end
    return pa
end

function op_work(p::Parser, s::Sentence, feats::Void=nothing)
    c = Array(Cost, p.nmove)
    totalcost = 0
    while anyvalidmoves(p)
        movecosts(p, s.head, s.deprel, c)
        (bestcost,bestmove) = findmin(c)
        totalcost += bestcost
        move!(p, bestmove)
    end
    if totalcost != truecost(p,s); warn("Cost mismatch"); end
    return p
end

# With feats specified, parse the sentences and compute the features for training

function op_work{T<:Parser}(pa::Vector{T}, c::Corpus, feats::Fvec, x=nothing, y=nothing, nx=0)
    (x,y) = initoparse(pa,c,feats,x,y,nx)
    for i=1:length(c)
        op_work(pa[i], c[i], feats, x, y, nx)
        nx += nmoves(pa[i], c[i])
    end
    return (pa,typeof(x)[x],typeof(y)[y])  # must return tuple of three vectors for pcat
end

function op_work(p::Parser, s::Sentence, feats::Fvec, x=nothing, y=nothing, nx=0)
    (x,y) = initoparse(p,s,feats,x,y,nx)
    c = Array(Cost, p.nmove)
    totalcost = 0; nx0 = nx
    while anyvalidmoves(p)
        movecosts(p, s.head, s.deprel, c)
        (bestcost,bestmove) = findmin(c)
        totalcost += bestcost
        features(p, s, feats, x, (nx+=1))
        y[:, nx] = zero(eltype(y))
        y[bestmove, nx] = one(eltype(y))
        move!(p, bestmove)
    end
    if nx0 + nmoves(p,s) != nx; error(); end
    if totalcost != truecost(p,s); error("Cost mismatch"); end
    return (p,x,y)
end

function initoparse(p, s, f, x, y, n)
    if f == nothing; error(); end
    (xrows,xcols) = xsize(p,s,f)
    (yrows,ycols) = ysize(p,s)
    if xcols != ycols; error("Bad size"); end
    ytype = wtype(s)
    xtype = (if isa(f,DFvec); wtype(s); elseif isa(f,SFvec); SFtype; else; error(); end)
    mincols = xcols + n
    if y==nothing; y = zeros(ytype, yrows, mincols); end
    if x == nothing; x = Array(xtype, xrows, mincols); end
    if !(isa(y,AbstractMatrix{ytype}) && (size(y,1)==yrows) && (size(y,2)>=mincols)); error("Bad y"); end
    if !(isa(x,AbstractMatrix{xtype}) && (size(x,1)==xrows) && (size(x,2)>=mincols)); error("Bad x"); end
    return (x,y)
end

function pcat(p)
    if isa(p[1], Vector)
        q = similar(p[1], 0)
        for pi in p
            append!(q, pi)
        end
        return q
    else
        pp = similar(p[1][1],0)
        xx = similar(p[1][2],0)
        yy = similar(p[1][3],0)
        for i=1:length(p)
            append!(pp, p[i][1])
            append!(xx, p[i][2])
            append!(yy, p[i][3])
        end
        return (pp, xx, yy)
    end
end

### DEAD CODE

# function oparse{T<:Parser}(pt::Type{T}, c::Corpus, ndeps::Integer, feats::Fvec)
#     pa = map(s->pt(wcnt(s), ndeps), c)
#     oparse(pa, c, ndeps, feats)
# end

# function oparse{T<:Parser}(pt::Type{T}, c::Corpus, ndeps::Integer)
#     pa = map(s->pt(wcnt(s), ndeps), c)
#     oparse(pa, c, ndeps)
# end

# function oparse{T<:Parser}(pt::Type{T}, c::Corpus, ndeps::Integer, ncpu::Integer)
#     @date Main.resetworkers(ncpu)
#     sa = distribute(c)                                  # distributed sentence array
#     pa = map(s->pt(wcnt(s), ndeps), sa)                 # distributed parser array
#     @sync for p in procs(sa)
#         @spawnat p oparse(localpart(pa), localpart(sa), ndeps)
#     end
#     pa = convert(Vector{pt}, pa)
#     @date Main.rmworkers()
#     return pa
# end

# function oparse{T<:Parser}(pt::Type{T}, c::Corpus, ndeps::Integer, ncpu::Integer, feats::DFvec)
#     # Sparse fvec not supported yet, we need SharedSparseArray
#     @date Main.resetworkers(ncpu)
#     sa = distribute(c)                                  # distributed sentence array
#     pa = map(s->pt(wcnt(s), ndeps), sa)                 # distributed parser array
#     xtype = wtype(c[1])
#     x = SharedArray(xtype, xsize(pa[1],c,feats))    # shared x array
#     y = SharedArray(xtype, ysize(pa[1],c))          # shared y array
#     fill!(y, zero(xtype))
#     nx = zeros(Int, length(c))                      # 1+nx[i] is the starting x column for i'th sentence
#     p1 = pt(1,ndeps)
#     for i=1:length(c)-1
#         nx[i+1] = nx[i] + nmoves(p1, c[i])
#     end
#     @sync for p in procs(sa)
#         @spawnat p oparse(localpart(pa), localpart(sa), ndeps, feats, x, y, nx[localindexes(sa)[1][1]])
#     end
#     pa = convert(Vector{pt}, pa)
#     @date Main.rmworkers()
#     return (pa, sdata(x), sdata(y))
# end

# function oparse{T<:Parser}(pt::Type{T}, s::Sentence, ndeps::Integer)
#     p = pt(wcnt(s), ndeps)
#     oparse(p, s, ndeps)
# end

# function oparse{T<:Parser}(pt::Type{T}, s::Sentence, ndeps::Integer, feats::Fvec)
#     p = pt(wcnt(s), ndeps)
#     oparse(p, s, ndeps, feats)
# end

# function oparse{T<:Parser}(pt::Type{T}, c::Corpus, ndeps::Integer; usepmap::Bool=false)
#     if usepmap
#         d = distribute(c)
#         p = pmap(procs(d)) do x
#             oparse(pt, localpart(d), ndeps)
#         end
#         pcat(p)
#     else
#         pa = map(s->pt(wcnt(s), ndeps), c)
#         oparse(pa, c, ndeps)
#     end
# end

# function oparse{T<:Parser}(pt::Type{T}, c::Corpus, ndeps::Integer, feats::DFvec; usepmap::Bool=false)
#     if usepmap
#         d = distribute(c)
#         p = pmap(procs(d)) do x
#             oparse(pt, localpart(d), ndeps, feats)
#         end
#         pcat(p)
#     else
#         pa = map(s->pt(wcnt(s), ndeps), c)
#         oparse(pa, c, ndeps, feats)
#     end
# end

# function op_work(p::Parser, s::Sentence, ndeps::Integer, feats::SFvec, x=nothing, y=nothing, nx=0)
#     (x,y) = initoparse(p,s,ndeps,feats,x,y,nx)
#     c = Array(Position, p.nmove)
#     totalcost = 0; nx0 = nx
#     while anyvalidmoves(p)
#         movecosts(p, s.head, s.deprel, c)
#         (bestcost,bestmove) = findmin(c)
#         totalcost += bestcost
#         # We let features directly write in x.rowval[x.colptr[nx]:x.colptr[nx+1]-1]
#         features(p, s, feats, x, (nx+=1))
#         # We do not need this check any more, x.m is set to typemax
#         # features() sorts rowval so the max can be found at the end
#         # maxrow = x.rowval[x.colptr[nx+1]-1]
#         # If max rowval exceeds matrix height, we update the height
#         # maxrow > x.m && (x.m = maxrow)
#         y[:, nx] = zero(eltype(y))
#         y[bestmove, nx] = one(eltype(y))
#         move!(p, bestmove)
#     end
#     if nx0 + nmoves(p,s) != nx; error(); end
#     if totalcost != truecost(p,s); error("Cost mismatch"); end
#     return (p,x,y)
# end


using Test, ColorSchemes, ColorSchemeTools, FileIO, Colors
using ImageMagick, QuartzImageIO

function run_all_tests()

    @testset "basic functions" begin
        # load existing scheme from ColorSchemes.jl
        hok = ColorSchemes.hokusai

        @test length(hok) == 32

        # image file located here in the test directory

        # create a colorscheme from image file, default is 10
        hokusai_test = ColorSchemeTools.extract(dirname(@__FILE__) * "/hokusai.jpg")

        @test length(hokusai_test) == 10

        # extract colors and weights
        c, w =  ColorSchemeTools.extract_weighted_colors(dirname(@__FILE__) * "/hokusai.jpg", 10, 10, 0.01; shrink = 4)

        @test length(c) == 10
        @test length(w) == 10

        # test that sampling schemes yield different values
        @test get(hokusai_test, 0.0) != get(hokusai_test, 0.5)

        # test sort
        @test ColorSchemeTools.sortcolorscheme(hokusai_test, rev=true) != ColorSchemeTools.sortcolorscheme(hokusai_test)

        # create weighted palette; there is some unpredictability here... :)
        csw = colorscheme_weighted(c, w, 37)
        @test 36 <= length(csw) <= 38

        # default is 50
        csw = colorscheme_weighted(c, w)
        @test length(csw) == 50

        # save as Julia text file
        colorscheme_to_text(ColorSchemes.hokusai, "hokusai_test_version", "hokusai_as_text.jl")

        # and read it in as text
        open("hokusai_as_text.jl") do f
            lines = readlines(f)
            @test occursin("loadcolorscheme(", lines[1])
            @test occursin("Colors.RGB{Float64}(0.1112499123425623", lines[4])
        end
    end

    @testset "getinverse tests" begin
        getinverse(ColorSchemes.colorschemes[:leonardo], RGB(1, 0, 0))
        getinverse(ColorScheme([Colors.RGB(0,0,0), Colors.RGB(1,1,1)]),  Colors.RGB(.5,.5,.5))
        cs = ColorScheme(range(Colors.RGB(0,0,0), stop=Colors.RGB(1,1,1), length=5))
        gi = getinverse(cs, cs[3])
        @test gi == 0.5
    end

    @testset "convert to scheme tests" begin
        # Add color to a grayscale image.
        # now using ColorScheme objects and .colors accessors
        red_cs = ColorScheme(range(RGB(0,0,0), stop=RGB(1,0,0), length=11))
        gray_cs = ColorScheme(range(RGB(0,0,0), stop=RGB(1,1,0), length=11))
        vs = [getinverse(gray_cs, p) for p in red_cs.colors]
        cs = ColorScheme([RGB(v, v, v) for v in vs])
        rcs = [get(red_cs, p) for p in vs]

        new_img = convert_to_scheme(red_cs, gray_cs.colors)
        # TODO
        # This is broken.. It should be way more specific. See next test.
        @test all(.≈(new_img, red_cs.colors, atol=0.5))

        # Should be able to uniquely match each increasing color with the next
        # increasing color in the new scale.
        red_cs = ColorScheme(range(RGB(0,0,0), stop=RGB(1,1,1)))
        blue_scale_img = range(RGB(0,0,0), stop=RGB(0,0,1))
        new_img = convert_to_scheme(red_cs, blue_scale_img)
        @test_broken unique(new_img) == new_img
    end

    @testset "make_colorscheme tests" begin
        cs = make_colorscheme([colorant"red", colorant"green", colorant"blue"], 20)
        @test cs[1] == RGB{Float64}(1.0,0.0,0.0)
        @test cs[end] == RGB{Float64}(0.0,0.0,1.0)
        colorscheme_to_text(cs, "rgb_scheme", "rgb_scheme.jl")
    end
end

function run_minimum_tests()
    @testset "basic minimum tests" begin
        # load scheme
        hok = ColorSchemes.hokusai

        @test length(hok) == 32

        # test sort
        @test ColorSchemeTools.sortcolorscheme(hok, rev=true) != ColorSchemeTools.sortcolorscheme(hok)

        # save as text
        ColorSchemeTools.colorscheme_to_text(hok, "hokusai_test_version", "hokusai_as_text.jl")

        @test filesize("hokusai_as_text.jl") > 2000

        open("hokusai_as_text.jl") do f
            lines = readlines(f)
            @test occursin("Colors.RGB{Float64}(0.1112499123425623", lines[4])
        end

        # convert an Array{T,2} to an RGB image
        tmp = get(ColorSchemes.leonardo, rand(10, 10))
        @test typeof(tmp) == Array{ColorTypes.RGB{Float64}, 2}

        # test conversion with default clamp
        x = [0.0 1.0 ; -1.0 2.0]
        y = get(ColorSchemes.leonardo, x)
        @test y[1,1] == y[2,1]
        @test y[1,2] == y[2,2]

        # test conversion with symbol clamp
        y2 = get(ColorSchemes.leonardo, x, :clamp)
        @test y2 == y

        # test conversion with symbol extrema
        y2 = get(ColorSchemes.leonardo, x, :extrema)
        @test y2[2,1] == y[1,1]   # Minimum now becomes one edge of ColorScheme
        @test y2[2,2] == y[1,2]   # Maximum now becomes other edge of ColorScheme
        @test y2[1,1] !== y2[2,1] # Inbetween values or now different

        # test conversion with manually supplied range
        y3 = get(ColorSchemes.leonardo, x, (-1.0, 2.0))
        @test y3 == y2

        # test with steplen (#17)
        r  = range(0, stop=5, length=10)
        y  = get(ColorSchemes.leonardo, r)
        y2 = get(ColorSchemes.leonardo, collect(r))
        @test y == y2

        # test for specific value
        val = 0.2
        y   = get(ColorSchemes.leonardo, [val])
        y2  = get(ColorSchemes.leonardo, val)
        @test y2 == y[1]
        end
end

if get(ENV, "ColorSchemeTools_KEEP_TEST_RESULTS", false) == "true"
    # they changed mktempdir in v1.3
    if VERSION <= v"1.2"
        cd(mktempdir())
    else
        cd(mktempdir(cleanup=false))
    end
    @info("...Keeping the results in: $(pwd())")
    @info("..running minimum tests")
    run_minimum_tests()
    @info("..running all tests")
    run_all_tests()
    @info("Test images saved in: $(pwd())")
else
    mktempdir() do tmpdir
    cd(tmpdir) do
        @info("..running tests in: $(pwd())")
        @info("..but not keeping the results")
        @info("..because you didn't do: ENV[\"ColorSchemeTools_KEEP_TEST_RESULTS\"] = \"true\"")
        @info("..running minimum tests")
        run_minimum_tests()
        @info("..running all tests")
        run_all_tests()
        @info("..Test images weren't saved. To see the test images, next time do this before running:")
        @info(" ENV[\"ColorSchemeTools_KEEP_TEST_RESULTS\"] = \"true\"")
        end
    end
end


"""
Type representing a third-order tensor
"""
struct ThirdOrderTensorValue{D1,D2,D3,T,L} <: MultiValue{Tuple{D1,D2,D3},T,3,L}
    data::NTuple{L,T}
    function ThirdOrderTensorValue{D1,D2,D3,T}(data::NTuple{L,T}) where {D1,D2,D3,T,L}
        @assert L == D1*D2*D3
        new{D1,D2,D3,T,L}(data)
    end
end

# Empty ThirdOrderTensorValue constructor

ThirdOrderTensorValue()                       = ThirdOrderTensorValue{0,0,0,Int}(NTuple{0,Int}())
ThirdOrderTensorValue{0,0,0}()                = ThirdOrderTensorValue{0,0,0,Int}(NTuple{0,Int}())
ThirdOrderTensorValue{0,0,0,T}() where{T}     = ThirdOrderTensorValue{0,0,0,T}(NTuple{0,T}())
ThirdOrderTensorValue(data::NTuple{0})        = ThirdOrderTensorValue{0,0,0,Int}(data)
ThirdOrderTensorValue{0,0,0}(data::NTuple{0}) = ThirdOrderTensorValue{0,0,0,Int}(data)

# ThirdOrderTensorValue single NTuple argument constructor

@generated function ThirdOrderTensorValue(data::NTuple{L,T}) where {L,T}
  D=Int(cbrt(L))
  quote
    ThirdOrderTensorValue{$D,$D,$D,T}(data)
  end
end
ThirdOrderTensorValue{D}(data::NTuple{L,T}) where {D,L,T}                         = ThirdOrderTensorValue{D,D,D,T}(data)
ThirdOrderTensorValue{D1,D2,D3}(data::NTuple{L,T}) where {D1,D2,D3,L,T}           = ThirdOrderTensorValue{D1,D2,D3,T}(data)
ThirdOrderTensorValue{D1,D2,D3,T1}(data::NTuple{L,T2}) where {D1,D2,D3,L,T1,T2}   = ThirdOrderTensorValue{D1,D2,D3,T1}(NTuple{L,T1}(data))
ThirdOrderTensorValue{D1,D2,D3,T1,L}(data::NTuple{L,T2}) where {D1,D2,D3,L,T1,T2} = ThirdOrderTensorValue{D1,D2,D3,T1}(NTuple{L,T1}(data))

# ThirdOrderTensorValue single Tuple argument constructor

ThirdOrderTensorValue(data::Tuple) = ThirdOrderTensorValue(promote(data...))
ThirdOrderTensorValue{D}(data::Tuple) where {D} = ThirdOrderTensorValue{D,D,D}(promote(data...))
ThirdOrderTensorValue{D1,D2,D3}(data::Tuple) where {D1,D2,D3} = ThirdOrderTensorValue{D1,D2,D3}(promote(data...))
ThirdOrderTensorValue{D1,D2,D3,T1}(data::Tuple) where {D1,D2,D3,T1}  = ThirdOrderTensorValue{D1,D2,D3,T1}(NTuple{length(data),T1}(data))
ThirdOrderTensorValue{D1,D2,D3,T1,L}(data::Tuple) where {D1,D2,D3,T1,L}  = ThirdOrderTensorValue{D1,D2,D3,T1}(NTuple{L,T1}(data))

# ThirdOrderTensorValue Vararg constructor

ThirdOrderTensorValue(data...) = ThirdOrderTensorValue(data)
ThirdOrderTensorValue{D}(data...) where {D} = ThirdOrderTensorValue{D}(data)
ThirdOrderTensorValue{D1,D2,D3}(data...) where {D1,D2,D3} = ThirdOrderTensorValue{D1,D2,D3}(data)
ThirdOrderTensorValue{D1,D2,D3,T1}(data...) where {D1,D2,D3,T1} =  ThirdOrderTensorValue{D1,D2,D3,T1}(data)
ThirdOrderTensorValue{D1,D2,D3,T1,L}(data...) where {D1,D2,D3,T1,L} =  ThirdOrderTensorValue{D1,D2,D3,T1}(data)

# ThirdOrderTensorValue single AbstractArray{3,T} argument constructor

ThirdOrderTensorValue(data::AbstractArray{T,3}) where {T} = ((D1,D2,D3)=size(data);L=length(data);ThirdOrderTensorValue{D1,D2,D3,T}(NTuple{L,T}(data)))
ThirdOrderTensorValue{D}(data::AbstractArray{T,3}) where {D,T} = (L=length(data);ThirdOrderTensorValue{D,D,D,T}(NTuple{L,T}(data)))
ThirdOrderTensorValue{D1,D2,D3}(data::AbstractArray{T,3}) where {D1,D2,D3,T} = (L=length(data);ThirdOrderTensorValue{D1,D2,D3,T}(NTuple{L,T}(data)))
ThirdOrderTensorValue{D1,D2,D3,T1}(data::AbstractArray{T2,3}) where {D1,D2,D3,T1,T2} = (L=length(data);ThirdOrderTensorValue{D1,D2,D3,T1}(NTuple{L,T1}(data)))
ThirdOrderTensorValue{D1,D2,D3,T1,L}(data::AbstractArray{T2,3}) where {D1,D2,D3,T1,T2,L} = ThirdOrderTensorValue{D1,D2,D3,T1}(NTuple{L,T1}(data))

###############################################################
# Conversions (ThirdOrderTensorValue)
###############################################################

# Direct conversion
convert(::Type{<:ThirdOrderTensorValue{D1,D2,D3,T}}, arg::AbstractArray) where {D1,D2,D3,T} = ThirdOrderTensorValue{D1,D2,D3,T}(arg)
convert(::Type{<:ThirdOrderTensorValue{D1,D2,D3,T}}, arg::Tuple) where {D1,D2,D3,T} = ThirdOrderTensorValue{D1,D2,D3,T}(arg)

# Inverse conversion
convert(::Type{<:SMatrix{D1,D2,D3,T}}, arg::ThirdOrderTensorValue) where {D1,D2,D3,T} = SMatrix{D1,D2,D3,T}(Tuple(arg))
convert(::Type{<:MMatrix{D1,D2,D3,T}}, arg::ThirdOrderTensorValue) where {D1,D2,D3,T} = MMatrix{D1,D2,D3,T}(Tuple(arg))
convert(::Type{<:NTuple{L,T1}}, arg::ThirdOrderTensorValue) where {L,T1} = NTuple{L,T1}(Tuple(arg))

# Internal conversion
convert(::Type{<:ThirdOrderTensorValue{D1,D2,D3,T}}, arg::ThirdOrderTensorValue{D1,D2,D3}) where {D1,D2,D3,T} = ThirdOrderTensorValue{D1,D2,D3,T}(Tuple(arg))
convert(::Type{<:ThirdOrderTensorValue{D1,D2,D3,T}}, arg::ThirdOrderTensorValue{D1,D2,D3,T}) where {D1,D2,D3,T} = arg

# other

change_eltype(::Type{ThirdOrderTensorValue{D1,D2,D3,T1,L}},::Type{T2}) where {D1,D2,D3,T1,T2,L} = ThirdOrderTensorValue{D1,D2,D3,T2,L}
change_eltype(::T,::Type{T2}) where {T<:ThirdOrderTensorValue,T2} = change_eltype(T,T2)

zero(::Type{<:ThirdOrderTensorValue{D1,D2,D3,T}}) where {D1,D2,D3,T} = ThirdOrderTensorValue{D1,D2,D3,T}(tfill(zero(T),Val{D1*D2*D3}()))
zero(::ThirdOrderTensorValue{D1,D2,D3,T}) where {D1,D2,D3,T} = zero(ThirdOrderTensorValue{D1,D2,D3,T})

Mutable(::Type{<:ThirdOrderTensorValue{D1,D2,D3,T}}) where {D1,D2,D3,T} = MArray{Tuple{D1,D2,D3},T}
Mutable(::ThirdOrderTensorValue{D1,D2,D3,T}) where {D1,D2,D3,T} = Mutable(ThirdOrderTensorValue{D1,D2,D3,T})
mutable(a::ThirdOrderTensorValue{D1,D2,D3}) where {D1,D2,D3} = MArray{Tuple{D1,D2,D3}}(a.data)

###############################################################
# Introspection (ThirdOrderTensorValue)
###############################################################

eltype(::Type{<:ThirdOrderTensorValue{D1,D2,D3,T}}) where {D1,D2,D3,T} = T
eltype(::ThirdOrderTensorValue{D1,D2,D3,T}) where {D1,D2,D3,T} = eltype(ThirdOrderTensorValue{D1,D2,D3,T})

size(::Type{<:ThirdOrderTensorValue{D1,D2,D3}}) where {D1,D2,D3} = (D1,D2,D3)
size(::ThirdOrderTensorValue{D1,D2,D3}) where {D1,D2,D3} = size(ThirdOrderTensorValue{D1,D2,D3})

length(::Type{<:ThirdOrderTensorValue{D1,D2,D3}}) where {D1,D2,D3} = D1*D2*D3
length(::ThirdOrderTensorValue{D1,D2,D3}) where {D1,D2,D3} = length(ThirdOrderTensorValue{D1,D2,D3})

num_components(::Type{<:ThirdOrderTensorValue{D1,D2,D3}}) where {D1,D2,D3} = length(ThirdOrderTensorValue{D1,D2,D3})
num_components(::ThirdOrderTensorValue{D1,D2,D3}) where {D1,D2,D3} = num_components(ThirdOrderTensorValue{D1,D2,D3})