using Resonance
using CairoMakie

md = CSV.read("data/wrangled.csv", DataFrame)

##

reads = CSV.read("data/read_counts.csv", DataFrame)
(fig, ax, hs) = hist(reads.count, axis=(
    xlabel="Total reads (single)",
    ylabel="# of samples"
))

save("slides/assets/kneaddatacounts.png", fig)

##

@rsubset!(md, !ismissing(:subject),
                    !ismissing(:sample),
                    )
@rsubset!(md, !startswith(:sample, "C"),
                    !startswith(:sample, "z"),
                    )

spec = metaphlan_profiles(filter(f-> contains(f, "profile") && any(s-> contains(f, s), md.sample),
    readdir("/grace/echo/analysis/biobakery3/links/metaphlan", join=true)), :species)

sns = map(samplenames(spec)) do s
    replace(s, r"_S\d+_profile"=>"")
end

idx = unique([findfirst(==(x), sns) for x in sns])

spec = CommunityProfile(abundances(spec)[:, idx], features(spec), MicrobiomeSample.(sns[idx]))

set!(spec, md)

for sampl in samples(spec)
    sn = name(sampl)
    set!(spec, sn, :richness, count(!=(0), abundances(spec[:, sampl])))
end

(fig, ax, hs) = hist([get(spec, sn, :richness) for sn in samplenames(spec)],
    axis=(
        xlabel="Richness (# of taxa)",
        ylabel="# of samples"
    ))

save("slides/assets/echo_richness.png", fig)
fig

##

fig = Figure()
ax = Axis(fig[1,1], ylabel="# of samples", xlabel = "Richness (# of taxa)")

idx = [any(col-> get(spec, s, col, false), [:age0to3mo, :age3to6mo, :age6to12mo]) for s in samplenames(spec)]
hist!(ax, [get(spec, sn, :richness, missing) for sn in samplenames(spec)][Not(idx)], label="Age > 12mo")
hist!(ax, [get(spec, sn, :richness, missing) for sn in samplenames(spec)][idx], label="Age < 12mo")
axislegend(ax)

save("slides/assets/echo_richness_byage.png", fig)
fig

##

yidx = [any(col-> get(spec, s, col, false), [:age0to3mo]) for s in samplenames(spec)]
oidx = [any(col-> get(spec, s, col, false), [:age12moplus]) for s in samplenames(spec)]

ratios = map(featurenames(spec)) do feature
    log(mean(abundances(spec[feature,oidx])) / mean(abundances(spec[feature, yidx])))
end

onlyyoung = count(isnan, ratios)
onlyold = count(isinf, ratios)

filter!(x-> !isinf(x) && !isnan(x), ratios)

#

fig = Figure(resolution=(1200, 800))
yngax = Axis(fig[1,1], title="Only in <3 mo", ylabel="# of species")
bothax = Axis(fig[1, 2:3], title="Present in both", xlabel="Log ratio abundances")
oldax = Axis(fig[1,4], title="Only in >12 mo")

barplot!(yngax, [1], [onlyyoung])
hist!(bothax, ratios)
barplot!(oldax, [1], [onlyold])

hidexdecorations!.([yngax, oldax])
bothax.ygridvisible = true
oldax.ygridvisible = true

tightlimits!(yngax, Bottom())
tightlimits!(bothax, Left(), Right(), Bottom())
tightlimits!(oldax, Bottom())

linkyaxes!(yngax, bothax, oldax)
save("slides/assets/age_taxon_ratios.png", fig)
fig
##

metabs = CSV.read("data/metabolites.csv", DataFrame)

##

humann_join("/grace/echo/analysis/biobakery3/links/humann/genefamilies",
            "/grace/echo/analysis/biobakery3/links/humann/all_genefamilies.tsv";
            file_name="genefamilies")

func = humann_profiles("/grace/echo/analysis/biobakery3/links/humann/all_genefamilies.tsv", stratified=false)

sns = map(samplenames(func)) do s
    replace(s, r"_S\d+_genefamilies"=>"")
end

idx = unique([findfirst(==(x), sns) for x in sns])

func = CommunityProfile(abundances(func)[:, idx], features(func), sns[idx])

set!(func, md)

for sampl in samples(func)
    sn = name(sampl)
    set!(func, sn, :richness, count(!=(0), abundances(func[:, sampl])))
end

##

(fig, ax, hs) = hist([get(func, sn, :richness) for sn in samplenames(func)],
    axis=(
        xlabel="Richness (# of gene functions)",
        ylabel="# of samples"
    ))

save("slides/assets/echo_richness_func.png", fig)
fig

##

fig = Figure()
ax = Axis(fig[1,1], ylabel="# of samples", xlabel = "Richness (# of gene functions)")

idx = [any(col-> get(func, s, col, false), [:age0to3mo, :age3to6mo, :age6to12mo]) for s in samplenames(func)]
hist!(ax, [get(func, sn, :richness, missing) for sn in samplenames(func)][Not(idx)], label="Age > 12mo")
hist!(ax, [get(func, sn, :richness, missing) for sn in samplenames(func)][idx], label="Age < 12mo")
axislegend(ax)

save("slides/assets/echo_richness__func_byage.png", fig)
fig


#Convenience for extracting positioner mod and mon with error checks
function pos_mod_mon(coms, recs)
    pos = getpositioners(coms)
    if length(pos) > 1 error("Provide only one positioner command.") end
    pos_mon = getpositionermonitors(recs)
    if length(pos_mon) > 1 error("Provide only one positioner monitor.") end
    return pos[1], pos_mon[1]
end

#take the mean MON cycle and calculate the number of sampls that MON is shifted from MOD
function mon_lag_nsamps(pos, pos_mon, ncycs_ignore)
    pos_cyc_name = first(sequence_names(pos))
    pos_cycle = get_samples(pos, pos_cyc_name)
    @assert all(sequence_names(pos).==pos_cyc_name)
    nsamps_cycle = Base.length(pos_cycle) #should always be even
    mon_samps = get_samples(pos_mon)
    mon_samps = view(mon_samps, (ncycs_ignore*nsamps_cycle+1):Base.length(mon_samps))
    cycs_mon = ImagineAnalyses.get_cycles(mon_samps, nsamps_cycle)
    mean_cyc = mean(ustrip.(cycs_mon); dims=2)[:] * unit(mon_samps[1])
    return ImagineAnalyses.mon_delay(pos_cycle, mean_cyc)
end

#function local_minima(mms)
#    min_is = Int[]
#    if mms[2] > mms[1]
#        push!(mms, 1)
#    end
#    for i = 2:(length(mms)-1)
#        if mms[i-1] > mms[i] < mms[i+1]
#            push!(min_is, i)
#        end
#    end
#    if mms[end] < mms[end-1]
#        push!(min_is,length(mms))
#        warn("Found local minimum at the last temporal offset.  You should use a wider range of offsets")
#    end
#    return min_is
#end
#
##Instead of just taking the lowest mismatch, find all local minima and take the highest-indexed minimum that satisfies thresh_fac
#function ind_best_offset(mms; thresh_fac = 0.1)
#    min_is = local_minima(mms)
#    min_mm = minimum(mms)
#    #thresh = min_mm + (maximum(mms) - min_mm) * thresh_fac
#    thresh = min_mm + min_mm * thresh_fac
#    for i = length(min_is):-1:1
#        if mms[min_is[i]] <= thresh
#            return min_is[i]
#        end
#    end
#end

#imageseq is a 3D image array where each 2D slice is an image acquired for calibration
#imageseq should be either forward or reverse slice trials, but not both (use exclude_fwd to separate)
#The first (slow) stack should not be included in imgseq (use exclude_template_stack to get that one)
get_slice_trials(imgseq, sliceidx, ntrials, ntoffsets) = view(imgseq, :, :, (sliceidx-1)*(ntrials*ntoffsets)+1:sliceidx*(ntrials*ntoffsets))
exclude_template_stack(imgseq, nslices) = view(imgseq, :, :, (nslices+1):size(imgseq,3))
get_template_stack(imgseq, nslices) = imgseq[:,:,1:nslices]
#assumes template stack is excluded
get_fwd_imgs(imgseq) = view(imgseq, :, :, 1:2:size(imgseq,3))
get_bck_imgs(imgseq) = view(imgseq, :, :, 2:2:size(imgseq,3))

#NOTE: the timings returned here are additional offsets relative to any lag already applied with a Calibration when
#generating the command signals.

function slicetimings(img, toffsets, nslices::Int; allow_shifts=true, allow_rotations=false, subpixel=true)
    timings, mms, tfms = timings_mms_tfms(img, toffsets, nslices; allow_shifts=allow_shifts, allow_rotations=allow_rotations, subpixel=subpixel)
    return timings
end

function timings_mms_tfms(img, toffsets, nslices; kwargs...)
    @assert ndims(img) == 3 # should be stored by Imagine as a 2D timeseries
    #first set of images is always the target / template image
    target = get_template_stack(img, nslices)
    testimgs = exclude_template_stack(img, nslices)
    fwdimgs = get_fwd_imgs(testimgs)
    backimgs = get_bck_imgs(testimgs)
    timings_mms_tfms(target, fwdimgs, backimgs, toffsets, nslices; kwargs...)
end

function safe_fetch(rr)
    rslt = fetch(rr)
    if isa(rslt, RemoteException)
        throw(rslt)
    else
        return rslt
    end
end

#Setting record_all will return arrays-of-arrays of mismatches and transforms (one for each condition) rather than just the best
function timings_mms_tfms(target, fwdimgs, backimgs, toffsets, nslices; allow_shifts=true, subpixel=false, allow_rotations=false, record_all=false)
    ntoffsets =length(toffsets)
    ntrials = convert(Int, size(fwdimgs,3) / (ntoffsets*nslices))

    fwd_timings = eltype(toffsets)[]
    back_timings = eltype(toffsets)[]
    fwd_tfms = []; back_tfms = []; fwd_mms = []; back_mms = []

    i_start = 1
    i_stop = 0
    nw = nworkers()
    ws = workers()
    if isodd(nw) && nw > 1
        nw-=1
        ws = ws[1:end-1]
        warn("Using one less worker (must be even number)")
    end
    print("Computing with $(nw) worker processes\n")
    while i_stop < nslices
        i_start = i_stop+1
        i_stop = min(nslices, i_stop+max(1, div(nw,2)))
        idxs = i_start:i_stop
        fwdrrs = []; bckrrs = [];
        for i = 1:length(idxs)
            widx_odd = 2*i-1
            fixed = view(target, :, :, idxs[i])
            movfwd = get_slice_trials(fwdimgs, idxs[i], ntrials, ntoffsets)
            #launch on odd-numbered worker
            push!(fwdrrs, remotecall(toffset_fits, ws[widx_odd], fixed, movfwd, toffsets; allow_shifts=allow_shifts, allow_rotations=allow_rotations, subpixel=subpixel))
            movback = get_slice_trials(backimgs, idxs[i], ntrials, ntoffsets)
            #launch on even-numbered worker if we have more than one worker
            if nw > 1
                push!(bckrrs, remotecall(toffset_fits, ws[widx_odd+1], fixed, movback, toffsets; allow_shifts=allow_shifts, allow_rotations=allow_rotations, subpixel=subpixel))
            else
                push!(bckrrs, remotecall(toffset_fits, ws[widx_odd], fixed, movback, toffsets; allow_shifts=allow_shifts, allow_rotations=allow_rotations, subpixel=subpixel))
            end
        end
        #fetch them all and store important results
        for i = 1:length(idxs)
            ftfms, fmms = safe_fetch(fwdrrs[i])
            #fwdi = ind_best_offset(fmms)
            fwdi = argmin(fmms)
            @show fwd_toffset = toffsets[fwdi]
            @show fwd_mm = fmms[fwdi]
            push!(fwd_timings, fwd_toffset)

            btfms, bmms = safe_fetch(bckrrs[i])
            #backi = ind_best_offset(bmms)
            backi = argmin(bmms)
            @show back_toffset = toffsets[backi]
            @show back_mm = bmms[backi]
            push!(back_timings, back_toffset)

            if record_all
                push!(fwd_tfms, ftfms)
                push!(back_tfms, btfms)
                push!(fwd_mms, fmms)
                push!(back_mms, bmms)
            else
                push!(fwd_tfms, ftfms[fwdi])
                push!(back_tfms, btfms[backi])
                push!(fwd_mms, fwd_mm)
                push!(back_mms, back_mm)
            end
            #print("Slice $(idxs[i]) complete\n")
        end
        print("------------------Finished $i_stop of $nslices slices-----------------------\n")
    end
    return (fwd_timings, back_timings), (fwd_mms, back_mms), (fwd_tfms, back_tfms)
end

function preprocess(img::AbstractArray{Float64,2}; sigmas=(1.0,1.0,), sqrt_tfm=true, bias=Float64(reinterpret(N0f16, UInt16(100))))
    img = imfilter(img, KernelFactors.IIRGaussian(Float64, sigmas))
    output = similar(img)
    for i in eachindex(img)
        temp = max(0.0, img[i] - bias)
        output[i] = ifelse(sqrt_tfm, sqrt(temp), temp)
    end
    return output
end

align2d(fixed, moving; kwargs...) = align2d(fixed, Float64.(moving); kwargs...)
function align2d(fixed, moving::AbstractMatrix{Float64}; thresh_fac=0.9, sigmas=(1.0,1.0,), allow_shifts = true, allow_rotations =false, subpixel=false)
    mxshift = (8,8,)
    moving = preprocess(moving; sigmas=sigmas, sqrt_tfm=false)
    thresh = (1-thresh_fac) * sum(abs2.(fixed[.!(isnan.(fixed))]))
    if allow_rotations
        maxradians = pi/180
        tfm, mm = RegisterQD.qd_rigid(fixed, moving, mxshift, [maxradians], [pi/10000]; thresh=thresh, initial_tfm=IdentityTransformation())
        return tfm, mm
    elseif allow_shifts
        if !subpixel
            shft, mm = RegisterQD.best_shift(fixed, moving, mxshift, thresh; normalization=:intensity, initial_tfm=IdentityTransformation())
            return Translation(shft...), mm
        else
            return RegisterQD.qd_translate(fixed, moving, mxshift; thresh=thresh, rtol=0.0, fvalue=0.0, maxevals=200)
        end
    else
        mm = RegisterMismatch.mismatch0(fixed, moving; normalization=:intensity)
        return Translation(0.0,0.0), ratio(mm, 0.0)
    end
end

#get_offset_trials returns the set of trials at a particular temporal offset
#Assumes that img_seq:
#   -does not include the slow "target" stack
#   -includes only forward or reverse images, not both
#   -includes data for only one slice
#(This means img_seq has size ntrials * ntoffsets in dimension 3)
get_toffset_trials(img_seq, itoffset, ntrials) = view(img_seq, :, :, (itoffset-1)*ntrials+1:itoffset*ntrials)

#consistent per-slice.  It depends on so many factors that we just do it emprically.
function toffset_fits(target, testimgs, toffsets::Vector; sigmas=(1.0,1.0), allow_shifts=true, subpixel=false, allow_rotations = false)
    ntoffsets = length(toffsets)::Int
    ntrials = convert(Int, size(testimgs,3) / ntoffsets)
    tfms = []
    mms = Float64[]
    target = preprocess(Float64.(target); sigmas=sigmas, sqrt_tfm=false)
    #Threads.@threads for i = 1:length(toffsets)  #Currently segfaults on julia 0.6.2
    for i = 1:length(toffsets)
        moving = dropdims(mean(Float64.(get_toffset_trials(testimgs, i, ntrials)); dims=3); dims=3)
        tfm, mm = align2d(target, moving; sigmas=sigmas, allow_shifts=allow_shifts, subpixel=subpixel, allow_rotations=allow_rotations)
        #@show mm
        push!(tfms, tfm)
        push!(mms, mm)
    end
    return tfms, mms
end

function find_bidi_centers(cyc::AbstractVector, sampr::HasInverseTimeUnits)
    @show pmin = minimum(cyc)
    @show pmax = maximum(cyc)
    pctr = pmin + (pmax-pmin)/2 #we will place the flash in the center
    find_bidi_locations(cyc, sampr, pctr)
end

function find_bidi_locations(cyc::AbstractVector, sampr::HasInverseTimeUnits, ploc::HasLengthUnits)
    stack_rate = 2 * (sampr / length(cyc))
    pad_secs = (1/stack_rate)/200 #wait 0.5% of a cycle before accepting threshold crossings (helps with noise)
    pad_nsamps = round(Int, pad_secs * sampr)
    mon_is = ImagineAnalyses.find_circular(cyc, [ploc;], pad_nsamps)[1]
    @assert Base.length(mon_is) == 2 #should have just two crossings, one fwd one back
    fwd_i = mon_is[1]
    back_i = mon_is[2]
    return fwd_i, back_i
end


module NextGenSeqUtils
    include("include_all.jl")

	export

    # align.jl

    usearch_filter,
    usearch_trim_fastq_with_phreds,

    nw_align,
    banded_nw_align,
    triplet_nw_align,
    local_align,
    kmer_seeded_align,
    triplet_kmer_seeded_align,
    loc_kmer_seeded_align,
    local_kmer_seeded_align,
    kmer_seeded_edit_dist,
    resolve_alignments,
    align_reference_frames,
    local_edit_dist,

    # hmm.jl

    viterbi_logs,
    trans_mat,
    obs_mat,
    initial_dist,
    get_obs_given_state,
    homopolymer_filter,
    markov_filter,
    forward_logs,
    backward_logs,
    forward_backward_logs,
    logsum,
    gen_seq_with_model,
    viterbiprint,

    # io.jl

    read_fasta,
    read_fasta_with_names,
    read_fasta_with_names_in_other_order,
    write_fasta,
    read_fastq,
    write_fastq,

    # kmers.jl

    KmerType,
    kmer_count,
    sparse_aa_kmer_count,
    corrected_kmer_dist,
    corrected_kmer_dist_full,

    # orient.jl

    orient_strands,
    orient_to_refs,
    orient_to_refs_file,

    # paths.jl

    Paths,
    PATHS,

    # phreds.jl

    Phred,
    Prob,
    LogProb,
    MIN_PHRED,
    MAX_PHRED,
    phred_to_log_p,
    phred_to_p,
    p_to_phred,
    error_probs_to_phreds,
    quality_filter,
    length_vs_qual,
    qual_hist,

    # simulation.jl

    simple_gen_seq,
    simple_evolve,
    fixed_diff_evolve,
    pb_seq_sim,
    env_pb_seq_sim,

    # utils.jl

    print_fasta,
    degap,
    dash_count,
    single_gap,
    single_mod_three_gap,
    seq_details,
    print_rgb,
    reverse_complement,
    print_diffs,
    trim_ends_indices,
    translate_to_aa,
    generate_aa_seqs,
    filter_by_length,
    length_filter,
    concat_fastas,
    maxfreq,
    freq,
    sorted_freqs,
    freq_dict_print,
    nl43env,
    dist_matrix,

    # demux.jl
    demux_fastx,
    IUPAC_equals,
    toIUPACnum,

    #FAD.jl
    FAD,

    #evodist
    estimate_distance

end # module


"""
    struct SkeletonTriangulation{Dc,Dp,B} <: Triangulation{Dc,Dp}
      left::B
      right::B
    end

The inner constructor enforces `B<:BoundaryTriangulation`
"""
struct SkeletonTriangulation{Dc,Dp,B} <: Triangulation{Dc,Dp}
  left::B
  right::B
  function SkeletonTriangulation(left::B,right::B) where B<:Triangulation
    Dc = num_cell_dims(left)
    Dp = num_point_dims(left)
    @assert Dc + 1 == Dp
    new{Dc,Dp,B}(left,right)
  end
end

"""
    SkeletonTriangulation(model::DiscreteModel,face_to_mask::Vector{Bool})
    SkeletonTriangulation(model::DiscreteModel)
"""
function SkeletonTriangulation(model::DiscreteModel,face_to_mask::Vector{Bool})
  left_cell_around = 1
  left = BoundaryTriangulation(model,face_to_mask,left_cell_around)
  right_cell_around = 2
  right = BoundaryTriangulation(model,face_to_mask,right_cell_around)
  SkeletonTriangulation(left,right)
end

function SkeletonTriangulation(model::DiscreteModel)
  topo = get_grid_topology(model)
  D = num_cell_dims(model)
  face_to_mask = collect(Bool, .!get_isboundary_face(topo,D-1))
  SkeletonTriangulation(model,face_to_mask)
end

function SkeletonTriangulation(
  model::DiscreteModel,
  reffe::ReferenceFE,
  face_own_dofs::Vector{Vector{Int}})

  ncells = num_cells(model)
  cell_to_reffe = Fill(1,ncells)
  SkeletonTriangulation(model,[reffe],cell_to_reffe,[face_own_dofs])
end

function SkeletonTriangulation(
  model::DiscreteModel,
  reffes::Vector{<:ReferenceFE},
  cell_to_reffe::AbstractVector,
  reffe_to_face_own_dofs::Vector{Vector{Vector{Int}}})

  reffe_to_polytope = map(get_polytope,reffes)
  reffe_to_faces = map(get_faces,reffe_to_polytope)
  reffe_to_range = map(p->get_dimrange(p,num_dims(p)-1),reffe_to_polytope)

  reffe_to_lfacet_to_mask = _compute_reffe_to_facet_to_mask(
    reffe_to_faces,reffe_to_range,reffe_to_face_own_dofs)

  topo = get_grid_topology(model)

  D = num_cell_dims(model)
  nfacets = num_facets(model)
  facet_to_isboundary = get_isboundary_face(topo,D-1)
  cell_to_facets = get_faces(topo,D,D-1)
  facet_to_mask = _compute_disc_facet_mask(
    facet_to_isboundary,cell_to_facets,cell_to_reffe,reffe_to_lfacet_to_mask)

  SkeletonTriangulation(model,facet_to_mask)

end

function _compute_disc_facet_mask(
  facet_to_isboundary, cell_to_facets::Table, cell_to_reffe, reffe_to_lfacet_to_mask)
  nfacets = length(facet_to_isboundary)
  facet_to_mask = Vector{Bool}(undef,nfacets)
  ncells = length(cell_to_reffe)
  for cell in 1:ncells
    pini = cell_to_facets.ptrs[cell]
    pend = cell_to_facets.ptrs[cell+1]-1
    reffe = cell_to_reffe[cell]
    lfacet_to_mask = reffe_to_lfacet_to_mask[reffe]
    for (lfacet,p) in enumerate(pini:pend)
      facet = cell_to_facets.data[p]
      mask = lfacet_to_mask[lfacet]
      isinterior = ! facet_to_isboundary[facet]
      facet_to_mask[facet] = mask && isinterior
    end
  end
  facet_to_mask
end

function _compute_reffe_to_facet_to_mask(
  reffe_to_faces,reffe_to_range,reffe_to_face_own_dofs)

  reffe_to_facet_to_mask = Vector{Bool}[]
  nreffes = length(reffe_to_faces)
  for reffe in 1:nreffes
    range = reffe_to_range[reffe]
    facet_to_faces = reffe_to_faces[reffe][range]
    facet_to_own_dofs = reffe_to_face_own_dofs[reffe][range]
    face_to_own_dofs = reffe_to_face_own_dofs[reffe]
    nfacets = length(facet_to_faces)
    facet_to_mask = fill(false,nfacets)
    for facet in 1:nfacets
      n = length(facet_to_own_dofs[facet])
      faces = facet_to_faces[facet]
      for face in faces
        n += length(face_to_own_dofs[face])
      end
      facet_to_mask[facet] = n == 0
    end
    push!(reffe_to_facet_to_mask,facet_to_mask)
  end

  reffe_to_facet_to_mask
end

const IN = -1
const OUT = 1

"""
"""
function InterfaceTriangulation(model::DiscreteModel,cell_to_is_in::Vector{Bool})
  cell_to_inout = fill(Int8(OUT),length(cell_to_is_in))
  cell_to_inout[cell_to_is_in] .= IN
  InterfaceTriangulation(model,cell_to_inout)
end

function InterfaceTriangulation(model::DiscreteModel,cells_in,cells_out)
  cell_to_inout = fill(Int8(OUT),num_cells(model))
  cell_to_inout[cells_in] .= IN
  cell_to_inout[cells_out] .= OUT
  InterfaceTriangulation(model,cell_to_inout)
end

function InterfaceTriangulation(model::DiscreteModel,cell_to_inout::AbstractVector{<:Integer})

  D = num_cell_dims(model)
  facet_grid = Grid(ReferenceFE{D-1},model)
  cell_grid = Grid(ReferenceFE{D},model)

  topo = get_grid_topology(model)
  facet_to_cells = Table(get_faces(topo,D-1,D))
  ifacet_to_facet, facet_to_lcell_left, facet_to_lcell_right = _find_interface_facets(
    cell_to_inout, facet_to_cells)

  ifacet_trian = TriangulationPortion(facet_grid,ifacet_to_facet)

  left = GenericBoundaryTriangulation(ifacet_trian,cell_grid,topo,ifacet_to_facet,facet_to_lcell_left)
  right = GenericBoundaryTriangulation(ifacet_trian,cell_grid,topo,ifacet_to_facet,facet_to_lcell_right)

  SkeletonTriangulation(left,right)
end

function _find_interface_facets( cell_to_inout, facet_to_cells::Table)

  nifacets = 0
  for facet in 1:length(facet_to_cells)
    a = facet_to_cells.ptrs[facet]
    b = facet_to_cells.ptrs[facet+1]
    if b-a == 2
      cell1 = facet_to_cells.data[a]
      cell2 = facet_to_cells.data[a+1]
      inout_1 = cell_to_inout[cell1]
      inout_2 = cell_to_inout[cell2]
      if (inout_1== IN && inout_2==OUT) || (inout_1== OUT && inout_2==IN)
        nifacets += 1
      end
    end
  end

  T = eltype(eltype(facet_to_cells))
  ifacet_to_facet = zeros(T,nifacets)
  nfacets = length(facet_to_cells)
  facet_to_lcell_left = fill(Int8(1),nfacets)
  facet_to_lcell_right = fill(Int8(2),nfacets)

  nifacets = 0
  for facet in 1:length(facet_to_cells)
    a = facet_to_cells.ptrs[facet]
    b = facet_to_cells.ptrs[facet+1]
    if b-a == 2
      cell1 = facet_to_cells.data[a]
      cell2 = facet_to_cells.data[a+1]
      inout_1 = cell_to_inout[cell1]
      inout_2 = cell_to_inout[cell2]
      if (inout_1== IN && inout_2==OUT) || (inout_1== OUT && inout_2==IN)
        nifacets += 1
        ifacet_to_facet[nifacets] = facet
        if inout_1 == IN
          facet_to_lcell_left[facet] = 1
          facet_to_lcell_right[facet] = 2
        else
          facet_to_lcell_left[facet] = 2
          facet_to_lcell_right[facet] = 1
        end
      end
    end
  end

   ifacet_to_facet, facet_to_lcell_left, facet_to_lcell_right
 end

# Triangulation interface

function get_cell_coordinates(trian::SkeletonTriangulation)
  get_cell_coordinates(trian.left)
end

function get_reffes(trian::SkeletonTriangulation)
    get_reffes(trian.left)
end

function get_cell_type(trian::SkeletonTriangulation)
  get_cell_type(trian.left)
end

function get_cell_id(trian::SkeletonTriangulation)
  left = get_cell_id(trian.left)
  right = get_cell_id(trian.right)
  SkeletonPair(left,right)
end

function restrict(f::AbstractArray, trian::SkeletonTriangulation)
  left = restrict(f,trian.left)
  right = restrict(f,trian.right)
  SkeletonPair(left,right)
end

# Delegating into the left side

"""
    get_volume_triangulation(trian::SkeletonTriangulation)
"""
function get_volume_triangulation(trian::SkeletonTriangulation)
  get_volume_triangulation(trian.left)
end

"""
    get_normal_vector(trian::SkeletonTriangulation)
"""
function get_normal_vector(trian::SkeletonTriangulation)
  get_normal_vector(trian.left)
end

function TriangulationPortion(oldtrian::SkeletonTriangulation,cell_to_oldcell::Vector{Int})
  left = TriangulationPortion(oldtrian.left,cell_to_oldcell)
  right = TriangulationPortion(oldtrian.right,cell_to_oldcell)
  SkeletonTriangulation(left,right)
end

# Specific API

"""
"""
get_left_boundary(trian::SkeletonTriangulation) = trian.left

"""
"""
get_right_boundary(trian::SkeletonTriangulation) = trian.right


"""
REWRITE:
back(r::Net,ygold,loss) computes the gradients of weights and
activations according to the given loss function and gold output.
ygold represents an individual item minibatch that may or may not be
an element of a sequence.  The seq keyword argument determines which:
initback sets incr=true for par if seq, back pops from stack if seq.
The loss gradient of the output, ygrad, is computed using
loss(ypred,ygold,ygrad).  ypred is retrieved from r.out[N] where N is
the index of the last op.  ygrad is written to r.dif[N].  If r.op[N]
has multiple outputs (toincr[N]), r.dif[N] is incremented.  If the
optional loss argument is not provided, ygold is used as the loss
gradient.  If ygold=nothing means the loss gradient from the output is
taken to be 0.  Gradient computation proceeds backwards from N..1.

"""
function back(f::Net, ygold=nothing, loss=copyloss; seq=false, getdx=false, o...)
    getdx = getdxbool(getdx, ninputs(f)); dx=Any[]
    initback(f, ygold, loss, getdx, seq)
    gotreturn = false
    gotinputs = 0
    for n = length(f):-1:1
        skipped(f,n,seq) && continue
        y = reg(f,n)
        getp(y,:grad) || continue # :grad means we need the gradient of the output of this operation, does not mean go back on this operation. but no :grad means no need to go back.
        yout = (!seq ? y.out : stack_output(f,n))
        if !gotreturn
            gotreturn = true
            if ygold == nothing # represents zero gradient
                getp(y,:incr) || (y.dif = nothing)
            else
                !isreturn(y) && Base.warn_once("ygold specified when there is no return")
                if getp(y,:incr)
                    loss(yout, ygold, y.tmp; o...) # loss needs o... for e.g. mask
                    y.dif = axpy!(1,y.tmp,y.dif0)
                else
                    y.dif = loss(yout, ygold, y.dif0; o...)
                end
            end
        elseif isreturn(y)
            error("Got return in non-final instruction")
        end
        # println(:back, (findfirst(regs(f), y), typeof(y.op), y.argv, Knet.vecnorm0(y.dif)))
        xx = (!seq ? inputregs(f,y) : stack_inputregs(f,n))
        if isempty(xx)
            # nothing to do
        elseif y.dif == nothing
            for x in xx
                getp(x,:grad) && !getp(x,:incr) && (x.dif = nothing)
            end
        else
            xdif = difs(xx)
            xout = get1(!seq ? outs(xx) : stack_inputs(f,n))
            back(y.op, y.dif, xdif...; x=xout, y=yout, o...)
            for x in xx
                x.dif = (getp(x,:incr) ? axpy!(1, x.tmp, x.dif0) :
                         getp(x,:grad) ? x.dif0 : nothing)
            end
            if getp(y,:incr) && !isa(y.op, Par)
                fillsync!(y.dif,0)
            end
        end
        if isa(y.op,Input)
            getdx[end-gotinputs] && unshift!(dx, y.dif)
            gotinputs += 1
        end
    end
    seq && (f.sp -= length(f))  # this way f.stack[f.sp] gives the output of n'th op next call
    return get1(dx)
end

sback(f::Net, ygold=nothing, loss=copyloss; o...)=back(f,ygold,loss;o...,seq=true)

# For FNNs we can tell whether an op was skipped using the :forw flag.
# For RNNs the :forw flag may change through the iterations, so we check the stack.
skipped(f::Net,n::Int,seq::Bool)=(seq ? !stack_forw(f,n) : !getp(reg(f,n),:forw))

# We assume f.reg[end].out is saved in f.stack[f.sp].out
# In the calls below n is the network index
stack_entry(f::Net,n::Int)=f.stack[f.sp - length(f) + n]
stack_output(f::Net,n::Int)=stack_entry(f,n).out
stack_argv(f::Net,n::Int)=stack_entry(f,n).argv
stack_forw(f::Net,n::Int)=stack_entry(f,n).forw

# For inputs the situation is a bit more complicated.
# We want to use their values prior to n'th op.
# And we want to use the indices from the argv on stack.

stack_inputregs(f::Net,n::Int)=f.reg[stack_argv(f,n)]

function stack_inputs(f::Net,n::Int)
    r = reg(f,n)
    back_reads_x(r.op) || return nothing
    a = stack_argv(f,n)
    x = cell(length(a))
    @inbounds for i=1:length(a)
        stack_index = f.sp - length(f) + a[i]
        a[i] >= n && (stack_index -= length(f)) # use prior value of read-before-write register
        if stack_index <= 0
            x[i] = nothing
        else
            s = f.stack[stack_index]
            s.forw || Base.warn_once("Stack input has forw=false.")
            x[i] = s.out
        end
    end
    return x
end

# this is used when no loss fn specified, in which case we assume ygold is actually ygrad
copyloss(ypred,ygold,ygrad;o...)=(ygrad===ygold ? ygrad : copysync!(ygrad,ygold))

# turn various forms of getdx into boolean vector
function getdxbool(getdx, n)
    (isa(getdx, Vector{Bool}) && length(getdx)==n ? getdx :
     isa(getdx, Bool) ? fill(getdx, n) :
     isa(getdx, Vector{Int}) ? (tmp=falses(n);tmp[getdx]=true;tmp) :
     isa(getdx, Int) ? (tmp=falses(n);tmp[getdx]=true;tmp) :
     error("getdx=$getdx ninputs=$(n)"))
end

# This is extremely slow:
#get1(x)=(!isempty(methods(length, (typeof(x),))) && length(x)==1?x[1]:x)
get1(x)=(x==nothing?x:length(x)==1?x[1]:x)

### DEAD CODE:

# function sback_delete(f::Net, ygold=nothing, loss=copyloss; getdx=false, o...)
#     getdx = getdxbool(getdx, ninputs(f))
#     initback(f, ygold, loss, getdx) # TODO: rethink lastforw==lastback in a seq context
#     gotreturn = false

#     for n = length(f):-1:1
#         s = pop!(f)
#         s == nothing && continue
#         (y, xsave, ysave) = s
#         @assert y === reg(f,n)
#         getp(y,:grad) || continue # :grad means we need the gradient of the output of this operation, does not mean go back on this operation. but no :grad means no need to go back.

#         if !gotreturn
#             gotreturn = true
#             if ygold == nothing # represents zero gradient
#                 getp(y,:incr) || (y.dif = nothing)
#             else
#                 !isreturn(y) && Base.warn_once("ygold specified when there is no return")
#                 ysave == nothing && error("return value was not saved")
#                 if getp(y,:incr)
#                     loss(ysave, ygold, y.tmp; o...) # loss needs o... for e.g. mask
#                     y.dif = axpy!(1,y.tmp,y.dif0)
#                 else
#                     y.dif = loss(ysave, ygold, y.dif0; o...)
#                 end
#             end
#         elseif isreturn(y)
#             error("Got return in non-final instruction")
#         end

#         xx = inputregs(f,y)
#         if y.dif == nothing
#             for x in xx
#                 getp(x,:grad) && !getp(x,:incr) && (x.dif = nothing)
#             end
#         else
#             xdif = inputdifs(f,y)  # map too slow? map(x->(!getp(x,:grad) ? nothing : getp(x,:incr) ? x.tmp : x.dif0), xx)
#             back(y.op, y.dif, xdif...; x=get1(xsave), y=ysave, o...)
#             for x in xx
#                 x.dif = (getp(x,:incr) ? axpy!(1, x.tmp, x.dif0) :
#                          getp(x,:grad) ? x.dif0 : nothing)
#             end
#             if getp(y,:incr) && !isa(y.op, Par)
#                 # what if y.op=Arr?  then it will have no inputs, thus :grad=:incr=false
#                 # where does Par.dif get zeroed out? at reset!
#                 fillsync!(y.dif,0)
#             end
#         end
#     end

#     if any(getdx)
#         dx = Any[]; nx = 0
#         for p in regs(f)
#             isa(p.op,Input) && getdx[nx+=1] && push!(dx, p.dif)
#         end
#         return get1(dx)
#     end
# end

# # back(r::Net,dy::Vector) for a sequence
# function back(r::Net, dy::Vector, dx...; a...)
#     dxi = map(x->(x==nothing ? x : x[end]), dx)
#     initback(r, dy[end], dxi...; seq=true, a...)
#     for i=length(dy):-1:1
#         dxi = map(x->(x==nothing ? x : x[i]), dx)
#         back(r, dy[i], dxi...; seq=true, a...)
#     end
# end

# DONE: truncated bptt
# - go forward k1 steps, run back for k2, update, recover state
# - if k1==k2 we just need the keepstate option to forw
# - if k1>k2 the stack won't be cleared
# - if k1<k2 the stack will be overdrawn

    # for i = ninputs(r):-1:1
    #     n = i+N
    #     r.tosave[n] && pop(r,n)                                    # ; r.tosave[n] && dbg(r,:out,n)
    #     dx == nothing || copysync!(dx[i], r.dif[n])
    # end

    # if dx != ()
    #     lastinput = 0
    #     for n = 1:N
    #         isa(r.op[n], Input) || continue
    #         copysync!(dx[lastinput += 1], r.dif[n])
    #     end
    # end
# function stack_output(f::Net,n::Int)
#     o = f.stack[f.sp - length(f) + n]
#     p = f.reg[n]
#     (o!=nothing) || !(isreturn(p) || back_reads_y(p.op)) || error("sp=$(f.sp) n=$n N=$(length(f)) op=$(typeof(f.reg[n].op))")
#     return o
# end


struct GPSL1{C <: AbstractMatrix} <: AbstractGNSS{C}
    codes::C
end

get_system_string(s::GPSL1) = "GPSL1"

function read_gpsl1_codes()
    read_in_codes(
        Int8,
        joinpath(dirname(pathof(GNSSSignals)), "..", "data", "codes_gps_l1.bin"),
        37,
        1023
    )
end

function GPSL1(;use_gpu = Val(false))
    _GPSL1(use_gpu)
end

function _GPSL1(::Val{false})
    GPSL1(Int16.(read_gpsl1_codes()))
end
function _GPSL1(::Val{true})
    GPSL1(CuMatrix{Float32}(read_gpsl1_codes()))
end

"""
$(SIGNATURES)

Get code length of GNSS system GPSL1.
```julia-repl
julia> get_code_length(gpsl1)
```
"""
@inline function get_code_length(gpsl1::GPSL1)
    1023
end

"""
$(SIGNATURES)

Get secondary code length of GNSS system GPSL1.
```julia-repl
julia> get_secondary_code_length(gpsl1)
```
"""
@inline function get_secondary_code_length(gpsl1::GPSL1)
    1
end

"""
$(SIGNATURES)

Get center frequency of GNSS system GPSL1.
```julia-repl
julia> get_center_frequency(gpsl1)
```
"""
@inline function get_center_frequency(gpsl1::GPSL1)
    1_575_420_000Hz
end

"""
$(SIGNATURES)

Get code frequency of GNSS system GPSL1.
```julia-repl
julia> get_code_frequency(gpsl1)
```
"""
@inline function get_code_frequency(gpsl1::GPSL1)
    1_023_000Hz
end
"""
$(SIGNATURES)

Get data frequency of GNSS system GPSL1.
```julia-repl
julia> get_data_frequency(gpsl1)
```
"""
@inline function get_data_frequency(gpsl1::GPSL1)
    50Hz
end

"""
$(SIGNATURES)
Get the spectral power of the GPSL1 CA code
"""
function get_code_spectrum(s::GPSL1, f)
    get_code_spectrum_BPSK(get_code_frequency(s), f)
end


import Base: size
import Base: stride
import Base: eachindex
import Base: getindex
import Base: setindex!
import Base: similar
import Base: copy
import Base: eltype
import Base: convert
import Base: collect
import Base.Broadcast: broadcast
import Base: hcat
import Base: vcat

"""
    Base.convert(::Type{SimpleSDMResponse}, layer::T) where {T <: SimpleSDMPredictor}

Returns a response with the same grid and bounding box as the predictor.
"""
function Base.convert(::Type{SimpleSDMResponse}, layer::T) where {T <: SimpleSDMPredictor}
   return copy(SimpleSDMResponse(layer.grid, layer.left, layer.right, layer.bottom, layer.top))
end

"""
    Base.convert(::Type{SimpleSDMPredictor}, layer::T) where {T <: SimpleSDMResponse}

Returns a predictor with the same grid and bounding box as the response.
"""
function Base.convert(::Type{SimpleSDMPredictor}, layer::T) where {T <: SimpleSDMResponse}
   return copy(SimpleSDMPredictor(layer.grid, layer.left, layer.right, layer.bottom, layer.top))
end

"""
    Base.convert(::Type{Matrix}, layer::T) where {T <: SimpleSDMLayer}

Returns the grid as an array.
"""
function Base.convert(::Type{Matrix}, layer::T) where {T <: SimpleSDMLayer}
   return copy(layer.grid)
end

"""
    Base.eltype(layer::SimpleSDMLayer{T}) where {T}

Returns the type of the values stored in the grid, where the `Nothing` type is
omitted.
"""
Base.eltype(::SimpleSDMResponse{T}) where {T} = T
Base.eltype(::SimpleSDMPredictor{T}) where {T} = T

"""
    Base.size(layer::T) where {T <: SimpleSDMLayer}

Returns the size of the grid.
"""
Base.size(layer::T) where {T <: SimpleSDMLayer} = size(layer.grid)

"""
    Base.size(layer::T, i...) where {T <: SimpleSDMLayer}

Returns the size of the grid alongside a dimension.
"""
Base.size(layer::T, i...) where {T <: SimpleSDMLayer} = size(layer.grid, i...)

"""
    Base.stride(layer::T; dims::Union{Nothing,Integer}=nothing) where {T <: SimpleSDMLayer}

Returns the stride, *i.e.* half the length, of cell dimensions, possibly
alongside a side of the grid. The first position is the length of the
*longitude* cells, the second the *latitude*.
"""
function Base.stride(layer::T; dims::Union{Nothing,Integer}=nothing) where {T <: SimpleSDMLayer}
   lon_stride = (layer.right-layer.left)/2.0size(layer, 2)
   lat_stride = (layer.top-layer.bottom)/2.0size(layer, 1)
   isnothing(dims) && return (lon_stride, lat_stride)
   dims == 1 && return lon_stride
   dims == 2 && return lat_stride
end
Base.stride(layer::T, i::Int) where {T<:SimpleSDMLayer} = stride(layer; dims=i)

"""
    Base.eachindex(layer::T) where {T <: SimpleSDMLayer}

Returns the index of the grid.
"""
Base.eachindex(layer::T) where {T <: SimpleSDMLayer} = eachindex(layer.grid)

"""
Extracts a  value from a layer by its grid position.
"""
Base.getindex(layer::T, i::Int64) where {T <: SimpleSDMLayer} = layer.grid[i]

"""
    Base.getindex(layer::T, i::R, j::R) where {T <: SimpleSDMLayer, R <: UnitRange}

Extracts a series of positions in a layer, and returns a layer corresponding to
the result. This is essentially a way to rapidly crop a layer to a given subset
of its extent. The `i` and `j` arguments are `UnitRange`s (of `Integer`).

The layer returned by this function will have the same type as the layer passed
as its argument, but this can be changed using `convert`. Note that this function
performs additional checks to ensure that the range is not empty, and to also
ensure that it does not overflows from the size of the layer.
"""
function Base.getindex(layer::T, i::R, j::R) where {T <: SimpleSDMLayer, R <: UnitRange}
   i_min = isempty(i) ? max(i.start-1, 1) : i.start
   i_max = isempty(i) ? max(i.stop+2, size(layer, 1)) : i.stop
   j_min = isempty(j) ? max(j.start-1, 1) : j.start
   j_max = isempty(j) ? max(j.stop+2, size(layer, 2)) : j.stop
   i_fix = i_min:i_max
   j_fix = j_min:j_max
   RT = T <: SimpleSDMResponse ? SimpleSDMResponse : SimpleSDMPredictor
   return RT(
            layer.grid[i_fix,j_fix],
            minimum(longitudes(layer)[j_fix])-stride(layer,1),
            maximum(longitudes(layer)[j_fix])+stride(layer,1),
            minimum(latitudes(layer)[i_fix])-stride(layer,2),
            maximum(latitudes(layer)[i_fix])+stride(layer,2)
           )
end

"""
Given a layer and a latitude, returns `nothing` if the latitude is outside the
range, or the grid index containing this latitude if it is within range
"""
function _match_latitude(layer::T, lat::K) where {T <: SimpleSDMLayer, K <: AbstractFloat}
   lat > layer.top && return nothing
   lat < layer.bottom && return nothing
   return last(findmin(abs.(lat .- latitudes(layer))))
end

"""
Given a layer and a longitude, returns `nothing` if the longitude is outside the
range, or the grid index containing this longitude if it is within range
"""
function _match_longitude(layer::T, lon::K) where {T <: SimpleSDMLayer, K <: AbstractFloat}
   lon > layer.right && return nothing
   lon < layer.left && return nothing
   return last(findmin(abs.(lon .- longitudes(layer))))
end

"""
    Base.getindex(layer::T, longitude::K, latitude::K) where {T <: SimpleSDMLayer, K <: AbstractFloat}

Extracts the value of a layer at a given latitude and longitude. If values
outside the range are requested, will return `nothing`.
"""
function Base.getindex(layer::T, longitude::K, latitude::K) where {T <: SimpleSDMLayer, K <: AbstractFloat}
   i = _match_longitude(layer, longitude)
   j = _match_latitude(layer, latitude)
   isnothing(i) && return nothing
   isnothing(j) && return nothing
   return layer.grid[j, i]
end

"""
    Base.getindex(layer::T; left=nothing, right=nothing, top=nothing, bottom=nothing) where {T <: SimpleSDMLayer, K <: Union{Nothing,AbstractFloat}}

Returns a subset of the argument layer, where the new limits are given by
`left`, `right`, `top`, and `bottom`. Up to three of these can be omitted, and
if so these limits will not be affected.
"""
function Base.getindex(layer::T; left=nothing, right=nothing, top=nothing, bottom=nothing) where {T <: SimpleSDMLayer}
   for limit in [left, right, top, bottom]
      if !isnothing(limit)
         @assert typeof(limit) <: AbstractFloat
      end
   end
   imax = _match_longitude(layer, isnothing(right) ? layer.right : right)
   imin = _match_longitude(layer, isnothing(left) ? layer.left : left)
   jmax = _match_latitude(layer, isnothing(top) ? layer.top : top)
   jmin = _match_latitude(layer, isnothing(bottom) ? layer.bottom : bottom)
   any(isnothing.([imin, imax, jmin, jmax])) && throw(ArgumentError("Unable to extract, coordinates outside of range"))
   # Note that this is LATITUDE first
   return layer[jmin:jmax, imin:imax]
end

"""
    Base.getindex(layer::T, n::NT) where {T <: SimpleSDMLayer, NT <: NamedTuple}

Returns a subset of the argument layer, where the new limits are given in
a NamedTuple by `left`, `right`, `top`, and `bottom`, in any order. Up to
three of these can be omitted, and if so these limits will not be affected.
"""
function Base.getindex(layer::T, n::NT) where {T <: SimpleSDMLayer, NT <: NamedTuple}
    l = isdefined(n, :left) ? n.left : nothing
    r = isdefined(n, :right) ? n.right : nothing
    t = isdefined(n, :top) ? n.top : nothing
    b = isdefined(n, :bottom) ? n.bottom : nothing
    Base.getindex(layer; left=l, right=r, top=t, bottom=b)
end

"""
    Base.getindex(layer1::T1, layer2::T2) where {T1 <: SimpleSDMLayer, T2 <: SimpleSDMLayer}

Extract a layer based on a second layer. Note that the two layers must be
*compatible*, which is to say they must have the same bounding box and grid
size.
"""
function Base.getindex(layer1::T1, layer2::T2) where {T1 <: SimpleSDMLayer, T2 <: SimpleSDMLayer}
   SimpleSDMLayers._layers_are_compatible(layer1, layer2)
   return layer1[left=layer2.left, right=layer2.right, bottom=layer2.bottom, top=layer2.top]
end

"""
     Base.setindex!(layer::SimpleSDMResponse{T}, v::T, i...) where {T}

Changes the value of a cell, or a range of cells, as indicated by their grid
positions.
"""
function Base.setindex!(layer::SimpleSDMResponse{T}, v::T, i...) where {T}
   typeof(v) <: eltype(layer.grid) || throw(ArgumentError("Impossible to set a value to a non-matching type"))
   layer.grid[i...] = v
end

"""
    Base.setindex!(layer::T, v, lon::Float64, lat::Float64) where {T <: SimpleSDMResponse}

Changes the values of the cell including the point at the requested latitude and
longitude.
"""
function Base.setindex!(layer::SimpleSDMResponse{T}, v::T, lon::Float64, lat::Float64) where {T}
   i = _match_longitude(layer, lon)
   j = _match_latitude(layer, lat)
   layer[j,i] = v
end

"""
    Base.similar(l::T) where {T <: SimpleSDMLayer}

Returns a `SimpleSDMResponse` of the same dimensions as the original layer, with
`nothing` in the same positions. The rest of the values are replaced by the
output of `zero(eltype(layer.grid))`, which implies that there must be a way to
get a zero for the type. If not, the same result can always be achieved through
the use of `copy`, manual update, and `convert`.
"""
function Base.similar(layer::T) where {T <: SimpleSDMLayer}
   emptygrid = similar(layer.grid)
   for i in eachindex(emptygrid)
      emptygrid[i] = isnothing(layer.grid[i]) ? nothing : zero(eltype(layer.grid[i]))
   end
   return SimpleSDMResponse(emptygrid, layer.left, layer.right, layer.bottom, layer.top)
end

"""
    Base.similar(::Type{TC}, l::T) where {TC <: Any, T <: SimpleSDMLayer}

Returns a `SimpleSDMResponse` of the same dimensions as the original layer, with
`nothing` in the same positions. The rest of the values are replaced by the
output of `zero(TC)`, which implies that there must be a way to get a zero for
the type. If not, the same result can always be achieved through the use of
`copy`, manual update, and `convert`.
"""
function Base.similar(::Type{TC}, layer::T) where {TC <: Any, T <: SimpleSDMLayer}
   emptygrid = convert(Matrix{Union{Nothing,TC}}, zeros(TC, size(layer)))
   emptygrid[findall(isnothing, layer.grid)] .= nothing
   return SimpleSDMResponse(emptygrid, layer.left, layer.right, layer.bottom, layer.top)
end

"""
    Base.copy(l::T) where {T <: SimpleSDMLayer}

Returns a new copy of the layer, which has the same type.
"""
function Base.copy(layer::T) where {T <: SimpleSDMLayer}
   copygrid = copy(layer.grid)
   RT = T <: SimpleSDMResponse ? SimpleSDMResponse : SimpleSDMPredictor
   return RT(copygrid, copy(layer.left), copy(layer.right), copy(layer.bottom), copy(layer.top))
end

"""
   Broadcast.broadcast(f, L::LT) where {LT <: SimpleSDMLayer}

TODO
"""
function Base.Broadcast.broadcast(f, L::LT) where {LT <: SimpleSDMLayer}
    newgrid = Array{Any}(nothing, size(L))
    N = SimpleSDMResponse(newgrid, L)
    v = filter(!isnothing, L.grid)
    fv = f.(v)
    N.grid[findall(!isnothing, L.grid)] .= fv

    internal_types = unique(typeof.(N.grid))

    RT = LT <: SimpleSDMResponse ? SimpleSDMResponse : SimpleSDMPredictor
    return RT(convert(Matrix{Union{internal_types...}}, N.grid), N)
end

"""
    Base.collect(l::T) where {T <: SimpleSDMLayer}

Returns the non-`nothing` values of a layer.
"""
function Base.collect(l::T) where {T <: SimpleSDMLayer}
    v = filter(!isnothing, l.grid)
    return convert(Vector{typeof(v[1])}, v)
end

"""
    Base.vcat(l1::T, l2::T) where {T <: SimpleSDMLayers}

Adds the second layer *under* the first one, assuming the strides and left/right
coordinates match. This will automatically re-order the layers if the second is
above the first.
"""
function Base.vcat(l1::T, l2::T) where {T <: SimpleSDMLayer}
    (l1.left == l2.left) || throw(ArgumentError("The two layers passed to vcat must have the same left coordinate"))
    (l1.right == l2.right) || throw(ArgumentError("The two layers passed to vcat must have the same right coordinate"))
    all(stride(l1) .≈ stride(l2)) || throw(ArgumentError("The two layers passed to vcat must have the same stride"))
    (l2.top == l1.bottom) && return vcat(l2, l1)
    new_grid = vcat(l1.grid, l2.grid)
    RT = T <: SimpleSDMPredictor ? SimpleSDMPredictor : SimpleSDMResponse
    return RT(new_grid, l1.left, l1.right, l1.top, l2.bottom)
end

"""
    Base.hcat(l1::T, l2::T) where {T <: SimpleSDMLayers}

Adds the second layer *to the right of* the first one, assuming the strides and
left/right coordinates match. This will automatically re-order the layers if the
second is to the left the first.
"""
function Base.hcat(l1::T, l2::T) where {T <: SimpleSDMLayer}
    (l1.top == l2.top) || throw(ArgumentError("The two layers passed to hcat must have the same top coordinate"))
    (l1.bottom == l2.bottom) || throw(ArgumentError("The two layers passed to hcat must have the same bottom coordinate"))
    all(stride(l1) .≈ stride(l2)) || throw(ArgumentError("The two layers passed to hcat must have the same stride"))
    (l2.right == l1.left) && return hcat(l2, l1)
    new_grid = hcat(l1.grid, l2.grid)
    RT = T <: SimpleSDMPredictor ? SimpleSDMPredictor : SimpleSDMResponse
    return RT(new_grid, l1.left, l2.right, l1.top, l1.bottom)
end

"""
    quantile(layer::T, p) where {T <: SimpleSDMLayer}

Returns the quantiles of `layer` at `p`, using `Statistics.quantile`.
"""
function Statistics.quantile(layer::T, p) where {T <: SimpleSDMLayer}
    return quantile(collect(layer), p)
end


abstract type AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec} end

    hasdensityles(a::Type{T}) where {L,HV,TT,α,dρdz,g,Gdirec,T<:AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}} =
        L === NoLESScalar ? false : true
    @inline hasdensityles(a::AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = hasdensityles(typeof(a))

    has_les_density_vorticity_model(a::Type{T}) where {L,HV,TT,α,dρdz,g,Gdirec,T<:AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}} =
        is_vorticity_model(L)
    @inline has_les_density_vorticity_model(a::AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = has_les_density_vorticity_model(typeof(a))
    @inline @par has_les_density_vorticity_model(s::Type{T}) where {T<:@par(AbstractSimulation)} = has_les_density_vorticity_model(DensityStratificationType)

    diffusivity(a::Type{T}) where {L,HV,TT,α,dρdz,g,Gdirec,T<:AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}} =
        α
    @inline diffusivity(a::AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = diffusivity(typeof(a))
    @inline @par diffusivity(s::Type{T}) where {T<:@par(AbstractSimulation)} = diffusivity(DensityStratificationType)
    @inline diffusivity(s::AbstractSimulation) = diffusivity(typeof(s))

    meangradient(a::Type{T}) where {L,HV,TT,α,dρdz,g,Gdirec,T<:AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}} =
        dρdz
    @inline meangradient(a::AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = meangradient(typeof(a))

    gravity(a::Type{T}) where {L,HV,TT,α,dρdz,g,Gdirec,T<:AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}} =
        g
    @inline gravity(a::AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = gravity(typeof(a))
    @inline @par gravity(s::Type{T}) where {T<:@par(AbstractSimulation)} = gravity(DensityStratificationType)
    @inline gravity(s::AbstractSimulation) = gravity(typeof(s))

    graddir(a::Type{T}) where {L,HV,TT,α,dρdz,g,Gdirec,T<:AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}} =
        Gdirec
    @inline graddir(a::AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = graddir(typeof(a))

    calculate_N(a::Type{T}) where {L,HV,TT,α,dρdz,g,Gdirec,T<:AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}} =
        sqrt(g.z*dρdz)
    @inline calculate_N(a::AbstractDensityStratification{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = calculate_N(typeof(a))

    initialize!(a::AbstractDensityStratification,s::AbstractSimulation) = initialize!(a.timestep,parent(real(a.rhs)),diffusivity(a),s)

    function statsheader(a::AbstractDensityStratification{L,HV}) where {L,HV}
        mles = L !== NoLESScalar ? ",rhopr" : ""
        mhvis = HV !== NoHyperViscosity ? ",rhohdiss" : ""
        return "rho,rhop2,drhodxp2,drhodyp2,drhodzp2,rhodiss,trhodiss,bflux"*mles*mhvis
    end

    function stats(a::AbstractDensityStratification{L,HV},s::A) where {L,HV,A<:AbstractSimulation}
        sstats, sstats_fil = scalar_stats(a.ρ,a,s)
        trhodiss = sstats[end]
        trhodiss_fil = sstats_fil[end]
        bf, bf_fil = buoyancy_flux(a,s)

        if L !== NoLESScalar
            if !STAT_FIL
                lesstats = (scalar_les_stats(a.reduction,a.flux,a.ρ),)
                lesstats_fil = (0.,)
                trhodiss += lesstats[1]
            else
                lesstatsv, lesstats_filv = scalar_les_stats(a.reduction,s.reductionh,a.flux,a.ρ)
                lesstats = (lesstatsv,)
                lesstats_fil = (lesstats_filv,)
                trhodiss += lesstats[1]
                trhodiss_fil += lesstats_fil[1]
            end
        else
            lesstats = ()
            lesstats_fil = ()
        end

        if HV !== NoHyperViscosity
            if !STAT_FIL
                hvstats = (scalar_hvis_stats(a.reduction,a.ρ,a.hyperviscosity),)
                hvsstats_fil = (0.,)
                trhodiss += hvstats[1]
            else
                hvstatsv, hvstats_filv = scalar_hvis_stats(a.reduction,s.reductionh,a.ρ,a.hyperviscosity)
                hvstats = (hvstatsv,)
                hvsstats_fil = (hvstats_filv,)
                trhodiss += hvstats[1]
                trhodiss_fil += hvstats_fil[1]
            end
        else
            hvstats = ()
            hvstats_fil = ()
        end

        return (sstats...,trhodiss,bf,lesstats...,hvstats...),(sstats_fil...,trhodiss_fil,bf_fil,lesstats_fil...,hvstats_fil...)
    end

struct NoDensityStratification <: AbstractDensityStratification{NoLESScalar,NoHyperViscosity,nothing,nothing,nothing,nothing,nothing} end

    initialize!(a::NoDensityStratification,s::AbstractSimulation) = nothing

    statsheader(a::NoDensityStratification) = ""

    stats(a::NoDensityStratification,s::AbstractSimulation) = (),()

    msg(a::NoDensityStratification) = "\nDensity Stratification: No density stratification\n"

struct BoussinesqApproximation{L,HV,TTimeStep, α #=Difusitivity = ν/Pr =#,
                   dρdz #=Linear mean profile=#, g #=This is actually g/ρ₀ =#,
                   Gdirec#=Gravity direction =#} <: AbstractDensityStratification{L,HV,TTimeStep,α,dρdz,g,Gdirec}
    ρ::ScalarField{Float64,3,2,false}
    rhs::ScalarField{Float64,3,2,false}
    flux::VectorField{Float64,3,2,false}
    lesmodel::L
    hyperviscosity::HV
    timestep::TTimeStep
    reduction::Vector{Float64}

    function BoussinesqApproximation{TT,α,dρdz,g,Gdirec}(ρ,timestep,les,hv) where {TT,α,dρdz,g,Gdirec}
        ρrhs = similar(ρ)
        flux = VectorField(size(ρ.field.r),(ρ.kx.l,ρ.ky.l,ρ.kz.l))
        reduction = zeros(THR ? Threads.nthreads() : 1)
        return new{typeof(les),typeof(hv),TT,α,dρdz,g,Gdirec}(ρ,ρrhs,flux,les,hv,timestep,reduction)
    end
end

initialize!(a::BoussinesqApproximation,s::AbstractSimulation) = initialize!(a.timestep,parent(real(a.rhs)),diffusivity(a),s)

msg(a::BoussinesqApproximation{L,HV,TT,α,dρdz,g,Gdirec}) where {L,HV,TT,α,dρdz,g,Gdirec} = """

Density Stratification: Boussinesq Approximation
Density diffusivity: $(α)
g/ρ₀ : $(g)
Density mean gradient : $(dρdz)
Density mean gradient direction: $(Gdirec)
Density time-stepping method: $(TT)
Density LES model: $(msg(a.lesmodel))
Density HyperViscosity model: {$(msg(a.hyperviscosity))\n}

"""

function stratificationtype(d::AbstractDict,start,integrator,nx,ny,nz,lx,ly,lz,variableTimeStep,idt)

    s = NoDensityStratification()

    if haskey(d,:densityHyperViscosity)
        if d[:densityHyperViscosity] == "spectralBarrier"
            initkp = haskey(d,:densityInitk) ? parse(Float64,d[:densityInitk]) : Globals.cutoffr*2*KX[end] / 3
            endK = haskey(d,:densityEndk) ? parse(Float64,d[:densityEndk]) : sqrt(Globals.cutoffr*3)*KX[end]
            hyperviscositytype = SpectralBarrier(initkp,endK)
        else
            νh = parse(Float64,d[:densityHyperViscosityCoefficient])
            m = haskey(d,:densityHyperViscosityM) ? parse(Int,d[:densityHyperViscosityM]) : 2
            hyperviscositytype = HyperViscosity{νh,m}()
        end
    else
        hyperviscositytype = NoHyperViscosity()
    end

    if haskey(d,:densityStratification)

        haskey(d,:gravityDirection) ? (gdir = Symbol(d[:gravityDirection])) : (gdir = :z)
        α = ν/Float64(eval(Meta.parse(d[:Pr])))
        dρdz = Float64(eval(Meta.parse(d[:densityGradient])))
        gval = Float64(eval(Meta.parse(d[:zAcceleration])))/Float64(eval(Meta.parse(d[:referenceDensity])))
        if gdir === :z
            g = Vec(0.0,0.0,gval)
        elseif gdir === :y
            g = Vec(0.0,gval,0.0)
        elseif gdir === :x
            g = Vec(gval,0.0,0.0)
        end

        @info("Reading initial density field rho.$start")
        rho = isfile("rho.$start") ? ScalarField{Float64}("rho.$start") : ScalarField{Float64}((nx,ny,nz),(lx,ly,lz))
        gdir = haskey(d,:gravityDirection) ? Symbol(d[:gravityDirection]) : :z
        densitytimestep = if integrator === :Euller
            Euller{variableTimeStep,idt}(Ref(idt))
        elseif integrator === :Adams_Bashforth3rdO
            Adams_Bashforth3rdO{variableTimeStep,idt}()
        else
            ETD3rdO(variableTimeStep,idt,hyperviscositytype,α)
        end

        lestypedensity = if haskey(d,:densityLesModel)
            if d[:densityLesModel] == "vorticityDiffusion"
                c = haskey(d,:densityVorticityCoefficient) ? parse(Float64,d[:densityVorticityCoefficient]) : 0.1
                VorticityDiffusion(c)
            else
                EddyDiffusion()
            end
        else
            NoLESScalar()
        end
        s = BoussinesqApproximation{typeof(densitytimestep),α,dρdz,g,gdir}(rho,densitytimestep,lestypedensity,hyperviscositytype)
    end

    return s
end

function buoyancy_flux(a::AbstractDensityStratification,s::AbstractSimulation)
    gdir = graddir(a)
    g = gravity(s)
    if !STAT_FIL
        if gdir === :z
           return -g.z*proj_mean(a.reduction,s.u.c.z,a.ρ), 0.0
        elseif gdir === :y
           return -g.y*proj_mean(a.reduction,s.u.c.y,a.ρ), 0.0
        elseif gdir === :x
           return -g.x*proj_mean(a.reduction,s.u.c.x,a.ρ), 0.0
        end
    else
        if gdir === :z
           return -g.z .* proj_mean(a.reduction,s.reductionh,s.u.c.z,a.ρ)
        elseif gdir === :y
           return -g.y .* proj_mean(a.reduction,s.reductionh,s.u.c.y,a.ρ)
        elseif gdir === :x
           return -g.x .* proj_mean(a.reduction,s.reductionh,s.u.c.x,a.ρ)
        end
    end
end


using IncrementalInference
using Test

##

@testset "saving to and loading from FileDFG" begin
##

fg = generateCanonicalFG_Kaess()
addVariable!(fg, :x4, ContinuousScalar)
addFactor!(fg, [:x2;:x3;:x4], LinearRelative(Normal()), multihypo=[1.0;0.6;0.4])

saveFolder = "/tmp/dfg_test"
saveDFG(fg, saveFolder)
# VERSION above 1.0.x hack required since Julia 1.0 does not seem to havfunction `splitpath`

retDFG = initfg()
retDFG = loadDFG!(retDFG, saveFolder)
Base.rm(saveFolder*".tar.gz")

@test symdiff(ls(fg), ls(retDFG)) == []
@test symdiff(lsf(fg), lsf(retDFG)) == []

@show getFactor(fg, :x2x3x4f1).solverData.multihypo
@show getFactor(retDFG, :x2x3x4f1).solverData.multihypo

# check for match
@test getFactor(fg, :x2x3x4f1).solverData.multihypo - getFactor(retDFG, :x2x3x4f1).solverData.multihypo |> norm < 1e-10
@test getFactor(fg, :x2x3x4f1).solverData.certainhypo - getFactor(retDFG, :x2x3x4f1).solverData.certainhypo |> norm < 1e-10

##
end

@testset "saving to and loading from FileDFG with nullhypo, eliminated, solveInProgress" begin
##

fg = generateCanonicalFG_Kaess()
getSolverParams(fg).attemptGradients = true

addVariable!(fg, :x4, ContinuousScalar)
addFactor!(fg, [:x2;:x3;:x4], LinearRelative(Normal()), multihypo=[1.0;0.6;0.4])
addFactor!(fg, [:x1;], Prior(Normal(10,1)), nullhypo=0.5)

solveTree!(fg)

#manually change a few fields to test if they are preserved
fa = getFactor(fg, :x2x3x4f1)
fa.solverData.eliminated = true
fa.solverData.solveInProgress = 1
fa.solverData.nullhypo = 0.5

saveFolder = "/tmp/dfg_test"
saveDFG(fg, saveFolder)

retDFG = initfg()
loadDFG!(retDFG, saveFolder)
Base.rm(saveFolder*".tar.gz")

@test issetequal(ls(fg), ls(retDFG))
@test issetequal(lsf(fg), lsf(retDFG))

@show getFactor(fg, :x2x3x4f1).solverData.multihypo
@show getFactor(retDFG, :x2x3x4f1).solverData.multihypo

# check for match
@test isapprox(getFactor(fg, :x2x3x4f1).solverData.multihypo, getFactor(retDFG, :x2x3x4f1).solverData.multihypo)
@test isapprox(getFactor(fg, :x2x3x4f1).solverData.certainhypo, getFactor(retDFG, :x2x3x4f1).solverData.certainhypo)

fb = getFactor(retDFG, :x2x3x4f1)
@test fa == fb

fa = getFactor(fg, :x1f2)
fb = getFactor(retDFG, :x1f2)

@test fa == fb

##
end


"""
    main_mode1
A main file to launch PSCOPF in MODE 1.

Parameters:
    instance_path : path to input data directory
    output_path : path to output directory
"""

using Dates

root_path = dirname(@__DIR__)
push!(LOAD_PATH, root_path);
cd(root_path)
include(joinpath(root_path, "src", "PSCOPF.jl"));

#########################
# INPUT & PARAMS
#########################

# instance_path is the path to the input data directory containing :
# pscopf_units : the description of the units
# pscopf_init : the description of the initial state of units (just before the first interest timepoit T)
# pscopf_limits : the flow capacity of each branch
# pscopf_ptdf : the ptdf coefficients per (branch, bus_id)
# pscopf_uncertainties : the nodal injections (for each bus and each limitable)
instance_path = ( length(ARGS) > 0 ? ARGS[1] :
                    joinpath(@__DIR__, "..", "data", "2buses_usecase", "instance") )

# output_path is the path where output files will be write_commitment_schedule
#NOTE: all files in output_path, except those starting with pscopf_, will be deleted
output_path = length(ARGS) > 1 ? ARGS[2] : instance_path

#########################
# EXECUTION
#########################

# load Data
network = PSCOPF.Data.pscopfdata2network(instance_path)
uncertainties = PSCOPF.PSCOPFio.read_uncertainties(instance_path)
generators_init_state = PSCOPF.PSCOPFio.read_initial_state(instance_path)

# Launch mode 1
mode = PSCOPF.PSCOPF_MODE_1

ts1 = Dates.DateTime("2015-01-01T11:00:00") #11h
TS = PSCOPF.create_target_timepoints(ts1) #T: 11h, 11h15, 11h30, 11h45
ECH = PSCOPF.generate_ech(network, TS, mode) #ech: -4h, -1h, -30mins, -15mins, 0h

sequence = PSCOPF.generate_sequence(network, TS, ECH, mode)

PSCOPF.rm_non_prefixed(output_path, "pscopf_")
exec_context = PSCOPF.PSCOPFContext(network, TS, mode,
                                    generators_init_state,
                                    uncertainties, nothing,
                                    output_path)

PSCOPF.run!(exec_context, sequence)


#=
pfile.jl
read GE MRI k-space data from .p file
based on https://gitlab.com/fMRI/toppe/+toppe/+utils/loadpfile.m

(c) 2016 The Regents of the University of Michigan
Jon-Fredrik Nielsen, jfnielse@umich.edu

Comments from Matlab version at
https://gitlab.com/fMRI/toppe/+toppe/+utils/loadpfile.m

This file was derived from part of the TOPPE development environment
for platform-independent MR pulse programming.

TOPPE is free software: you can redistribute it and/or modify
it under the terms of the GNU Library General Public License as published by
the Free Software Foundation version 2.0 of the License.

TOPPE is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU Library General Public License for more details.

You should have received a copy of the GNU Library General Public License
along with TOPPE.
If not, see <http://www.gnu.org/licenses/old-licenses/lgpl-2.0.html>.

2019-05-22 Julia version by Jeff Fessler
=#

export loadpfile

# using MIRTio: read_rdb_hdr, header_init, header_size, header_string, header_write

"""
    (dat, rdb_hdr) = loadpfile(fid::IOStream ; ...)

Load data for one or more echoes from GE MRI scan Pfile.

in
- `fid::IOStream`

option
- `coils::AbstractVector{Int}`
 only get data for these coils; default: all coils
- `echoes::AbstractVector{Int}`
 only get data for these echoes; default: all echoes
- `slices::AbstractVector{Int}`
 only get data for these slices; default: `2:nslices` (NB!)
 because first slice (dabslice=0 slot) may contain corrupt data.
- `views::AbstractVector{Int}`
 only get data for these views; default: all views
- `quiet::Bool` non-verbosity, default `false`

out
- `dat::Array{Complex{Int16}}` `[ndat, ncoil, nslice, necho, nview]`
- `rdb_hdr::NamedTuple` header information

To save memory the output type is complex-valued Int16.
"""
function loadpfile(
    fid::IOStream;
    filesize::Int = 0,
    coils::AbstractVector{Int} = Int[],
    echoes::AbstractVector{Int} = Int[],
    slices::AbstractVector{Int} = Int[],
    views::AbstractVector{Int} = Int[],
    quiet::Bool = false,
)

    rdb_hdr = read_rdb_hdr(fid) # read header

    # Header parameters
    ndat = Int(rdb_hdr.frame_size)
    nslices = rdb_hdr.nslices
    ptsize = rdb_hdr.point_size # 2: data stored in short int format, int16)
    ptsize != 2 && # 4 (extended precision) unsupported here
        throw("ptsize = $ptsize unsupported")
    nechoes = rdb_hdr.nechoes
    nviews = rdb_hdr.nframes
    ncoils = rdb_hdr.dab[2] - rdb_hdr.dab[1] + 1

    # Calculate size of data chunks
    # See pfilestruct.jpg, and rhrawsize calculation in .e file.
    # number of data points per 'echo' loaddab slot. Includes baseline (0) view:
    # Apparent file data order is: [ndat, nview, necho, nslice, ncoil]
    echores = ndat * (nviews + 1)
    sliceres = nechoes * echores # number of data points per 'slice'
    coilres = nslices * sliceres # number of data points per receive coil

    # This size should match the Pfile size exactly:
    pfilesize =
        rdb_hdr.off_data + 2 * ptsize * ncoils * nslices * nechoes * (nviews + 1) * ndat

    # File size check
    (filesize != 0) &&
        (pfilesize != filesize) &&
        throw("Expected $(pfilesize/1e6) MB but see $(filesize/1e6) MB file")

    # Check if second view is empty
    # This happens when there's only one view, but the scanner sets nviews = 2
    if nviews == 2
        # Seek to view slice 2, echo 1, view 2, coil 1:
        seek(fid, rdb_hdr.off_data + (2 * ptsize * (sliceres + 2 * ndat)))
        view2tmp = Array{Int16}(undef, 2 * ndat)
        view2tmp = read!(fid, view2tmp)
        if all(view2tmp .== Int16(0))
            nviews = 1 # set nviews to 1 so we don't read in all the empty data
            !quiet && @warn("View 2 appears to be empty, only loading view 1.")
        end
    end

    # Determine which portions to load
    # NB: default is to skip first slice (sometimes contains corrupted data)
    isempty(coils) && (coils = 1:ncoils)
    isempty(echoes) && (echoes = 1:nechoes)
    isempty(slices) && (slices = 2:nslices)
    isempty(views) && (views = 1:nviews)

    # output dimensions:
    dims = (ndat, length(coils), length(slices), length(echoes), length(views))

    # Warn if output size will exceed system memory.
    # Exceeding free RAM may cause a system freeze due to attempted mem swap
    memneeded = 4 * prod(dims) # 4 bytes per complex value because Int16
    memfree = Sys.total_memory() # system total memory
    mempercentuse = 100 * memneeded / memfree
    (mempercentuse > 100) &&
        @warn("Loading data ($(memneeded/1e9) GB) may exceed available RAM and freeze system")
    # fprintf('Press enter to continue anyway...');
    # input('');
    (mempercentuse > 90) && # Warn if we will we use 90% of memory
        @warn("Loading data ($(memneeded/1e9) GB) will use $mempercentuse % of your available RAM. Proceed with caution!")

    !quiet && (@info("ndat = $ndat, memory = $(memneeded/1e9) GB"))
    !quiet && (@info("dims = $dims"))
    !quiet && (@info("slices $(slices[1]):$(slices[end]) / $nslices"))
    !quiet && (@info("echoes $(echoes[1]):$(echoes[end]) / $nechoes"))
    !quiet && (@info("views $(views[1]):$(views[end]) / $nviews"))
    !quiet && (@info("coils $(coils[1]):$(coils[end]) / $ncoils"))

    # Read data from file
    # Julia stores complex data as real1,imag1, real2,imag2, ... unlike matlab!
    # This simplifies the IO here!
    data = Array{Complex{Int16}}(undef, dims)
    dtmp = Array{Complex{Int16}}(undef, ndat) # one readout
    # !quiet && textprogressbar('Loading data: ')
    for icoil in coils
        !quiet && (@show icoil)
        # !quiet && textprogressbar(icoil/ncoils*100)
        for islice in slices
            for iecho in echoes
                for iview in views
                    # !quiet && (@info "icoil=$icoil islice=$islice iecho=$iecho iview=$iview")
                    offsetres =
                        (icoil - 1) * coilres +
                        (islice - 1) * sliceres +
                        (iecho - 1) * echores +
                        iview * ndat
                    offsetbytes = 2 * ptsize * offsetres
                    seek(fid, rdb_hdr.off_data + offsetbytes)
                    read!(fid, dtmp)
                    data[
                        :,
                        icoil-coils[1]+1,
                        islice-slices[1]+1,
                        iecho-echoes[1]+1,
                        iview-views[1]+1,
                    ] .= dtmp
                end
            end
        end
    end

    # !quiet && textprogressbar(' done.')

    return (data, rdb_hdr)
end

"""
    (dat, rdb_hdr) = loadpfile(file::String ; ...)

Load from `file`.
"""
function loadpfile(file::String; kwargs...)
    open(file, "r") do fid
        return loadpfile(fid; filesize = filesize(file), kwargs...)
    end
end

"""
    (dat, rdb_hdr) = loadpfile(file, echo::Integer ; ...)

Load a single echo from `file`.
"""
function loadpfile(pfile::String, echo::Integer; kwarg...)
    return loadpfile(pfile, echoes = [echo]; kwarg...)
end


#====================================================

    generate_SFC_models.jl

    generates definitions for models
    with same formula for each cycle (SFC models)

====================================================#

import DataStructures.OrderedDict
using JuMP

## function definitions

"Generate prediction function given SFC model definition."
function add_funcs_pred!(
    md          ::SFCModelDef,
    verbose     ::Bool = false
)
    _x_args_str = join(map(_x_str -> "$_x_str::Real", md._x_strs), ", ")
    coefs_str = join(map(str -> "$str::Real", md.coef_strs), ", ")
    #
    for func_key in MD_func_keys
        func_name = "$(md.name)_$func_key"
        func_str = join([
            "function $func_name($_x_args_str, $coefs_str)", ## In v0.4.5, `;` can't be parsed correctly for anonymous function lambda, because the signature will be parsed as a tuple not allowing `;`; default values are not allowed either. Heard it'll be fixed in v0.5
                md.pred_strs[func_key],
            "end",
            "function $func_name(X_cases::AbstractVector, $coefs_str)", ## X_cases is a vector (indexed by cases) of vectors (indexed by features)
                "map(X_case -> $func_name(X_case..., $coefs_str), X_cases)",
            "end"
        ], "; ")
        md.func_pred_strs[func_key] = func_str
        func_expr = Base.parse(func_str) ## not JSON.parse
        md.funcs_pred[func_key] = @eval $func_expr
    end
    #
    return nothing ## side effects only
end

"Generate fit function given SFC model definition."
function add_func_fit!( ## vco = variable constraints objective
    md          ::SFCModelDef;
    Y_str       ::String = "Y",
    obj_algrt   ::Symbol = :RSS,
    sense       ::Symbol = :Min, ## :Min, :Max
)
    ## X_strs
    X_strs = md.X_strs
    num_fts = length(X_strs)
    X_args_str = join(map(X_str -> "$X_str::AbstractVector", X_strs), ", ")
    #
    ## function signature
    sig_str = "function $(md.name)_func_fit(
        $X_args_str,
        $Y_str::AbstractVector,
        wts::AbstractVector=ones(length($Y_str));
        kwargs_Model...
    )"
    #
    ## initiate model
    mod_init_str = "jmp_model = Model(;kwargs_Model...)"
    #
    ## define coefficients as variables
    coef_init_str = "$X_args_str, $Y_str = map(abs_vec -> Array(abs_vec), ($X_args_str, $Y_str)); init_coefs = $(md.name)_func_init_coefs($X_args_str, $Y_str)" ## `Array` because `linreg` doesn't work on `DataArray`
    var_str = join(
        map(md.coef_strs) do coef_str
            "@variable(
                jmp_model,
                $coef_str,
                start=init_coefs[\"$coef_str\"])"
        end,
        "; ")
    #
    ## add constraints for the coefficents
    cnstrnts_str = join(
        map(md.coef_cnstrnts) do coef_constrnt
            "@constraint(jmp_model, $coef_constrnt)"
        end,
        "; ")
    #
    ## set objective
    obj_macro_str = md.linear ? "@objective" : "@NLobjective" ## `a1 = :(@some_macro); :($a1(arg1, arg2))` is equivalent to :((@some_macro()(arg1,arg2))), both of which raises "syntax: invalid macro use \"@($a1)\"". This why obj_macro need to be string, and other expressions are started as strings too for convenience.
    #
    func_str_replaced = md.pred_strs[:f]
    for j in 1:num_fts
        func_str_replaced = replace(
            func_str_replaced,
            md._x_strs[j],
            "$(X_strs[j])[i]")
    end
    #
    residual_str = "($func_str_replaced - $Y_str[i])"
    iter_str = "for i in eachindex($(X_strs[1]))" ## assuming X and Y have the same length
    #
    if obj_algrt == :RSS
        obj_expr_str = "sum(wts[i] * $residual_str^2 $iter_str)"
        # sumabs2(map(eval(x_symbol) -> eval(func_expr), X) .- Y)
    elseif obj_algrt == :l1_norm
        obj_expr_str = "sum(abs($residual_str) $iter_str)" ## `abs()` may cause "Ipopt finished with status Restoration_Faild"
    end
    #
    obj_str = "$obj_macro_str(jmp_model, $sense, $obj_expr_str)"
    #
    ## return
    return_str = join([
        "status = solve(jmp_model)",
        "coef_strs = [\"$(join(md.coef_strs, "\", \""))\"]",
        "coefs = map(getvalue, [$(join(md.coef_strs, ", "))])",
        "obj_val = getobjectivevalue(jmp_model)",
        "return SFCFit(coef_strs, coefs, status, obj_val, jmp_model, init_coefs); end"
    ], "; ")
    #
    ## add definition of func_fit
    func_str = join(
        [sig_str, mod_init_str, coef_init_str, var_str, cnstrnts_str, obj_str, return_str],
        "; ")
    md.func_fit_str = func_str
    #
    func_expr = Base.parse(func_str) ## not JSON.parse
    md.func_fit = @eval $func_expr
    #
    return nothing ## side effects only
end

## constants (MD_ = model definition)

## when `num_fts > 1`, "d*" are partial derivatives in vector of length `num_fts`
const MD_func_keys = [:f, :inv, :bl, :dr1, :dr2]

# function empty_func() end
# function empty_func(arg1::Any=0, args...; kwargs...) end
function empty_func(args...; kwargs...) end

## `EMPTY_fp` for `func_pred_strs` and `funcs_pred`
const EMPTY_fp = map(("", empty_func)) do empty_val
    # OrderedDict(map(MD_func_keys) do func_key ## v0.4, `supertype` not defined, `typeof(some_function) == Function`
    OrderedDict{Symbol,supertype(typeof(empty_val))}(map(MD_func_keys) do func_key ## v0.5, `super` becomes `supertype`, `typeof(some_function) == some_function AND supertype(typeof(some_function)) == Function`
        func_key => empty_val
    end) ## do func_key
end ## do empty_val

## generate generic model definition objects
const MD_EMPTY_vals = (
    EMPTY_fp...,    ## :func_pred_strs, :funcs_pred
    "",             ## func_fit_str
    empty_func)     ## func_fit
const SFC_MODEL_DEFS = OrderedDict(map(SFC_MODEL_BASES) do SFC_model_base
    SFC_model_base[1] => SFCModelDef(
        SFC_model_base...,
        deepcopy(MD_EMPTY_vals)...)
end)
for md_ in values(SFC_MODEL_DEFS)
    add_funcs_pred!(md_)
    add_func_fit!(md_)
end


module Graphene

using StatsBase: counts
using CSV: read
using Images: load, Gray, erode, label_components, component_centroids
using HDF5: h5read
using DataFrames
using NearestNeighbors: KDTree, knn, inrange
using GeometricalPredicates: Point2D, Line, orientation, getx, gety, length2
using JuliaDB: IndexedTable
using AutoHashEquals
using Statistics: mean
using SparseArrays: sparse, findnz
using Base.Iterators: Stateful, take, cycle
using Plots: plot, plot!, scatter, scatter!, @recipe, @colorant_str
using Plots
using LazySets: convex_hull, VPolygon
using IterTools: subsets

import Base.isless

export AbstractGEntry, AbstractGPrimitive, AbstractGDefect
export GAtom, GBond, GPolygon, GDefect
export get_id, get_x, get_y, get_relatives, get_signature, get_frame, get_dataset, get_noa, get_type, get_members
export isatom, isbond, ispolygon, istype

abstract type AbstractGEntry end
get_id(g::AbstractGEntry) = g._id
get_x(g::AbstractGEntry) = g._x
get_y(g::AbstractGEntry) = g._y
get_relatives(g::AbstractGEntry) = g._relatives
get_signature(g::AbstractGEntry) = g._signature
get_frame(g::AbstractGEntry) = g._frame
get_dataset(g::AbstractGEntry) = g._dataset

isatom(g::AbstractGEntry) = isequal(get_type(g), "Atom")
isbond(g::AbstractGEntry) = isequal(get_type(g), "Bond")
ispolygon(g::AbstractGEntry) = isequal(get_type(g), "Polygon")
istype(g::AbstractGEntry, types...) = |([isequal(get_type(g), t) for t in types]...)
isless(a::AbstractGEntry, b::AbstractGEntry) = isless(get_noa(a), get_noa(b))

abstract type AbstractGPrimitive <: AbstractGEntry end
get_members(g::AbstractGPrimitive) = Set(g._id)

abstract type AbstractGDefect <: AbstractGEntry end

@auto_hash_equals mutable struct GAtom <: AbstractGPrimitive
    _id       ::Int
    _x        ::Float64
    _y        ::Float64
    _relatives::Set{Int}
    _signature::String
    _frame    ::Int
    _dataset  ::String
end
GAtom(id::Int, x::Float64, y::Float64, frame::Int, dataset::String) = GAtom(id, x, y, Set([]), "", frame, dataset)
GAtom(id::Int, x::Float64, y::Float64, frame::Int) = GAtom(id, x, y, Set([]), "", frame, "dataset")
GAtom(id::Int, x::Float64, y::Float64, relatives::Set) = GAtom(id, x, y, relatives, "", 0, "dataset")
GAtom(id::Int, x::Float64, y::Float64) = GAtom(id, x, y, Set([]), "", 0, "dataset")
GAtom(id::Int, p::Point2D) = GAtom(id, getx(p), gety(p), Set([]), "", 0, "dataset")
get_noa(g::GAtom) = 1
get_type(g::GAtom) = "Atom"

@auto_hash_equals mutable struct GBond <: AbstractGPrimitive
    _id       ::Int
    _x        ::Float64
    _y        ::Float64
    _relatives::Set{Int}
    _signature::String
    _frame    ::Int
    _dataset  ::String
end
GBond(id::Int, x::Float64, y::Float64, frame::Int, dataset::String) = GBond(id, x, y, Set([]), "", frame, dataset)
GBond(id::Int, x::Float64, y::Float64, frame::Int) = GBond(id, x, y, Set([]), "", frame, "dataset")
GBond(id::Int, x::Float64, y::Float64, relatives::Set) = GBond(id, x, y, relatives, "", 0, "dataset")
GBond(id::Int, x::Float64, y::Float64) = GBond(id, x, y, Set([]), "", 0, "dataset")
get_noa(g::GBond) = 2
get_type(g::GBond) = "Bond"

@auto_hash_equals mutable struct GPolygon <: AbstractGPrimitive
    _id       ::Int
    _x        ::Float64
    _y        ::Float64
    _relatives::Set{Int}
    _signature::String
    _frame    ::Int
    _dataset  ::String

    _noa      ::Int
end
GPolygon(id::Int, x::Float64, y::Float64, frame::Int, dataset::String, noa::Int) = GPolygon(id, x, y, Set([]), "", frame, dataset, noa)
GPolygon(id::Int, x::Float64, y::Float64, frame::Int, noa::Int) = GPolygon(id, x, y, Set([]), "", frame, "dataset", noa)
GPolygon(id::Int, x::Float64, y::Float64, noa::Int) = GPolygon(id, x, y, Set([]), "", 0, "dataset", noa)
get_noa(g::GPolygon) = g._noa
get_type(g::GPolygon) = "Polygon"

@auto_hash_equals mutable struct GEntry<: AbstractGEntry
    _id       ::Int
    _x        ::Float64
    _y        ::Float64
    _relatives::Set{Int}
    _signature::String
    _frame    ::Int
    _dataset  ::String

    _noa      ::Int
    _members  ::Set{Int}
end
get_noa(g::GEntry) = g._noa
get_members(g::GEntry) = g._members
get_type(g::GEntry) = "Undetermined"

@auto_hash_equals mutable struct GDefect<: AbstractGDefect
    _id       ::Int
    _x        ::Float64
    _y        ::Float64
    _relatives::Set{Int}
    _signature::String
    _frame    ::Int
    _dataset  ::String

    _noa      ::Int
    _members  ::Set{Int}
    _type     ::String
end
get_noa(g::GDefect) = g._noa
get_members(g::GDefect) = g._members
get_type(g::GDefect) = g._type

include("fileio.jl")
include("atoms.jl")
include("bonds.jl")
include("polygons.jl")
include("defects.jl")
include("plotting.jl")

end # module


#
println("Ci) Test of the DecayYield module with ASF from an internally generated multiplet.")
#
setDefaults("print summary: open", "zzz-DecayYield.sum")
wa = Atomic.Computation(Atomic.Computation(), name="xx", grid=JAC.Radial.Grid(false), nuclearModel=Nuclear.Model(12.),
                        properties=[Yields()],
                        configs=[Configuration("1s 2s^2 2p^6")],
                        yieldSettings=DecayYield.Settings("SCA", true, false, Int64[]) )

wb = perform(wa)
setDefaults("print summary: close", "")


function evolve_neutral!( pop::Population, generations::Int64 = 1 )
    # function to evolve the population under neutral selection
    # that is, role models are selected uniformly at random; fitnesses have no impact
    # generations specifies the total number of generations for which to run the simulation
    for i::Int64 in 1:generations
        if pop.verbose println("\ninitiating generation $(pop.generation)") end

        # first, update payoffs
        if pop.verbose println("updating actions only") end
        update_actions_neutral!(pop)  # method that updates actions only

        # then, select a learner and a role model and perform update
        if pop.verbose println("") end
        if pop.verbose println("evolving, generation $(pop.generation)") end
        update_strategies_and_opinions_neutral!(pop)

        if pop.verbose println("") end
        if pop.verbose println("ending generation $(pop.generation)") end
        pop.generation += 1
    end
end

function update_actions_neutral!( pop::Population )
    # update individual actions
    # for neutral selection (β = 0), so payoffs are not needed

    if pop.sets_changed
        # if (previous) learner's sets have changed, re-compute all actions
        if pop.verbose println("updating all actions because set_changed = $(pop.sets_changed)") end
        pop.prev_actions .= zero(Int64)   # reset all prev_actions

        # then, recompute all payoffs
        for k in 1:pop.sets.M  # for each set
            for (i, j) in pop.sets.set_pairs[k]   # for each pair within each set
                compute_actions_neutral!( pop, i, j, k )  # update action between i & j when intereacting in set k
            end
        end
    elseif pop.strategies_changed
        # if (previous) learner's strategy has changed, recompute actions involving that individual
        if pop.verbose println("updating actions involving $(pop.prev_learner) b/c strategies_changed = $(pop.strategies_changed)") end
        pop.prev_actions[:,pop.prev_learner,:] .= zero(Int64)
        pop.prev_actions[:,:,pop.prev_learner] .= zero(Int64)

        # then, recompute payoffs involving prev_learner
        for k in 1:pop.sets.M  # for each set
            for (i, j) in pop.sets.set_pairs[k]  # for each pair within each set
                if i == pop.prev_learner || j == pop.prev_learner
                    compute_actions_neutral!( pop, i, j, k )  # update action between i & j when intereacting in set k
                end
            end
        end
    else
        if pop.verbose println("not updating actions b/c sets_changed = $(pop.sets_changed) and strategies_changed = $(pop.strategies_changed)") end
        return # no need to update prev_interactions or payoffs
    end
    # sum rows to get total individual payoffs
    pop.prev_interactions = 2 * sum([length(x) for x in pop.sets.set_pairs])

    if pop.verbose println( "\npayoff matrix is now $(pop.payoffs_mat)" ) end
    if pop.verbose println( "payoffs are now $(pop.payoffs)" ) end
    if pop.verbose println( "there were $(pop.prev_interactions) interactions, $(sum(pop.prev_actions)) of which were C" ) end
end

function compute_actions_neutral!( pop::Population, i::Int64, j::Int64, k::Int64 )
    # function to compute payoffs for individuals i and j when they interact in set k
    pop.sim.i_action, pop.sim.j_action = -1, -1  # set to -1 to catch any errors in action assignment

    # actions depend on whether i and j agree on issue k or not
    if pop.sets.h[i,k] == 0 || pop.sets.h[j,k] == 0
        # if either i or j does not belong to set k, something's wrong
        # leave pop.sim.i_action, pop.sim.j_action as -1 so that the compute_payoffs! throws an error when assigning payoffs
        @warn("$i and $j should not be interacting in set $k"); return
    elseif pop.sets.h[i,k] == pop.sets.h[j,k]
        pop.sim.i_action, pop.sim.j_action = pop.strategies[i,1], pop.strategies[j,1] # if they agree, play s_agree
    else
        pop.sim.i_action, pop.sim.j_action = pop.strategies[i,2], pop.strategies[j,2] # if they disagree, play s_disagree
    end

    if pop.verbose
        println("\n\tupdating actions of $i and $j in set $k; agree on issue $k? $(pop.sets.h[i,k] == pop.sets.h[j,k])")
        println("\t\t$i has strats $(pop.strategies[i,:]); $i plays strat $(pop.sim.i_action)")
        println("\t\t$j has strats $(pop.strategies[j,:]); $j plays strat $(pop.sim.j_action)")
    end
    # track actions; no need to track payoffs under neutral selection
    pop.prev_actions[k,i,j] = pop.sim.i_action
    pop.prev_actions[k,j,i] = pop.sim.j_action
end

function update_strategies_and_opinions_neutral!( pop::Population )
    # function to perform death-birth update
    if pop.verbose println("Strategies before updating:\t $(pop.strategies)") end
    if pop.verbose println("Set memberships before updating: $(pop.sets.h)") end

    # choose a learner uniformly at random
    pop.sim.learner = sample( 1:pop.sets.N )
    # choose a role model also uniformly at random (for neutral selection)
    pop.sim.role_model = sample( 1:pop.sets.N )

    # learner updates
    if pop.verbose println("\n\trandomly chosen learner is $(pop.sim.learner), role_model is $(pop.sim.role_model)") end
    if pop.verbose println("\t\tall payoffs are $(pop.payoffs)") end
    if pop.verbose println("\t\tfitnesses are $(pop.sim.fitnesses)") end

    # compute probability of imitation: 1 if same party, 1-p if different parties
    pop.sets.affils[pop.sim.learner] == pop.sets.affils[pop.sim.role_model] ? imit_prob = 1.0 : imit_prob = 1-pop.game.ps[pop.sets.affils[pop.sim.learner]]

    if pop.verbose
        println("\t\tlearner and role model in the same party? $(pop.sets.affils[ pop.sim.learner ] == pop.sets.affils[ pop.sim.role_model ]), imitate with prob $(imit_prob)")
    end

    # update strategies and sets: imitation / mutation occurs with probability imit_prob
    if rand() < imit_prob
        if pop.verbose println("\twill consider imitating") end
        update_strategies!( pop ) # includes mutation
        update_opinions!( pop )   # includes mutation
        pop.num_imitating += 1
    else
        if pop.verbose println("\tnah won't imitate") end
        no_strategy_or_opinion_update!( pop )
        pop.num_not_imitating += 1
    end

    # track updates
    pop.prev_learner = pop.sim.learner
    if pop.sets_changed == true
        # h is updated above; need to update h_bar, h_base10, h_bar_base10, set_members, set_pairs
        pop.sets.set_members, pop.sets.set_pairs = set_members_and_pairs(pop.sets.h)  # update set_members and set_pairs
        pop.sets.h_bar        = [abs(x) for x in pop.sets.h]
        pop.sets.h_base10     = [vectobase3(pop.sets.h[i,:]) for i in 1:pop.sets.N]
        pop.sets.h_bar_base10 = [vectobase3(pop.sets.h_bar[i,:]) for i in 1:pop.sets.N]
        pop.num_sets_changed += 1
    end
    if pop.strategies_changed == true
        # strategies are updated above; need to update strategies_base10
        pop.strategies_base10 = [2 * pop.strategies[i,1] + pop.strategies[i,2] for i in 1:pop.sets.N]
        pop.num_strategies_changed += 1
    end
    if pop.strategies_changed && pop.sets_changed
        pop.num_both_changed += 1
    end

    if pop.verbose println("\nStrategies after updating:\t $(pop.strategies)") end
    if pop.verbose println("Set memberships after updating:\t $(pop.sets.h)") end
    if pop.verbose println("tracking: prev_learner = $(pop.prev_learner), strategies_changed = $(pop.strategies_changed), sets_changed = $(pop.sets_changed)") end
end


#
println("Dd) Test of the PhotoRecombination module with ASF from an internally generated initial- and final-state multiplet.")
#
setDefaults("print summary: open", "zzz-PhotoRecombination.sum")
setDefaults("method: continuum, Galerkin")           ## setDefaults("method: continuum, Galerkin")  "method: continuum, asymptotic Coulomb"
setDefaults("method: normalization, pure sine")      ## setDefaults("method: normalization, pure Coulomb")    setDefaults("method: normalization, pure sine")

if  false
    grid = Radial.Grid(Radial.Grid(true), rnt = 2.0e-5,h = 5.0e-2, hp = 1.0e-2, NoPoints = 900)

    wa = Atomic.Computation(Atomic.Computation(), name="xx", grid=grid, nuclearModel=Nuclear.Model(12.),
                            initialConfigs=[Configuration("1s^2 2s^2 2p^5"), Configuration("1s^2 2s 2p^6") ],
                            finalConfigs  =[Configuration("1s^2 2s^2 2p^6")],
                            process = Rec(),
                            processSettings=PhotoRecombination.Settings([E1, M1], [JAC.UseCoulomb, JAC.UseBabushkin], [10., 30., 50.], [0.],
                                                                        false, true, true, true, true, Tuple{Int64,Int64}[(1,1)]) )

    wb = perform(wa)

elseif  false
    # REC calculations for the capture into 1s, 2s, 2p, 3s, 3p, 3d and Z=5: Comparison with Eichler (ADNDT, 2000)
    setDefaults("unit: energy", "eV") # then, give photonEnergies in eV
    setDefaults("unit: cross section", "barn")

    recSettings = PhotoRecombination.Settings([E1, M1], [JAC.UseCoulomb, JAC.UseBabushkin], [1., 10., 100., 1000., 10000.],
                                              Float64[], false, false, false, true, LineSelection())

    grid = Radial.Grid(Radial.Grid(false), rnt = 4.0e-6, h = 5.0e-2, hp = 1.0e-2, rbox = 10.0)
    wa = Atomic.Computation(Atomic.Computation(), name="xx", grid=grid, nuclearModel=Nuclear.Model(9.),
                            ## initialConfigs = [Configuration("1s^2")],
                            ## finalConfigs   = [Configuration("1s^2 2s"), Configuration("1s^2 2p"),
                            ##                   Configuration("1s^2 3s"), Configuration("1s^2 3p"), Configuration("1s^2 3d")],
                            initialConfigs = [Configuration("1s")],
                            finalConfigs   = [Configuration("1s^2")],
                            process = Rec(), processSettings=recSettings )

    wb = perform(wa)

elseif  true
    # REC calculations for the capture into 2s at given ion energies and Z=92: Comparison with Fritzsche (PRA, 2005)
    setDefaults("unit: energy", "eV") # then, give photonEnergies in eV
    setDefaults("unit: cross section", "barn")

    recSettings = PhotoRecombination.Settings([E1, M1, E2, M2, E3, M3], [JAC.UseBabushkin], Float64[],
                                              [2.18, 21.8, 218.0], true, false, false, true, LineSelection())

    grid = Radial.Grid(Radial.Grid(false), rnt = 4.0e-6, h = 5.0e-2, hp = 0.3e-2, rbox = 10.0)
    wa = Atomic.Computation(Atomic.Computation(), name="xx", grid=grid, nuclearModel=Nuclear.Model(92.),
                            ## initialConfigs = [Configuration("1s^2")],
                            ## finalConfigs   = [Configuration("1s^2 2s"), Configuration("1s^2 2p"),
                            ##                   Configuration("1s^2 3s"), Configuration("1s^2 3p"), Configuration("1s^2 3d")],
                            initialConfigs = [Configuration("1s^2")],
                            finalConfigs   = [Configuration("1s^2 2s")],
                            process = Rec(), processSettings=recSettings )

    wb = perform(wa)

end
setDefaults("print summary: close", "")


# TODO: Make tests pass for
# - T ∈ [Float32, Float64]
# - Setup ∈ [PlateSetup, CylinderSetup, SphereSetup, NeuronSetup]
# - Gradient ∈ [GeneralGradient, ScalarGradient] (all time profiles)

# Floating point type for simulations
T = Float64

@testset "Setup recipes" begin
    @testset "PlateSetup" begin
        setup = get_setup(PlateSetup{T})
        coeffs = get_coeffs(setup)
        femesh, = create_geometry(setup; recreate = true)
        model = Model(; mesh = femesh, coeffs...)
    end

    @testset "CylinderSetup" begin
        setup = get_setup(CylinderSetup{T})
        coeffs = get_coeffs(setup)
        femesh, = create_geometry(setup; recreate = true)
        model = Model(; mesh = femesh, coeffs...)
    end

    @testset "SphereSetup" begin
        setup = get_setup(SphereSetup{T})
        coeffs = get_coeffs(setup)
        @test_broken femesh, = create_geometry(setup; recreate = true)
        @test_broken model = Model(; mesh = femesh, coeffs...)
    end

    @testset "NeuronSetup" begin
        setup = get_setup(NeuronSetup{T})
        coeffs = get_coeffs(setup)
        femesh, = create_geometry(setup; recreate = true)
        model = Model(; mesh = femesh, coeffs...)
    end
end

# Geometrical setup
setup = get_setup(CylinderSetup{T})
coeffs = get_coeffs(setup)

# Get compartimentalized coefficient vectors
femesh, = create_geometry(setup; recreate = true)
model = Model(; mesh = femesh, coeffs...)
volumes = get_cmpt_volumes(model.mesh)
D_avg = 1 / 3 * tr.(model.D)' * volumes / sum(volumes)
ncompartment = length(model.mesh.points)
matrices = assemble_matrices(model);

## Gradients
dir = [1.0, 1.0, 1.0]
profile = PGSE(2000.0, 6000.0)
b = 1000
g = √(b / int_F²(profile)) / coeffs.γ
pgse_gradient = ScalarGradient(dir, profile, g)

dir = [1.0, 1.0, 1.0]
profile = CosOGSE(5000.0, 5000.0, 2)
b = 1000
g = √(b / int_F²(profile)) / coeffs.γ
ogse_gradient = ScalarGradient(dir, profile, g)

TE = 5000
φ = -π / 6
R = [cos(φ) sin(φ) 0; -sin(φ) cos(φ) 0; 0 0 1]
g⃗(t) = 1.0 * R * [sin(2π * t / TE), sin(20π * t / TE) / 5, cos(2π * t / TE)]
general_gradient = GeneralGradient{T,typeof(g⃗)}(; g⃗, TE)

@testset "Signal" begin
    ρ = initial_conditions(model)
    @test compute_signal(matrices.M, ρ) isa Complex{T}
    @test compute_signal.(matrices.M_cmpts, split_field(model.mesh, ρ)) isa
          Vector{Complex{T}}
end

@testset "ADC STA" begin
    @test compute_adc_sta(model, pgse_gradient) isa Vector{T}
    @test compute_adc_sta(model, ogse_gradient) isa Vector{T}
    @test_throws MethodError compute_adc_sta(model, general_gradient)
end

@testset "HADC" begin
    hadc = HADC(;
        model,
        matrices,
        odesolver = QNDF(autodiff = false),
        reltol = 1e-4,
        abstol = 1e-6,
    )
    @test_throws MethodError solve(hadc, general_gradient)
    @test solve(hadc, pgse_gradient) isa Vector{T}
    @test solve(hadc, ogse_gradient) isa Vector{T}
end

@testset "BTPDE" begin
    btpde = GeneralBTPDE(;
        model,
        matrices,
        reltol = 1e-4,
        abstol = 1e-6,
        odesolver = QNDF(autodiff = false),
    )
    @test solve(btpde, general_gradient) isa Vector{Complex{T}}
    @test solve(btpde, ogse_gradient) isa Vector{Complex{T}}

    btpde = IntervalConstanBTPDE{T}(; model, matrices, θ = 0.5, timestep = 5)
    @test_throws MethodError solve(btpde, general_gradient) isa Vector{Complex{T}}
    @test_throws ErrorException solve(btpde, ogse_gradient) isa Vector{Complex{T}}
    @test solve(btpde, pgse_gradient) isa Vector{Complex{T}}
end

@testset "Karger" begin
    # Compute HADC and fit difftensors
    directions = unitsphere(10)
    gradients = [
        ScalarGradient(collect(d), pgse_gradient.profile, pgse_gradient.amplitude) for
        d ∈ eachcol(directions)
    ]
    hadc = HADC(;
        model,
        matrices,
        odesolver = QNDF(autodiff = false),
        reltol = 1e-4,
        abstol = 1e-6,
    )
    adcs, = solve_multigrad(hadc, gradients)
    difftensors = fit_tensors(directions, adcs)

    # Solve Karger
    karger = Karger(; model, difftensors, odesolver = MagnusGL6(), timestep = 5.0)
    signal = solve(karger, pgse_gradient)
end

@testset "Matrix formalism" begin
    # Perform Laplace eigendecomposition
    laplace = Laplace{T}(; model, matrices, neig_max = 400)
    lap_eig = solve(laplace)
    length_scales = eig2length.(lap_eig.values, D_avg)

    # Truncate basis at minimum length scale
    length_scale = 3
    λ_max = length2eig(length_scale, D_avg)
    lap_eig = limit_lengthscale(lap_eig, λ_max)
    @test all(λ -> eig2length(λ, D_avg) ≥ length_scale, lap_eig.values)

    mf = MatrixFormalism(; model, matrices, lap_eig, ninterval = 500)
    @test solve(mf, general_gradient) isa Vector{Complex{T}}
    @test solve(mf, pgse_gradient) isa Vector{Complex{T}}
    @test solve(mf, ogse_gradient) isa Vector{Complex{T}}
end

@testset "Analytical" begin
    length_scale = 1.0
    eigstep = 1e-8
    eiglim = length2eig(length_scale, D_avg)
    analytical_coeffs = analytical_coefficients(setup, coeffs)
    analytical_laplace = AnalyticalLaplace(; analytical_coeffs..., eiglim, eigstep)
    lap_mat = solve(analytical_laplace)
    analytical_mf = AnalyticalMatrixFormalism(; analytical_laplace, lap_mat, volumes)
    @test_throws MethodError solve(analytical_mf, general_gradient)
    @test_throws ErrorException solve(analytical_mf, ogse_gradient)
    @test solve(analytical_mf, pgse_gradient) isa Complex{T}
end