# Clayton, N. S., Yu, K. S. and Dickinson, A. (2003), Interacting Cache
# Memories: Evidence for Flexible Memory Use by Western Scrub-Jays (Aphelocoma
# californica) Journal of Experimental Psychology: Animal Behavior Processes,
# 1:14-22

function _summarize(exp::Experiment{:Clayton03_exp1}, results)
    res = similar(exp.data, 0)
    for g in groupby(@where(results, :action .== "cache", :set .== 7),
                     [:foodtype, :set])
        push!(res, [missing, g.foodtype[1], "cache", missing, missing, g.set[1],
                    mean(g.counts), sem(g.counts), missing, length(g.counts)])
    end
    res = [res; combine(groupby(@where(results, :action .== "inspect", :foodtype .== "other", :set .== 7),
                                [:group, :foodtype, :action, :RI, :trial, :set]),
             df -> DataFrame(μ = missing, sem = missing,
                             firstinspection = mean(df.firstinspection),
                             n = length(df.counts)))]
    res = [res; combine(groupby(cricketproportions(results), [:group, :RI]),
                    d -> DataFrame(foodtype = "cricketproportion", action = "inspect",
                                   set = 7, μ = mean(d.counts), sem = sem(d.counts),
                                   n = length(d.counts), trial = "pc",
                                   firstinspection = missing))]
    res
end
function statistical_tests(exp::Experiment{:Clayton03_exp1}, data)
    tests = Test[]
    datasetinspectClayton03_exp1 = @where(data, (|).(:set .== 4, :set .== 7),
                                         :action .== "inspect", :trial .== "pc",
                                         :RI .> 4)
    for g in groupby(datasetinspectClayton03_exp1, :RI)
        push!(tests, Test("first inspectes RI $(g.RI[1]) Clayton03 exp1",
                          fisherexacttest(g, cond1 = :group, cond2 = :foodtype,
                                          data = :firstinspection),
                          exp.tests))
    end
    cricketprop = cricketproportions(data)
    for g in groupby(cricketprop, :RI)
        id = "inspect RI $(g.RI[1]) Clayton03 exp1"
        push!(tests, Test(id,
                          anova(g, @formula(counts ~ group), exp.tests[id].locals),
                          exp.tests))
    end
#   How did they analyse this in the paper: RI 76 was measured twice which
#   yields unbalanced within factors.
    id = "cache Clayton03 exp1"
    push!(tests, Test(id,
                      splitplotanova(@where(data, :action .== "cache", :set .== 7),
                                     exp.tests[id].locals,
                                     betweenfactors = :group,
                                     withinfactors = [:foodtype, :RI],
                                     id = :id, data = :counts),
                      exp.tests))
    id = "inspect Clayton03 exp1"
    push!(tests, Test(id,
                      splitplotanova(cricketprop,
                                     exp.tests[id].locals,
                                     betweenfactors = :group,
                                     withinfactors = :RI, id = :id,
                                     data = :counts),
                      exp.tests))
#   TODO: "inspect degrade Clayton03 exp1" "inspect replenish Clayton03 exp1"
    TestCollection(tests)
end
function run!(::Experiment{:Clayton03_exp1}, models)
    results = DataFrame(group = Union{String,Missing}[],
                        foodtype = Union{String,Missing}[],
                        id = Int[], set = Int[], trial = Union{String,Missing}[],
                        RI = Union{Int,Missing}[],
                        action = String[], counts = Int[],
                        firstinspection = Union{Missing,Int}[])
    id = 0
    for group in ["degrade", "replenish"]
        for i in 1:8
            id += 1
            m = models[id]
            # training before Clayton 2003, experiment 1, January 2000
            wait!(m, 30u"d")
            for RI in (52u"hr", 76u"hr")
                Clayton01_trial!(m, results, group, id, 7, Peanut, Cricket, RI,
                                 degrade = false, pilfer = true, bothtrayspresent = true)
                #interleaved retraining
                Clayton01_trial!(m, results, group, id, 0, Peanut, Cricket, 4u"hr",
                                 degrade = false, pilfer = false)
            end
            wait!(m, 30u"d")
            #interleaved retraining
            Clayton01_trial!(m, results, group, id, 0, Peanut, Cricket, 100u"hr",
                             degrade = isdegrade(group, Cricket, 100u"hr"), pilfer = false)
            for RI in (76u"hr", 124u"hr")
                Clayton01_trial!(m, results, group, id, 7, Peanut, Cricket, RI,
                                 degrade = false, pilfer = true, bothtrayspresent = true)
                #interleaved retraining
                Clayton01_trial!(m, results, group, id, 0, Peanut, Cricket, 4u"hr",
                                 degrade = false, pilfer = false)
            end
        end
    end
    results
end

function _summarize(::Experiment{:Clayton03_exp2}, results)
    res = combine(groupby(@where(results, :action .== "inspect"),
                          [:group, :foodtype, :action, :condition, :RI]),
             d -> DataFrame(action = "inspect", μ = mean(d.counts),
                            sem = sem(d.counts),
                            firstinspection = mean(d.firstinspection),
                            n = length(d.counts)))
    res = [res; combine(groupby(@where(results, :action .== "cache"),
                                [:group, :foodtype, :action]),
                   d -> DataFrame(action = "cache", μ = mean(d.counts),
                                  sem = sem(d.counts),
                                  firstinspection = missing,
                                  condition = missing, RI = missing,
                                  n = length(d.counts)))]
end
function statistical_tests(exp::Experiment{:Clayton03_exp2}, data)
    tests = Test[]
    for g in groupby(@where(data, :action .== "inspect"), :group)
        group = g.group[1]
        id = "inspect $group"
        push!(tests, Test(id,
                          splitplotanova(g,
                                         exp.tests[id].locals,
                                         betweenfactors = :condition,
                                         withinfactors = [:foodtype, :RI],
                                         id = :id, data = :counts),
                         exp.tests))
        for subg in groupby(g, :RI)
            RI = subg.RI[1]
            id = "inspect $group RI $RI"
            push!(tests, Test(id,
                              splitplotanova(subg,
                                             exp.tests[id].locals,
                                             betweenfactors = :condition,
                                             withinfactors = :foodtype,
                                             id = :id, data = :counts),
                              exp.tests))
            if group == "degrade" && RI == 76
                for subsubg in groupby(subg, :condition)
                    id = "inspect $group RI $RI $(subsubg.condition[1])"
                    push!(tests, Test(id,
                                      anova(subsubg, @formula(counts ~ foodtype + id), exp.tests[id].locals),
                                      exp.tests))
                end
                for subsubg in groupby(subg, :foodtype)
                    id = "inspect $group RI $RI $(subsubg.foodtype[1])"
                    push!(tests, Test(id,
                                      anova(subsubg, @formula(counts ~ condition), exp.tests[id].locals),
                                      exp.tests))
                end
            end
        end
        if group == "replenish"
            for subg in groupby(g, :condition)
                id = "inspect $group $(subg.condition[1])"
                push!(tests, Test(id,
                                  splitplotanova(subg,
                                                 exp.tests[id].locals,
                                                 betweenfactors = [],
                                                 withinfactors = [:foodtype, :RI],
                                                 id = :id, data = :counts),
                                  exp.tests))
            end
            for subg in groupby(g, :foodtype)
                id = "inspect $group $(subg.foodtype[1])"
                push!(tests, Test(id,
                                  splitplotanova(subg,
                                                 exp.tests[id].locals,
                                                 betweenfactors = :condition,
                                                 withinfactors = [:RI],
                                                 id = :id, data = :counts),
                                  exp.tests))
            end
        end
        gsorted = sort(g, [:id, :RI, :foodtype])
        gsortedconsistent = @where(gsorted, :condition .== "consistent")
        gsortedreversed = @where(gsorted, :condition .== "reversed")
        push!(tests, Test("first inspectes $group",
                          chisqtest(firstinspect_dist(gsortedconsistent,
                                                      n = group == "degrade" ? 4 : 6),
                                    firstinspect_dist(gsortedreversed,
                                                      n = group == "degrade" ? 4 : 6)),
                          exp.tests))
    end
    for g in groupby(@where(data, :action .== "cache"), :group)
        id = "cache $(g.group[1])"
        push!(tests, Test(id,
                          splitplotanova(g,
                                         exp.tests[id].locals,
                                         betweenfactors = :condition,
                                         withinfactors = [:foodtype, :RI],
                                         id = :id, data = :counts),
                          exp.tests))
    end
    TestCollection(tests)
end
hack(i) = i
@inline function Clayton03_trial!(results, m, group, id, condition, RI)
    traypairs = [(Tray(hack(1)), Tray(hack(2))) for d in 1:3]
    # caching
    for (i, (tray1, tray2)) in enumerate(traypairs)
        remove!(m, Any)
        wait!(m, 4u"hr")
        cover!(tray2)
        add!(m, tray1); add!(m, tray2)
        add!(m, Peanut, 50)
        wait!(m, 15u"minute")
        remove!(m, Any)
        cover!(tray1); uncover!(tray2)
        add!(m, tray1); add!(m, tray2)
        add!(m, Cricket, 50)
        wait!(m, 15u"minute")
        remove!(m, Any)
        uncover!(tray1)
        if i == 3
            push!(results, [group, "peanut", id, condition, value(RI), "cache",
                            countcache(tray1), missing])
            push!(results, [group, "cricket", id, condition, value(RI), "cache",
                            countcache(tray2), missing])
        end
        add!(m, MaintenanceDiet)
        wait!(m, 25u"minute" + 15u"hr")
    end
    # recovery
    wait!(m, RI - 3u"d" - 4u"hr")
    for (i, (tray1, tray2)) in enumerate(traypairs)
        if i == 3
            pilfer!(tray1); pilfer!(tray2)
        elseif condition == "reversed"
            degrade!(tray2)
        end
        remove!(m, Any)
        wait!(m, 4u"hr")
        add!(m, tray1)
        add!(m, tray2)
        inspectionobserver = InspectionObserver()
        add!(m, inspectionobserver)
        wait!(m, 5u"minute")
        remove!(m, Any)
        if i == 3
            push!(results, [group, "peanut", id, condition, value(RI), "inspect",
                            countinspections(inspectionobserver, tray1),
                            firstinspection(inspectionobserver, tray1)])
            push!(results, [group, "cricket", id, condition, value(RI), "inspect",
                            countinspections(inspectionobserver, tray2),
                            firstinspection(inspectionobserver, tray2)])
        end
        add!(m, MaintenanceDiet)
        wait!(m, 55u"minute" + 15u"hr")
    end
end
function run!(::Experiment{:Clayton03_exp2}, models)
    results = DataFrame(group = Union{String,Missing}[],
                        foodtype = Union{String,Missing}[],
                        id = Int[], condition = Union{String,Missing}[],
                        RI = Union{Int,Missing}[],
                        action = String[], counts = Int[],
                        firstinspection = Union{Missing,Int}[])
    id = 0
    for group in ["degrade", "replenish"]
        for i in 1:8
            id += 1
            m = models[id]
            # retraining Feb 2000
            for wait in (4u"hr", 100u"hr")
                Clayton01_trial!(m, [], group, id, 0, Peanut, Cricket, wait,
                       degrade = isdegrade(group, Cricket, wait), pilfer = false)
            end
            condition = i > 4 ? "reversed" : "consistent"
            Clayton03_trial!(results, m, group, id, condition, 76u"hr")
            group == "replenish" && Clayton03_trial!(results, m, group, id, condition, 124u"hr")
            tmpres = [] # since Clayton01_trial! expects another form of results table
            Clayton01_trial!(m, tmpres, group, id, 1, Peanut, Cricket, 28u"hr",
                             degrade = false, pilfer = true, specificothername = true)
            for row in tmpres
                push!(results, [row[1:3]; condition; row[end-3:end]])
            end
        end
    end
    results
end


using AstroTime
using Test
using ERFA
using SPICE: furnsh, et2utc, utc2et

furnsh(joinpath("data", "naif0012.tls"))
AstroTime.load_test_eop()

const speed_of_light = 299792458.0 # m/s
const astronomical_unit = 149597870700.0 # m

@timescale GMT UTC
AstroTime.Epochs.getoffset(::GMTScale, ::CoordinatedUniversalTime, _, _) = 0.0
AstroTime.Epochs.getoffset(::CoordinatedUniversalTime, ::GMTScale, _, _) = 0.0

@timescale SCET UTC
function AstroTime.Epochs.getoffset(::SCETScale, ::CoordinatedUniversalTime,
                                    _, _, distance)
    return distance / speed_of_light
end
function AstroTime.Epochs.getoffset(::CoordinatedUniversalTime, ::SCETScale,
                                    _, _, distance)
    return -distance / speed_of_light
end

@timescale Dummy TDB

@timescale Lonely
@timescale Together Lonely
function AstroTime.Epochs.getoffset(::LonelyScale, ::TogetherScale, _, _)
    return 5.0
end
function AstroTime.Epochs.getoffset(::TogetherScale, ::LonelyScale, _, _)
    return -5.0
end

@timescale OneWay TDB true

@testset "AstroTime" begin
    include("periods.jl")
    include("astrodates.jl")
    include("epochs.jl")
    @testset "Custom Time Scales" begin
        utc = UTCEpoch(2000, 1, 1, 0, 0, 0.1)
        gmt = GMTEpoch(2000, 1, 1, 0, 0, 0.1)
        @test utc == UTCEpoch(gmt)
        @test gmt == GMTEpoch(utc)

        @test string(GMT) == "GMT"
        @test typeof(GMT) == GMTScale
        @test string(gmt) == "2000-01-01T00:00:00.100 GMT"
        @test find_path(GMT, UTC) == [GMT, UTC]

        @test string(SCET) == "SCET"
        @test typeof(SCET) == SCETScale
        @test find_path(SCET, UTC) == [SCET, UTC]
        scet = SCETEpoch(2000, 1, 1, 0, 0, 0.1)
        utc_exp = UTCEpoch(2000, 1, 1, 0, 8, 19.10478383615643)
        @test UTCEpoch(scet, astronomical_unit) ≈ utc_exp
        @test SCETEpoch(utc_exp, astronomical_unit) ≈ scet

        @test string(Dummy) == "Dummy"
        @test typeof(Dummy) == DummyScale
        @test find_path(Dummy, UTC) == [Dummy, TDB, TT, TAI, UTC]
        dummy = DummyEpoch(2000, 1, 1)
        @test_throws NoOffsetError UTCEpoch(dummy)
        @test_throws NoOffsetError TDBEpoch(dummy)

        lonely = LonelyEpoch(2000, 1, 1)
        together = TogetherEpoch(lonely)
        @test together == TogetherEpoch(2000, 1, 1, 0, 0, 5.0)
        @test lonely == LonelyEpoch(together)
        @test find_path(Lonely, TDB) == []

        @test find_path(TDB, OneWay) == [TDB, OneWay]
        @test find_path(OneWay, TDB) == []
        @test_throws NoPathError TDBEpoch(OneWayEpoch(2000, 1, 1))
    end
end


function run_life_view!(results::Array{Float64,3},environment::Array{Float64,3},
    grn::NamedTuple,diff_adhesion::Array{Float64,2},diffusion_constants::Array{Float64,1},
    Ax::Array{Float64,2},Ay::Array{Float64,2},
    dc::Array{Float64,1},cell_config::CellularConfig,exp_config::ExperimentalConfig,grn_config::FGRNConfig)

    target_cell_areas = Dict{Float64,Float64}()
    n_div = Dict{Float64,Integer}()

    all_cells::Array{Float64,1} = [1.]
    n_div[1.] = 0
    target_cell_areas[1.] = 2*0.25*cell_config.cell_size_upper_limit

    if exp_config.est_strength == "evolve"
        affinity_1 = calculate_affinity(grn.rec_prom[:,:,1],grn_config.substrate_protein)
        affinity_2 = calculate_affinity(grn.rec_prom[:,:,2],grn_config.substrate_protein)
        affinity_3 = calculate_affinity(grn.rec_prom[:,:,3],grn_config.substrate_protein)
        affinity_4 = calculate_affinity(grn.rec_prom[:,:,4],grn_config.substrate_protein)
    else
        affinity_1 = 1 - exp_config.est_strength
        affinity_2 = 1 - exp_config.est_strength
        affinity_3 = 1 - exp_config.est_strength
        affinity_4 = 1 - exp_config.est_strength
    end

    left_segment = Int.(floor(cell_config.grid_size/3))
    middle_segment = Int.(floor(2*cell_config.grid_size/3))

    est_setup = ESTSetup(affinity_1,affinity_2,affinity_3,affinity_4,left_segment,middle_segment)

    for step in 1:size(results,3) # development

        results[:,:,step] = environment[:,:,2]

        environment_update2!(grn,environment,all_cells,diff_adhesion,target_cell_areas,cell_config,exp_config)

        if size(all_cells,1) == 0
            break
        end

        cell_mitosis_and_apoptosis!(environment,all_cells,n_div,cell_config)

        protein_dynamics!(environment,est_setup,grn,all_cells,target_cell_areas,diffusion_constants,Ax,Ay,dc,
            exp_config,cell_config,grn_config)

        for cell in all_cells
            if n_div[cell] > cell_config.max_split
                target_cell_areas[cell] = 0
            end
        end

    end
end

function run_life_view!(results::Array{Float64,4},environment::Array{Float64,3},
    grn::NamedTuple,diff_adhesion::Array{Float64,2}, diffusion_constants::Array{Float64,1},
    Ax::Array{Float64,2},Ay::Array{Float64,2},dc::Array{Float64,1},cell_config::CellularConfig,exp_config::ExperimentalConfig,grn_config::FGRNConfig)

    target_cell_areas = Dict{Float64,Float64}()
    n_div = Dict{Float64,Integer}()

    all_cells::Array{Float64,1} = [1.]
    n_div[1.] = 0
    target_cell_areas[1.] = 2*0.25*cell_config.cell_size_upper_limit

    if exp_config.est_strength == "evolve"
        affinity_1 = calculate_affinity(grn.rec_prom[:,:,1],grn_config.substrate_protein)
        affinity_2 = calculate_affinity(grn.rec_prom[:,:,2],grn_config.substrate_protein)
        affinity_3 = calculate_affinity(grn.rec_prom[:,:,3],grn_config.substrate_protein)
        affinity_4 = calculate_affinity(grn.rec_prom[:,:,4],grn_config.substrate_protein)
    else
        affinity_1 = 1 - exp_config.est_strength
        affinity_2 = 1 - exp_config.est_strength
        affinity_3 = 1 - exp_config.est_strength
        affinity_4 = 1 - exp_config.est_strength
    end

    left_segment = Int.(floor(cell_config.grid_size/3))
    middle_segment = Int.(floor(2*cell_config.grid_size/3))

    est_setup = ESTSetup(affinity_1,affinity_2,affinity_3,affinity_4,left_segment,middle_segment)

    for step in 1:size(results,4)

        results[:,:,:,step] = environment[:,:,:]

        environment_update2!(grn,environment,all_cells,diff_adhesion,target_cell_areas,cell_config,exp_config)

        if size(all_cells,1) == 0
            break
        end

        cell_mitosis_and_apoptosis!(environment,all_cells,n_div,cell_config)

        protein_dynamics!(environment,est_setup,grn,all_cells,target_cell_areas,diffusion_constants,Ax,Ay,dc,
            exp_config,cell_config,grn_config)

        for cell in all_cells
            if n_div[cell] > cell_config.max_split
                target_cell_areas[cell] = 0
            end
        end

    end
end

function run_life_view!(damage_times::Vector{Int64},damage_percentage::Float64,results::Array{Float64,3},environment::Array{Float64,3},
    grn::NamedTuple,diff_adhesion::Array{Float64,2}, diffusion_constants::Array{Float64,1},
    Ax::Array{Float64,2},Ay::Array{Float64,2},dc::Array{Float64,1},
    cell_config::CellularConfig,exp_config::ExperimentalConfig,grn_config::FGRNConfig)

    target_cell_areas = Dict{Float64,Float64}()
    n_div = Dict{Float64,Integer}()

    all_cells::Array{Float64,1} = [1.]
    n_div[1.] = 0
    target_cell_areas[1.] = 2*0.25*cell_config.cell_size_upper_limit

    if exp_config.est_strength == "evolve"
        affinity_1 = calculate_affinity(grn.rec_prom[:,:,1],grn_config.substrate_protein)
        affinity_2 = calculate_affinity(grn.rec_prom[:,:,2],grn_config.substrate_protein)
        affinity_3 = calculate_affinity(grn.rec_prom[:,:,3],grn_config.substrate_protein)
        affinity_4 = calculate_affinity(grn.rec_prom[:,:,4],grn_config.substrate_protein)
    else
        affinity_1 = 1 - exp_config.est_strength
        affinity_2 = 1 - exp_config.est_strength
        affinity_3 = 1 - exp_config.est_strength
        affinity_4 = 1 - exp_config.est_strength
    end

    left_segment = Int.(floor(cell_config.grid_size/3))
    middle_segment = Int.(floor(2*cell_config.grid_size/3))

    est_setup = ESTSetup(affinity_1,affinity_2,affinity_3,affinity_4,left_segment,middle_segment)

    for step in 1:size(results,3)

        if step in damage_times
            d_int = Int64.(floor(damage_percentage*size(all_cells,1)))
            damage_pixels = sample(all_cells,d_int,replace = false)
            for dp in damage_pixels
                environment[environment[:,:,1] .== dp,:] .= 0
            end

        end

        results[:,:,step] = environment[:,:,2]

        environment_update2!(grn,environment,all_cells,diff_adhesion,target_cell_areas,cell_config,exp_config)

        if size(all_cells,1) == 0
            break
        end

        cell_mitosis_and_apoptosis!(environment,all_cells,n_div,cell_config)

        protein_dynamics!(environment,est_setup,grn,all_cells,target_cell_areas,diffusion_constants,Ax,Ay,dc,
            exp_config,cell_config,grn_config)

        for cell in all_cells
            if n_div[cell] > cell_config.max_split
                target_cell_areas[cell] = 0
            end
        end

    end
end

function run_life_view!(damage_times::Vector{Int64},damage_percentage::Float64,energies::Array{Float64,1},
    results::Array{Float64,3},environment::Array{Float64,3},
    grn::NamedTuple,diff_adhesion::Array{Float64,2}, diffusion_constants::Array{Float64,1},
    Ax::Array{Float64,2},Ay::Array{Float64,2},dc::Array{Float64,1},
    cell_config::CellularConfig,exp_config::ExperimentalConfig,grn_config::FGRNConfig)

    target_cell_areas = Dict{Float64,Float64}()
    n_div = Dict{Float64,Integer}()

    all_cells::Array{Float64,1} = [1.]
    n_div[1.] = 0
    target_cell_areas[1.] = 2*0.25*cell_config.cell_size_upper_limit

    if exp_config.est_strength == "evolve"
        affinity_1 = calculate_affinity(grn.rec_prom[:,:,1],grn_config.substrate_protein)
        affinity_2 = calculate_affinity(grn.rec_prom[:,:,2],grn_config.substrate_protein)
        affinity_3 = calculate_affinity(grn.rec_prom[:,:,3],grn_config.substrate_protein)
        affinity_4 = calculate_affinity(grn.rec_prom[:,:,4],grn_config.substrate_protein)
    else
        affinity_1 = 1 - exp_config.est_strength
        affinity_2 = 1 - exp_config.est_strength
        affinity_3 = 1 - exp_config.est_strength
        affinity_4 = 1 - exp_config.est_strength
    end

    left_segment = Int.(floor(cell_config.grid_size/3))
    middle_segment = Int.(floor(2*cell_config.grid_size/3))

    est_setup = ESTSetup(affinity_1,affinity_2,affinity_3,affinity_4,left_segment,middle_segment)

    for step in 1:size(results,3)

        if step in damage_times
            d_int = Int64.(floor(damage_percentage*size(all_cells,1)))
            damage_pixels = sample(all_cells,d_int,replace = false)
            for dp in damage_pixels
                environment[environment[:,:,1] .== dp,:] .= 0
            end

        end

        results[:,:,step] = environment[:,:,2]

        energies[step] = measure_energy(grn,environment,all_cells,diff_adhesion,target_cell_areas,cell_config,exp_config)

        environment_update2!(grn,environment,all_cells,diff_adhesion,target_cell_areas,cell_config,exp_config)

        if size(all_cells,1) == 0
            break
        end

        cell_mitosis_and_apoptosis!(environment,all_cells,n_div,cell_config)

        protein_dynamics!(environment,est_setup,grn,all_cells,target_cell_areas,diffusion_constants,Ax,Ay,dc,
            exp_config,cell_config,grn_config)

        for cell in all_cells
            if n_div[cell] > cell_config.max_split
                target_cell_areas[cell] = 0
            end
        end

    end
end

function run_life_fitness!(time_steps::Int64,environment::Array{Float64,3},
    all_cells::Array{Float64,1},target_cell_areas::Dict{Float64,Float64},n_div::Dict{Float64,Integer},
    grn::NamedTuple,diff_adhesion::Array{Float64,2}, diffusion_constants::Array{Float64,1},
    Ax::Array{Float64,2},Ay::Array{Float64,2},dc::Array{Float64,1},cell_config::CellularConfig,
    exp_config::ExperimentalConfig,grn_config::FGRNConfig)

    if exp_config.est_strength == "evolve"
        affinity_1 = calculate_affinity(grn.rec_prom[:,:,1],grn_config.substrate_protein)
        affinity_2 = calculate_affinity(grn.rec_prom[:,:,2],grn_config.substrate_protein)
        affinity_3 = calculate_affinity(grn.rec_prom[:,:,3],grn_config.substrate_protein)
        affinity_4 = calculate_affinity(grn.rec_prom[:,:,4],grn_config.substrate_protein)
    else
        affinity_1 = 1 - exp_config.est_strength
        affinity_2 = 1 - exp_config.est_strength
        affinity_3 = 1 - exp_config.est_strength
        affinity_4 = 1 - exp_config.est_strength
    end

    left_segment = Int.(floor(cell_config.grid_size/3))
    middle_segment = Int.(floor(2*cell_config.grid_size/3))

    est_setup = ESTSetup(affinity_1,affinity_2,affinity_3,affinity_4,left_segment,middle_segment)

    for step in 1:time_steps

        environment_update2!(grn,environment,all_cells,diff_adhesion,target_cell_areas,cell_config,exp_config)

        if size(all_cells,1) == 0
            break
        end

        cell_mitosis_and_apoptosis!(environment,all_cells,n_div,cell_config)

        protein_dynamics!(environment,est_setup,grn,all_cells,target_cell_areas,diffusion_constants,Ax,Ay,dc,
            exp_config,cell_config,grn_config)

        for cell in all_cells
            if n_div[cell] > cell_config.max_split
                target_cell_areas[cell] = 0
            end
        end

    end
end


using StatProfilerHTML
using ThreadPools
using ProgressMeter

"""
    compute_batched_pt2(ci_vector::ClusteredState{T,N,R}, cluster_ops, clustered_ham::ClusteredOperator;
        nbody=4,
        H0="Hcmf",
        E0=nothing, #pass in <0|H0|0>, or compute it
        thresh_foi=1e-8,
        prescreen=true,
        verbose=1) where {T,N,R}
"""
function compute_batched_pt2(ci_vector_in::ClusteredState{T,N,R}, cluster_ops, clustered_ham::ClusteredOperator;
        nbody=4,
        H0="Hcmf",
        E0=nothing, #pass in <0|H0|0>, or compute it
        thresh_foi=1e-9,
        prescreen=true,
        verbose=1) where {T,N,R}
    #={{{=#

    println()
    println(" |........................do batched PT2............................")
    println(" thresh_foi    :", thresh_foi   )
    println(" prescreen     :", prescreen   )
    println(" H0            :", H0   )
    println(" nbody         :", nbody   )

    e2 = zeros(T,R)

    ci_vector = deepcopy(ci_vector_in)
    clusters = ci_vector.clusters
    norms = norm(ci_vector);
    @printf(" Norms of input states:\n")
    [@printf(" %12.8f\n",i) for i in norms]
    orthonormalize!(ci_vector)

    clustered_ham_0 = extract_1body_operator(clustered_ham, op_string = H0)
    if E0 == nothing
        @printf(" %-50s", "Compute <0|H0|0>:")
        @time E0 = compute_expectation_value_parallel(ci_vector, cluster_ops, clustered_ham_0)
        #E0 = diag(E0)
        flush(stdout)
    end
    @printf(" %-50s", "Compute <0|H|0>:")
    @time Evar = compute_expectation_value_parallel(ci_vector, cluster_ops, clustered_ham)
    #Evar = diag(Evar)
    flush(stdout)

    #
    # define batches (FockConfigs present in resolvant)
    jobs = Dict{FockConfig{N},Vector{Tuple}}()
    for (fock_ket, configs_ket) in ci_vector.data
        for (ftrans, terms) in clustered_ham
            fock_x = ftrans + fock_ket

            #
            # check to make sure this fock config doesn't have negative or too many electrons in any cluster
            all(f[1] >= 0 for f in fock_x) || continue
            all(f[2] >= 0 for f in fock_x) || continue
            all(f[1] <= length(clusters[fi]) for (fi,f) in enumerate(fock_x)) || continue
            all(f[2] <= length(clusters[fi]) for (fi,f) in enumerate(fock_x)) || continue

            job_input = (terms, fock_ket, configs_ket)
            if haskey(jobs, fock_x)
                push!(jobs[fock_x], job_input)
            else
                jobs[fock_x] = [job_input]
            end

        end
    end

    #
    # prepare scratch arrays to help cut down on allocation in the threads
    jobs_vec = []
    for (fock_x, job) in jobs
        push!(jobs_vec, (fock_x, job))
    end

    scr_f = Vector{Vector{Vector{Float64}} }()
    scr_i = Vector{Vector{Vector{Int16}} }()
    scr_m = Vector{Vector{MVector{N,Int16}} }()
    nscr = 20

    scr1 = Vector{Vector{Float64}}()
    scr2 = Vector{Vector{Float64}}()
    scr3 = Vector{Vector{Float64}}()
    scr4 = Vector{Vector{Float64}}()
    tmp1 = Vector{MVector{N,Int16}}()
    tmp2 = Vector{MVector{N,Int16}}()

    e2_thread = Vector{Vector{Float64}}()
    for tid in 1:Threads.nthreads()
        push!(e2_thread, zeros(R))
        push!(scr1, zeros(1000))
        push!(scr2, zeros(1000))
        push!(scr3, zeros(1000))
        push!(scr4, zeros(1000))
        push!(tmp1, zeros(Int16,N))
        push!(tmp2, zeros(Int16,N))

        tmp = Vector{Vector{Float64}}()
        [push!(tmp, zeros(Float64,10000)) for i in 1:nscr]
        push!(scr_f, tmp)

        tmp = Vector{Vector{Int16}}()
        [push!(tmp, zeros(Int16,10000)) for i in 1:nscr]
        push!(scr_i, tmp)

        tmp = Vector{MVector{N,Int16}}()
        [push!(tmp, zeros(Int16,N)) for i in 1:nscr]
        push!(scr_m, tmp)
    end

    println(" Number of jobs:    ", length(jobs_vec))
    println(" Number of threads: ", Threads.nthreads())
    BLAS.set_num_threads(1)
    flush(stdout)

    tmp = Int(round(length(jobs_vec)/100))
    verbose < 1 || println("   |----------------------------------------------------------------------------------------------------|")
    verbose < 1 || println("   |0%                                                                                100%|")
    verbose < 1 || print("   |")
    #@profilehtml @Threads.threads for job in jobs_vec
    t = @elapsed begin
        #@qthreads for job in jobs_vec
        #@time for job in jobs_vec

        @Threads.threads for (jobi,job) in collect(enumerate(jobs_vec))
        #for (jobi,job) in collect(enumerate(jobs_vec))
            fock_bra = job[1]
            tid = Threads.threadid()
            e2_thread[tid] .+= _pt2_job(job[2], fock_bra, cluster_ops, nbody, thresh_foi,
                                        scr_f[tid], scr_i[tid], scr_m[tid],  prescreen, verbose,
                                        ci_vector, clustered_ham_0, E0)
            if verbose > 0
                if  jobi%tmp == 0
                    print("-")
                    flush(stdout)
                end
            end
        end
    end
    verbose < 1 || println("|")
    flush(stdout)

    @printf(" Time spent computing E2 %12.1f (s)\n",t)
    e2 = sum(e2_thread)

    #BLAS.set_num_threads(Threads.nthreads())

    @printf(" %5s %12s %12s\n", "Root", "E(0)", "E(2)")
    for r in 1:R
        @printf(" %5s %12.8f %12.8f\n",r, Evar[r], Evar[r] + e2[r])
    end
    println(" ..................................................................|")

    return e2
end
#=}}}=#

function _pt2_job(job, fock_x, cluster_ops, nbody, thresh,
                  scr_f, scr_i, scr_m, prescreen, verbose,
                  ci_vector::ClusteredState{T,N,R}, clustered_ham_0, E0) where {T,N,R}
    #={{{=#

    sig = ClusteredState(ci_vector.clusters, T=T, R=R)
    add_fockconfig!(sig, fock_x)

    for jobi in job

        terms, fock_ket, configs_ket = jobi

        for term in terms

            length(term.clusters) <= nbody || continue

            for (config_ket, coeff_ket) in configs_ket

                #term isa ClusteredTerm2B || continue
                #if (length(sig[fock_x]) > 0) && (term isa ClusteredTerm4B)
                if (term isa ClusteredTerm4B)
                    #println("1: ", length(sig[fock_x]))
                    #@btime contract_matvec_thread($term, $cluster_ops, $fock_x, $fock_ket, $config_ket, $coeff_ket,
                    #                       $sig[$fock_x],
                    #                       $scr_f, $scr_i, $scr_m,
                    #                       thresh=$thresh, prescreen=$prescreen)
                    #contract_matvec_thread(term, cluster_ops, fock_x, fock_ket, config_ket, coeff_ket,
                    #                   sig[fock_x],
                    #                   scr_f, scr_i, scr_m,
                    #                   thresh=thresh, prescreen=prescreen)
                    #println("2: ", length(sig[fock_x]))
                    #@profilehtml contract_matvec_thread(term, cluster_ops, fock_x, fock_ket, config_ket, coeff_ket,
                    #                   sig[fock_x],
                    #                   scr_f, scr_i, scr_m,
                    #                   thresh=thresh, prescreen=prescreen)
                    #error("we good?")
                end

                contract_matvec_thread(term, cluster_ops, fock_x, fock_ket, config_ket, coeff_ket,
                                       sig[fock_x],
                                       scr_f, scr_i, scr_m,
                                       thresh=thresh, prescreen=prescreen)
                #if term isa ClusteredTerm3B
                    #@code_warntype contract_matvec_thread(term, cluster_ops, fock_x, fock_ket, config_ket,
                    #                                coeff_ket,  sig[fock_x],scr1, scr2, thresh=thresh)
                    #@btime contract_matvec_thread($term, $cluster_ops, $fock_x, $fock_ket, $config_ket,
                    #                        $coeff_ket, $sig[$fock_x], $scr_f, $scr_i, $scr_m, thresh=$thresh)
                #end
            end
        end
    end

    #@btime project_out!($sig, $ci_vector)
    #@time project_out!(sig, ci_vector)

    project_out!(sig, ci_vector)
    #verbose > 0 || println(" Fock(X): ", fock_x)
    #verbose > 1 || println(" Length of FOIS vector: ", length(sig))
    #verbose > 1 || println(" Compute diagonal")

    nx = length(sig)

    Hd = scr_f[9]
    resize!(Hd, nx)
    Hd = reshape2(Hd, (nx, 1))

    #Hd = compute_diagonal(sig, cluster_ops, clustered_ham_0)
    fill!(Hd,0.0)
    compute_diagonal!(Hd, sig, cluster_ops, clustered_ham_0)
    #@btime compute_diagonal!($Hd, $sig, $cluster_ops, $clustered_ham_0)

    sig_v = scr_f[10]
    resize!(sig_v, nx*R)
    sig_v = reshape2(sig_v, size(sig))
    fill!(sig_v,0.0)

    get_vectors!(sig_v, sig)

    #sig_v = get_vectors(sig)

    e2 = zeros(T,R)

    _sum_pt2(sig_v, e2, Hd, E0, R)

    return e2
end
#=}}}=#

function _sum_pt2(sig_v, e2, Hd, E0, R)
    # put into a function to let compiler specialize
    nx = length(Hd)
    sig_vx = 0.0
    Hdx = 0.0
    @inbounds for x in 1:nx
        Hdx = Hd[x]
        @simd for r in 1:R
            sig_vx = sig_v[x,r]
            e2[r] += sig_vx*sig_vx / (E0[r] - Hdx)
            #verbose > 0 || @printf(" %5s %12.8f\n",r, e2[r])
        end
    end
end

"""
    open_matvec_thread2(ci_vector::ClusteredState{T,N,R}, cluster_ops, clustered_ham; thresh=1e-9, nbody=4) where {T,N,R}

Compute the action of the Hamiltonian on a tpsci state vector. Open here, means that we access the full FOIS
(restricted only by thresh), instead of the action of H on v within a subspace of configurations.
This is essentially used for computing a PT correction outside of the subspace, or used for searching in TPSCI.

This parallellizes over FockConfigs in the output state, so it's not the most fine-grained, but it avoids data races in
filling the final vector
"""
function open_matvec_thread2(ci_vector::ClusteredState{T,N,R}, cluster_ops, clustered_ham;
                             thresh=1e-9,
                             prescreen=true,
                             nbody=4) where {T,N,R}
#={{{=#
    println(" In open_matvec_thread2")
    sig = deepcopy(ci_vector)
    zero!(sig)
    clusters = ci_vector.clusters
    jobs = Dict{FockConfig{N},Vector{Tuple}}()
    #sig = ClusteredState(clusters)
    #sig = OrderedDict{FockConfig{N}, OrderedDict{NTuple{N,Int16}, MVector{T} }}()

    @printf(" %-50s", "Setup threaded jobs: ")
    @time for (fock_ket, configs_ket) in ci_vector.data
        for (ftrans, terms) in clustered_ham
            fock_bra = ftrans + fock_ket

            #
            # check to make sure this fock config doesn't have negative or too many electrons in any cluster
            all(f[1] >= 0 for f in fock_bra) || continue
            all(f[2] >= 0 for f in fock_bra) || continue
            all(f[1] <= length(clusters[fi]) for (fi,f) in enumerate(fock_bra)) || continue
            all(f[2] <= length(clusters[fi]) for (fi,f) in enumerate(fock_bra)) || continue

            job_input = (terms, fock_ket, configs_ket)
            if haskey(jobs, fock_bra)
                push!(jobs[fock_bra], job_input)
            else
                jobs[fock_bra] = [job_input]
            end

        end
    end

    jobs_vec = []
    for (fock_bra, job) in jobs
        push!(jobs_vec, (fock_bra, job))
    end

    scr_f = Vector{Vector{Vector{Float64}} }()
    scr_i = Vector{Vector{Vector{Int16}} }()
    scr_m = Vector{Vector{MVector{N,Int16}} }()
    nscr = 20

    scr1 = Vector{Vector{Float64}}()
    scr2 = Vector{Vector{Float64}}()
    scr3 = Vector{Vector{Float64}}()
    scr4 = Vector{Vector{Float64}}()
    tmp1 = Vector{MVector{N,Int16}}()
    tmp2 = Vector{MVector{N,Int16}}()

    jobs_out = Vector{ClusteredState{T,N,R}}()
    for tid in 1:Threads.nthreads()
        push!(jobs_out, ClusteredState(clusters, T=T, R=R))
        push!(scr1, zeros(1000))
        push!(scr2, zeros(1000))
        push!(scr3, zeros(1000))
        push!(scr4, zeros(1000))
        push!(tmp1, zeros(Int16,N))
        push!(tmp2, zeros(Int16,N))

        tmp = Vector{Vector{Float64}}()
        [push!(tmp, zeros(Float64,10000)) for i in 1:nscr]
        push!(scr_f, tmp)

        tmp = Vector{Vector{Int16}}()
        [push!(tmp, zeros(Int16,10000)) for i in 1:nscr]
        push!(scr_i, tmp)

        tmp = Vector{MVector{N,Int16}}()
        [push!(tmp, zeros(Int16,N)) for i in 1:nscr]
        push!(scr_m, tmp)
    end

    println(" Number of jobs:    ", length(jobs_vec))
    println(" Number of threads: ", Threads.nthreads())
    BLAS.set_num_threads(1)
    #Threads.@threads for job in jobs_vec

    flush(stdout)

    #@time for job in jobs_vec
    #@qthreads for job in jobs_vec
    @printf(" %-50s", "Compute matrix-vector: ")
    @time @Threads.threads for job in jobs_vec
        fock_bra = job[1]
        tid = Threads.threadid()
        _open_matvec_thread2_job(job[2], fock_bra, cluster_ops, nbody, thresh,
                                 jobs_out[tid], scr_f[tid], scr_i[tid], scr_m[tid], prescreen)
    end
    flush(stdout)

    @printf(" %-50s", "Now collect thread results: ")
    flush(stdout)
    @time for threadid in 1:Threads.nthreads()
        add!(sig, jobs_out[threadid])
    end

    #BLAS.set_num_threads(Threads.nthreads())
    return sig
end
#=}}}=#

function open_matvec_serial2(ci_vector::ClusteredState{T,N,R}, cluster_ops, clustered_ham;
                             thresh=1e-9,
                             prescreen=true,
                             nbody=4) where {T,N,R}
#={{{=#
    println(" In open_matvec_serial2")
    sig = deepcopy(ci_vector)
    zero!(sig)
    clusters = ci_vector.clusters
    jobs = Dict{FockConfig{N},Vector{Tuple}}()

    @printf(" %-50s", "Setup threaded jobs: ")
    @time for (fock_ket, configs_ket) in ci_vector.data
        for (ftrans, terms) in clustered_ham
            fock_bra = ftrans + fock_ket

            #
            # check to make sure this fock config doesn't have negative or too many electrons in any cluster
            all(f[1] >= 0 for f in fock_bra) || continue
            all(f[2] >= 0 for f in fock_bra) || continue
            all(f[1] <= length(clusters[fi]) for (fi,f) in enumerate(fock_bra)) || continue
            all(f[2] <= length(clusters[fi]) for (fi,f) in enumerate(fock_bra)) || continue

            job_input = (terms, fock_ket, configs_ket)
            if haskey(jobs, fock_bra)
                push!(jobs[fock_bra], job_input)
            else
                jobs[fock_bra] = [job_input]
            end

        end
    end

    jobs_vec = []
    for (fock_bra, job) in jobs
        push!(jobs_vec, (fock_bra, job))
    end

    scr_f = Vector{Vector{Vector{Float64}} }()
    scr_i = Vector{Vector{Vector{Int16}} }()
    scr_m = Vector{Vector{MVector{N,Int16}} }()
    nscr = 20

    scr1 = Vector{Vector{Float64}}()
    scr2 = Vector{Vector{Float64}}()
    scr3 = Vector{Vector{Float64}}()
    scr4 = Vector{Vector{Float64}}()
    tmp1 = Vector{MVector{N,Int16}}()
    tmp2 = Vector{MVector{N,Int16}}()

    jobs_out = Vector{ClusteredState{T,N,R}}()
    for tid in 1:1
        push!(jobs_out, ClusteredState(clusters, T=T, R=R))
        push!(scr1, zeros(1000))
        push!(scr2, zeros(1000))
        push!(scr3, zeros(1000))
        push!(scr4, zeros(1000))
        push!(tmp1, zeros(Int16,N))
        push!(tmp2, zeros(Int16,N))

        tmp = Vector{Vector{Float64}}()
        [push!(tmp, zeros(Float64,10000)) for i in 1:nscr]
        push!(scr_f, tmp)

        tmp = Vector{Vector{Int16}}()
        [push!(tmp, zeros(Int16,10000)) for i in 1:nscr]
        push!(scr_i, tmp)

        tmp = Vector{MVector{N,Int16}}()
        [push!(tmp, zeros(Int16,N)) for i in 1:nscr]
        push!(scr_m, tmp)
    end

    println(" Number of jobs:    ", length(jobs_vec))
    flush(stdout)
    @printf(" %-50s", "Compute matrix-vector: ")
    @time for job in jobs_vec
        fock_bra = job[1]
        tid = 1
        _open_matvec_thread2_job(job[2], fock_bra, cluster_ops, nbody, thresh,
                                 jobs_out[tid], scr_f[tid], scr_i[tid], scr_m[tid],prescreen)
    end
    flush(stdout)

    flush(stdout)
    @printf(" %-50s", "Now collect thread results: ")
    @time for threadid in 1:1
        add!(sig, jobs_out[threadid])
    end

    return sig
end
#=}}}=#

function _open_matvec_thread2_job(job, fock_bra, cluster_ops, nbody, thresh, sig, scr_f, scr_i, scr_m, prescreen)
#={{{=#

    haskey(sig, fock_bra) || add_fockconfig!(sig, fock_bra)
    #sizehint!(sig[fock_bra],1000)

    for jobi in job

        terms, fock_ket, configs_ket = jobi

        for term in terms

            length(term.clusters) <= nbody || continue

            for (config_ket, coeff_ket) in configs_ket

                #term isa ClusteredTerm2B || continue

                contract_matvec_thread(term, cluster_ops, fock_bra, fock_ket, config_ket, coeff_ket, sig[fock_bra],
                                       scr_f, scr_i, scr_m, thresh=thresh, prescreen=prescreen)
                if term isa ClusteredTerm3B
                    #@code_warntype contract_matvec_thread(term, cluster_ops, fock_bra, fock_ket, config_ket, coeff_ket,
                    #                              sig[fock_bra],scr1, scr2, thresh=thresh)
                    #@btime contract_matvec_thread($term, $cluster_ops, $fock_bra, $fock_ket, $config_ket, $coeff_ket,
                    #                              $sig[$fock_bra], $scr_f, $scr_i, $scr_m, thresh=$thresh)
                end
            end
        end
    end
    return
end
#=}}}=#

reshape2(a, dims) = invoke(Base._reshape, Tuple{AbstractArray,typeof(dims)}, a, dims)

"""
    contract_matvec_thread(   term::ClusteredTerm1B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9) where {T,R,N}
"""
function contract_matvec_thread(   term::ClusteredTerm1B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9, prescreen=true) where {T,R,N}
#={{{=#
    c1 = term.clusters[1]

    #
    # determine sign from rearranging clusters if odd number of operators
    state_sign = compute_terms_state_sign(term, fock_ket)

    #
    # <:|p'|J> h(pq) <:|q|L>
    #
    # <:|p'|J> h(pq) <:|q|L>
    g1a = cluster_ops[c1.idx][term.ops[1]]

    # the following type declarations are needed to prevent allocations. This ultimately is due to the fact
    # that the ClusterOps type isn't formed in an ideal concrete manner. We just currently use Array.
    # This should be cleaned up eventually, preferably with a distinction between contracted, and uncontracted
    # operators.
    g1::Array{Float64,2} = g1a[(fock_bra[c1.idx],fock_ket[c1.idx])]

    @views gamma1 = g1[:,conf_ket[c1.idx]]

    scr1 = scr_f[1]
    resize!(scr1, size(gamma1,1))

    scr1 .= gamma1 .* state_sign
    newI = 1:size(scr1,1)
    #new_coeffs = scr1
    #newI = 1:size(new_coeffs,1)

    _collect_significant_thread!(sig, conf_ket, scr1, coef_ket, c1.idx,  newI,  thresh, scr_m[1])
    #_collect_significant_thread!(sig, conf_ket, new_coeffs, coef_ket, c1.idx,  newI,  thresh)

    return
end
#=}}}=#

"""
    contract_matvec_thread(   term::ClusteredTerm2B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9) where {T,R,N}
"""
function contract_matvec_thread(   term::ClusteredTerm2B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9, prescreen=true) where {T,R,N}
#={{{=#

    c1 = term.clusters[1]
    c2 = term.clusters[2]

    #
    # determine sign from rearranging clusters if odd number of operators
    state_sign = compute_terms_state_sign(term, fock_ket)

    #
    # <:|p'|J> h(pq) <:|q|L>
    g1a = cluster_ops[c1.idx][term.ops[1]]
    g2a = cluster_ops[c2.idx][term.ops[2]]

    # the following type declarations are needed to prevent allocations. This ultimately is due to the fact
    # that the ClusterOps type isn't formed in an ideal concrete manner. We just currently use Array.
    # This should be cleaned up eventually, preferably with a distinction between contracted, and uncontracted
    # operators.

    haskey(g1a, (fock_bra[c1.idx],fock_ket[c1.idx])) || return
    haskey(g2a, (fock_bra[c2.idx],fock_ket[c2.idx])) || return

    g1::Array{Float64,3} = g1a[(fock_bra[c1.idx],fock_ket[c1.idx])]
    g2::Array{Float64,3} = g2a[(fock_bra[c2.idx],fock_ket[c2.idx])]

    @views gamma1 = g1[:,:,conf_ket[c1.idx]]
    @views gamma2 = g2[:,:,conf_ket[c2.idx]]

    scr1 = scr_f[1]
    scr2 = scr_f[2]

    resize!(scr1, size(term.ints,2) * size(gamma1,2))
    resize!(scr2, size(gamma1,2) * size(gamma2,2))

    scr1 = reshape2(scr1, (size(term.ints,2), size(gamma1,2)))
    scr2 = reshape2(scr2, (size(gamma1,2), size(gamma2,2)))

    mul!(scr1, term.ints', gamma1)
    mul!(scr2, scr1', gamma2)
    new_coeffs = scr2

    newI = 1:size(new_coeffs,1)
    newJ = 1:size(new_coeffs,2)

    _collect_significant_thread!(sig, conf_ket, new_coeffs, coef_ket, c1.idx, c2.idx, newI, newJ, thresh, scr_m[1], state_sign)
    #@btime _collect_significant_thread!($sig, $conf_ket, $new_coeffs, $coef_ket, $c1.idx, $c2.idx, $newI, $newJ, $thresh, $scr3)

    return
end
#=}}}=#

"""
    contract_matvec_thread(   term::ClusteredTerm3B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9, prescreen=true) where {T,R,N}

This version should only use M^2N^2 storage, and n^4 scaling n={MN}
"""
function contract_matvec_thread(   term::ClusteredTerm3B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9, prescreen=true) where {T,R,N}
#={{{=#
    c1 = term.clusters[1]
    c2 = term.clusters[2]
    c3 = term.clusters[3]

    #
    # determine sign from rearranging clusters if odd number of operators
    state_sign = compute_terms_state_sign(term, fock_ket)

    #
    # h(pqr) <I|p'|_> <J|q|_> <K|r|_>
    #
    # X(p,J,K) = h(pqr) <J|q|_> <K|r|_>
    #
    #
    haskey(cluster_ops[c1.idx][term.ops[1]],  (fock_bra[c1.idx],fock_ket[c1.idx])) || return
    haskey(cluster_ops[c2.idx][term.ops[2]],  (fock_bra[c2.idx],fock_ket[c2.idx])) || return
    haskey(cluster_ops[c3.idx][term.ops[3]],  (fock_bra[c3.idx],fock_ket[c3.idx])) || return

    g1::Array{Float64,3} = cluster_ops[c1.idx][term.ops[1]][(fock_bra[c1.idx],fock_ket[c1.idx])]
    g2::Array{Float64,3} = cluster_ops[c2.idx][term.ops[2]][(fock_bra[c2.idx],fock_ket[c2.idx])]
    g3::Array{Float64,3} = cluster_ops[c3.idx][term.ops[3]][(fock_bra[c3.idx],fock_ket[c3.idx])]
    @views gamma1 = g1[:,:,conf_ket[c1.idx]]
    @views gamma2 = g2[:,:,conf_ket[c2.idx]]
    @views gamma3 = g3[:,:,conf_ket[c3.idx]]

    scr1 = scr_f[1]
    scr2 = scr_f[2]
    scr3 = scr_f[3]
    tmp1 = scr_m[1]
    tmp2 = scr_m[2]

    newI = UnitRange{Int16}(1,size(g1,2))
    newJ = UnitRange{Int16}(1,size(g2,2))
    newK = UnitRange{Int16}(1,size(g3,2))

    if prescreen
        up_bound = upper_bound_thread(term.ints, gamma1, gamma2, gamma3,
                              scr1, scr2, scr3,
                              c=maximum(abs.(coef_ket)))
        if up_bound < thresh
            return
        end
        #@btime newI, newJ, newK = upper_bound2($term.ints, $gamma1, $gamma2, $gamma3, $thresh, c=maximum(abs.($coef_ket)))
        newI, newJ, newK = upper_bound2_thread(term.ints, gamma1, gamma2, gamma3,
                                                     scr_f, scr_i, thresh, c=maximum(abs, coef_ket))
        length(newI) > 0 || return
        length(newJ) > 0 || return
        length(newK) > 0 || return

        if true
            gamma1 = scr_f[4]
            gamma2 = scr_f[5]
            gamma3 = scr_f[6]

            resize!(gamma1, size(g1,1)*length(newI))
            resize!(gamma2, size(g2,1)*length(newJ))
            resize!(gamma3, size(g3,1)*length(newK))

            gamma1 = reshape2(gamma1, (size(g1,1), length(newI)))
            gamma2 = reshape2(gamma2, (size(g2,1), length(newJ)))
            gamma3 = reshape2(gamma3, (size(g3,1), length(newK)))

            _fill1!(gamma1, newI, g1, conf_ket[c1.idx])
            _fill1!(gamma2, newJ, g2, conf_ket[c2.idx])

            #gamma1 .= g1[:,newI,conf_ket[c1.idx]]
            #gamma2 .= g2[:,newJ,conf_ket[c2.idx]]
            #gamma3 .= g3[:,newK,conf_ket[c3.idx]]
        else
            @views gamma1 = g1[:,newI,conf_ket[c1.idx]]
            @views gamma2 = g2[:,newJ,conf_ket[c2.idx]]
            @views gamma3 = g3[:,newK,conf_ket[c3.idx]]
        end
    end

    #
    # for K in G3
    #   scr1(p,q) = h(p,q,r) * G3(r;K)          N^3M^1
    #   scr2(p,J) = scr1(p,q) * G2(q,J)
    #   scr3(I,J) = G1(p,I)' * scr2(p,J)
    #
    #   collect(scr3(I,J))

    np = size(term.ints,1)
    nq = size(term.ints,2)
    nr = size(term.ints,3)
    nI = length(newI)
    nJ = length(newJ)
    nK = length(newK)

    resize!(scr1, np*nq)
    resize!(scr2, np*nJ)
    resize!(scr3, nI*nJ)

    scr2 = reshape2(scr2, (np,nJ))
    scr3 = reshape2(scr3, (nI,nJ))

    v = reshape2(term.ints, (np*nq,nr))

    #a = @allocated begin
    #end; if a > 0 println(a); error("here we are"); end

    tmp1 .= conf_ket.config

    for K::Int16 in newK

        @views gamma3 = g3[:,K,conf_ket[c3.idx]]

        tmp1[c3.idx] = K

        tmp_conf = ClusterConfig(SVector(tmp1))

        mul!(scr1, v, gamma3)
        mul!(scr2, reshape2(scr1, (np,nq)), gamma2)
        #@allocated mul!(scr2, reshape2(scr1, (np,nq)), gamma2) == 0 || error(" we allocated")

        if prescreen
            #println("3b")
            #@btime upper_bound_thread($gamma1, $scr2, c=maximum(abs.($coef_ket)))
            #println(size(gamma1))
            #println(size(scr2))
            #println(typeof(gamma1))
            #println(typeof(scr2))
            upper_bound_thread(gamma1, scr2, c=maximum(abs.(coef_ket))) > thresh || continue
        end

        mul!(scr3, gamma1', scr2)
        #@btime mul!($scr3, $gamma1', $scr2)
        #a = @allocated begin
        #    mul!(scr3, gamma1', scr2)
        #end
        #a == 0 || error(" we allocated: ", a)

        #@btime _collect_significant_thread!($sig, $tmp_conf, $scr3, $coef_ket, $c1.idx, $c2.idx, $newI, $newJ, $thresh, $tmp2, $state_sign)
        _collect_significant_thread!(sig, tmp_conf, scr3, coef_ket, c1.idx, c2.idx, newI, newJ, thresh, tmp2, state_sign)
    end

    return
end
#=}}}=#

"""
    contract_matvec_thread(   term::ClusteredTerm4B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9, prescreen=true) where {T,R,N}

This version should only use M^2N^2 storage, and n^5 scaling n={MN}
"""
function contract_matvec_thread(   term::ClusteredTerm4B,
                                    cluster_ops::Vector{ClusterOps},
                                    fock_bra::FockConfig{N},
                                    fock_ket::FockConfig{N}, conf_ket::ClusterConfig{N}, coef_ket::MVector{R,T},
                                    sig,
                                    scr_f::Vector{Vector{Float64}},
                                    scr_i::Vector{Vector{Int16}},
                                    scr_m::Vector{MVector{N,Int16}};
                                    thresh=1e-9, prescreen=true) where {T,R,N}
#={{{=#
    c1 = term.clusters[1]
    c2 = term.clusters[2]
    c3 = term.clusters[3]
    c4 = term.clusters[4]

    #
    # determine sign from rearranging clusters if odd number of operators
    state_sign = compute_terms_state_sign(term, fock_ket)

    #
    # h(pqr) <I|p'|_> <J|q|_> <K|r|_>
    #
    # X(p,J,K) = h(pqr) <J|q|_> <K|r|_>
    #
    #
    haskey(cluster_ops[c1.idx][term.ops[1]],  (fock_bra[c1.idx],fock_ket[c1.idx])) || return
    haskey(cluster_ops[c2.idx][term.ops[2]],  (fock_bra[c2.idx],fock_ket[c2.idx])) || return
    haskey(cluster_ops[c3.idx][term.ops[3]],  (fock_bra[c3.idx],fock_ket[c3.idx])) || return
    haskey(cluster_ops[c4.idx][term.ops[4]],  (fock_bra[c4.idx],fock_ket[c4.idx])) || return

    g1::Array{Float64,3} = cluster_ops[c1.idx][term.ops[1]][(fock_bra[c1.idx],fock_ket[c1.idx])]
    g2::Array{Float64,3} = cluster_ops[c2.idx][term.ops[2]][(fock_bra[c2.idx],fock_ket[c2.idx])]
    g3::Array{Float64,3} = cluster_ops[c3.idx][term.ops[3]][(fock_bra[c3.idx],fock_ket[c3.idx])]
    g4::Array{Float64,3} = cluster_ops[c4.idx][term.ops[4]][(fock_bra[c4.idx],fock_ket[c4.idx])]
    @views gamma1 = g1[:,:,conf_ket[c1.idx]]
    @views gamma2 = g2[:,:,conf_ket[c2.idx]]
    @views gamma3 = g3[:,:,conf_ket[c3.idx]]
    @views gamma4 = g4[:,:,conf_ket[c4.idx]]

    np = size(term.ints,1)
    nq = size(term.ints,2)
    nr = size(term.ints,3)
    ns = size(term.ints,4)

    scr1 = scr_f[1]
    scr2 = scr_f[2]
    scr3 = scr_f[3]
    scr4 = scr_f[4]
    tmp1 = scr_m[1]
    tmp2 = scr_m[2]

    newI = UnitRange{Int16}(1,size(g1,2))
    newJ = UnitRange{Int16}(1,size(g2,2))
    newK = UnitRange{Int16}(1,size(g3,2))
    newL = UnitRange{Int16}(1,size(g4,2))
    if prescreen
        up_bound = upper_bound_thread(term.ints, gamma1, gamma2, gamma3, gamma4,
                              scr1, scr2, scr3, scr4,
                              c=maximum(abs.(coef_ket)))
        if up_bound < thresh
            return
        end
        #println("a")
        #
        # screen phase 2: ignore indices for each cluster which will produce discarded terms

        #@btime newI, newJ, newK, newL = upper_bound2_thread($term.ints, $gamma1, $gamma2, $gamma3, $gamma4,
        #                                             $scr_f, $scr_i, $thresh, c=maximum(abs, $coef_ket))

        newI, newJ, newK, newL = upper_bound2_thread(term.ints, gamma1, gamma2, gamma3, gamma4,
                                                     scr_f, scr_i, thresh, c=maximum(abs, coef_ket))
        length(newI) > 0 || return
        length(newJ) > 0 || return
        length(newK) > 0 || return
        length(newL) > 0 || return
        #println("b")

        #
        # While we could simply use a view, this creates a non-contiguous array and BLAS then does allocations

        if true
            g1t = typeof(gamma1)
            g2t = typeof(gamma1)
            g3t = typeof(gamma1)
            g4t = typeof(gamma1)
            gamma1 = scr_f[5]
            gamma2 = scr_f[6]
            gamma3 = scr_f[7]
            gamma4 = scr_f[8]

            #g1t == typeof(gamma1) || println(" here: ", g1t, " vs ", typeof(gamma1))
            #g2t == typeof(gamma2) || println(" here: ", g2t, " vs ", typeof(gamma2))
            #g3t == typeof(gamma3) || println(" here: ", g3t, " vs ", typeof(gamma3))
            #g4t == typeof(gamma4) || println(" here: ", g4t, " vs ", typeof(gamma4))

            resize!(gamma1, size(g1,1)*length(newI))
            resize!(gamma2, size(g2,1)*length(newJ))
            resize!(gamma3, size(g3,1)*length(newK))
            resize!(gamma4, size(g4,1)*length(newL))

            gamma1 = reshape2(gamma1, (size(g1,1), length(newI)))
            gamma2 = reshape2(gamma2, (size(g2,1), length(newJ)))
            gamma3 = reshape2(gamma3, (size(g3,1), length(newK)))
            gamma4 = reshape2(gamma4, (size(g4,1), length(newL)))

            _fill1!(gamma1, newI, g1, conf_ket[c1.idx])
            _fill1!(gamma2, newJ, g2, conf_ket[c2.idx])
            #_fill1!(gamma3, newK, g3, conf_ket[c3.idx])
            #_fill1!(gamma4, newL, g4, conf_ket[c4.idx])

            #gamma1 .= g1[:,newI,conf_ket[c1.idx]]
            #gamma2 .= g2[:,newJ,conf_ket[c2.idx]]
            #gamma3 .= g3[:,newK,conf_ket[c3.idx]]
            #gamma4 .= g4[:,newL,conf_ket[c4.idx]]
        else
            @views gamma1 = g1[:,newI,conf_ket[c1.idx]]
            @views gamma2 = g2[:,newJ,conf_ket[c2.idx]]
            @views gamma3 = g3[:,newK,conf_ket[c3.idx]]
            @views gamma4 = g4[:,newL,conf_ket[c4.idx]]
        end
    end

    #
    #   for L in G4
    #       scr1(p,q,r) = h(p,q,r,s) * G4(s;L)          N^4M^1
    #
    #       for K in G3
    #           scr2(p,q) = scr1(p,q,r) * G3(r;K)       N^3M^2
    #           scr3(p,J) = scr2(p,q) * G2(q,J)         N^2M^3
    #           scr4(I,J) = G1'(p,I) * scr3(p,J)        N^1M^4
    #
    #           collect(scr4(I,J))

    nI = length(newI)
    nJ = length(newJ)
    nK = length(newK)
    nL = length(newL)

    resize!(scr1, np*nq*nr)
    resize!(scr2, np*nq)
    resize!(scr3, np*nJ)
    resize!(scr4, nI*nJ)

    scr3 = reshape2(scr3, (np,nJ))
    scr4 = reshape2(scr4, (nI,nJ))

    v = reshape2(term.ints, (np*nq*nr,ns))
    #@btime v = reshape2($term.ints, ($np*$nq*$nr,$ns))

    tmp1 .= conf_ket.config

    a = 0.0
    #@btime begin
    #    for L::Int16 in $newL
    #        $a += 1.0
    #    end
    #end

    for L::Int16 in newL

        @views gamma4 = g4[:,L,conf_ket[c4.idx]]
        #@btime @views gamma4 = $g4[:,$L,$conf_ket[$c4.idx]]

        tmp1[c4.idx] = L
        #@btime $tmp1[$c4.idx] = $L

        mul!(scr1, v, gamma4)
        #@btime mul!($scr1, $v, $gamma4)

        for K::Int16 in newK

            @views gamma3 = g3[:,K,conf_ket[c3.idx]]
            #@btime @views $gamma3 = $g3[:,$K,$conf_ket[$c3.idx]]

            tmp1[c3.idx] = K
            #@btime $tmp1[$c3.idx] = $K

            tmp_conf = ClusterConfig(SVector(tmp1))
            #@btime $tmp_conf = ClusterConfig(SVector($tmp1))

            mul!(scr2, reshape2(scr1, (np*nq,nr)), gamma3)
            #@btime mul!($scr2, reshape2($scr1, ($np*$nq,$nr)), $gamma3)

            #@views gamma2 = g2[:,:,conf_ket[c2.idx]]

            #
            # this is allocating :(  .... but only when threaded? actually @allocated doesn't work when threaded
            mul!(scr3, reshape2(scr2, (np,nq)), gamma2)
            #a = @allocated begin
            #    mul!(scr3, reshape2(scr2, (np,nq)), gamma2)
            #end
            #@allocated mul!(scr3, reshape2(scr2, (np,nq)), gamma2) == 0 || error("why allocate?")
            #@btime mul!($scr3, reshape2($scr2, ($np,$nq)), $gamma2)

            #display(typeof(gamma3))
            #display(typeof(gamma2))
            #println()
            if prescreen
                #println("4b")
                #@btime upper_bound_thread($gamma1, $scr3, c=maximum(abs.($coef_ket)))
                #println(size(gamma1))
                #println(size(scr3))
                #println(typeof(gamma1))
                #println(typeof(scr3))
                upper_bound_thread(gamma1, scr3, c=maximum(abs.(coef_ket))) > thresh || continue
            end
            #println("c")

            mul!(scr4, gamma1', scr3)
            #a = @allocated begin
            #    mul!(scr4, gamma1', scr3)
            #end
            #a == 0 || error(" we allocated: ", a)
            #@btime mul!($scr4, $gamma1', $scr3)

            _collect_significant_thread!(sig, tmp_conf, scr4, coef_ket, c1.idx, c2.idx, newI, newJ, thresh, tmp2, state_sign)
        end
    end

    return
end
#=}}}=#

function _fill1!(gamma, idx, g, k)
    @inbounds for i in 1:size(g,1)
        for (II,I) in enumerate(idx)
            gamma[i,II] = g[i,I,k]
            #gamma[i,II] = deepcopy(g[i,I,k])
        end
    end
end

function _collect_significant_thread!(out, conf_ket, new_coeffs, coeff, c1idx, newI, thresh, tmp1)
    #={{{=#
    tmp1 .= conf_ket.config
    thresh_curr = thresh / maximum(abs.(coeff))
    for i::Int16 in newI
        if (new_coeffs[i] > thresh_curr) || (new_coeffs[i] < -thresh_curr)
            tmp1[c1idx] = i

            tmp = ClusterConfig(SVector(tmp1))
            if haskey(out, tmp)
                out[tmp] .+= new_coeffs[i].*coeff
            else
                out[tmp] = new_coeffs[i].*coeff
            end
        end
    end
end
#=}}}=#

function _collect_significant_thread!(out, conf_ket, new_coeffs, coeff, c1idx, c2idx, newI, newJ, thresh, tmp1, sign)
#={{{=#
    tmp1 .= conf_ket.config
    thresh_curr = thresh / maximum(abs.(coeff))

    ii::Int = 1
    nI = length(newI)
    nJ = length(newJ)
    @inbounds for j::Int16 in 1:nJ
        tmp1[c2idx] = newJ[j]
        for i::Int16 in 1:nI
            if abs(new_coeffs[i,j]) > thresh_curr
                tmp1[c1idx] = newI[i]
                tmp = ClusterConfig(SVector(tmp1))
                #@btime haskey($out, $tmp)
                if haskey(out, tmp)
                    ii += 1
                    if sign == 1
                        out[tmp] .+= new_coeffs[i,j] .* coeff
                    elseif sign == -1
                        out[tmp] .-= new_coeffs[i,j] .* coeff
                    else
                        error(" only 1 or -1")
                    end
                else
                    if sign == 1
                        out[tmp] = new_coeffs[i,j].*coeff
                    elseif sign == -1
                        out[tmp] = -new_coeffs[i,j].*coeff
                    else
                        error(" only 1 or -1")
                    end
                end
            end
        end
    end
end
#=}}}=#

"""
    upper_bound_thread(g1, g2; c::Float64=1.0)

Return upper bound on the size of matrix elements resulting from matrix multiply

    V[I,J] =  g1[i,I] * g2[i,J] * c

    max(|V|) <= sum_i max|g1[i,:]| * max|g2[i,:]| * |c|
"""
function upper_bound_thread(g1, g2; c::Float64=1.0)
#={{{=#
    bound::Float64 = 0.0
    n1 = size(g1,1)
    n2 = size(g2,1)
    n1 == n2 || throw(DimensionMismatch)

    absc = abs(c)
    @inbounds @simd for p in 1:n1
        @views pmax = maximum(abs, g1[p,:])
        @views qmax = maximum(abs, g2[p,:])
        bound += pmax * qmax
    end
    bound *= absc
    return bound
end
#=}}}=#

"""
    upper_bound_thread(v::AbstractArray{Float64,3}, g1, g2, g3, scr1, scr2, scr3; c::Float64=1.0)

Return upper bound on the size of tensor elements resulting from the following contraction

    V[I,J,K] = v[i,j,k] * g1[i,I] * g2[j,J] * g3[k,K]

    max(|V|) <= sum_ijk |v[ijk]| * |g1[i,:]|_8 * |g2[j,:]|_8 * |g3[k,:]|_8
"""
function upper_bound_thread(v::AbstractArray{Float64,3}, g1, g2, g3, scr1, scr2, scr3; c::Float64=1.0)
    #={{{=#
        bound = 0
        n1 = size(g1,1)
        n2 = size(g2,1)
        n3 = size(g3,1)

        resize!(scr1, n1)
        resize!(scr2, n2)
        resize!(scr3, n3)
        pmax = scr1
        qmax = scr2
        rmax = scr3

        fill!(pmax, 0.0)
        fill!(qmax, 0.0)
        fill!(rmax, 0.0)

        for p in 1:n1
            @views pmax[p] = maximum(abs(i) for i in g1[p,:])
            #@views pmax[p] = maximum(abs.(g1[p,:]))
        end
        for p in 1:n2
            @views qmax[p] = maximum(abs(i) for i in g2[p,:])
        end
        for p in 1:n3
            @views rmax[p] = maximum(abs(i) for i in g3[p,:])
        end

        tmp = 0.0
        @inbounds for r in 1:n3
            for q in 1:n2
                tmp = abs(c) * qmax[q] * rmax[r]
                @simd for p in 1:n1
                    bound += tmp * abs(v[p,q,r]) * pmax[p]
                end
            end
        end
    return bound
end
#=}}}=#

"""
    upper_bound_thread(v::Array{Float64,4}, g1, g2, g3, g4, scr1, scr2, scr3, scr4; c::Float64=1.0)

Return upper bound on the size of tensor elements resulting from the following contraction

    V[I,J,K,L] = v[i,j,k,l] * g1[i,I] * g2[j,J] * g3[k,K] * g4[l,L]

    max(|V|) <= sum_ijkl |v[ijkl]| * |g1[i,:]|_8 * |g2[j,:]|_8 * |g3[k,:]|_8 * |g4[l,:]|_8
"""
function upper_bound_thread(v::Array{Float64,4}, g1, g2, g3, g4, scr1, scr2, scr3, scr4; c::Float64=1.0)
    #={{{=#
        bound = 0
        n1 = size(g1,1)
        n2 = size(g2,1)
        n3 = size(g3,1)
        n4 = size(g4,1)

        resize!(scr1, n1)
        resize!(scr2, n2)
        resize!(scr3, n3)
        resize!(scr4, n4)
        pmax = scr1
        qmax = scr2
        rmax = scr3
        smax = scr4

        fill!(pmax, 0.0)
        fill!(qmax, 0.0)
        fill!(rmax, 0.0)
        fill!(smax, 0.0)

        for p in 1:n1
            @views pmax[p] = maximum(abs(i) for i in g1[p,:])
        end
        for p in 1:n2
            @views qmax[p] = maximum(abs(i) for i in g2[p,:])
        end
        for p in 1:n3
            @views rmax[p] = maximum(abs(i) for i in g3[p,:])
        end
        for p in 1:n4
            @views smax[p] = maximum(abs(i) for i in g4[p,:])
        end

        tmp = 0.0
        @inbounds for s in 1:n4
            for r in 1:n3
                for q in 1:n2
                    tmp = abs(c) * qmax[q] * rmax[r] * smax[s]
                    @simd for p in 1:n1
                        bound += tmp * abs(v[p,q,r,s]) * pmax[p]
                    end
                end
            end
        end
    return bound
end
#=}}}=#

"""
    upper_bound2_thread(v::Array{Float64,4}, g1, g2, g3,
        scr_f::Vector{Vector{Float64}}, scr_i::Vector{Vector{Int16}}, thresh;
        c::Float64=1.0)

    max(H_IJ(K)|_K <= sum_s (sum_pqr vpqrs max(g1[p,:]) * max(g2[q,:]) * |c| ) * |g3(r,K)|
"""
function upper_bound2_thread(v::Array{Float64,3}, g1, g2, g3,
        scr_f::Vector{Vector{Float64}}, scr_i::Vector{Vector{Int16}}, thresh;
        c::Float64=1.0)
    #={{{=#

        newI = scr_i[1]
        newJ = scr_i[2]
        newK = scr_i[3]
        resize!(newI,0)
        resize!(newJ,0)
        resize!(newK,0)

        n1 = size(v,1)
        n2 = size(v,2)
        n3 = size(v,3)

        n1 == size(g1,1) || throw(DimensionMismatch)
        n2 == size(g2,1) || throw(DimensionMismatch)
        n3 == size(g3,1) || throw(DimensionMismatch)

        pmax = scr_f[5]
        qmax = scr_f[6]
        rmax = scr_f[7]

        resize!(pmax,n1)
        resize!(qmax,n2)
        resize!(rmax,n3)

        fill!(pmax, 0.0)
        fill!(qmax, 0.0)
        fill!(rmax, 0.0)

        for p in 1:n1
            @views pmax[p] = maximum(abs, g1[p,:])
        end
        for p in 1:n2
            @views qmax[p] = maximum(abs, g2[p,:])
        end
        for p in 1:n3
            @views rmax[p] = maximum(abs, g3[p,:])
        end

        tmp = 0.0

        mI = scr_f[9]
        resize!(mI,size(g1,2))
        fill!(mI, 0.0)
        @inbounds for r in 1:n3
            for q in 1:n2
                tmp = qmax[q] * rmax[r] * abs(c)
                for p in 1:n1
                    @views @. mI += abs(v[p,q,r]) * abs.(g1[p,:]) * tmp
                end
            end
        end

        mJ = scr_f[10]
        resize!(mJ,size(g2,2))
        fill!(mJ, 0.0)
        @inbounds for r in 1:n3
            for p in 1:n1
                tmp = pmax[p] * rmax[r] * abs(c)
                for q in 1:n2
                    @views @. mJ += abs(v[p,q,r]) * abs.(g2[q,:])  * tmp
                end
            end
        end

        mK = scr_f[11]
        resize!(mK,size(g3,2))
        fill!(mK, 0.0)
        @inbounds for q in 1:n2
            for p in 1:n1
                tmp = pmax[p] * qmax[q] * abs(c)
                for r in 1:n3
                    @views @. mK += abs(v[p,q,r]) * abs.(g3[r,:]) * tmp
                end
            end
        end

        for I in 1:size(g1,2)
            if abs(mI[I]) > thresh
                push!(newI,I)
            end
        end

        for J in 1:size(g2,2)
            if abs(mJ[J]) > thresh
                push!(newJ,J)
            end
        end

        for K in 1:size(g3,2)
            if abs(mK[K]) > thresh
                push!(newK,K)
            end
        end

    return newI, newJ, newK
end
#=}}}=#

"""
    upper_bound2_thread(v::Array{Float64,4}, g1, g2, g3, g4,
        scr_f::Vector{Vector{Float64}}, scr_i::Vector{Vector{Int16}}, thresh;
        c::Float64=1.0)

    max(H_IJK(L)|_L <= sum_s (sum_pqr vpqrs max(g1[p,:]) * max(g2[q,:]) * max(g3[r,:]) * |c| ) * |g4(s,L)|
"""
function upper_bound2_thread(v::Array{Float64,4}, g1, g2, g3, g4,
        scr_f::Vector{Vector{Float64}}, scr_i::Vector{Vector{Int16}}, thresh;
        c::Float64=1.0)
    #={{{=#

        newI = scr_i[1]
        newJ = scr_i[2]
        newK = scr_i[3]
        newL = scr_i[4]
        resize!(newI,0)
        resize!(newJ,0)
        resize!(newK,0)
        resize!(newL,0)

        n1 = size(v,1)
        n2 = size(v,2)
        n3 = size(v,3)
        n4 = size(v,4)

        n1 == size(g1,1) || throw(DimensionMismatch)
        n2 == size(g2,1) || throw(DimensionMismatch)
        n3 == size(g3,1) || throw(DimensionMismatch)
        n4 == size(g4,1) || throw(DimensionMismatch)

        pmax = scr_f[5]
        qmax = scr_f[6]
        rmax = scr_f[7]
        smax = scr_f[8]

        resize!(pmax,n1)
        resize!(qmax,n2)
        resize!(rmax,n3)
        resize!(smax,n4)

        fill!(pmax, 0.0)
        fill!(qmax, 0.0)
        fill!(rmax, 0.0)
        fill!(smax, 0.0)

        for p in 1:n1
            @views pmax[p] = maximum(abs, g1[p,:])
        end
        for p in 1:n2
            @views qmax[p] = maximum(abs, g2[p,:])
        end
        for p in 1:n3
            @views rmax[p] = maximum(abs, g3[p,:])
        end
        for p in 1:n4
            @views smax[p] = maximum(abs, g4[p,:])
        end

        tmp = 0.0

        mI = scr_f[9]
        resize!(mI,size(g1,2))
        fill!(mI, 0.0)
        @inbounds for s in 1:n4
            for r in 1:n3
                for q in 1:n2
                    tmp = qmax[q] * rmax[r] * smax[s] * abs(c)
                    for p in 1:n1
                        @views @. mI += abs(v[p,q,r,s]) * abs.(g1[p,:]) * tmp
                    end
                end
            end
        end

        mJ = scr_f[10]
        resize!(mJ,size(g2,2))
        fill!(mJ, 0.0)
        @inbounds for s in 1:n4
            for r in 1:n3
                for p in 1:n1
                    tmp = pmax[p] * rmax[r] * smax[s] * abs(c)
                    for q in 1:n2
                        @views @. mJ += abs(v[p,q,r,s]) * abs.(g2[q,:])  * tmp
                    end
                end
            end
        end

        mK = scr_f[11]
        resize!(mK,size(g3,2))
        fill!(mK, 0.0)
        @inbounds for s in 1:n4
            for q in 1:n2
                for p in 1:n1
                    tmp = pmax[p] * qmax[q] * smax[s] * abs(c)
                    for r in 1:n3
                        @views @. mK += abs(v[p,q,r,s]) * abs.(g3[r,:]) * tmp
                    end
                end
            end
        end

        mL = scr_f[12]
        resize!(mL,size(g4,2))
        fill!(mL, 0.0)
        @inbounds for r in 1:n3
            for q in 1:n2
                for p in 1:n1
                    tmp =  pmax[p] * qmax[q] * rmax[r] * abs(c)
                    for s in 1:n4
                        @views @. mL += abs(v[p,q,r,s]) * abs.(g4[s,:]) * tmp
                    end
                end
            end
        end

        for I in 1:size(g1,2)
            if abs(mI[I]) > thresh
                push!(newI,I)
            end
        end

        for J in 1:size(g2,2)
            if abs(mJ[J]) > thresh
                push!(newJ,J)
            end
        end

        for K in 1:size(g3,2)
            if abs(mK[K]) > thresh
                push!(newK,K)
            end
        end

        for L in 1:size(g4,2)
            if abs(mL[L]) > thresh
                push!(newL,L)
            end
        end

    return newI, newJ, newK, newL
end
#=}}}=#


using Test, Seis, SeisTau

@testset "SeisTau" begin
    @testset "Travel time" begin
        let t = Trace(0, 1, 2)
            @testset "Phase supplied" begin
                @test_throws ArgumentError travel_time(t, "P")
                t.evt.lon, t.evt.lat, t.evt.dep = 0, 0, 0
                t.sta.lon, t.sta.lat = 10, 10
                @test travel_time(t, "P") == travel_time(t.evt, t.sta, "P")
                arr = travel_time(t, "P")
                @test length(arr) == 1
                @test arr[1].time ≈ 200.405669 atol=0.0001
            end
            @testset "Using picks" begin
                @test travel_times(t) == []
                t.picks.P = (0, "P")
                @test travel_times(t) == [travel_time(t, "P")]
                t′ = deepcopy(t)
                # Ignore unnamed picks
                t′.picks.X = 1
                @test travel_times(t′) == travel_times(t)
                # Ignore empty but not missing names
                t′.picks.S = 2, ""
                @test travel_times(t′) == travel_times(t)
                # Ignore picks with the wrong kind of name
                t′.picks.sS = 3, "asS"
                @test travel_times(t′) == travel_times(t)
            end
            @testset "Default phases" begin
                @test travel_time(t) == travel_time(t, "ttall")
            end
        end
    end

    @testset "Path" begin
        let t = Trace(0, 1, 2)
            @test_throws ArgumentError path(t)
            @test_throws ArgumentError path(t.evt, t.sta)
            t.evt.lon, t.evt.lat, t.evt.dep = 0, 0, 0
            t.sta.lon, t.sta.lat = 0, 80
            @test path(t) == path(t, "ttall")
            @test path(t) == path(t.evt, t.sta)
            arr = path(t, "P")
            @test length(arr) == 1
            @test arr[1].name == "P"
            @test !isempty(arr[1].radius)
            @test !isempty(arr[1].lon)
        end
    end

    @testset "Picks" begin
        # Adding travel time picks from TauPy
        let t = Trace(0, 1, rand(2))
            # Can't add travel times without correct geometry
            @test_throws ArgumentError add_picks!(t, "S")
            t.evt.lon, t.evt.lat, t.evt.dep = 0, 0, 0
            t.sta.lon, t.sta.lat = 10, 10
            add_picks!(t, "Sn", model="ak135")
            @test picks(t)[end].time ≈ 358.4306 atol=0.0001
            @test picks(t)[end].name == "Sn"
        end
    end

    @testset "Sphere" begin
        # Calculating geographic travel times using elliptical and spherical Earth
        let t = Trace(0, 1, 2)
            t.evt.lon = t.evt.lat = t.sta.lon = t.evt.dep = 0
            t.sta.lat = 45
            tt = travel_time(t, "P") # WGS84 ellipsoid
            @test tt[1].delta ≈ 44.78 atol=0.01
            tt′ = travel_time(t, "P", sphere=true) # Sphere
            @test tt′[1].delta == 45.0
        end
    end

    @testset "Checking" begin
        let e = Seis.Event(), s = Seis.Station(), t = Trace(0, 1, 2)
            t.evt = e
            t.sta = s
            @test_throws ArgumentError SeisTau._check_headers_taup(e, s)
            @test_throws ArgumentError SeisTau._check_headers_taup(t)
            e.lon, e.lat, e.dep = 0, 0, 0
            s.lon, s.lat = 10, 10
            @test SeisTau._check_headers_taup(e, s) === nothing
            @test SeisTau._check_headers_taup(t) === nothing
        end
    end
end


module MetacommunityDynamics
using DynamicGrids
using DynamicGrids:
    CellRule, SetCellRule, applyrule, Rule, Ruleset, Neighborhood, Moore, Chain
using Dispersal: OutwardsDispersal, DispersalKernel
using Distributions
using Crayons
using EcologicalNetworks

include(joinpath("types.jl"))
export Measurement

include(joinpath("layers", "layer.jl"))
export Layer
include(joinpath("layers", "occupancy.jl"))
export Occupancy
include(joinpath("layers", "biomass.jl"))
export Biomass

include(joinpath("generators", "poissonprocess.jl"))
export generate, PoissonProcess, grid, pointstogrid
include(joinpath("layers", "traits.jl"))
export StaticTraitLayer

include(joinpath("generators", "environment.jl"))
export StaticEnvironmentalLayer

include(joinpath("mechanisms", "_combinemechanisms.jl"))
include(joinpath("mechanisms", "selection.jl"))
export FitnessFunction,
    GaussianFitness,
    ExponentialFitness,
    DensityDependentFitness,
    AbioticSelection,
    BioticSelection

include(joinpath("mechanisms", "colonization.jl"))
export Colonization,
    RandomColonization, SpatiallyExplicitLevinsColonization, LevinsColonization

include(joinpath("mechanisms", "extinction.jl"))
export RandomExtinction, AbioticExtinction

include(joinpath("mechanisms", "eating.jl"))
export Eating,
    FunctionalResponse,
    HollingTypeI,
    HollingTypeII,
    HollingTypeIII,
    LinearFunctionalResponse,
    LotkaVolterra,
    MichaelisMenten,
    CrowleyMartin,
    FoodWebEating

include(joinpath("mechanisms", "mortality.jl"))
export Mortality, LinearMortality

include(joinpath("mechanisms", "dispersal.jl"))
export AdjacentBernoulliDispersal

include(joinpath("mechanisms", "growth.jl"))
export LogisticGrowth

include(joinpath("species", "speciespool.jl"))
export ContinuousSpeciesPool, DiscreteSpeciesPool
export SingleSpecies,
    DiscreteUnipartiteSpeciesPool,
    DiscreteKpartiteSpeciesPool,
    ContinuousUnipartiteSpeciesPool,
    ContinuousKpartiteSpeciesPool
export speciespool, specieslayers, nspecies, species, metaweb
include(joinpath("species", "printing.jl"))

end # module


"""
    train!(gen_fn::GenerativeFunction, data_generator::Function,
           update::ParamUpdate,
           num_epoch, epoch_size, num_minibatch, minibatch_size; verbose::Bool=false)

Train the given generative function to maximize the expected conditional log probability (density) that `gen_fn` generates the assignment `constraints` given inputs, where the expectation is taken under the output distribution of `data_generator`.

The function `data_generator` is a function of no arguments that returns a tuple `(inputs, constraints)` where `inputs` is a `Tuple` of inputs (arguments) to `gen_fn`, and `constraints` is an `ChoiceMap`.
`conf` configures the optimization algorithm used.
`param_lists` is a map from generative function to lists of its parameters.
This is equivalent to minimizing the expected KL divergence from the conditional distribution `constraints | inputs` of the data generator to the distribution represented by the generative function, where the expectation is taken under the marginal distribution on `inputs` determined by the data generator.
"""
function train!(gen_fn::GenerativeFunction, data_generator::Function,
                update::ParamUpdate;
                num_epoch=1, epoch_size=1, num_minibatch=1, minibatch_size=1,
                evaluation_size=epoch_size, verbose=false)

    history = Vector{Float64}(undef, num_epoch)
    for epoch=1:num_epoch

        # generate data for epoch
        if verbose
            println("epoch $epoch: generating $epoch_size training examples...")
        end
        epoch_inputs = Vector{Tuple}(undef, epoch_size)
        epoch_choice_maps = Vector{ChoiceMap}(undef, epoch_size)
        for i=1:epoch_size
            (epoch_inputs[i], epoch_choice_maps[i]) = data_generator()
        end

        # train on epoch data
        if verbose
            println("epoch $epoch: training using $num_minibatch minibatches of size $minibatch_size...")
        end
        for minibatch=1:num_minibatch
            permuted = Random.randperm(epoch_size)
            minibatch_idx = permuted[1:minibatch_size]
            minibatch_inputs = epoch_inputs[minibatch_idx]
            minibatch_choice_maps = epoch_choice_maps[minibatch_idx]
            for (inputs, constraints) in zip(minibatch_inputs, minibatch_choice_maps)
                (trace, _) = generate(gen_fn, inputs, constraints)
                accumulate_param_gradients!(trace)
            end
            apply!(update)
        end

        # evaluate score on held out data
        if verbose
            println("epoch $epoch: evaluating on $evaluation_size examples...")
        end
        avg_score = 0.
        for i=1:evaluation_size
            (inputs, constraints) = data_generator()
            (_, weight) = generate(gen_fn, inputs, constraints)
            avg_score += weight
        end
        avg_score /= evaluation_size

        history[epoch] = avg_score

        if verbose
            println("epoch $epoch: est. objective value: $avg_score")
        end
    end
    return history
end

"""
    score = lecture!(
        p::GenerativeFunction, p_args::Tuple,
        q::GenerativeFunction, get_q_args::Function)

Simulate a trace of p representing a training example, and use to update the gradients of the trainable parameters of q.

Used for training q via maximum expected conditional likelihood.
Random choices will be mapped from p to q based on their address.
get_q_args maps a trace of p to an argument tuple of q.
score is the conditional log likelihood (or an unbiased estimate of a lower bound on it, if not all of q's random choices are constrained, or if q uses non-addressable randomness).
"""
function lecture!(
        p::GenerativeFunction, p_args::Tuple,
        q::GenerativeFunction, get_q_args::Function)
    p_trace = simulate(p, p_args)
    q_args = get_q_args(p_trace)
    q_trace, score = generate(q, q_args, get_choices(p_trace)) # NOTE: q won't make all the random choices that p does
    accumulate_param_gradients!(q_trace)
    score
end

"""
    score = lecture_batched!(
        p::GenerativeFunction, p_args::Tuple,
        q::GenerativeFunction, get_q_args::Function)

Simulate a batch of traces of p representing training samples, and use them to update the gradients of the trainable parameters of q.

Like `lecture!` but q is batched, and must make random choices for training sample i under hierarchical address namespace i::Int (e.g. i => :z).
get_q_args maps a vector of traces of p to an argument tuple of q.
"""
function lecture_batched!(
        p::GenerativeFunction, p_args::Tuple,
        q_batched::GenerativeFunction, get_q_args::Function,
        batch_size::Int)
    p_traces = Vector{Any}(undef, batch_size)
    constraints = choicemap()
    for i=1:batch_size
        p_trace = simulate(p, p_args)
        set_submap!(constraints, i, get_choices(p_trace))
        p_traces[i] = p_trace
    end
    q_args = get_q_args(p_traces)
    q_trace, score = generate(q_batched, q_args, constraints) # NOTE: q won't make all the random choices that p does
    accumulate_param_gradients!(q_trace)
    score / batch_size
end

export train!
export lecture!
export lecture_batched!


#######################################################################
# Slice
#######################################################################
struct Slice{H<:Union{Hamiltonian,ParametricHamiltonian},A<:AbstractSparseMatrix,A´<:AbstractSparseMatrix,X,X´}
    h::H
    fullmat::A
    fullmatflat::A´
    axes::X
    axesflat::X´
end

(s::Slice{<:ParametricHamiltonian})(; params...) =
    Slice(s.h(; params...), s.fullmat, s.fullmatflat, s.axes, s.axesflat)

function Base.show(io::IO, m::MIME"text/plain", s::Slice)
    i = get(io, :indent, "")
    print(io, i, "Slice of ")
    show(io, m, parent(s))
    print(io, i, "\n",
"$i  Sliced size      : $(size(s))
$i  Flat size        : $(sizeflat(s))")
end

orbitalstructure(s::Slice) = orbitalstructure(parent(s))

matrixtype(s::Slice) = matrixtype(parent(s))

blocktype(s::Slice) = blocktype(parent(s))

blockeltype(s::Slice) = blockeltype(parent(s))

axesflat(s::Slice) = s.axesflat

axesflat(s::Slice, n) = axesflat(s)[n]

Base.parent(s::Slice) = s.h

Base.axes(s::Slice) = s.axes

Base.axes(s::Slice, n) = axes(s)[n]

Base.size(s::Slice, n...) = size(view(s.fullmat, axes(s)...), n...)

sizeflat(s::Slice, n...) = size(view(s.fullmatflat, axesflat(s)...), n...)

# bloch of slices

# This is faster than bloch!(similarmatrix(s), parent(s), args...), because
# copy!(sparse, view(sparse,...)) is slower than copy(view(sparse, ...))
function bloch(s::Slice, args...)
    h = parent(s)
    m = bloch(h, args...)
    T = eltype(m)
    axes = axes_for_eltype(s, T)
    m´ = copy(view(m, axes...))
    return m´
end

function bloch!(matrix::AbstractMatrix{T}, s::Slice, args...) where {T}
    bfull = bloch!(fullmat_for_eltype(s, T), s.h, args...)
    axes = axes_for_eltype(s, T)
    fast_sparse_copy!(matrix, bfull, axes)
    return matrix
end

# fastpath for dense arrays
similarmatrix(s::Slice, ::Type{S}) where {T<:SMatrix,S<:StridedMatrix{T}} = S(undef, size(s)...)
similarmatrix(s::Slice, ::Type{S}) where {T<:Number, S<:StridedMatrix{T}} = S(undef, sizeflat(s)...)
similarmatrix(s::Slice, ::Type{<:StridedMatrix}) = Matrix{blocktype(s)}(undef, size(s)...)

# in any other case (sparse), we need to add structural zeros
function similarmatrix(s::Slice, args...)
    mat = similarmatrix(s.h, args...)
    axes = axes_for_eltype(s, eltype(mat))
    v = view(mat, axes...)
    mat´ = copy(v)
    return mat´
end

axes_for_eltype(s, ::Type{<:SMatrix}) = s.axes
axes_for_eltype(s, ::Type{<:Number}) = s.axesflat

fullmat_for_eltype(s, ::Type{<:SMatrix}) = s.fullmat
fullmat_for_eltype(s, ::Type{<:Number}) = s.fullmatflat

# indexing into a slice
Base.getindex(s::Slice{<:Hamiltonian}, dn...) = parent(s)[dn...][axes(s)...]

# indexing of Hamiltonian and ParametricHamiltonian
function Base.getindex(ph::Union{Hamiltonian,ParametricHamiltonian}, is, js)
    h = parent(ph)
    fullmatflat, fullmat = similarmatrix_flat_unflat(h)
    rows = all_unflat_indices(h, is)
    cols = all_unflat_indices(h, js)
    flatrows = all_flat_indices(h, is)
    flatcols = all_flat_indices(h, js)
    axes = (rows, cols)
    axesflat = (flatrows, flatcols)
    return Slice(ph, fullmat, fullmatflat, axes, axesflat)
end

# axes and axesflat need not be vectors of indices. Can be Colon or other containers.
all_unflat_indices(h, i::Integer) = all_unflat_indices(h, i:i)
all_unflat_indices(h, is::Colon) = is
all_unflat_indices(h, is::AbstractRange) = is
all_unflat_indices(h, is::SiteSelector) = collect(siteindices(h.lattice, is))
all_unflat_indices(h, is) = siteindices(h.lattice, is)

all_flat_indices(::Hamiltonian{<:Any,<:Any, <:Number}, is) = is
all_flat_indices(h::Hamiltonian{<:Any,<:Any, <:Number}, is::SiteSelector) = collect(siteindices(h.lattice, is))
all_flat_indices(h::Hamiltonian{<:Any,<:Any, <:Number}, i::Integer) = all_flat_indices(h, i:i)
all_flat_indices(h::Hamiltonian{<:Any,<:Any, <:SMatrix}, i::Integer) = all_flat_indices(h, i:i)
all_flat_indices(::Hamiltonian{<:Any,<:Any, <:SMatrix}, is::Colon) = is

function all_flat_indices(h::Hamiltonian{<:Any,<:Any, <:SMatrix}, is)
    o = orbitalstructure(h)
    is´ = Int[]
    for i in siteindices(h.lattice, is)
        append!(is´, flatindices(i, o))
    end
    return is´
end

function all_flat_indices(h::Hamiltonian{<:Any,<:Any, <:SMatrix}, is::AbstractRange)
    o = orbitalstructure(h)
    is´ = intersect(is, 1:nsites(h))
    is == is´ || throw(ArgumentError("Indices $is are out of range $(1:nsites(h))"))
    istart = flatoffset_site(first(is´), o) + 1
    oend, N = flatoffsetorbs_site(last(is´), o)
    iend = oend+N
    return istart:iend
end

similarmatrix_flat_unflat(h::Hamiltonian{<:Any,<:Any, <:SMatrix}) =
    similarmatrix(h, flatten), similarmatrix(h)

function similarmatrix_flat_unflat(h::Hamiltonian{<:Any,<:Any, <:Number})
    mat = similarmatrix(h)
    return mat, mat
end