module TestScenes

using ..GfxBase
using ..Scenes
using ..Materials
using ..WWUMeshes
using ..Cameras

make_diffuse(color) = Material(0.0, 0.0, color)

black = Vec3(0,0,0)
red = Vec3(1,0,0)
green = Vec3(0,1,0)
blue = Vec3(0,0,1)
white = Vec3(1,1,1)
purple = Vec3(1,0,1)

""" Return an Array of all Triangles belonging to the given mesh, assigning
each one the given material. """
function create_triangles(mesh::OBJMesh, material)
    triangles = []

    for t in mesh.triangles
        positions = [mesh.positions[t.positions[1]], mesh.positions[t.positions[2]], mesh.positions[t.positions[3]]]

        normals = nothing
        if t.normals !== nothing && length(t.normals) == 3
            normals = [mesh.normals[t.normals[1]], mesh.normals[t.normals[2]], mesh.normals[t.normals[3]]]
        end

        uvs = nothing
        if t.uvs !== nothing && length(t.uvs) == 3
            uvs = [mesh.uvs[t.uvs[1]], mesh.uvs[t.uvs[2]], mesh.uvs[t.uvs[3]]]
        end

        push!(triangles, Triangle(positions, normals, uvs, material))
    end

    triangles
end

function mesh_helper(mesh, material, scale=Vec3(1.0,1.0,1.0), translation=Vec3(0,0,0), rotation=Vec3(0, 0, 0))
    Rx = [1 0                0
          0 cos(rotation[1]) -sin(rotation[1])
          0 sin(rotation[1]) cos(rotation[1])]

    Ry = [cos(rotation[2])  0 sin(rotation[2])
          0                 1 0
          -sin(rotation[2]) 0 cos(rotation[2])]

    Rz = [cos(rotation[3]) -sin(rotation[3]) 0
          sin(rotation[3]) cos(rotation[3])  0
          0                0                 1]

    R = Rx * Ry * Rz

    for i in 1:length(mesh.positions)
        mesh.positions[i] = (R * mesh.positions[i]) .* scale + translation
    end

    create_triangles(mesh, material)
end

function camera_1(img_height, img_width)
    CanonicalCamera(img_height, img_width)
end

function camera_2(img_height, img_width)
    eye = Vec3(20, 4, 10)
    view = Vec3(-1, 0, -5) - eye
    up = Vec3(0, 1, 0)
    focal = 8.0
    Cameras.PerspectiveCamera(eye, view, up, focal, img_height, img_width)
end

function camera_3(img_height, img_width)
    Cameras.PerspectiveCamera(Vec3(-1, 0.8, -1.2), Vec3(1, -1, -1), Vec3(0, 1, 0), 0.3, img_height, img_width)
end

cameras = [camera_1, camera_2, camera_3]

function get_camera(i, img_height, img_width)
    cameras[i](img_height, img_width)
end

function bunny()
    bunny = read_obj("data/bunny.obj")
    bunny = mesh_helper(bunny, 1, 1.0, Vec3(0.2, 0, -5))

    cube = mesh_helper(cube_mesh(), 2, 10.0, Vec3(-11.2, 0, 0))

    lights = [ Light(Float32(1.3) * white, Vec3(1,2,-5), 1),
               Light(Float32(0.8) * white, Vec3(0,0,10), 3),
               Light(Float32(0.8) * white, Vec3(0,10,10), 3),
               Light(Float32(0.8) * white, Vec3(10,10,10), 3),
               Light(Float32(0.8) * white, Vec3(0,10,0), 3) ]

    materials = [Material(0.0, 0.0, Vec3(0.6, 0.5, 0.5)), Material(0.6, 0.0, white)]
    triangles = vcat(cube, bunny)
    Scene(black, materials, triangles, lights, bvh(triangles))
end

function teapot()
    teapot = read_obj("data/teapot.obj")
    estimate_normals(teapot)
    teapot = mesh_helper(teapot, 1, 0.5, Vec3(0, -1, -5))

    cube = mesh_helper(cube_mesh(), 1, 10.0, Vec3(0, -11, 0))

    lights = [ Light(4 * white, Vec3(1,2,-5), 1) ]

    materials = [Material(0.0, 0.0, white)]
    triangles = vcat(cube, teapot)
    Scene(black, materials, triangles, lights, bvh(triangles))
end

function cornell()
    cube_mat = Material(0, 0.0, .7 * white)

    cube_back = mesh_helper(cube_mesh(), 1, 2.0, Vec3(0, 0, -8))
    cube_left = mesh_helper(cube_mesh(), 2, 2.0, Vec3(-4, 0, -6))
    cube_right = mesh_helper(cube_mesh(), 3, 2.0, Vec3(4, 0, -6))
    cube_top = mesh_helper(cube_mesh(), 1, 2.0, Vec3(0, 4, -6))
    cube_bottom = mesh_helper(cube_mesh(), 1, 2.0, Vec3(0, -4, -6))

    cube_1 = mesh_helper(cube_mesh(), 1, Vec3(0.5, 1.0, 0.5), Vec3(-0.75, -1, -5), Vec3(0, pi / 8, 0))
    cube_2 = mesh_helper(cube_mesh(), 1, 0.5, Vec3(0.75, -1.5, -5), Vec3(0, -pi / 8, 0))

    lights = [ Light(4 * white, Vec3(0,1.49,-4), 0.5) ]

    materials = [Material(0, 0.0, .7 * white), Material(0, 0.0, .7 * red), Material(0, 0.0, .7 * green)]
    triangles = vcat(cube_back, cube_left, cube_right, cube_top, cube_bottom, cube_1, cube_2)
    Scene(.7 * white, materials, triangles, lights, bvh(triangles))
end

function cubes()
    cube_bottom = mesh_helper(cube_mesh(), make_diffuse(white), 6.0, Vec3(0, -9, -12))
    cube_back = mesh_helper(cube_mesh(), make_diffuse(blue), 6.0, Vec3(0, 0, -16))
    cube_opaque = mesh_helper(cube_mesh(), make_diffuse(green), 0.5, Vec3(-2, -1, -7))
    cube_semi_transparent = mesh_helper(cube_mesh(), Material(0.0, 0.75, white), 0.5, Vec3(0, -1, -7))

    lights = [ Light(4 * white, Vec3(0, 10, 10), 3)]

    triangles = vcat(cube_bottom, cube_back, cube_opaque, cube_semi_transparent)
    Scene(black, triangles, lights, bvh(triangles))
end

function glass()
    glass_mat = Material(0, 0.5, white)
    glass = read_obj("data/glass.obj")
    estimate_normals(glass)
    glass = mesh_helper(glass, glass_mat, 0.01, Vec3(0, 0.1, -3))

    cube_mat = Material(0.6, 0.0, white)
    cube = mesh_helper(cube_mesh(), cube_mat, 10.0, Vec3(0, -11, 0))

    lights = [ Light(4 * white, Vec3(1,2,-2), 1) ]

    triangles = vcat(cube, glass)
    Scene(black, triangles, lights, bvh(triangles))
end

function get_scene(name)
    scenes = Dict("bunny" => bunny(), "cornell" => cornell(), "teapot" => teapot())
    scenes[name]
end

end


# This file is a part of Julia. License is MIT: https://julialang.org/license

using Base: @deprecate, depwarn

# 1.0 deprecations
function (+)(x::AbstractArray{<:TimeType}, y::GeneralPeriod)
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x .+ y
end
function (+)(x::StridedArray{<:GeneralPeriod}, y::TimeType)
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x .+ y
end
function (+)(y::GeneralPeriod, x::AbstractArray{<:TimeType})
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x .+ y
end
function (+)(y::TimeType, x::StridedArray{<:GeneralPeriod})
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x .+ y
end
function (-)(x::AbstractArray{<:TimeType}, y::GeneralPeriod)
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x .- y
end
function (-)(x::StridedArray{<:GeneralPeriod}, y::TimeType)
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x .- y
end

# TimeType, AbstractArray{TimeType}
function (-)(x::AbstractArray{T}, y::T) where {T<:TimeType}
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x .- y
end
function (-)(y::T, x::AbstractArray{T}) where {T<:TimeType}
    # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
    y .- x
end

for (op, Ty, Tz) in ((:*, Real, :P),
                   (:/, :P, Float64), (:/, Real, :P))
    @eval begin
        function ($op)(X::StridedArray{P}, y::$Ty) where P<:Period
            # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
            Z = similar(X, $Tz)
            for (Idst, Isrc) in zip(eachindex(Z), eachindex(X))
                @inbounds Z[Idst] = ($op)(X[Isrc], y)
            end
            return Z
        end
    end
end

function (+)(x::StridedArray{<:GeneralPeriod})
    # depwarn("non-broadcasted operations are deprecated for Dates.TimeType; use broadcasting instead", nothing)
    x
end

for op in (:+, :-)
    @eval begin
        function ($op)(X::StridedArray{<:GeneralPeriod}, Y::StridedArray{<:GeneralPeriod})
            # depwarn("non-broadcasted arithmetic is deprecated for Dates.TimeType; use broadcasting instead", nothing)
            reshape(CompoundPeriod[($op)(x, y) for (x, y) in zip(X, Y)], promote_shape(size(X), size(Y)))
        end
    end
end


const U = UInt8

"""
    encode(x) -> Vector{$U}

Encode some [ETF](http://erlang.org/doc/apps/erts/erl_ext_dist.html) data.
"""
encode(x) = U[VERSION_MAGIC; _encode(x)]

_encode(x) = error("Unknown ETF type $(typeof(x))")

# Integers

# Note: Encoding with precision does not include the tag byte.
_encode(x::Integer, p::Integer) = reverse(reinterpret(U, [x]))[end-p+1:end]
function _encode(x::Integer)
     return if 0 <= x <= 255
         U[97, x]
     elseif typemin(Int32) <= x <= typemax(Int32)
         U[98; _encode(x, 4)]
     else
         n = Int(x < 0)
         digs = digits(U, x; base=256)
         if -(BigInt(256)^255) < x < BigInt(256)^255
             U[110; round(U, log(256, abs(x))) + 1; n; digs]
         else
             U[111; _encode(round(UInt32, log(256, abs(x))) + 1, 4); n; digs]
         end
     end
end

# Floats

_encode(x::AbstractFloat) = U[V, 70, reverse(reinterpret(U, [x]))...]

# Tuples

function _encode(x::Tuple)
    len = length(x)
    tag = len <= typemax(U) ? 104 : 105
    return U[tag; _encode(len)[2:end]; vcat(_encode.(x)...)]
end

# Atoms

function _encode(x::Symbol)
    # This one is a weird, ATOM_EXT is deprecated but latest OTP still uses it.
    # Also the length limit for atoms is supposedly 255, which would fit in SMALL_ATOM_EXT.
    # However atoms of any length up to 255 are encoded as ATOM_EXT.
    # We're just going to do what the spec actually says, except for the atom length limit.
    s = Vector{U}(string(x))
    len = length(s)
    return len <= typemax(U) ? U[119; len; s] : U[118; _encode(len, 2); s]
end
_encode(x::Bool) = _encode(Symbol(x))
_encode(::Nothing) = _encode(:nil)

# Lists

function _encode(x::AbstractArray)
    return isempty(x) ? U[106] : U[108; _encode(length(x), 4); _encode.(x)...; 106]
end
_encode(x::Vector{U}) = U[107; _encode(length(x), 2); x]

# Maps

function _encode(x::AbstractDict)
    return U[
        116;
        _encode(length(x), 4);
        vcat(_encode.(vcat([[p.first, p.second] for p in x]...))...);
    ]
end

# Strings

_encode(x::AbstractString) = U[109; _encode(length(x), 4); Vector{U}(x)]


using DFTK
using JLD2
import DFTK: ScfDefaultCallback, ScfSaveCheckpoints
using Test
include("testcases.jl")

function test_scfres_agreement(tested, ref)
    @test tested.basis.model.lattice           == ref.basis.model.lattice
    @test tested.basis.model.smearing          == ref.basis.model.smearing
    @test tested.basis.model.symmetries        == ref.basis.model.symmetries
    @test tested.basis.model.spin_polarization == ref.basis.model.spin_polarization

    @test length(tested.basis.model.atoms) == length(ref.basis.model.atoms)
    @test tested.basis.model.atoms[1][2]   == ref.basis.model.atoms[1][2]

    @test tested.basis.Ecut      == ref.basis.Ecut
    @test tested.basis.kweights  == ref.basis.kweights
    @test tested.basis.ksymops   == ref.basis.ksymops
    @test tested.basis.fft_size  == ref.basis.fft_size

    kcoords_test = getproperty.(tested.basis.kpoints, :coordinate)
    kcoords_ref  = getproperty.(ref.basis.kpoints, :coordinate)
    @test kcoords_test == kcoords_ref

    @test tested.n_iter         == ref.n_iter
    @test tested.energies.total ≈  ref.energies.total atol=1e-13
    @test tested.eigenvalues    == ref.eigenvalues
    @test tested.occupation     == ref.occupation
    @test tested.ψ              == ref.ψ
    @test tested.ρ.real         == ref.ρ.real
    @test tested.ρspin.real     == ref.ρspin.real
end

if mpi_nprocs() == 1  # Not yet implemented for MPI-parallel runs
    @testset "Test checkpointing" begin
        O = ElementPsp(o2molecule.atnum, psp=load_psp("hgh/pbe/O-q6.hgh"))
        magnetic_moments = [O => [1., 1.]]
        model = model_PBE(o2molecule.lattice, [O => o2molecule.positions],
                          temperature=0.02, smearing=smearing=Smearing.Gaussian(),
                          magnetic_moments=magnetic_moments)
        basis  = PlaneWaveBasis(model, 4; kgrid=[1, 1, 1])
        ρspin  = guess_spin_density(basis, magnetic_moments)

        # Run SCF and do checkpointing along the way
        scfres = mktempdir() do tmpdir
            checkpointfile = joinpath(tmpdir, "scfres.jld2")
            callback = ScfDefaultCallback() ∘ ScfSaveCheckpoints(checkpointfile; keep=true)
            scfres = self_consistent_field(basis, tol=5e-2, ρspin=ρspin, callback=callback)
            test_scfres_agreement(scfres, load_scfres(checkpointfile))
            scfres
        end

        # Save & load scfres
        mktempdir() do tmpdir
            checkpointfile = joinpath(tmpdir, "scfres.jld2")
            save_scfres(checkpointfile, scfres)
            test_scfres_agreement(scfres, load_scfres(checkpointfile))
        end
    end
end


export NeuralNetworkApproximator, ActorCritic

using Flux

"""
    NeuralNetworkApproximator(;kwargs)

Use a DNN model for value estimation.

# Keyword arguments

- `model`, a Flux based DNN model.
- `optimizer=Descent()`
"""
Base.@kwdef struct NeuralNetworkApproximator{M,O} <: AbstractApproximator
    model::M
    optimizer::O = Descent()
end

(app::NeuralNetworkApproximator)(x) = app.model(x)

# !!! watch https://github.com/FluxML/Functors.jl/blob/master/src/functor.jl#L2
Flux.functor(x::NeuralNetworkApproximator) =
    (model = x.model,), y -> NeuralNetworkApproximator(y.model, x.optimizer)

device(app::NeuralNetworkApproximator) = device(app.model)

RLBase.update!(app::NeuralNetworkApproximator, gs) =
    Flux.Optimise.update!(app.optimizer, params(app), gs)

Base.copyto!(dest::NeuralNetworkApproximator, src::NeuralNetworkApproximator) =
    Flux.loadparams!(dest.model, params(src))

Flux.testmode!(app::NeuralNetworkApproximator, mode = true) = testmode!(app.model, mode)

#####
# ActorCritic
#####

"""
    ActorCritic(actor, critic)

The `actor` part must return a **normalized** vector representing the action values, and the `critic` part must return a state value.

TODO: remove the **normalized** restriction!!!
"""
Base.@kwdef struct ActorCritic{A,C}
    actor::A
    critic::C
end

Flux.@functor ActorCritic

function RLBase.update!(app::ActorCritic, gs)
    update!(app.actor, gs)
    update!(app.critic, gs)
end

function Flux.testmode!(app::ActorCritic, mode = true)
    testmode!(app.actor, mode)
    testmode!(app.critic, mode)
end


#
println("Ea) Two-step cascade computations and simulations following the 1s-3p photo-excitation of Si^-: AverageSCA model.")
using JLD
#
setDefaults("method: continuum, asymptotic Coulomb")    ## setDefaults("method: continuum, Galerkin")
setDefaults("method: normalization, pure sine")         ## setDefaults("method: normalization, pure Coulomb")    setDefaults("method: normalization, pure sine")
#
if  true    ## Stepwise decay cascade
setDefaults("print summary: open", "zzz-Cascade-computation.sum")

name = "Cascade after Si- 1s-3p excitation"
grid = Radial.Grid(false)
wa   = Cascade.Computation(Cascade.Computation(); name=name, nuclearModel=Nuclear.Model(14.), grid=grid, approach=Cascade.AverageSCA(),
                           scheme=Cascade.StepwiseDecayScheme([Auger, Radiative], 2, 0, Shell[], Shell[]),
                           initialConfigs=[Configuration("1s^1 2s^2 2p^6 3s^2 3p^4")] )
println(wa)
wb = perform(wa; output=true)
setDefaults("print summary: close", "")

else
setDefaults("print summary: open", "zzz-Cascade-simulation.sum")

## data = [JLD.load("zzz-cascade-decay-computations-2020-01-25T11.jld"), JLD.load("zzz-cascade-decay-computations-2020-01-25T11.jld")]
data = [JLD.load("zzz-cascade-decay-computations-2020-01-25T21.jld")]
name = "Simulation after Si- 1s-3p excitation"

wc   = Cascade.Simulation(Cascade.Simulation(), name=name, properties=Cascade.AbstractSimulationProperty[Cascade.IonDistribution()], ## , Cascade.FinalLevelDistribution()],
                          settings=Cascade.SimulationSettings(0., 0., 0., 0., [(1, 2.0), (2, 1.0), (3, 0.5)]), computationData=data )
println(wc)
wd = perform(wc; output=true)
setDefaults("print summary: close", "")

end


# Copyright (c) 2018: Matthew Wilhelm & Matthew Stuber.
# This code is licensed under MIT license (see LICENSE.md for full details)
#############################################################################
# EAGO
# A development environment for robust and global optimization
# See https://github.com/PSORLab/EAGO.jl
#############################################################################
# src/eago_optimizer/optimize/optimize.jl
# Load subproblem type optimization routines and define general optimize!
# throw_optimize_hook!.
#############################################################################

include("optimize_lp_cone.jl")
include("optimize_convex.jl")
include("optimize_nonconvex.jl")

throw_optimize_hook!(m::Optimizer) = optimize_hook!(m.ext_type, m)

function MOI.optimize!(m::Optimizer)

    m._start_time = time()

    # Runs the branch and bound routine
    if !m.enable_optimize_hook

        # Need to eliminate fixed variables prior to checking
        # for convex quadratic constraints
        initial_parse!(m)

        # Determines if the problem is an LP, MILP, SOCP, MISCOP,
        # CONVEX, OF MINCVX PROBLEM TYPE
        parse_classify_problem!(m)

        m._parse_time = m._start_time - time()

        # Throws the problem to the appropriate solution routine
        optimize!(Val{m._working_problem._problem_type}(), m)
    else

        # throws to user-defined optimization hook
        throw_optimize_hook!(m)

    end

    return nothing
end


"""
    synthesis(f0::Vector{Float64}, spectrogram::Matrix{Float64}, aperiodicity::Matrix{Float64}, fs::Int, frame_period::Float64 = DEFAULT_FRAME_PERIOD)

Synthesize an audio signal `y::Vector{Float64}` using WORLD from and `f0` track, a spectrogram, and an aperiodicity map, at the sampling rate `fs` per second.
Optionally specify a `frame_period` in milliseconds.
"""
function synthesis(f0::Vector{Float64}, spectrogram::Matrix{Float64}, aperiodicity::Matrix{Float64}, fs::Real, frame_period::Float64 = DEFAULT_FRAME_PERIOD)

    fs = convert(Int, fs)

    if size(spectrogram) != size(aperiodicity)
        error("Sizes of spectrogram $(size(spectrogram)) and aperiodicity $(size(aperiodicity)) don't match.")
    end

    nfreqs = size(spectrogram, 1)

    aperiodicity_pointer_vector = [Ref(aperiodicity, offset)
        for offset in 1:nfreqs:length(aperiodicity)]

    spectrogram_pointer_vector = [Ref(spectrogram, offset)
        for offset in 1:nfreqs:length(spectrogram)]

    fft_size = (nfreqs - 1) * 2

    f0_length = length(f0)
    y_length = trunc(Int, f0_length * frame_period * fs / 1000)
    y = Vector{Float64}(undef, y_length)

    @ccall libworld.Synthesis(
        f0::Ptr{Cdouble},
        f0_length::Cint,
        spectrogram_pointer_vector::Ptr{Ptr{Cdouble}},
        aperiodicity_pointer_vector::Ptr{Ptr{Cdouble}},
        fft_size::Cint,
        frame_period::Cdouble,
        fs::Cint,
        y_length::Cint,
        y::Ptr{Cdouble}
    )::Cvoid

    y
end


"""
Tests for the function `estimate_h2`.
Tests all fail because `ld_score_regression` is only half-implemented.
"""

include("../src/LDScore.jl")

using Main
using Main.LDScore
using Test

approx(x, y, eps) = abs(x - y) <= eps

function vector_approx(x, y, eps)
    if size(x) != size(y)
        return false
    end
    for (i, j) in enumerate(x)
        if !approx(x[i], y[i], eps) return false end
    end
    return true
end

LDScore.args["ref-ld"] = "test/test_ldscore/oneld_onefile"
LDScore.args["w-ld"] = "test/test_ldscore/w"
LDScore.args["h²"] = "test/test_sumstats/1"
LDScore.args["out"] = "test/test_out/1"

# Test case from original ldsc repo
x = LDScore.estimate_h2()

# TODO finish implementing `ld_score_regression`


#
# This file is part of module
#   ModiaMedia (ModiaMedia/src/ModiaMedia.jl)
#

### Data structures that are additionally available for media <: PartialPureSubstance -----------------------

"""
    state = setState_pT(medium, p,T)

Generate a state object for medium `medium::PureSubstance` for
pressure `p` [Pa] and temperature `T` [K].
"""
setState_pT( m::PureSubstance,p,T) = setState_pTX(m,p,T,fill(0.0,0))

"""
    state = setState_ph(medium, p,h)

Generate a state object for medium `medium::PureSubstance` for
pressure `p` [Pa] and specific enthalpy `h` [J/kg].
"""
setState_ph( m::PureSubstance,p,h) = setState_phX(m,p,h,fill(0.0,0))

"""
    state = setState_ps(medium, p,s)

Generate a state object for medium `medium::PureSubstance` for
pressure `p` [Pa] and specific entropy `s` [J/(kg*K)].
"""
setState_ps( m::PureSubstance,p,s) = setState_psX(m,p,s,fill(0.0,0))

"""
    state = setState_dT(medium, d,T)

Generate a state object for medium `medium::PureSubstance` for
density `d` [kg/m^3] and temperature `T` [K].
"""
setState_dT( m::PureSubstance,d,T) = setState_dTX(m,d,T,fill(0.0,0))

### Medium functions that are additionally available for media <: PartialPureSubstance -----------------------

"d = density_ph(medium,p,h) - return density in [kg/m^3] for `medium::PureSubstance` from pressure `p` [Pa] and specific enthalpy `h` [J/kg]."
density_ph(m::PureSubstance, p,h) = density(m, setState_ph(m,p,h))

"T = temperature_ph(medium,p,h) - return temperature in [K] for `medium::PureSubstance` from pressure `p` [Pa] and specific enthalpy `h` [J/kg]."
temperature_ph(m::PureSubstance, p,h) = temperature(m, setState_ph(m,p,h))

"p = pressure_dT(medium,d,T) - return pressure in [Pa] for `medium::PureSubstance` from density `c` in [kg/m^3] and temperature `T` [K]."
pressure_dT(m::PureSubstance, d,T) = pressure(m, setState_dT(m,d,T))

"h = specificEnthalpy_dT(medium,d,T) - return specific enthalpy in [J/kg] for `medium` from density `d` [kg/m^3] and and temperature `T` [K]."
specificEnthalpy_dT(m::PureSubstance, d,T) = specificEnthalpy(m, setState_dT(m,d,T))

### Derivatives needed for mass and energy balance ------------------------------------------------------

density_ph_der_1(m::PureSubstance, p, h) = 0.0
density_ph_der_2(m::PureSubstance, p, h) = undefinedFunction("density_ph_der_2")
density_ph_der_3(m::PureSubstance, p, h) = undefinedFunction("density_ph_der_3")


"""
struct containing grids used in a simulation

# Examples

```jldoctest
julia> using UltraDark

julia> len = 1;

julia> resol = 16;

julia> PencilGrids(len, resol);

```
"""
struct PencilGrids{K, RX, RK, CX, CK, FFT, RFFT, M}
    "Array of x positions"
    x::Array{Float64,3}
    "Array of y positions"
    y::Array{Float64,3}
    "Array of z positions"
    z::Array{Float64,3}
    "Array of x Fourier modes"
    kx::Array{Float64,3}
    "Array of y Fourier modes"
    ky::Array{Float64,3}
    "Array of z Fourier modes"
    kz::Array{Float64,3}
    "Fourier space postition array"
    k::K
    "Array of x Fourier modes for use with `rfft`"
    rkx::Array{Float64,3}
    "Array of y Fourier modes for use with `rfft`"
    rky::Array{Float64,3}
    "Array of z Fourier modes for use with `rfft`"
    rkz::Array{Float64,3}
    "Fourier space postition array for use with `rfft`"
    rk::K
    "ψ field"
    ψx::CX
    "ψ field in Fourier space"
    ψk::CK
    "density field ρ"
    ρx::RX
    "density field ρ in Fourier space"
    ρk::RK
    "gravitational potential field Φ"
    Φx::RX
    "gravitational potential field Φ in fourier space"
    Φk::RK

    "FFT plan for complex-to-complex transforms"
    fft_plan::FFT
    "FFT plan for real-to-complex transforms"
    rfft_plan::RFFT

    "Indices at which k==0.0"
    k_vanish_indices::Vector{CartesianIndex{3}}
    "Indices at which rk==0.0"
    rk_vanish_indices#::Vector{CartesianIndex{3}}

    "MPI communicator"
    MPI_COMM::M

    function PencilGrids(x, y, z, kx, ky, kz, k, rkx, rky, rkz, rk, ψx, ψk, ρx, ρk, Φx, Φk, fft_plan, rfft_plan, MPI_COMM)
        n_dims = 3
        resol_tuple = (size(x)[1], size(y)[2], size(z)[3])
        resol_tuple_realfft = (size(x)[1] ÷ 2 + 1, size(y)[2], size(z)[3])

        for var in [x, y, z, k, rk, ψx, ψk, ρx, ρk, Φx, Φk]
            @assert(ndims(var) == n_dims)
        end

        for var in [k, ψx, ψk, ρx, Φx]
            @assert(size_global(var) == resol_tuple)
        end

        for var in [rk, ρk, Φk]
            @assert(size_global(var) == resol_tuple_realfft)
        end

        @assert(size(x) == (resol_tuple[1], 1, 1))
        @assert(size(y) == (1, resol_tuple[2], 1))
        @assert(size(z) == (1, 1, resol_tuple[3]))

        k_vanish_indices = findall(x -> x==0.0, k)
        rk_vanish_indices = findall(x -> x==0.0, rk)

        K = typeof(k)
        RX = typeof(Φx)
        RK = typeof(Φk)
        CX = typeof(ψx)
        CK = typeof(ψk)
        FFT = typeof(fft_plan)
        RFFT = typeof(rfft_plan)
        M = typeof(MPI_COMM)

        new{K, RX, RK, CX, CK, FFT, RFFT, M}(x, y, z, kx, ky, kz, k, rkx, rky, rkz, rk, ψx, ψk, ρx, ρk, Φx, Φk, fft_plan, rfft_plan, k_vanish_indices, rk_vanish_indices, MPI_COMM)
    end
end

"""
    PencilGrids(length, resol)

Constructor for `PencilGrids`

Create an empty grid with length `length` and resolution `resol`.  Uses `PencilFFTs` to create `PencilArrays`.

# Examples

```jldoctest
julia> using UltraDark

julia> PencilGrids(1.0, 64);

```
"""
function PencilGrids(length, resol::Int)::PencilGrids

    PencilGrids((length, length, length), (resol, resol, resol))

end

"""
    PencilGrids(length_tuple, resol_tuple::Tuple{Int, Int, Int})

Constructor for `PencilGrids`

Create an empty `length[1]`x`length[2]`x`length[3]` grid with resolution
`resol[1]`x`resol[2]`x`resol[3]`.
Each grid is a `PencilArray`, allowing multiprocess FFTs.

# Examples

```jldoctest
julia> using UltraDark

julia> PencilGrids((1.0, 1.0, 0.5), (64, 64, 32));

```
"""
function PencilGrids(length_tuple, resol_tuple::Tuple{Int, Int, Int})::PencilGrids

    resol_tuple_realfft = (resol_tuple[1] ÷ 2 + 1, resol_tuple[2], resol_tuple[3])

    @assert length_tuple[1] / resol_tuple[1] ≈ length_tuple[2] / resol_tuple[2]
    @assert length_tuple[1] / resol_tuple[1] ≈ length_tuple[3] / resol_tuple[3]

    if ~MPI.Initialized()
        MPI.Init()
    end

    # MPI topology information
    comm = MPI.COMM_WORLD
    Nproc = MPI.Comm_size(comm)

    # Let MPI_Dims_create choose the decomposition.
    proc_dims = let pdims = zeros(Int, 2)
        MPI.Dims_create!(Nproc, pdims)
        pdims[1], pdims[2]
    end

    FFTW.set_num_threads(Threads.nthreads())
    # Plan a 3D complex-to-complex (c2c) FFT.
    fft_plan = PencilFFTPlan(resol_tuple, Transforms.FFT(), proc_dims, comm)

    # Allocate ψx and ψk arrays
    ψx = allocate_input(fft_plan)
    ψx .= 0
    ψk = allocate_output(fft_plan)

    # Plan a 3D real-to-complex (r2c) FFT.
    rfft_plan = PencilFFTPlan(resol_tuple, Transforms.RFFT(), proc_dims, comm)

    ρx = allocate_input(rfft_plan)
    ρk = allocate_output(rfft_plan)

    Φx = allocate_input(rfft_plan)
    Φk = allocate_output(rfft_plan)

    x = reshape(
                range(
                      -length_tuple[1] / 2 + length_tuple[1] / 2resol_tuple[1],
                      +length_tuple[1] / 2 - length_tuple[1] / 2resol_tuple[1],
                      length=resol_tuple[1]
                     ),
                resol_tuple[1], 1, 1
    )

    y = reshape(
                range(
                      -length_tuple[2] / 2 + length_tuple[2] / 2resol_tuple[2],
                      +length_tuple[2] / 2 - length_tuple[2] / 2resol_tuple[2],
                      length=resol_tuple[2]
                     ),
                1, resol_tuple[2], 1
    )

    z = reshape(
                range(
                      -length_tuple[3] / 2 + length_tuple[3] / 2resol_tuple[3],
                      +length_tuple[3] / 2 - length_tuple[3] / 2resol_tuple[3],
                      length=resol_tuple[3]
                     ),
                1, 1, resol_tuple[3]
    )

    kvec = k_vec(length_tuple, resol_tuple)

    kx = reshape(kvec[1], size(kvec[1])[1], 1, 1)
    ky = reshape(kvec[2], 1, size(kvec[2])[1], 1)
    kz = reshape(kvec[3], 1, 1, size(kvec[3])[1])
    k_norm = (kx.^2 .+ ky.^2 .+ kz.^2).^0.5  #TODO: don't allocate full array on every node

    k = similar(ψk, Float64)
    k_glob = global_view(k)
    _k = k_norm
    for I in CartesianIndices(k_glob)
        k_glob[I] = _k[I]
    end

    rkvec = rk_vec(length_tuple, resol_tuple)

    rkx = reshape(rkvec[1], size(rkvec[1])[1], 1, 1)
    rky = reshape(rkvec[2], 1, size(rkvec[2])[1], 1)
    rkz = reshape(rkvec[3], 1, 1, size(rkvec[3])[1])
    rk_norm = (rkx.^2 .+ rky.^2 .+ rkz.^2).^0.5

    rk = similar(ρk, Float64)
    rk_glob = global_view(rk)
    _rk = rk_norm
    for I in CartesianIndices(rk_glob)
        rk_glob[I] = _rk[I]
    end

    PencilGrids(
        x, y, z,
        kx, ky, kz, k,
        rkx, rky, rkz, rk,
        ψx, ψk,
        ρx, ρk,
        Φx, Φk,
        fft_plan, rfft_plan,
        comm
    )
end


module CodeMathOverrides
  #
  # Julia requires these imports
  #
  import Base.+
  import Base.-
  import Base.*
  import Base./
  import Base.&
  import Base.|
  import Base.$
  import Base.>>
  import Base.<<
  import Base.%

  export +
  export -
  export *
  export /
  export &
  export |
  export $
  export >>
  export <<
  export %
  #
  # Shorthand for Float16 type
  #
  const f16 = Float16::DataType
  #
  # Max positive Float16 value
  #
  const F16_MAX = f16(65504.0)
  #
  # Max negative Float16 value
  #
  const F16_MIN = f16(-65504.0)
  #
  # Max positive Float64 value
  #
  const F16_MAX_FLOAT = 65504.0::Float64
  #
  # Max negative Float64 value
  #
  const F16_MIN_FLOAT = -65504.0::Float64
  #
  # Max positive Int value
  #
  const F16_MAX_INT = 65504::Int
  #
  # Max negative Int value
  #
  const F16_MIN_INT = -65504::Int
  #
  # Override for + operator for Float16 type. Checks type
  # overflow and returns max or min value in this case.
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function +(left::f16, right::f16)
    local res::Float64 = Float64(left) + Float64(right)

    if res > F16_MAX_FLOAT return F16_MAX
    elseif res < F16_MIN_FLOAT return F16_MIN end
    #
    # We have to use + operator for Float64, because
    # + for Float16 we have already overridden. This is
    # how we prevent StackOverflow error
    #
    f16(res)
  end
  #
  # Override for - operator for Float16 type. Checks type
  # overflow and returns max or min value in this case.
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function -(left::f16, right::f16)
    local res::Float64 = Float64(left) - Float64(right)

    if res > F16_MAX_FLOAT return F16_MAX
    elseif res < F16_MIN_FLOAT return F16_MIN end
    #
    # We have to use - operator for Float64, because
    # - for Float16 we have already overridden. This is
    # how we prevent StackOverflow error
    #
    f16(res)
  end
  #
  # Override for * operator for Float16 type. Checks type
  # overflow and returns max or min value in this case.
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function *(left::f16, right::f16)
    local res::Float64 = Float64(left) * Float64(right)

    if res > F16_MAX_FLOAT return F16_MAX
    elseif res < F16_MIN_FLOAT return F16_MIN end
    #
    # We have to use * operator for Float64, because
    # * for Float16 we have already overridden. This is
    # how we prevent StackOverflow error
    #
    f16(res)
  end
  #
  # Override for / operator for Float16 type. Checks type
  # overflow and returns max or min value in this case.
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function /(left::f16, right::f16)
    local res::Float64 = Float64(left) / Float64(right)

    if res > F16_MAX_FLOAT return F16_MAX
    elseif res < F16_MIN_FLOAT return F16_MIN end
    #
    # We have to use / operator for Float64, because
    # / for Float16 we have already overridden. This is
    # how we prevent StackOverflow error
    #
    f16(res)
  end
  #
  # Override for & operator for Float16 type. We have to
  # convert Float16 values into Int, because & operator
  # works only with integers
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function (&)(left::f16, right::f16)
    f16(round(Int, left) & round(Int, right))
  end
  #
  # Override for | operator for Float16 type. We have to
  # convert Float16 values into Int, because | operator
  # works only with integers
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function (|)(left::f16, right::f16)
    f16(round(Int, left) | round(Int, right))
  end
  #
  # Override for $ operator for Float16 type. We have to
  # convert Float16 values into Int, because $ operator
  # works only with integers
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function ($)(left::f16, right::f16)
    f16(round(Int, left) $ round(Int, right))
  end
  #
  # Override for >> operator for Float16 type. Checks type
  # overflow and returns max or min value in this case.
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function (>>)(left::f16, right::f16)
    local res::Int = round(Int, left) >> round(Int, right)

    if res > F16_MAX_INT return F16_MAX
    elseif res < F16_MIN_INT return F16_MIN end

    f16(res)
  end
  #
  # Override for << operator for Float16 type. Checks type
  # overflow and returns max or min value in this case.
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function (<<)(left::f16, right::f16)
    local res::Int = round(Int, left) << round(Int, right)

    if res > F16_MAX_INT return F16_MAX
    elseif res < F16_MIN_INT return F16_MIN end

    f16(res)
  end
  #
  # Override for << operator for Float16 type. Checks type
  # overflow and returns max or min value in this case.
  # @param left Left operand
  # @param right Right operand
  # @return {Float16}
  #
  function (%)(left::f16, right::f16)
    local r::Int = round(Int, right)
    f16(round(Int, left) % (r === 0 ? 1 : r))
  end
end


using NamedTupleTools: namedtuple, merge

# -------------- Implementation of Product Basis

struct Product1pBasis{NB, TB <: Tuple} <: OneParticleBasis{Any}
   bases::TB
   indices::Vector{NTuple{NB, Int}}
end

function Product1pBasis(bases)
   NB = length(bases)
   return Product1pBasis(bases, NTuple{NB, Int}[])
end

import Base.*
*(B1::OneParticleBasis, B2::OneParticleBasis) =
      Product1pBasis((B1, B2))
*(B1::Product1pBasis, B2::OneParticleBasis) =
      Product1pBasis((B1.bases..., B2))
*(B1::OneParticleBasis, B2::Product1pBasis) =
      Product1pBasis((B1, B2.bases...))
*(B1::Product1pBasis, B2::Product1pBasis) =
      Product1pBasis((B1.bases..., B2.bases...))
*(B1::Product1pBasis, B2::B1pComponent) =
      Product1pBasis((B1.bases..., B2))

_numb(b::Product1pBasis{NB}) where {NB} = NB

Base.length(basis::Product1pBasis) = length(basis.indices)

function Base.show(io::IO, basis::Product1pBasis)
   print(io, "Product1pBasis")
   print(io, basis.bases)
end

Base.getindex(basis::Product1pBasis, i::Integer) = basis.bases[i]

function Base.getindex(basis::Product1pBasis, label::AbstractString)
   inds = findall(getlabel.(basis.bases) .== label)
   if length(inds) == 0
      error("label not found amongst 1p basis components")
   elseif length(inds) > 1
      error("label not unique amongst 1p basis components")
   end
   return basis.bases[inds[1]]
end

# ------------------------- FIO CODES

==(B1::Product1pBasis, B2::Product1pBasis) =
      ( all(B1.bases .== B2.bases) &&
        B1.indices == B2.indices )

write_dict(B::Product1pBasis) =
      Dict("__id__" => "ACE_Product1pBasis",
            "bases" => write_dict.(B.bases),
          "indices" => B.indices )

function read_dict(::Val{:ACE_Product1pBasis}, D::Dict)
   bases = tuple( read_dict.(D["bases"])... )
   indices = [ tuple(v...) for v in D["indices"] ]
   return Product1pBasis(bases, indices)
end

# ----------------- evaluation of the basis

import Base.Cartesian: @nexprs

function _write_A_code(VA, NB)
   prodBi = Meta.parse("B_1[ϕ[1]]" * prod(" * B_$i[ϕ[$i]]" for i = 2:NB))
   if VA == Nothing
      getVT = "promote_type(" * prod("eltype(B_$i), " for i = 1:NB) * ")"
      getA = Meta.parse("_A = zeros($(getVT), length(basis))")
   else
      getA = :(_A = A)
   end
   return prodBi, getA
end

"""
`add_into_A!` : this is an internal function implementing the main evaluation
for the one-particle basis and possibly add it into the A basis.

There are two ways to call it.
* Use `A = nothing` for the first argument to allocate the necessary memory to
evaluate the 1p basis into it.
* Use `A::Vector{T}` to evaluate the 1p basis and add it into `A` directly
without additional allocation.
"""
@generated function add_into_A!(A::VA, basis::Product1pBasis{NB}, X) where {NB, VA}
   prodBi, getA = _write_A_code(VA, NB)
   quote
      # evaluate the 1p basis components
      @nexprs $NB i -> begin
         bas_i = basis.bases[i]
         B_i = evaluate(bas_i, X)
      end
      # allocate A if necessary or just name _A = A if A is a buffer
      $(getA)
      # evaluate the 1p product basis functions and add/write into _A
      for (iA, ϕ) in enumerate(basis.indices)
         @inbounds _A[iA] += $prodBi
      end
      # release the memory allocated by the 1p basis components they normally
      # use preallocated chache to avoid too many small allocations.
      @nexprs $NB i -> release!(B_i)
      return _A
   end
end

evaluate(basis::Product1pBasis, X::AbstractState) =
      add_into_A!(nothing, basis, X)

function evaluate(basis::Product1pBasis, cfg::UConfig)
   @assert length(cfg) > 0 "Product1pBasis can only be evaluated with non-empty configurations"
   # evaluate the first item "manually", then so we know the output types
   # but then write directly into the allocated array to avoid additional
   # allocations.
   A = evaluate(basis, first(cfg))
   for (i, X) in enumerate(cfg)
      i == 1 && continue;
      add_into_A!(A, basis, X)
   end
   return A
end

function evaluate!(A::AbstractVector, basis::Product1pBasis, X::AbstractState)
   fill!(A, zero(eltype(A)))
   add_into_A!(A, basis, X)
   return A
end

function evaluate!(A, basis::Product1pBasis, cfg::UConfig)
   fill!(A, zero(eltype(A)))
   for X in cfg
      add_into_A!(A, basis, X)
   end
   return A
end

# -------------------- jacobian codes, forward rule

function _write_dA_code(VDA, NB)
   if VDA == Nothing
      getDVT = "promote_type(eltype(_A), " * prod("eltype_dB_$i, " for i = 1:NB) * ")"
      getdA = Meta.parse("_dA = zeros($(getDVT), length(basis))")
   else
      getdA = :(_dA = dA)
   end

end

# args... could be a symbol to enable partial derivatives
# at the moment this is just passed into evaluate_ed!(...)
# to not evaluate 1p basis derivatives that aren't needed, but
# in the future a more complex construction could be envisioned that
# might considerably reduce the computational cost here...
@generated function _add_into_A_dA!(A::VA, dA::VDA, basis::Product1pBasis{NB}, X,
                                    args...) where {VA, VDA, NB}
   prodBi, getA = _write_A_code(VA, NB)
   getdA = _write_dA_code(VDA, NB)
   quote
      Base.Cartesian.@nexprs($NB, i -> begin   # for i = 1:NB
         bas_i = basis.bases[i]
         if !(bas_i isa Discrete1pBasis)
            # only evaluate basis gradients for a continuous basis
            B_i, dB_i = evaluate_ed(bas_i, X, args...)
            eltype_dB_i = eltype(dB_i)
         else
            # we still need the basis values for the discrete basis though
            # TODO: maybe the d part should be a no-op and remove this
            # case distinction ...
            B_i = evaluate(bas_i, X)
            eltype_dB_i = Bool
         end
      end)
      # allocate A if necessary or just name _A = A if A is a buffer
      $(getA)
      $(getdA)

      for (iA, ϕ) in enumerate(basis.indices)
         # evaluate A
         @inbounds _A[iA] += $prodBi

         # evaluate dA
         # TODO: redo this with adjoints!!!!
         #     also reverse order of operations to make fewer multiplications!
         _dA[iA] = zero(eltype(_dA))
         Base.Cartesian.@nexprs($NB, a -> begin  # for a = 1:NB
            if !(basis.bases[a] isa Discrete1pBasis)
               dt = dB_a[ϕ[a]]
               Base.Cartesian.@nexprs($NB, b -> begin  # for b = 1:NB
                  if b != a
                     dt *= B_b[ϕ[b]]
                  end
               end)
               _dA[iA] += dt
            end
         end)
      end
      Base.Cartesian.@nexprs($NB, i -> ( begin   # for i = 1:NB
         # release_B!(bas_i, B_i)
         release!(B_i)
         if !(basis.bases[i] isa Discrete1pBasis)
            release!(dB_i)
         end
      end))
      return _A, _dA
   end
end

# this is a hack to resolve a method ambiguity.
add_into_A_dA!(A, dA, basis::Product1pBasis, X) =
               _add_into_A_dA!(A, dA, basis, X)
add_into_A_dA!(A, dA, basis::Product1pBasis, X, sym::Symbol) =
               _add_into_A_dA!(A, dA, basis, X, sym)

evaluate_ed(basis::Product1pBasis, X::AbstractState) =
         _add_into_A_dA!(nothing, nothing, basis, X)

function evaluate_ed(basis::Product1pBasis, Xs::UConfig)
   A, dA1 = evaluate_ed(basis, first(Xs))
   dA = zeros(eltype(dA1), length(basis), length(Xs))
   dA[:, 1] .= dA1[:]
   release!(dA1)
   for (i, X) in enumerate(Xs)
      i == 1 && continue;
      _add_into_A_dA!(A, (@view dA[:, i]), basis, X)
   end
   return A, dA
end

function evaluate_d(basis::Product1pBasis, X::Union{AbstractState, UConfig})
   A, dA = evaluate_ed(basis, X)
   release!(A)
   return dA
end

# ------------- Partial derivative functionality

_check_args_is_sym() = true
_check_args_is_sym(::Symbol) = true

# args... may be empty or a symbol  for partial derivatives
function evaluate_ed!(A, dA, basis::OneParticleBasis,
                     cfg::UConfig, args...)
   @assert _check_args_is_sym(args...)
   fill!(A, 0)
   for (j, X) in enumerate(cfg)
      add_into_A_dA!(A, (@view dA[:, j]), basis, X, args...)
   end
   return A, dA
end

# args... may be empty or a symbol for partial derivatives
function evaluate_ed!(A, dA, basis::Product1pBasis, X::AbstractState, args...)
   @assert _check_args_is_sym(args...)
   fill!(A, 0)
   add_into_A_dA!(A, dA, basis, X, args...)
   return A, dA
end

# ----------------------------------------

_symbols_prod(bases) = tuple(union( symbols.(bases)... )...)

symbols(basis::Product1pBasis) = _symbols_prod(basis.bases)

function indexrange(basis::Product1pBasis)
   allsyms = tuple(symbols(basis)...)
   rg = Dict{Symbol, Vector{Any}}([ sym => [] for sym in allsyms]...)
   for b in basis.bases
      rgb = indexrange(b)
      for sym in allsyms
         if haskey(rgb, sym)
            rg[sym] = union(rg[sym], rgb[sym])
         end
      end
   end
   # HACK: fix the m range based on the maximal l-range
   #       this needs to be suitably generalised if we have multiple
   #       (l, m) pairs, e.g. (l1, m1), (l2, m2)
   if haskey(rg, :m)
      maxl = maximum(rg[:l])
      rg[:m] = collect(-maxl:maxl)
   end

   # convert the range into a named tuple so that we remember the order!!
   return NamedTuple{allsyms}(ntuple(i -> rg[allsyms[i]], length(allsyms)))
end

isadmissible(b, basis::Product1pBasis) = all(isadmissible.(Ref(b), basis.bases))

function set_spec!(basis::Product1pBasis{NB}, spec) where {NB}
   empty!(basis.indices)
   for b in spec
      inds = ntuple(i -> get_index(basis.bases[i], b), NB)
      push!(basis.indices, inds)
   end
   return basis
end

get_spec(basis::Product1pBasis) = [ get_spec(basis, i) for i = 1:length(basis) ]

function get_spec(basis::Product1pBasis, i::Integer)
   inds = basis.indices[i]
   specs = get_spec.(basis.bases, inds)
   # TODO: here we should check that we are only merging compatible tuples,
   #       e.g. (n = 5, l = 2), (l = 2, m = -1) is ok
   #       but  (n = 5, l = 2), (l = 3, m = -1) is forbidden!
   return merge(specs...)
end

degree(b, basis::Product1pBasis) = sum( degree(b, B) for B in basis.bases )

degree(b::NamedTuple, basis::Product1pBasis, weight::Dict) =
      sum( degree(b, B, weight) for B in basis.bases )

# TODO: this looks like a horrible hack ...
function rand_radial(basis::Product1pBasis)
   for B in basis.bases
      if B isa ScalarACEBasis
         return rand_radial(B)
      end
   end
   return nothing
end

# -------------- sparsification

function sparsify!(basis1p::Product1pBasis, keep::AbstractVector{<: NamedTuple})
   # spec, keep, new_spec will be lists of named tuples,
   #                    e.g. [ (n = , l = , m = ), ... ]
   spec = get_spec(basis1p)
   new_spec = eltype(spec)[]
   new_inds = Vector{Int}(undef, length(spec))
   for (ib, b) in enumerate(spec)
      if b in keep
         push!(new_spec, b)
         new_inds[ib] = length(new_spec)
      end
   end

   # now we need to recompute the indices array, this can be easily done via
   # set_spec!(basis::Product1pBasis{NB}, spec), but before we do that
   # we should sparsify the basis components as well
   #  .... but it is not so clear that his is a good idea, maybe the
   #       1p basis components should just remain frozen???
   #       => turn this off for now
   # TODO - return to this point?!?!?
   # for bas_i in basis1p.bases
   #    _sparsify_component!(bas_i, new_spec)
   # end

   # finally fix the basis1pspec internally:
   set_spec!(basis1p, new_spec)

   # return the old to new index mapping so that the pibasis can fix itself.
   return basis1p, new_inds
end

using NamedTupleTools: select

# """
# this performs some generic work to sparsify a 1p-basis component.
# but the actual sparsificatin happens in the individual basis implementations
# """
# function _sparsify_component!(basis1p, keep)
#    # if basis1p has no symbols (e.g. a multiplier) then it means it must
#    # be a one-component basis, so there is nothing to sparsify.
#    syms = symbols(basis1p)
#    if isempty(syms)
#       return basis1p
#    end
#    # get rid of all info we don't need
#    keep1 = unique( select.(keep, Ref(syms)) )
#    # double-check that keep1 is compatible
#    spec = get_spec(basis1p)
#    @assert all(b in spec for b in keep1)
#    # now get the basis spec and get the list of indices to keep
#    if length(keep1) < length(spec)
#       # Ikeep = findall( [b in keep1 for b in spec] )
#       # sparsify!(basis1p, Ikeep)
#       sparsify!(basis1p, keep1)
#    end
#    return basis1p
# end

# --------------- AD codes

# import ChainRules: rrule, NoTangent, ZeroTangent

# _evaluate_bases(basis::Product1pBasis{NB}, X::AbstractState) where {NB} =
#       ntuple(i -> evaluate(basis.bases[i], X), NB)

# _evaluate_A(basis::Product1pBasis{NB}, BB) where {NB} =
#       [ prod(BB[i][ϕ[i]] for i = 1:NB) for ϕ in basis.indices ]

# evaluate(basis::Product1pBasis, X::AbstractState) =
#       _evaluate_A(basis, _evaluate_bases(basis, X))

# function _rrule_evaluate(basis::Product1pBasis{NB}, X::AbstractState,
#                          w::AbstractVector{<: Number},
#                          BB = _evaluate_bases(basis, X)) where {NB}
#    VT = promote_type(valtype(basis, X), eltype(w))

#    # dB = evaluate_d(basis, X)
#    # return sum( (real(w) * real(db) + imag(w) * imag(db))
#    #             for (w, db) in zip(w, dB) )

#    # Compute the differentials for the individual sub-bases
#    Wsub = ntuple(i -> zeros(VT, length(BB[i])), NB)
#    for (ivv, vv) in enumerate(basis.indices)
#       for t = 1:NB
#          _A = one(VT)
#          for s = 1:NB
#             if s != t
#                _A *= BB[s][vv[s]]
#             end
#          end
#          Wsub[t][vv[t]] += w[ivv] * conj(_A)
#       end
#    end

#    # now these can be propagated into the inner basis
#    #  -> type instab to be fixed here
#    g = sum( _rrule_evaluate(basis.bases[t], X, Wsub[t] )
#             for t = 1:NB )
#    return g
# end

# function rrule(::typeof(evaluate), basis::Product1pBasis, X::AbstractState)
#    BB = _evaluate_bases(basis, X)
#    A = _evaluate_A(basis, BB)
#    return A,
#       w -> (NoTangent(), NoTangent(), _rrule_evaluate(basis, X, w, BB))
# end

#    function _rrule_evaluate(basis::Scal1pBasis, X::AbstractState,
#       w::AbstractVector{<: Number})
# x = _val(X, basis)
# a = _rrule_evaluate(basis.P, x, w)
# TDX = ACE.dstate_type(a, X)
# return TDX( NamedTuple{(_varsym(basis),)}( (a,) ) )
# end


using Gadfly, Compose

set_default_plot_size(4inch,9inch)

p=plot(y=[1,2,3], Geom.line)
c=Compose.context()
vstack(Union{Plot,Compose.Context}[p,c,p,c,p])


export timer, TimerObservable

import Base: ==
import Base: show

"""
    timer(delay::Int)
    timer(delay::Int, period::Int)

Creation operator for the `TimerObservable`. Its like `interval`(@ref), but you can specify when should the emissions start.
`timer` returns an Observable that emits an infinite sequence of ascending integers,
with a constant interval of time, period of your choosing between those emissions.
The first emission happens after the specified `delay`.
If `period` is not specified, the output Observable emits only one value, 0.
Otherwise, it emits an infinite sequence.

# Arguments
- `delay`: the initial delay time specified as an integer denoting milliseconds to wait before emitting the first value of 0.
- `period`: the minimum period of time between emissions of the subsequent numbers (in milliseconds).

# Examples
```
using Rocket

source = timer(0, 50)

sleep(0.075)
subscription = subscribe!(source, logger())
sleep(0.105)
unsubscribe!(subscription)

close(source)
;

# output

[LogActor] Data: 2
[LogActor] Data: 3

```

See also: [`interval`](@ref), [`TimerObservable`](@ref), [`subscribe!`](@ref), [`logger`](@ref)
"""
function timer(delay::Int)
    @assert delay >= 0 "'delay' argument should be positive"
    return TimerObservable(delay, 0)
end

function timer(delay::Int, period::Int)
    @assert delay >= 0 "'delay' argument should be positive"
    @assert period >= 0 "'period' argument should be positive"
    return TimerObservable(delay, period)
end

"""
    TimerObservable

An Observable that starts emitting after an `delay` and emits
ever increasing numbers after each `period` of time thereafter.

# Parameters
- `delay`: The initial delay time specified as an integer denoting milliseconds to wait before emitting the first value of 0`.
- `period`: The minimum period of time in milliseconds between emissions of the subsequent numbers.

See also: [`timer`](@ref), [`Subscribable`](@ref)
"""
struct TimerObservable <: Subscribable{Int}
    delay  :: Int
    period :: Int
end

getdelay_ms(observable::TimerObservable)  = observable.delay
getperiod_ms(observable::TimerObservable) = observable.period

getdelay_sec(observable::TimerObservable)  = getdelay_ms(observable) / MILLISECONDS_IN_SECOND
getperiod_sec(observable::TimerObservable) = getperiod_ms(observable) / MILLISECONDS_IN_SECOND

struct TimerSubscription <: Teardown
    timer  :: Timer
end

as_teardown(::Type{ <: TimerSubscription }) = UnsubscribableTeardownLogic()

mutable struct TimerActorProps
    counter :: Int
end

struct TimerActor{A} <: Actor{ Nothing }
    actor :: A
    once  :: Bool
    props :: TimerActorProps
end

TimerActor(actor::A, once::Bool) where A = TimerActor{A}(actor, once, TimerActorProps(0))

getcounter(actor::TimerActor) = actor.props.counter
itcounter!(actor::TimerActor) = actor.props.counter = actor.props.counter + 1

isonce(actor::TimerActor) = actor.once

function on_next!(actor::TimerActor, ::Nothing)
    next!(actor.actor, getcounter(actor))
    if isonce(actor)
        complete!(actor)
    else
        itcounter!(actor)
    end
end

on_error!(actor::TimerActor, err) = error!(actor.actor, err)
on_complete!(actor::TimerActor)   = complete!(actor.actor)

function on_subscribe!(observable::TimerObservable, actor)

    tactor   = TimerActor(actor, getperiod_ms(observable) === 0)

    callback = let tactor = tactor
        (timer) -> begin
            try
                next!(tactor, nothing)
            catch err
                error!(tactor, err)
                close(timer)
            end
        end
    end

    timer = Timer(callback, getdelay_sec(observable), interval = getperiod_sec(observable))

    return TimerSubscription(timer)
end

function on_unsubscribe!(subscription::TimerSubscription)
    close(subscription.timer)
    return nothing
end

Base.:(==)(t1::TimerObservable, t2::TimerObservable) = getdelay_ms(t1) === getdelay_ms(t1) && getperiod_ms(t1) === getperiod_ms(t2)

Base.show(io::IO, observable::TimerObservable)   = print(io, "TimerObservable($(getdelay_ms(observable)), $(getperiod_ms(observable)))")
Base.show(io::IO, ::TimerSubscription)           = print(io, "TimerSubscription()")


# Combines two 1-output circuits c1 and c2 with the same paramterers and number of gates into a 2-output circuit
# The output values of the result is the 2-output goal whose components are the output values of c1 and of c2.
function combine_circuits( c1::Chromosome, c2::Chromosome )
  # Two functions used only within this function
  inp1(i,numinputs) = i <= numinputs ? i : 2*(i-numinputs)+numinputs-1
  inp2(i,numinputs) = i <= numinputs ? i : 2*(i-numinputs)+numinputs
  @assert c1.params.numinputs == c2.params.numinputs
  @assert c1.params.numoutputs == 1
  @assert c2.params.numoutputs == 1
  @assert c1.params.numinteriors == c2.params.numinteriors
  @assert c1.params.numlevelsback == c2.params.numlevelsback
  if length(c1.interiors) < length(c2.interiors)
    println("  c1ints: ",length(c1.interiors),"  c2ints: ",length(c2.interiors))
    ctemp = c1
    c1 =c2
    c2 = ctemp
  end
  #println("c1ints: ",length(c1.interiors),"  c2ints: ",length(c2.interiors))
  p = Parameters( c1.params.numinputs, 2, c1.params.numinteriors+c2.params.numinteriors,
    c1.params.numlevelsback+c2.params.numlevelsback )
  inputs = c1.inputs
  interiors = InteriorNode[]
  new_numinteriors = 2*length(c2.interiors)
  #println("new_numints: ",new_numinteriors)
  #print_parameters(p)
  for i = 1:length(c2.interiors)
    int_node1 = deepcopy(c1.interiors[i])
    input1 = inp1(c1.interiors[i].inputs[1],p.numinputs)
    input2 = inp1(c1.interiors[i].inputs[2],p.numinputs)
    int_node1.inputs = [input1,input2]
    #println("i: ",i,"  int_node1: ",int_node1)
    push!(interiors,int_node1)
    int_node2 = deepcopy(c2.interiors[i])
    input1 = inp2(c2.interiors[i].inputs[1],p.numinputs)
    input2 = inp2(c2.interiors[i].inputs[2],p.numinputs)
    int_node2.inputs = [input1,input2]
    #println("i: ",i,"  int_node2: ",int_node2)
    push!(interiors,int_node2)
  end
  #println("interiors: ",interiors)
  outputs = [OutputNode(p.numinputs+new_numinteriors-1),OutputNode(p.numinputs+new_numinteriors)]
  Chromosome( p, c1.inputs, interiors, outputs, 0.0, 0.0 )
end

# Starting with circuit c (and its parameters), try to evolve a circuit whose output is the same as c
#   and whose number of active nodes is reduced.
# Version of 5/7/21
function evolve_reduce_numactive( c::Chromosome, maxsteps::Int64; num_mutations::Int64=1, funcs::Vector{Func}=default_funcs(c.params.numinputs)  )
  p = c.params
  goal = output_values(c)
  current_num_active = number_active_gates( c )
  for i = 1:maxsteps
    new_c = c
    for _ = 1:num_mutations
      new_c = deepcopy(c)
      mutate_chromosome!( new_c, funcs )
    end
    if output_values(new_c) == goal && number_active_gates( new_c ) <= current_num_active
      c = new_c
      if number_active_gates( c ) < current_num_active
        #println("number active reduced from ",current_num_active," to ",number_active_gates( new_c )," on step ",i,".")
        current_num_active = number_active_gates( c )
      end
    end
  end
  (c, current_num_active )
end

# Use neutral evolution to find a chromosome c that maximizes funct(c)
# funct( c::Chromosome ) returns a Float64.
function evolve_function( funct, p::Parameters, funcs::Vector{Func}, max_steps::Int64;
    goallist::Vector{Vector{MyInt}}=[MyInt[]], max_evolve_steps::Int64=10000 )
  println("goallist: ",goallist)
  use_goal = !(goallist==[MyInt[]])
  c = random_chromosome( p, funcs )
  orig_c = deepcopy(c)
  goal = goallist[1]
  println("goal: ",goal)
  if use_goal
    result = mut_evolve( c, goallist, funcs, max_evolve_steps, print_improvements = true )
    steps = result[2]
    if steps == max_evolve_steps
      error("evolution to goal failed in function evolve_function()")
    else
      println("evolution to goal suceeded in function evolve_function()")
    end
    c = result[1]
    goal = output_values(c)
    orig_c = deepcopy(c)
  end
  current_fitness = funct(c)
  println("starting fitness: ",current_fitness)
  for i = 1:max_steps
    prev_c = deepcopy(c)
    (c,active) = mutate_chromosome!(c,funcs)
    if use_goal && output_values(c) != goal
      c = prev_c
      continue
    end
    new_fitness = funct(c)
    if new_fitness < current_fitness
      c = prev_c
    elseif new_fitness > current_fitness
      println("i: ",i,"  new_fitness: ", new_fitness,"  funct(c): ",funct(c) )
      current_fitness = new_fitness
    end
    println("i: ",i,"  current_fitness: ", current_fitness, "  funct(c): ",funct(c)  )
  end
  (c,orig_c)
end

# to test circuit_evolve()
function test_evolve()
  p = Parameters(numinputs=2,numoutputs=1,numinteriors=3,numlevelsback=3);
  funcs=default_funcs(p.numinputs);c = random_chromosome(p,funcs); goal=output_values(c)
  res = mut_evolve(c,[goal],funcs,1000); c_dest=res[1]
  circuit_evolve(c,c_dest, 1000 )
end

# Attempts to reduce the number active of the chromosome c by mutational neutral evolution
# Older version written prior to 7/18/20.
# The chomosome c is assumed to be at optimal fitness, and this function will terminate with
#   an error if it is not optimal.
# Then mutation is done nreps times, and the mutation is accepted if number_active(c) is reduced.
function mut_reduce_numactive( c::Chromosome, goallist::GoalList, funcs::Vector{Func}, nreps::Integer;
      hamming_sel::Bool=true, num_mutations::Int64=1, print_improvements::Bool=false )
  output = output_values(c)   # Executes c if it has not already been executed
  goals_matched = hamming_sel ? goals_matched_hamming : goals_matched_exact
  ( fitness, matched_goals, matched_goals_list ) = goals_matched( output, goallist, c.params.numinputs )
  if sort(output) != sort(goallist[matched_goals[1]])
    error("input chromosome must be have optimal fitness in function mut_reduce_numactive()")
  end
  best_num_active = number_active(c)
  #println("init num_active  num_active: ",best_num_active)
  orig_c = deepcopy(c)
  #println("initial fitness: ",fitness)
  step = 0
  worse = 0
  same = 0
  better = 0
  prev_c = deepcopy(c)  # previous generation c
  best_c = c
  @assert output_values(prev_c) == output_values(orig_c)
  (c, matched_goals, matched_goals_list ) = next_chromosome!(c, goallist, funcs,
      hamming_sel=hamming_sel, use_robustness=false, num_mutations=num_mutations )
  new_num_active = number_active(c)
  #println("before while new_num_active: ",new_num_active)
  #while step < nreps && new_fitness < c.params.numoutputs
  while step < nreps
    #println("step: ",step,"  output: ",output,"   ")
    if c.fitness > fitness  # This should never happen
      error("c.fitness > fitness")
      if print_improvements
        matched_goal_components = [map( x->x[1], mgl) for mgl in matched_goals_list]
        println("fitness improved from ",fitness," to ",c.fitness," at step: ",step, "  goals_matched: ", matched_goals, "  matched goal components: ", matched_goal_components)
      end
      fitness = c.fitness
      worse = 0
      same = 0
      better += 1
    elseif c.fitness == fitness && new_num_active < best_num_active
      #println("new chromosome with fitness: ",fitness, "  and num_active: ", new_num_active )
      if print_improvements
        matched_goal_components = [map( x->x[1], mgl) for mgl in matched_goals_list]
        println("num_active decreased from ",best_num_active," to ",new_num_active," at step: ",step,"  goals_matched: ", matched_goals, "  matched goal componets: ", matched_goal_components)
      end
      worse = 0
      same = 0
      better += 1
      best_c = deepcopy(c)
      @assert new_num_active == number_active(c)
      best_num_active = new_num_active
      #println("num_active_improved  new_num_active: ",new_num_active)
    elseif c.fitness == fitness && new_num_active == best_num_active
      #println("new chromosome with fitness: ",fitness, "  and num_active: ", new_num_active )
      same += 1
    elseif c.fitness == fitness && new_num_active > best_num_active
      #println("discarded chromosome with fitness: ",fitness, " and num_active: ", new_num_active )
      c = prev_c
      worse += 1
    elseif c.fitness < fitness
      #println("discarded chromosome with new_fitness: ",c.fitness)
      c = prev_c
      worse += 1
    end
    step += 1
    prev_c = deepcopy(c)
    if step < nreps
      (c, matched_goals, matched_goals_list ) = next_chromosome!(c, goallist, funcs,
            hamming_sel=hamming_sel, use_robustness=false, num_mutations=num_mutations )
      #output = output_values(c)   # Executes c if it has not already been executed
      new_num_active = number_active(c)
      #println("end while new_num_active: ",new_num_active)
    end
  end
  if step == nreps
    #println("mut_reduce_numactive finished at step limit ",nreps," with fitness: ", best_c.fitness, " and num_active: ", best_num_active )
  else
    error("step == nreps")
    println("mut_reduce_numactive finished in ",step," steps with fitness: ", c.fitness, " and num_active: ", new_num_active )
  end
  if orig_c.fitness > c.fitness
    error("orig_c.fitness > c.fitness")
    #println("(orig_c.fitness,c.fitness): ",(orig_c.fitness,c.fitness))
    output = output_values(orig_c)
    ( fitness, matched_goals, matched_goals_list ) = goals_matched( output, goallist, c.params.numinputs )
    return(orig_c,step,0,0,0,output,matched_goals,matched_goals_list)
  end
  ##println("worse: ",worse,"  same: ",same,"  better: ",better)
  #println("matched_goals: ",matched_goals)
  if step < nreps
    if print_improvements
      matched_goal_components = [map( x->x[1], mgl) for mgl in matched_goals_list]
      println("num_active decreased from ",num_active," to ",new_num_active," at step: ",step, "  goals_matched: ", matched_goals, "  matched goal componets: ", matched_goal_components)
    end
    @assert sort(output) == sort(goallist[matched_goals[1]])
  end
  ( fitness, matched_goals, matched_goals_list ) = goals_matched( output, goallist, best_c.params.numinputs )
  #println("assert  best_num_active: ",best_num_active,"  number_active(best_c): ",number_active(best_c))
  @assert best_num_active == number_active(best_c)
  (best_c,step,worse,same,better,output,matched_goals,matched_goals_list)
end

# Evaluate the performance of function evolve_reduce_numactive() which is in evolve_funct.jl.
# n_repeats is the maximum number of tries in mut_evolve_repeat()
# For each pair of chromosomes
#   Creates two chromosomes using parameter settings p.
#   Combines them into one chromosome with one more input
#   For each trial run evolve_reduce_numactive() and record average and minimum numactive.
function test_reduce_numactive( p::Parameters, numpairs::Int64, numtrials::Int64, n_repeats::Int64, maxsteps::Int64;
      num_mutations::Int64=1, csvfile::String="" )
  df = DataFrame()
  df.combined_goal = Goal[]
  df.numinputs = Int64[]
  df.numgates = Int64[]
  df.numlevelsback = Int64[]
  df.numpairs = Int64[]
  df.numtrials = Int64[]
  df.maxsteps = Int64[]
  df.nactive0 = Float64[]
  df.nactive1_min = Float64[]
  df.nactive1_mean = Float64[]
  df.nactive1_std = Float64[]
  if num_mutations >= 2
    df.nactive2_min = Float64[]
    df.nactive2_mean = Float64[]
    df.nactive2_std = Float64[]
  end
  if num_mutations == 3
    df.nactive3_min = Float64[]
    df.nactive3_mean = Float64[]
    df.nactive3_std = Float64[]
  end
  funcs = default_funcs(p.numinputs)
  for npair = 1:numpairs
    #c1 = random_chromosome(p); print(output_values(c1),"  "); print_circuit(c1)
    #c2 = random_chromosome(p); print(output_values(c2),"  "); print_circuit(c2)
    goal1 = randgoal(p)
    c = random_chromosome(p)
    (nc1,step,worse,same,better,output,matched_goals,matched_goals_list) = mut_evolve_repeat(n_repeats, p, [goal1], funcs, maxsteps )
    goal2 = randgoal(p)
    (nc2,step,worse,same,better,output,matched_goals,matched_goals_list) = mut_evolve_repeat(n_repeats, p, [goal2], funcs, maxsteps )
    c = combine_circuits(nc1,nc2)
    rowlist = [ output_values(c), p.numinputs, p.numinteriors, p.numlevelsback, numpairs, numtrials, maxsteps ]
    #@assert [output_values(nc1)[1],output_values(nc1)[1]] == output_values(c)
    println("output_values(c): ",output_values(c))
    println("goal: ",[output_values(nc1)[1],output_values(nc2)[1]])
    #nactive_lists = map(i->reduce_nactive_helper(c,maxsteps,num_mutations=num_mutations), collect(1:numtrials ) )
    nactive_lists = pmap(i->reduce_nactive_helper(c,maxsteps,num_mutations=num_mutations), collect(1:numtrials ) )
    nactive0 = vcat( [ na[1] for na in nactive_lists]... )
    nactive1 = vcat( [ na[2] for na in nactive_lists]... )
    if num_mutations == 1
      row_results = [mean(nactive0),minimum(nactive1),mean(nactive1),std(nactive1)]
    end
    if num_mutations >= 2
      nactive2 = vcat( [ na[3] for na in nactive_lists]... )
      row_results = [mean(nactive0),minimum(nactive2),mean(nactive1),std(nactive1),minimum(nactive2),mean(nactive2),std(nactive2)]
    end
    if num_mutations >= 3
      nactive3 = vcat( [ na[4] for na in nactive_lists]... )
      row_results = [mean(nactive0),minimum(nactive2),mean(nactive1),std(nactive1),minimum(nactive2),mean(nactive2),std(nactive2),minimum(nactive3),mean(nactive3),std(nactive3)]
    end
    row = deepcopy(rowlist)
    println("length(row): ",length(row),"  length(row_results): ",length(row_results),"  size(df): ",size(df))
    #println("npair: ",npair,"  mean(nactive1): ",mean(nactive1),"  mean(nactive2): ",mean(nactive2),"  mean(nactive3): ",mean(nactive3))
    push!(df,vcat(row,row_results))
  end
  hostname = chomp(open("/etc/hostname") do f read(f,String) end)
  if length(csvfile) > 0
    open( csvfile, "w" ) do f
      println(f,"# date and time: ",Dates.now())
      println(f,"# host: ",hostname," with ",nprocs()-1,"  processes: " )
      println(f,"# funcs: ", Main.CGP.default_funcs(p.numinputs))
      print_parameters(f,p,comment=true)
      println(f,"# numpairs: ", numpairs)
      println(f,"# numtrials: ", numtrials)
      println(f,"# maxsteps: ", maxsteps)
      println(f,"# n_repeats: ", n_repeats)
      CSV.write( f, df, append=true, writeheader=true )
    end
  end
  df
end

# nactive0 is number_active_gates() after evolution to find circuit that maps to goal
#    but before running evolve_reduce_numactive()
function reduce_nactive_helper( c::Chromosome, maxsteps::Int64; num_mutations::Int64=1 )
  #println("num_mutations: ",num_mutations)
  goal = output_values( c )
  (ec,steps) = neutral_evolution( random_chromosome(c.params), goal, maxsteps )
  nactive0 = number_active_gates(ec)
  #println("nactive0: ",nactive0)
  nactive1 = Int64[]
  nc = deepcopy(c)
  (ec,cur_nactive) = evolve_reduce_numactive( nc, maxsteps, num_mutations=1 )
  @assert cur_nactive == number_active_gates(ec)
  push!(nactive1,number_active_gates(ec))
  if num_mutations == 1
    return(nactive0, nactive1)
  end
  nactive2 = Int64[]
  nc = deepcopy(c)
  (ec,cur_nactive) = evolve_reduce_numactive( nc, maxsteps, num_mutations=2 )
  push!(nactive2,number_active_gates(ec))
  if num_mutations == 2
    return(nactive0, nactive1, nactive2)
  end
  nactive3 = Int64[]
  nc = deepcopy(c)
  (ec,cur_nactive) = evolve_reduce_numactive( nc, maxsteps, num_mutations=3 )
  push!(nactive3,number_active_gates(ec))
  #println("trial: ",trial,"  mean(nactive1): ",mean(nactive1),"  mean(nactive2): ",mean(nactive2),"  mean(nactive2): ",mean(nactive2))
  (nactive0, nactive1, nactive2, nactive3)
end

# folder_path is relative to data folder
#function compare_dataframe( folder_path::String, suffix::String; csvfile::String="" )
function compare_dataframe( folder_path::String, suffix::String )
  data_prefix = "../data/"
  rdf = read_dataframe("$(data_prefix)$(folder_path)/reduce_numactive_$(folder_path)_$(suffix).csv")
  kdf = read_dataframe("$(data_prefix)$(folder_path)/k_complexity$(folder_path)$(suffix).csv")
  cdf = DataFrame()
  cdf.goal = kdf.goal
  cdf.kng = kdf.num_gates
  cdf.rng = rdf.nactive1_min
  cdf.k_minus_r_diff = cdf.kng .- cdf.rng
  csvfile = "$(data_prefix)$(folder_path)/compare_$(folder_path)_$(suffix).csv"
  lines = String[]
  open( "$(data_prefix)$(folder_path)/reduce_numactive_$(folder_path)_$(suffix).csv", "r" ) do infile
    line = readline(infile)
    while line[1] == '#'
      #println("line: ",line)
      push!(lines,line)
      line = readline(infile)
    end
  end
  hostname = chomp(open("/etc/hostname") do f read(f,String) end)
  if length(csvfile) > 0
    open( csvfile, "w" ) do f
      for line in lines
        println(f, line)
      end
      CSV.write( f, df, append=true, writeheader=true )
    end
  end
  cdf
end

# goal1 is the first uncombined goal
# goal2 is the first uncombined goal
# p is the parameters for these goals
function compare_two_vs_one_output( p::Parameters, numtrials::Int64, maxsteps::Int64 )
  @assert p.numoutputs == 1
  goal1 = randgoal(p)
  goal2 = randgoal(p)
  pc = Parameters( p.numinputs+1, 1, 2*p.numinteriors, 2*p.numlevelsback )
  pb = Parameters( p.numinputs, 2, 2*p.numinteriors, 2*p.numlevelsback )
  println("pc: ",pc)
  numbits = 2^p.numinputs
  goal = [(deepcopy(goal1)[1] <<= numbits) | goal2[1]]
  println("[goal]: ",[goal],"  [goal1,goal2]: ",[goal1[1],goal2[1]])
  csteps_sum = 0
  c_successes = 0
  bsteps_sum = 0
  b_successes = 0
  for i = 1:numtrials
    cr = random_chromosome(pc)
    #println("[goal]: ",[goal])
    (cnc,csteps) = neutral_evolution( cr, goal, maxsteps )
    if csteps < maxsteps
      csteps_sum += csteps
      c_successes += 1
    end
    cb = random_chromosome(pb)
    #println("[goal1,goal2]: ",[goal1[1],goal2[1]])
    (bnc,bsteps) = neutral_evolution( cb, [goal1[1],goal2[1]], maxsteps )
    if bsteps < maxsteps
      bsteps_sum += bsteps
      b_successes += 1
    end
    #println("(csteps,bsteps): ",(csteps,bsteps))
  end
  println("csteps_avg: ",c_successes > 0 ? csteps_sum/c_successes : maxsteps )
  println("bsteps_avg: ",b_successes > 0 ? bsteps_sum/b_successes : maxsteps )
  (goal, numtrials, c_successes,c_successes > 0 ? csteps_sum/c_successes : maxsteps, b_successes, b_successes > 0 ? bsteps_sum/b_successes : maxsteps )
end

function run_compare_two_vs_one_output( p::Parameters, numruns::Int64, numtrials::Int64, maxsteps::Int64; csvfile::String="" )
  df = DataFrame()
  df.goal = Goal[]
  df.numtrials = Int64[]
  df.c_successes = Int64[]
  df.c_avg = Float64[]
  df.b_successes = Int64[]
  df.b_avg = Float64[]
  rows = pmap( x->compare_two_vs_one_output( p, numtrials, maxsteps ), collect(1:numruns) )
  for r in rows
    push!( df, r )
  end
  if length(csvfile) > 0
    open( csvfile, "w" ) do f
      hostname = chomp(open("/etc/hostname") do f read(f,String) end)
      println(f,"# date and time: ",Dates.now())
      println(f,"# host: ",hostname," with ",nprocs()-1,"  processes: " )
      println(f,"# funcs: ", Main.CGP.default_funcs(p.numinputs))
      print_parameters(f,p,comment=true)
      println(f,"# numruns: ",numruns)
      println(f,"# numtrials: ",numtrials)
      println(f,"# maxsteps: ",maxsteps)
      CSV.write( f, df, append=true, writeheader=true )
    end
  end
  df
end


using Test
using ForestBiometrics

@test typeof(sdi_chart(500,8)) == Plots.Plot{Plots.GRBackend}
@test typeof(sdi_chart(500,8;maxsdi=600)) == Plots.Plot{Plots.GRBackend}


"""
### function plot_bode(gain, phase, w, num_u, num_y)

Plot the bode diagram.

##### Args

* gain: Gain [dB].
* phase: Phase [deg].
* w: Frequencies [rad/s].
* num_u: Number of inputs.
* num_y: Number of outputs.

"""

function plot_bode(gain, phase, w, num_u, num_y)
    if (plot_system == :pyplot)
        # Get the number of graphics.
        num_plots = length(gain[1])

        fig, ax = plt.subplots(num_y*2, num_u, sharex=true, sharey=true)

        for i = 1:num_plots
            gain_i  = map(x->x[i], gain)
            phase_i = map(x->x[i], phase)

            ax[2(i-1)+1][:semilogx](w, gain_i,  linewidth=plw)
            ax[2(i-1)+2][:semilogx](w, phase_i,  linewidth=plw)
            ax[2(i-1)+1][:grid]("on", which="major",
                         color="#AAAAAA", linewidth=glw)
            ax[2(i-1)+2][:grid]("on", which="major",
                         color="#AAAAAA", linewidth=glw)

            # Change figure border width.
            ax[2(i-1)+1][:spines]["top"][:set_linewidth](blw)
            ax[2(i-1)+1][:spines]["bottom"][:set_linewidth](blw)
            ax[2(i-1)+1][:spines]["left"][:set_linewidth](blw)
            ax[2(i-1)+1][:spines]["right"][:set_linewidth](blw)

            ax[2(i-1)+2][:spines]["top"][:set_linewidth](blw)
            ax[2(i-1)+2][:spines]["bottom"][:set_linewidth](blw)
            ax[2(i-1)+2][:spines]["left"][:set_linewidth](blw)
            ax[2(i-1)+2][:spines]["right"][:set_linewidth](blw)
        end

        for i = 1:num_u
            (num_u > 1) && ax[2*num_y*(i-1)+1][:set_title]("From Input $i")
            ax[2*num_y*i][:set_xlabel]("Freq. (rad/s)")
        end

        for i = 1:num_y
            ax[2(i-1)+1][:set_ylabel]("Gain (dB)")
            ax[2(i-1)+2][:set_ylabel]("Phase (deg)")

            ax_aux = ax[2(num_u-1)*num_y+2(i-1)+1][:twinx]()
            ax_aux[:yaxis][:set_label_position]("right")
            ax_aux[:set_yticks]([])
            ax_aux[:set_ylabel]("To Output $i")

            ax_aux = ax[2(num_u-1)*num_y+2(i-1)+2][:twinx]()
            ax_aux[:yaxis][:set_label_position]("right")
            ax_aux[:set_yticks]([])
            ax_aux[:set_ylabel]("To Output $i")
        end

        fig[:suptitle]("Bode Diagram", fontsize=tifsz)
    end
end


function accept(obj::StorelessUnivariateStatistic, arg0::jdouble)
    return jcall(obj, "accept", void, (jdouble,), arg0)
end

function and_then(obj::DoubleConsumer, arg0::DoubleConsumer)
    return jcall(obj, "andThen", DoubleConsumer, (DoubleConsumer,), arg0)
end

function clear(obj::FirstMoment)
    return jcall(obj, "clear", void, ())
end

function copy(obj::FirstMoment)
    return jcall(obj, "copy", FirstMoment, ())
end

function equals(obj::AbstractStorelessUnivariateStatistic, arg0::Object)
    return jcall(obj, "equals", jboolean, (Object,), arg0)
end

function evaluate(obj::StorelessUnivariateStatistic, arg0::Vector{jdouble}, arg1::jint, arg2::jint)
    return jcall(obj, "evaluate", jdouble, (Vector{jdouble}, jint, jint), arg0, arg1, arg2)
end

function evaluate(obj::UnivariateStatistic, arg0::Vector{jdouble})
    return jcall(obj, "evaluate", jdouble, (Vector{jdouble},), arg0)
end

function get_class(obj::Object)
    return jcall(obj, "getClass", Class, ())
end

function get_n(obj::FirstMoment)
    return jcall(obj, "getN", jlong, ())
end

function get_result(obj::FirstMoment)
    return jcall(obj, "getResult", jdouble, ())
end

function hash_code(obj::AbstractStorelessUnivariateStatistic)
    return jcall(obj, "hashCode", jint, ())
end

function increment(obj::FirstMoment, arg0::jdouble)
    return jcall(obj, "increment", void, (jdouble,), arg0)
end

function increment_all(obj::StorelessUnivariateStatistic, arg0::Vector{jdouble})
    return jcall(obj, "incrementAll", void, (Vector{jdouble},), arg0)
end

function increment_all(obj::StorelessUnivariateStatistic, arg0::Vector{jdouble}, arg1::jint, arg2::jint)
    return jcall(obj, "incrementAll", void, (Vector{jdouble}, jint, jint), arg0, arg1, arg2)
end

function notify(obj::Object)
    return jcall(obj, "notify", void, ())
end

function notify_all(obj::Object)
    return jcall(obj, "notifyAll", void, ())
end

function to_string(obj::AbstractStorelessUnivariateStatistic)
    return jcall(obj, "toString", JString, ())
end

function wait(obj::Object)
    return jcall(obj, "wait", void, ())
end

function wait(obj::Object, arg0::jlong)
    return jcall(obj, "wait", void, (jlong,), arg0)
end

function wait(obj::Object, arg0::jlong, arg1::jint)
    return jcall(obj, "wait", void, (jlong, jint), arg0, arg1)
end


import Colors: Colorant, Color, RGB, red, green, blue, parse, convert
import PyCall: @pyimport, PyObject
@pyimport cycler
import PyPlot: plt

# I need a way to convert the types from Colors.jl (RGB{8}(r, g, b) -> python tuple)
# this code is directly from PyCall.jl -> conversions.jl for tuple conversion
function PyObject(t::Color)
    trgb = convert(RGB, t)
    ctup = map(float, (red(trgb), green(trgb), blue(trgb)))
    o = PyObject(ctup)
    return o
end

#TODO: add RGBA versions

# string color names
# the order will matter ... or maybe I should have it return multiple
color_defs = [svg_rgb, crayons, xkcd_rgb]

function colorname2rgb(name::String)
    for def in color_defs
        if haskey(def, name)
            return parse(Colorant, def[name])
        end
    end
    throw(ArgumentError("Not a Color name: $name"))
end

#NC for Named Color
macro NC_str(name::String)
    c = colorname2rgb(name)
    return :($c)
end

#HC for Hex Color
macro HC_str(hexstr::String)
    #TODO: test to ensure actually a hex string
    c = parse(Colorant, hexstr)
    return :($c)
end

# Named color palettes
SEABORN_PALETTES = Dict(
:deep => [HC"#4C72B0", HC"#55A868", HC"#C44E52", HC"#8172B2", HC"#CCB974", HC"#64B5CD"],
:muted => [HC"#4878CF", HC"#6ACC65", HC"#D65F5F", HC"#B47CC7", HC"#C4AD66", HC"#77BEDB"],
:pastel => [HC"#92C6FF", HC"#97F0AA", HC"#FF9F9A", HC"#D0BBFF", HC"#FFFEA3", HC"#B0E0E6"],
:bright => [HC"#003FFF", HC"#03ED3A", "#E8000B", HC"#8A2BE2", HC"#FFC400", HC"#00D7FF"],
:dark => [HC"#001C7F", HC"#017517", "#8C0900", HC"#7600A1", HC"#B8860B", HC"#006374"],
:colorblind => [HC"#0072B2", HC"#009E73", HC"#D55E00", HC"#CC79A7", HC"#F0E442", HC"#56B4E9"])

color_cycle(carr::AbstractArray) = cycler.cycler("color", carr)

#TODO: this needs a more a more julian name something like style!, palette!
function set_palette(name::Symbol)
    colprop = color_cycle(SEABORN_PALETTES[name])
    plt[:rc]("axes", prop_cycle=colprop)
end


"""
    InternalTranslation <: InternalTransition

Saves a translation as described by [`Translation`](@ref) for the current frame.
Is part of the [`Action`](@ref) struct.
"""
mutable struct InternalTranslation <: InternalTransition
    by::Point
end

"""
    InternalRotation <: InternalTransition

Saves a rotation as described by [`Rotation`](@ref) for the current frame.
Is part of the [`Action`](@ref) struct.
"""
mutable struct InternalRotation <: InternalTransition
    angle::Float64
    center::Point
end

"""
    InternalScaling <: InternalTransition

Saves a scaling as described by [`Scaling`](@ref) for the current frame.
Is part of the [`Action`](@ref) struct.
"""
mutable struct InternalScaling <: InternalTransition
    scale::Tuple{Float64,Float64}
end

"""
    Translation <: Transition

Stores the `Point` or a link for the start and end position of the translation

# Fields
`from::Union{Point, Symbol}`: The start position or a link to the start position.
    See `:red_ball` in [`javis`](@ref)
`to::Union{Point, Symbol}`: The end position or a link to the end position
"""
struct Translation <: Transition
    from::Union{Point,Symbol}
    to::Union{Point,Symbol}
end

"""
    Translation(p::Union{Point, Symbol})

Create a `Translation(O, p)` such that a translation is done from the origin.
"""
Translation(p::Union{Point,Symbol}) = Translation(O, p)

"""
    Translation(x::Real, y::Real)

Create a `Translation(O, Point(x,y))` such that a translation is done from the origin.
Shorthand for writing `Translation(Point(x,y))`.
"""
Translation(x::Real, y::Real) = Translation(Point(x, y))

"""
    Rotation <: Transition

Stores the rotation similar to [`Translation`](@ref) with `from` and `to`
but also the rotation point.

# Fields
- `from::Union{Float64, Symbol}`: The start rotation or a link to it
- `to::Union{Float64, Symbol}`: The end rotation or a link to it
- `center::Union{Point, Symbol}`: The center of the rotation or a link to it.
"""
struct Rotation <: Transition
    from::Union{Float64,Symbol}
    to::Union{Float64,Symbol}
    center::Union{Point,Symbol}
end

"""
    Rotation(r::Union{Float64, Symbol})

Rotation as a transition from 0.0 to `r` .
Can be used as a short-hand.
"""
Rotation(r::Union{Float64,Symbol}) = Rotation(0.0, r)

"""
    Rotation(r::Union{Float64, Symbol}, center::Union{Point, Symbol})

Rotation as a transition from `0.0` to `r` around `center`.
Can be used as a short-hand for rotating around a `center` point.
"""
Rotation(r::Union{Float64,Symbol}, center::Union{Point,Symbol}) = Rotation(0.0, r, center)

"""
    Rotation(from, to)

Rotation as a transition from `from` to `to` (in radians) around the origin.
"""
Rotation(from, to) = Rotation(from, to, O)

"""
    Scaling <: Transition

Stores the scaling similar to [`Translation`](@ref) with `from` and `to`.

# Example
- Can be called with different constructors like:
```
Scaling(10) -> Scaling(CURRENT_SCALING, (10.0, 10.0))
Scaling(10, :my-scale) -> Scaling((10.0, 10.0), :my_scale)
Scaling(10, 2) -> Scaling((10.0, 10.0), (2.0, 2.0))
Scaling(10, (1,2)) -> Scaling((10.0, 10.0), (1.0, 2.0))
```

# Fields
- `from::Union{Tuple{Float64, Float64}, Symbol}`: The start scaling or a link to it
- `to::Union{Tuple{Float64, Float64}, Symbol}`: The end scaling or a link to it
- `compute_from_once::Bool`: Saves whether the from is computed for the first frame or
    every frame. Is true if from is `:_current_scale`.
"""
mutable struct Scaling <: Transition
    from::Union{Tuple{Float64,Float64},Symbol}
    to::Union{Tuple{Float64,Float64},Symbol}
    compute_from_once::Bool
end

Scaling(to::Tuple) = Scaling(:_current_scale, to, true)
Scaling(to::Real) = Scaling(:_current_scale, convert(Float64, to), true)
Scaling(to::Symbol) = Scaling(:_current_scale, to, true)

function Scaling(from::Real, to::Real, compute_from_once = false)
    from_flt = convert(Float64, from)
    to_flt = convert(Float64, to)
    Scaling((from_flt, from_flt), (to_flt, to_flt), compute_from_once)
end

function Scaling(from::Real, to, compute_from_once = false)
    flt = convert(Float64, from)
    Scaling((flt, flt), to, compute_from_once)
end

function Scaling(from, to::Real, compute_from_once = false)
    flt = convert(Float64, to)
    Scaling(from, (flt, flt), compute_from_once)
end


module Gunrock
    include("config.jl")
	include("bfs.jl")
	include("cc.jl")
	include("sssp.jl")
	include("pagerank.jl")
	include("bc.jl")
	export bfs, cc, sssp, bc, pagerank
end


# utility functions

"""
`update_bin_weights!`: Update the bin weights from the ensemble

### Arguments
* `B` - bin data structure
* `E` - particle ensemble
"""
function update_bin_weights!(B::TB, E::TE) where {TB<:Bins, TE<:Ensemble}

   n_bins = length(B);

   # this loops over bins
   for i in 1:n_bins
      particle_ids = findall(isequal(i), E.b);

      B.ν[i] = sum(E.ω[particle_ids]);
      B.n[i] = length(particle_ids);
   end
   B
end

"""
`Points_to_Bins`: Convenience function for constructing a bin structure using a
sequence of points

### Arguments
`points` - An array of points defining the Voronoi cells
"""
function Points_to_Bins(points; d::TD = Nothing()) where {TD}

   B = Bins{typeof(points[1]), Float64, Int, Int, TD}([],[],[],[],[]);
   for point in points
      push!(B, point , 0, 0, 0, deepcopy(d));
   end
   return B
end

"""
`Voronoi_to_Bins`: Convenience function for constructing a bin structure using a
sequence of points as the Voronoi sites

### Arguments
`sites` - An array of points defining the Voronoi cells
"""
function Voronoi_to_Bins(sites; d::TD = Nothing()) where {TD}

   B = Bins{typeof(sites[1]), Float64, Int, Int, TD}([],[],[],[],[]);
   for site in sites
      push!(B, site , 0, 0, 0, deepcopy(d));
   end
   return B
end

"""
`Voronoi_bin_id`: Convenience function for bin id in Voronoi based bins

### Arguments
`X` - Point thats bin is to be determined
`tree` - A nearest neighbors tree structure constructed with `KDTree`
"""
function Voronoi_bin_id(X, tree)
   return knn(tree, X, 1)[1][1]
end

"""
`setup_Voronoi_bins` - Convenience function for constructing bins, a bin id function,
and a rebinning function based on a set of Voronoi points

### Arguments
`voronoi_pts` - User specified Voronoi cell centers
"""
function setup_Voronoi_bins(voronoi_pts; d::TD = Nothing()) where {TD}
   # construct the bins
   B = Voronoi_to_Bins(voronoi_pts, d = d);
   # construct the bin id function
   tree = KDTree(hcat(voronoi_pts...));
   bin_id = x-> Voronoi_bin_id(x,tree);
   # define the rebinning function
   function rebin!(E, B, t)
       @. E.b = bin_id(E.ξ);
      update_bin_weights!(B, E);
      E, B
   end

   return B, bin_id, rebin!

end

"""
`Dirac_to_Ensemble`: Convenience function for construction an ensemble from a
single initial walker.  This hard codes the weights to be Float64.

### Arguments
`X` - Starting state of all walkers
`n_particles` - Number of walkers in the ensemble
"""
function Dirac_to_Ensemble(X::TP, n_particles::TI; d::TD = Nothing()) where {TP, TI<:Integer, TD}
   ω = 1.0/n_particles
   E = Ensemble{TP, Float64, TI, TD}([deepcopy(X) for _ in 1:n_particles],
                                    [deepcopy(X) for _ in 1:n_particles],
                                    ω * ones(n_particles),ω * ones(n_particles),
                                    zeros(TI, n_particles),zeros(TI, n_particles),
                                    zeros(TI, n_particles),
                                    [deepcopy(d) for _ in 1:n_particles],
                                    [deepcopy(d) for _ in 1:n_particles]);
   return E
end

function trivial_analysis!(E, B, t)
   E, B
end


function link_epcdata(env,project,activeTiles)

    # Link EPC data
    DataSource = "EPC"
    selected_cols = [:OSG_UPRN,:Dwelling_Type,:Property_Type,:Historical,
        :Total_floor_area_m2_,:Prop_Summ_Total_Floor_Area,:Top_Storey,
        :Habitable_Room_Count,:Heated_Room_Count,
        :Wall_Summary,:Roof_Summary,:Floor_Summary,:Window_Summary,
        :Main_Heating_Summary,:Main_Heating_Controls_Summary,:Secondary_Heating_Summary,
        :Hot_Water_Summary,:Lighting_Summary,:Air_Tightness_Summary,
        :Part_1_Construction_Age_Band,:Multiple_Glazing_Type,
        ]
    project = ExtractData(project,DataSource,activeTiles,selected_cols;method="uprn")

    # Remove historical records (for dwellings with more than one entry)
    project.dat["EPC"] = project.dat["EPC"][project.dat["EPC"].Historical.==false,:]
    epcdata = project.dat["EPC"]

    # Find indices in EPC register that correspond to target UPRNs
    link_uprns(uprns,uprn) = findfirst(uprns.==uprn)
    function avoidmissings(vec)
        vec1 = fill!(Array{Int64}(undef,length(vec)),0)
        i_nonmissing = ismissing.(vec).==false
        vec1[i_nonmissing] .= vec[i_nonmissing]
        return vec1
    end
    epc_uprns = avoidmissings(project.dat["EPC"].OSG_UPRN)
    i_linked_uprns = link_uprns.((epc_uprns,),project.dat["master"].UPRN)

    # Find indices (in master data) which have an EPC
    i_epc = findall(isnothing.(i_linked_uprns).==false)

    # Add 'EPC_' suffix to all EPC column headings, except UPRN (i=1)
    colnames = names(project.dat["EPC"])
    suffix(str) = "EPC_$str"
    newcolnames = suffix.(colnames)

    # Create new EPC cols to append master stock data
    L = length(i_linked_uprns)
    function newcol(project,i_linked_uprns,i_epc,L,colname,newcolname)
        df = DataFrame(Symbol(newcolname) => fill!(Array{Union{Missing,Any}}(undef,L),missing))
        df[!,Symbol(newcolname)][i_epc] .= project.dat["EPC"][!,colname][i_linked_uprns[i_epc]]
        return df
    end
    newcols = newcol.((project,),(i_linked_uprns,),(i_epc,),(L,),colnames,newcolnames)
    newcols = hcat(newcols...)

    # Combine master and EPC data
    project.dat["master"] = hcat(project.dat["master"],newcols)

    # Area ratio, between EPC data and OS-evaluated. 1.0 assumed where EPC missing
    arearatio = project.dat["master"].EPC_Total_floor_area_m2_./project.dat["master"].OS_area_eval
    project.dat["master"][!,:arearatio] = ones(nrow(project.dat["master"]))
    project.dat["master"].arearatio[ismissing.(arearatio).==false] = arearatio[ismissing.(arearatio).==false]

    return project
end


module LyceumCore

using Base: @pure, @_inline_meta

using StaticNumbers

export @mustimplement, propertytype, argerror
include("util.jl")

export Maybe
export TupleN
export AbsArr, AbsMat, AbsVec
export AbsNestedArr, AbsSimilarNestedArr
export StaticOrVal
include("aliases.jl")

export front, tail, tuple_split, tuple_length, tuple_prod, tuple_minimum
include("tuple.jl")

end # module


#### LeveSpec types for various types of level specification

## TODO: Heads => True

abstract type LevelSpec end

struct LevelSpecToDepth  <: LevelSpec
    level::Int
end

struct LevelSpecAtDepth  <: LevelSpec
    level::Int
end

struct LevelSpecRange  <: LevelSpec
    start::Int
    stop::Int
end

struct LevelSpecAll  <: LevelSpec
    includezero::Bool
end

LevelSpecAll() = LevelSpecAll(false)

#### Create LevelSpec instances from Mxpr expressions

## Interpretation of negative index is different here than from Mma
## In Mma, a negative index counts up from each leaf.
## TODO: or FIXME
## 1. Implement both depth-first and breadth-first traversal. Only
## breadth-first is implmented now.
## 2. For negative indicies. Traverse the whole tree and build a flat
## Array of of the depth of each leaf. Then count negative levels from
## each leaf.

# Form n.  levels 1 through n
function make_level_specification(expr,n::Integer)
    if n < 0 n += depth(expr) end
    LevelSpecToDepth(n)
end

# Form [n] or [m,n]  level n only, or levels m through n
function make_level_specification(expr,spec::Mxpr{:List})
    _make_level_specification(expr,spec,margs(spec)...)
end

# Form Infinity or else error. levels 1 through Infinity
function make_level_specification(expr,x)
    if x == Infinity     # could use singlton type
        return LevelSpecRange(1,depth(expr))
#        return LevelSpecAll(false)
    end
    symerror("Level specification $x is not of the form n, {n}, or {m, n}.")
end

# Form [n]
function _make_level_specification(expr,spec::Mxpr{:List}, n::Integer)
    if n < 0 n += depth(expr) end
    LevelSpecAtDepth(n)
end

# Form [m,n]
function _make_level_specification(expr, spec::Mxpr{:List}, start::Integer, stop::Integer)
    if start < 0 start += depth(expr) end
    if stop < 0 stop += depth(expr) end
    LevelSpecRange(start, stop)
end

function _make_level_specification(expr, spec::Mxpr{:List}, start::Integer, stop)
    stop != Infinity && symerror("Level specification $spec is not of the form n, [n], or [m, n].")
    d = depth(expr)
    if start < 0 start += d end
    LevelSpecRange(start,d)
end

function _make_level_specification(expr, spec::Mxpr{:List},args...)
    symerror("Level specification $spec is not of the form n, [n], or [m, n].")
end

#### Structure for performing action at levels

mutable struct LevelAction
    data::Any            # application-specific data
    doaction::Function   # function takes arguments (data,expr)
    levelind::Int
    subind::Int
    parent::Any
    levelbreak::Bool
end

LevelAction(data,doaction) = LevelAction(data,doaction,0,0,Null,false)

has_level_zero(spec::LevelSpecAtDepth) = spec.level == 0
has_level_zero(spec::LevelSpecToDepth) = spec.level == 0
has_level_zero(spec::LevelSpecRange) = spec.start == 0
has_level_zero(spec::LevelSpecAll) = spec.includezero

# This is length for mapping purposes
function length_for_level(x)
    return 0
end

function length_for_level(mx::Mxpr)
    return length(mx)
end

function checkbreak(action)
    if action.levelbreak
        true
    else
        false
    end
end

# macro travcode()
#     code = quote
#         elen = length_for_level(expr)
#         if elen > 0
#             action.parent = expr
#             for i in 1:elen
#                 action.parent = expr
#                 action.levelind += 1
#                 action.subind = i

#                 traverse_levels!(action,spec,expr[i])
#                 if  checkbreak(action)
#                     return
#                 end
#                 action.levelind -= 1
#             end
#             #            action.parent = Null
#         end
#     end
#     return code
# end

const traversecode =
    quote
        elen = length_for_level(expr)
        if elen > 0
            action.parent = expr
            for i in 1:elen
                action.parent = expr
                action.levelind += 1
                action.subind = i

                traverse_levels!(action,spec,expr[i])
                if  checkbreak(action)
                    return
                end
                action.levelind -= 1
            end
            #            action.parent = Null
        end
    end

@eval function traverse_levels!(action::LevelAction, spec::LevelSpecAtDepth, expr)
    if action.levelind == spec.level
        action.doaction(action.data, expr)
        action.levelbreak && return
    elseif action.levelind < spec.level
        $traversecode
    end
end

@eval function traverse_levels!(action::LevelAction, spec::LevelSpecToDepth, expr)
    if action.levelind <= spec.level && action.levelind > 0
        action.doaction(action.data, expr)
        checkbreak(action) && return
    end
    if action.levelind < spec.level
        $traversecode
    end
end

@eval function traverse_levels!(action::LevelAction, spec::LevelSpecRange, expr)
    if action.levelind <= spec.stop && action.levelind >= spec.start
        action.doaction(action.data, expr)
        action.levelbreak && return
    end
    if action.levelind < spec.stop
        $traversecode
    end
end

@eval function traverse_levels!(action::LevelAction, spec::LevelSpecAll, expr)
    action.doaction(action.data, expr)
    action.levelbreak && return
    $traversecode
end


module Types
struct Bucket
end

struct SamplingLimit
	min::UInt16
	center::UInt16
	max::UInt16
end

struct SamplingDescription
	encoded_FOV::NTuple{Float32,3}
	recon_FOV::NTuple{Float32,3}
	encoded_matrix::NTuple{UInt16,3}
	recon_matrix::NTuple{UInt16,3}
	sampling_limits::NTuple{SamplingLimit,3}
end

struct ReconBuffer
	data::Array{ComplexF32}
	trajectory::Optional{Array{Float32}}
	density::Optiona{Array{Float32}}
	headers::Vector{MRD.AcquisitionHeader}
	sampling::SamplingDescription
end

struct ReconBit
	data::ReconBuffer
	reference::Optional{ReconBuffer}
end
end