const MAX_COLOUR = 255
const NUM_COLOURS = MAX_COLOUR + 1

struct Palette
    p::Vector{RGB{N0f8}}
end

const palettes = Dict{Symbol, Palette}()

Base.getindex(pal::Palette, idx::Integer) = pal.p[idx]

"""
    readpalette(stream::IO)
Read palette data from `stream`. The data should be
formatted as up to $NUM_COLOURS lines with three integers
each, separated by white space. The integers must fit into
a `UInt8`; they represent the RGB values of a 24-bit colour.
'#' marks the beginning of a comment which runs to the
end of a line and is ignored.
"""
function readpalette(stream::IO)
    i = 0
    pal = Vector{RGB{N0f8}}(undef, NUM_COLOURS)
    for line in readlines(stream)
        line = replace(strip(line), r"\s*#.*" => "")
        if isempty(line)
            continue
        end
        i += 1
        if i > NUM_COLOURS
            break
        end
        r, g, b = split(line)
        pal[i] = RGB{N0f8}(parse(Float32, r) / MAX_COLOUR,
                           parse(Float32, g) / MAX_COLOUR,
                           parse(Float32, b) / MAX_COLOUR)
    end
    while i < NUM_COLOURS
        i += 1
        pal[i] = RGB{N0f8}(0, 0, 0)
    end
    return Palette(pal)
end

"""
    palette(filename::AbstractString)
Reads and returns a `Palette` from `filename`.
    palette(name::Symbol)
Reads and returns a palette from a user supplied file
or from a standard palette file. Defaults to a palette
of $NUM_COLOURS greys.
"""
palette(filename::AbstractString) = palette(Symbol(filename))

function palette(name::Symbol)
    paths = [
        "$name",
        joinpath(dirname(@__FILE__), "maps", "$name.map")
    ]
    if haskey(palettes, name)
        return palettes[name]
    elseif any(ispath, paths)
        path = paths[findfirst(ispath, paths)]
        if !haskey(palettes, name)
            pal = readpalette(open(path))
            palettes[name] = pal
        end
        return palettes[name]
    else
        if !haskey(palettes, :grey)
            pal = Vector{RGB{N0f8}}(undef, NUM_COLOURS)
            for i in 1:NUM_COLOURS
                g = (i - 1) / MAX_COLOUR
                pal[i] = RGB{N0f8}(g, g, g)
            end
            palettes[:gray] = palettes[:grey] = Palette(pal)
        end
        return palettes[:grey]
    end
end


using OneClassSampling

import JuMP
import Ipopt
import MLKernels
import SVDD

using Random
using Test

Random.seed!(0)

TEST_SOLVER =  JuMP.with_optimizer(Ipopt.Optimizer, print_level=0)

@testset "OneClassSampling" begin
    include("sampler_test.jl")
    include("density_test.jl")
end


@testset "properties" begin

@testset "topology" begin
    cubane = smilestomol("C12C3C4C1C1C4C3C12")
    @test smallestsssr(cubane)[4] == 4
    @test sssrcount(cubane)[6] == 3
    biphenyl = smilestomol("C1CCCCC1C1CCCCC1")
    @test isringatom(biphenyl)[6]
    @test isringatom(biphenyl)[7]
    @test !isringbond(biphenyl)[7]
end

@testset "elements" begin
    sodiumsulfate = smilestomol("O=S(=O)([O-])[O-].[Na+].[Na+]")
    @test atomsymbol(sodiumsulfate) == [:O, :S, :O, :O, :O, :Na, :Na]
    @test bondorder(sodiumsulfate) == [2, 2, 1, 1]
    @test charge(sodiumsulfate) == [0, 0, 0, -1, -1, 1, 1]
    @test nodedegree(sodiumsulfate) == [1, 4, 1, 1, 1, 0, 0]

    isocyanurate = smilestomol("ClN1C(=O)N(Cl)C(=O)N(Cl)C1=O")
    @test atomsymbol(isocyanurate) == [
        :Cl, :N, :C, :O, :N, :Cl, :C, :O, :N, :Cl, :C, :O]
    @test bondorder(isocyanurate) == [
        1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 2]
    @test all(charge(isocyanurate) .== 0)
    @test nodedegree(isocyanurate) == [1, 3, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1]
end

@testset "formula" begin
    hexose = smilestomol("OCC(O)C(O)C(O)C(O)C=O")
    @test molecularformula(hexose) == "C6H12O6"
    @test empiricalformula(hexose) == "CH2O"

    sulfalicacid = smilestomol("O=S(=O)(O)O")
    @test molecularformula(sulfalicacid) == "H2O4S"
    @test empiricalformula(sulfalicacid) == "H2O4S"

    order = smilestomol("[Na].[Ba].B.Br.B.[Ar].Br")
    @test molecularformula(order) == "ArB2BaBr2H8Na"
end

@testset "valence" begin
    atoms = smilestomol("B.C.N.O.F.[Si].P.S.Cl.[As].[Se].Br.I")
    @test valence(atoms) == [3, 4, 3, 2, 1, 4, 3, 2, 1, 3, 2, 1, 1]
    @test lonepair(atoms) == [-1, 0, 1, 2, 3, 0, 1, 2, 3, 1, 2, 3, 3]
    @test implicithconnected(atoms) == [3, 4, 3, 2, 1, 4, 3, 2, 1, 3, 2, 1, 1]

    pyridineoxide = smilestomol("[N+]1([O-])=CC=CC=C1")
    @test valence(pyridineoxide) == [4, 1, 4, 4, 4, 4, 4]
    @test lonepair(pyridineoxide) == [0, 3, 0, 0, 0, 0, 0]
    @test implicithconnected(pyridineoxide) == [0, 0, 1, 1, 1, 1, 1]

    trifluoroborate = smilestomol("C[B-](F)(F)F")
    @test valence(trifluoroborate) == [4, 4, 1, 1, 1]
    @test lonepair(trifluoroborate) == [0, 0, 3, 3, 3]
    @test implicithconnected(trifluoroborate) == [3, 0, 0, 0, 0]

    etmacl = smilestomol("[H]C([H])([H])C[Mg][Cl]")
    @test valence(etmacl) == [1, 4, 1, 1, 4, nothing, 1]
    @test lonepair(etmacl) == [0, 0, 0, 0, 0, nothing, 3]
    @test implicithconnected(etmacl) == [0, 0, 0, 0, 2, 0, 0]
    @test explicithconnected(etmacl) == [0, 3, 0, 0, 0, 0, 0]
    @test hydrogenconnected(etmacl) == [0, 3, 0, 0, 2, 0, 0]
    @test heavyatomconnected(etmacl) == [1, 1, 1, 1, 2, 2, 1]
    @test connectivity(etmacl) == [1, 4, 1, 1, 4, 2, 1]
end

@testset "hybridization" begin
    alkyne = smilestomol("CC=CC#C")
    @test pielectron(alkyne) == [0, 1, 1, 2, 2]
    @test hybridization(alkyne) == [:sp3, :sp2, :sp2, :sp, :sp]

    anilinium = smilestomol("C1=CC=CC=C1[N+]")
    @test pielectron(anilinium) == [1, 1, 1, 1, 1, 1, 0]
    @test hybridization(anilinium) == [:sp2, :sp2, :sp2, :sp2, :sp2, :sp2, :sp3]

    azide = smilestomol("CC(=O)N=[N+]=[N-]")
    @test pielectron(azide) == [0, 1, 1, 1, 2, 1]
    @test hybridization(azide) == [:sp3, :sp2, :sp2, :sp2, :sp, :sp2]

    pyrrole = smilestomol("C1=CC=CN1")
    @test pielectron(pyrrole) == [1, 1, 1, 1, 2]
    @test hybridization(pyrrole) == [:sp2, :sp2, :sp2, :sp2, :sp2]

    nacl = smilestomol("[Na+][Cl-]")
    @test pielectron(nacl) == [0, 0]
    @test hybridization(nacl) == [:none, :none]
end

@testset "rotatable" begin
    Phe = smilestomol("N[C@@H](CC1=CC=CC=C1)C(O)=O")
    @test rotatablecount(Phe) == 3
    KCl = smilestomol("[K+].[Cl-]")
    @test rotatablecount(KCl) == 0
    dipyridamole = smilestomol(
        "n3c(nc2c(nc(nc2N1CCCCC1)N(CCO)CCO)c3N4CCCCC4)N(CCO)CCO")
    @test rotatablecount(dipyridamole) == 12
    paclitaxel = smilestomol(
        "CC1=C2[C@@]([C@]([C@H]([C@@H]3[C@]4([C@H](OC4)C[C@@H]([C@]3(C(=O)[C@@H]2OC(=O)C)C)O)OC(=O)C)OC(=O)c5ccccc5)(C[C@@H]1OC(=O)[C@H](O)[C@@H](NC(=O)c6ccccc6)c7ccccc7)O)(C)C"
    )
    @test rotatablecount(paclitaxel) == 15
end

@testset "aromatic" begin
    phenol = smilestomol("C=1C=CC=CC=1O")  # not carbonyl O
    @test count(isaromatic(phenol)) == 6

    furan = smilestomol("o1cccc1")
    @test count(isaromatic(furan)) == 5

    quinone = smilestomol("C1(=O)C=CC(=O)C=C1")
    @test count(isaromatic(quinone)) == 0

    tropone = smilestomol("C1(=O)C=CC=CC=C1")
    @test count(isaromatic(tropone)) == 7

    azepine = smilestomol("N1C=CC=CC=C1")
    @test count(isaromatic(azepine)) == 0

    thiopheneoxide = smilestomol("C1=CC=CS1=O")
    @test count(isaromatic(thiopheneoxide)) == 0

    dihydroazaborine = smilestomol("C1=CC=CBN1")
    @test count(isaromatic(dihydroazaborine)) == 6

    fulvene = smilestomol("C1=CC=CC1=C")
    @test count(isaromatic(fulvene)) == 0

    cyclopropenyl = smilestomol("C1=C[C+]1")
    @test count(isaromatic(cyclopropenyl)) == 3

    hexatriene = smilestomol("C=CC=CC=C")
    @test count(isaromatic(hexatriene)) == 0

    borazine = smilestomol("B1NBNBN1")
    @test count(isaromatic(borazine)) == 6

    methylenedioxybenzene = smilestomol("C1=CC=CC2=C1OCO2")
    @test count(isaromatic(methylenedioxybenzene)) == 6

    naphthalene = smilestomol("C=1C=CC=C2C1C=CC=C2")
    @test count(isaromatic(naphthalene)) == 10

    coumarin = smilestomol("C1=CC(=O)OC2=C1C=CC=C2")
    @test count(isaromatic(coumarin)) == 10

    pyrene = smilestomol("C12=CC=C3C=CC=C4C=CC(C2=C34)=CC=C1")
    @test count(isaromatic(pyrene)) == 16

    caffeine = smilestomol("CN1C=NC2=C1C(=O)N(C)C(=O)N2C")
    @test count(isaromatic(caffeine)) == 9

    # Difficult aromaticities
    quinodimethane = smilestomol("C1=CC=CC(=C)C1=C")
    @test_broken count(isaromatic(azulene)) == 10

    dihydronaphthalene = smilestomol("C=1CCC=C2C1C=CC=C2")
    @test_broken count(isaromatic(dihydronaphthalene)) == 10

    azulene = smilestomol("C=1C=CC=2C1C=CC=CC2")
    @test_broken count(isaromatic(azulene)) == 10

    pyromellitimide = smilestomol("C1(=O)NC(=O)C=2C1=CC=3C(=O)NC(=O)C=3C=2")
    @test count(isaromatic(pyromellitimide)) == 6

    # TODO: how to deal with tautomerism
    isocyanurate = smilestomol("ClN1C(=O)N(Cl)C(=O)N(Cl)C1=O")
end

@testset "properties" begin
    amide = smilestomol("CCC(=O)N")
    @test hacceptorcount(amide) == 2
    @test hdonorcount(amide) == 1
    fluoro = smilestomol("CCN(CO)CF")
    @test hacceptorcount(fluoro) == 3
    @test hdonorcount(fluoro) == 1
end

end # properties


export setup!, solve!, estimate_penalty_min!

"""
    setup!( model::GRAMPC.Model; settings... )

Initialize the GRAMPC problem so it is ready to solve.

Settings for the solver can be specified by passing in keyword argument pairs.
"""
function setup!( model::GRAMPC.Model; settings... )
    _is_initialized( model ) && error( "Model already initialized" )

    # The GRAMPC library takes a double pointer to create the problem, so we must create a temp
    # reference to handle this
    problem = Ref{Ptr{CGRAMPC}}()
    ccall( (:grampcjl_init, libgrampc_julia), Cvoid, (Ptr{Ptr{CGRAMPC}}, Ref{Cgrampcjl_callbacks}, Ref{Model}), problem, model.Ccallbacks, model )
    model.problem = problem[]

    # Enable/disable solver parts based on specified callbacks
    _set_settings!( model, OptimControl=model.optimizeu,
                           OptimParam=model.optimizep,
                           OptimTime=model.optimizeT,
                           IntegralCost=model.integralCost,
                           TerminalCost=model.terminalCost,
                           EqualityConstraints=model.eqCon,
                           InequalityConstraints=model.ineqCon,
                           TerminalEqualityConstraints=model.termEqCon,
                           TerminalInequalityConstraints=model.termIneqCon )

    # The user can provide options here as kwargs, so pass them to the solver
    _set_settings!( model; settings...)
end

function solve!( model::Model )
    _is_initialized( model ) || error( "Model not initialized" )

    ccall( (:grampc_run, libgrampc_julia), Cvoid, (Ptr{CGRAMPC},), model.problem )

    return Solution( model )
end

function estimate_penalty_min!( model::GRAMPC.Model; runonce=true )
    _is_initialized( model ) || error( "Model not initialized" )

    run = runonce ? 1 : 0

    ret = ccall( (:grampc_estim_penmin, libgrampc_julia), Cint, (Ptr{CGRAMPC}, Cint), model.problem, run )

    return PenaltyEstimationStatus( ret )
end


using Test
using MPI

if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
    import CUDA
    ArrayType = CUDA.CuArray
else
    ArrayType = Array
end

MPI.Init()

comm = MPI.COMM_WORLD
size = MPI.Comm_size(comm)
rank = MPI.Comm_rank(comm)
root = 0
isroot = rank == root

counts = Cint[mod(i,2) + 1 for i in 0:(size-1)]
check = collect(Iterators.flatten([fill(r, counts[r+1]) for r = 0:size-1]))

for T in Base.uniontypes(MPI.MPIDatatype)
    A = ArrayType(fill(T(rank), mod(rank,2) + 1))

    # Test passing the output buffer
    B = ArrayType{T}(undef, sum(counts))
    MPI.Gatherv!(A, isroot ? VBuffer(B, counts) : nothing, comm; root=root)
    if isroot
        @test B == ArrayType{T}(check)
    end

    # Test assertion when output size is too small
    B = ArrayType{T}(undef, sum(counts)-1)
    if isroot
        @test_throws AssertionError MPI.Gatherv!(A, VBuffer(B, counts), comm; root=root)
    end

    # Test explicit MPI_IN_PLACE
    if isroot
        B = ArrayType(fill(T(rank), sum(counts)))
        MPI.Gatherv!(MPI.IN_PLACE, VBuffer(B, counts), comm; root=root)
    else
        B = ArrayType(fill(T(rank), counts[rank+1]))
        MPI.Gatherv!(B, nothing, comm; root=root)
    end
    if isroot
        @test B == ArrayType{T}(check)
    end
end

MPI.Finalize()
@test MPI.Finalized()


"""
    RaibertHopper <: Environment

    hopping robot, inspired by: "Dynamically Stable Legged Locomotion"
"""
struct RaibertHopper end

function raiberthopper(;
    representation=:minimal,
    timestep=0.05,
    gravity=[0.0; 0.0; -9.81],
    spring=0.0,
    damper=0.1,
    control_scaling=Diagonal(ones(3)),
    seed=1,
    contact_foot=true,
    contact_body=true,
    vis=Visualizer(),
    name=:robot,
    info=nothing,
    opts_step=SolverOptions(rtol=3.0e-4, btol=3.0e-4, undercut=1.5),
    opts_grad=SolverOptions(rtol=3.0e-4, btol=3.0e-4, undercut=1.5),
    T=Float64)

    mechanism = get_raiberthopper(;
        timestep,
        gravity,
        spring,
        damper,
        contact_foot,
        contact_body)
    initialize_raiberthopper!(mechanism)

    if representation == :minimal
        nx = minimal_dimension(mechanism)
    elseif representation == :maximal
        nx = maximal_dimension(mechanism)
    end

    nu = 3
    no = nx

    aspace = BoxSpace(nu,
        low=(-1.0e8 * ones(nu)),
        high=(1.0e8 * ones(nu)))
    ospace = BoxSpace(no,
        low=(-Inf * ones(no)),
        high=(Inf * ones(no)))

    rng = MersenneTwister(seed)

    z = get_maximal_state(mechanism)
    x = representation == :minimal ? maximal_to_minimal(mechanism, z) : z

    fx = zeros(nx, nx)
    fu = zeros(nx, nu)

    u_prev = zeros(nu)
    control_mask = raiberthopper_control_mask()
    build_robot(mechanism,
        vis=vis,
        name=name)

    TYPES = [RaibertHopper, T, typeof(mechanism), typeof(aspace), typeof(ospace), typeof(info)]
    env = Environment{TYPES...}(mechanism, representation, aspace, ospace,
        x, fx, fu,
        u_prev, control_mask' * control_scaling,
        nx, nu, no,
        info,
        [rng], vis,
        opts_step, opts_grad)

    return env
end

function raiberthopper_control_mask()
    [0 0 0 1 0 0 0;
	 0 0 0 0 1 0 0;
	 0 0 0 0 0 0 1]
end

function raiberthopper_nominal_max(;
    foot_radius=0.05)

    # initial state
    x1b1 = [0.0; 0.0; 0.5 + foot_radius]
    v1b1 = [0.0; 0.0; 0.0]
    q1b1 = [1.0; 0.0; 0.0; 0.0]
    ω1b1 = [0.0; 0.0; 0.0]
    z1b1 = [x1b1; v1b1; q1b1; ω1b1]

    x1b2 = [0.0; 0.0; 0.0 + foot_radius]
    v1b2 = [0.0; 0.0; 0.0]
    q1b2 = [1.0; 0.0; 0.0; 0.0]
    ω1b2 = [0.0; 0.0; 0.0]
    z1b2 = [x1b2; v1b2; q1b2; ω1b2]

    z1 = [z1b1; z1b2]
end

function raiberthopper_offset_max(x_shift, y_shift, z_shift)
    z = raiberthopper_nominal_max()
    shift = [x_shift; y_shift; z_shift]
    z[1:3] += shift
    z[13 .+ (1:3)] += shift
    return z
end

function visualize(env::Environment{RaibertHopper}, traj::Vector{Vector{T}};
    name=:robot,
    axes=false,
    grid=true) where T

	@assert size(traj[1]) == size(env.state)

    # convert to maximal representation
    z = [env.representation == :minimal ? minimal_to_maximal(env.mechanism, x) : x for x in traj]

    body_color = magenta
    foot_color = magenta
    env.mechanism.bodies[1].shape.color = body_color
    env.mechanism.bodies[2].shape.color = foot_color

    # build system
    build_robot(env.mechanism,
        vis=env.vis,
        name=name)

    n_leg = 100
    r_leg = 0.025
    for i = 1:n_leg
        setobject!(env.vis["leg$i"], GeometryBasics.Sphere(Point3f0(0),
            convert(Float32, r_leg)),
            MeshPhongMaterial(color=magenta))
    end

    # animate
    anim = MeshCat.Animation(convert(Int, floor(1.0 / env.mechanism.timestep)))
    for (t, x) in enumerate(z)
        x_body = x[1:3]
        x_foot = x[13 .+ (1:3)]
        dir = x_body - x_foot
        dir_norm = dir ./ norm(dir)

        MeshCat.atframe(anim, t) do
            set_robot(env.vis, env.mechanism, x,
                name=name)
            step = range(0.0, stop=norm(dir), length=n_leg)
            for i = 1:n_leg
                MeshCat.settransform!(env.vis["leg$i"], MeshCat.compose(MeshCat.Translation(step[i] .* dir_norm + x_foot), MeshCat.LinearMap(Dojo.RotY(0.0))))
            end
        end
    end
    MeshCat.setanimation!(env.vis, anim)

    set_camera!(env.vis, zoom=1.5)
    setvisible!(env.vis["/Axes"], axes)
    setvisible!(env.vis["/Grid"], grid)
end

function ghost(env::Environment{RaibertHopper}, traj::Vector{Vector{T}};
    timesteps=[t for t = 1:length(traj)],
    axes=false,
    grid=true,
    line=false) where T

    # convert to maximal representation
    z = [env.representation == :minimal ? minimal_to_maximal(env.mechanism, x) : x for x in traj]

    # color
    body_color = magenta
    foot_color = magenta
    env.mechanism.bodies[1].shape.color = body_color
    env.mechanism.bodies[2].shape.color = foot_color

    for t in timesteps
        # build system
        build_robot(env.vis, env.mechanism,
            name=Symbol("robot_$t"),
            color=(t == length(z) ? magenta : magenta_light))
        set_robot(env.vis, env.mechanism, z[t],
            name=Symbol("robot_$t"))

        x_body = z[t][1:3]
        x_foot = z[t][13 .+ (1:3)]

        leg = GeometryBasics.Cylinder(Point3f0(x_foot...), Point3f0(x_body...), convert(Float32, 0.025))
        setobject!(env.vis["leg_$t"], leg, MeshPhongMaterial(color=(t == length(z) ? magenta : magenta_light)))
    end
    setvisible!(env.vis[:robot], false)

    # body
    if line
        line_mat = LineBasicMaterial(color=color=RGBA(51.0 / 255.0, 1.0, 1.0, 1.0), linewidth=25.0)
        points = Vector{Point{3,Float64}}()
        for (i, xt) in enumerate(z)
            k = xt[1:3]
            push!(points, Point(k[1], k[2], k[3]))
        end
        setobject!(env.vis[:body_traj], MeshCat.Line(points, line_mat))

        # foot
        line_mat = LineBasicMaterial(color=color=RGBA(1.0, 153.0 / 255.0, 51.0 / 255.0, 1.0), linewidth=10.0)
        points = Vector{Point{3,Float64}}()
        for (i, xt) in enumerate(z)
            k = xt[13 .+ (1:3)]
            push!(points, Point(k[1], k[2], k[3]))
        end
        setobject!(env.vis[:foot_traj], MeshCat.Line(points, line_mat))
    end

    set_camera!(env.vis,
        zoom=1.5)
    setvisible!(env.vis["/Axes"], axes)
    setvisible!(env.vis["/Grid"], grid)

    open(env.vis)
end


verbose = true
 if verbose && !isdefined(Main,:RvSpectMLBase)  println("# Loading RvSpecMLBase")    end
 using RvSpectMLBase
 if verbose && !isdefined(Main,:EchelleInstruments)  println("# Loading EchelleInstruments")    end
 using EchelleInstruments, EchelleInstruments.NEID
 using DataFrames, Query

# USER: You must create a data_paths.jl file in one of the default_paths_to_search listed below. It need only contain one line:
# solar_data_path = "/home/eford/Data/SolarSpectra/NEID_solar/"
target_subdir = "good_days/DRPv0.7/"   # USER: Replace with directory of your choice
 #fits_target_str = "Solar"
 fits_target_str = "Sun"
 output_dir = "examples/output"
 paths_to_search_for_param = [pwd(),"examples",joinpath(pkgdir(RvSpectMLBase),"..","RvSpectML","examples"), "/gpfs/group/ebf11/default/ebf11/neid_solar"]
 # NOTE: make_manifest does not update its paths_to_search when default_paths_to_search is defined here, so if you change the line above, you must also include "paths_to_search=default_paths_to_search" in the make_manifest() function call below
 pipeline_plan = PipelinePlan()

reset_all_needs!(pipeline_plan)
if need_to(pipeline_plan,:read_spectra)
   if verbose println("# Finding what data files are avaliable.")  end
   eval(read_data_paths(paths_to_search=paths_to_search_for_param))
   @assert isdefined(Main,:solar_data_path)
   df_files = make_manifest(solar_data_path, target_subdir, NEID )
   #df_files = EchelleInstruments.make_manifest(solar_data_path, target_subdir, NEID )

   if verbose println("# Reading in customized parameters from param.jl.")  end
   eval(code_to_include_param_jl(paths_to_search=paths_to_search_for_param))
   end

if verbose println("# Reading in ", size(df_files_use,1), " FITS files.")  end
   @time all_spectra = map(NEID.read_data,eachrow(df_files_use))
   dont_need_to!(pipeline_plan,:read_spectra)

#=
# Pre-ship corrections
if verbose println("# Applying wavelength corrections.")  end
   @assert isdefined(Main,:ancilary_solar_data_path)
   NEID.read_drift_corrections!(joinpath(ancilary_solar_data_path,"SolarRV20190918_JD_SciRV_CalRV.txt"), df_files_use)
   NEID.read_barycentric_corrections!(joinpath(ancilary_solar_data_path,"SolarTelescope2019-09-18_inclGravRedshiftAirMassAltitude.csv"), df_files_use)
   NEID.read_differential_extinctions!(joinpath(ancilary_solar_data_path,"20190918_diff_ex_full_fixed.txt"), df_files_use)
   apply_doppler_boost!(all_spectra,df_files_use)
   all_spectra
end
=#


################################################################################
#                              BASIC MODEL DEFINITION
################################################################################
"""
    TranscriptionData

A DataType for storing the data mapping an [`InfiniteOpt.InfiniteModel`](@ref)
that has been transcribed to a regular [`JuMP.Model`](@ref) that contains the
transcribed variables. This is stored in the `ext` field of a `JuMP.Model` to
make what is called a `TranscriptionModel` via the [`TranscriptionModel`](@ref)
constructor.

**Fields**
- `infvar_lookup::Dict{InfiniteOpt.GeneralVariableRef, Dict{Vector{Float64}, Int}}`:
   A lookup table of infinite variable transcriptions via support value.
- `infvar_mappings::Dict{InfiniteOpt.GeneralVariableRef, Vector{JuMP.VariableRef}}`:
   Map infinite variables to their transcription variables.
- `infvar_supports::Dict{InfiniteOpt.GeneralVariableRef, Vector{Tuple}}`:
   Map infinite variables to their support values.
- `infvar_support_labels::Dict{InfiniteOpt.GeneralVariableRef, Vector{Set{DataType}}}`:
   Map the infinite variables to their support labels.
- `finvar_mappings::Dict{InfiniteOpt.GeneralVariableRef, JuMP.VariableRef}`:
   Map finite variables to their transcription variables.
- `reduced_vars::Vector{InfiniteOpt.ReducedVariable{InfiniteOpt.GeneralVariableRef}}`:
   Store the core reduced variable objects of reduced variables formed on transcription.
- `last_point_index::Int`: The last internal point variable index added.
- `measure_lookup::Dict{InfiniteOpt.GeneralVariableRef, Dict{Vector{Float64}, Int}}`:
   A lookup table of measure transcriptions via support value.
- `measure_mappings::Dict{InfiniteOpt.GeneralVariableRef, Vector{JuMP.AbstractJuMPScalar}}`:
   Map measures to transcription expressions.
- `measure_supports::Dict{InfiniteOpt.GeneralVariableRef, Vector{Tuple}}`:
   Map measures to their supports values (if the transcribed measure is still infinite).
- `measure_support_labels::Dict{InfiniteOpt.GeneralVariableRef, Vector{Set{DataType}}}`:
   Map measures to their support labels if they have any.
- `constr_mappings::Dict{InfiniteOpt.InfOptConstraintRef, Vector{JuMP.ConstraintRef}}`:
   Map constraints to their transcriptions.
- `constr_supports::Dict{InfiniteOpt.InfOptConstraintRef, Vector{Tuple}}`:
   Map constraints to their support values.
- `constr_support_labels::Dict{InfiniteOpt.InfOptConstraintRef, Vector{Set{DataType}}}`:
   Map constraints to their support labels.
- `supports::Tuple`: Store the collected parameter supports here.
- `support_labels::Tuple`: Store the collected parameter labels here.
- `has_internal_supports::Bool`: Where any internal supports collected?
"""
mutable struct TranscriptionData
    # Variable information
    infvar_lookup::Dict{InfiniteOpt.GeneralVariableRef, Dict{Vector{Float64}, Int}}
    infvar_mappings::Dict{InfiniteOpt.GeneralVariableRef, Vector{JuMP.VariableRef}}
    infvar_supports::Dict{InfiniteOpt.GeneralVariableRef, Vector{Tuple}}
    infvar_support_labels::Dict{InfiniteOpt.GeneralVariableRef, Vector{Set{DataType}}}
    finvar_mappings::Dict{InfiniteOpt.GeneralVariableRef, JuMP.VariableRef}

    # Internal variables (created via internal measure expansions)
    reduced_vars::Vector{InfiniteOpt.ReducedVariable{InfiniteOpt.GeneralVariableRef}}
    last_point_index::Int

    # Measure information
    measure_lookup::Dict{InfiniteOpt.GeneralVariableRef, Dict{Vector{Float64}, Int}}
    measure_mappings::Dict{InfiniteOpt.GeneralVariableRef, Vector{JuMP.AbstractJuMPScalar}}
    measure_supports::Dict{InfiniteOpt.GeneralVariableRef, Vector{Tuple}}
    measure_support_labels::Dict{InfiniteOpt.GeneralVariableRef, Vector{Set{DataType}}}

    # Constraint information
    constr_mappings::Dict{InfiniteOpt.InfOptConstraintRef,
                          Vector{JuMP.ConstraintRef}}
    constr_supports::Dict{InfiniteOpt.InfOptConstraintRef,
                          Vector{Tuple}}
    constr_support_labels::Dict{InfiniteOpt.InfOptConstraintRef,
                               Vector{Set{DataType}}}

    # Collected Supports
    supports::Tuple
    support_labels::Tuple
    has_internal_supports::Bool

    # Default constructor
    function TranscriptionData()
        return new( # variable info
                   Dict{InfiniteOpt.GeneralVariableRef, Dict{Vector{Float64}, Int}}(),
                   Dict{InfiniteOpt.GeneralVariableRef, Vector{JuMP.VariableRef}}(),
                   Dict{InfiniteOpt.GeneralVariableRef, Vector{Tuple}}(),
                   Dict{InfiniteOpt.GeneralVariableRef, Vector{Set{DataType}}}(),
                   Dict{InfiniteOpt.GeneralVariableRef, JuMP.VariableRef}(),
                   # internal variables
                   Vector{InfiniteOpt.ReducedVariable{InfiniteOpt.GeneralVariableRef}}(),
                   0,
                   # measure info
                   Dict{InfiniteOpt.GeneralVariableRef, Dict{Vector{Float64}, Int}}(),
                   Dict{InfiniteOpt.GeneralVariableRef, Vector{JuMP.AbstractJuMPScalar}}(),
                   Dict{InfiniteOpt.GeneralVariableRef, Vector{Tuple}}(),
                   Dict{InfiniteOpt.GeneralVariableRef, Vector{Set{DataType}}}(),
                   # constraint info
                   Dict{InfiniteOpt.InfOptConstraintRef, Vector{JuMP.ConstraintRef}}(),
                   Dict{InfiniteOpt.InfOptConstraintRef, Vector{Vector{Float64}}}(),
                   Dict{InfiniteOpt.InfOptConstraintRef, Vector{Set{DataType}}}(),
                   # support storage
                   (), (), false)
    end
end

"""
    TranscriptionModel([optimizer_constructor;
                       caching_mode::MOIU.CachingOptimizerMode = MOIU.AUTOMATIC,
                       bridge_constraints::Bool = true])::JuMP.Model

Return a [`JuMP.Model`](@ref) with [`TranscriptionData`](@ref) included in the
`ext` data field. Accepts the same arguments as a typical JuMP `Model`.
More detailed variable and constraint naming can be enabled via `verbose_naming`.

**Example**
```julia-repl
julia> TranscriptionModel()
A JuMP Model
Feasibility problem with:
Variables: 0
Model mode: AUTOMATIC
CachingOptimizer state: NO_OPTIMIZER
Solver name: No optimizer attached.
```
"""
function TranscriptionModel(; kwargs...)::JuMP.Model
    model = JuMP.Model(; kwargs...)
    model.ext[:TransData] = TranscriptionData()
    return model
end
# Accept optimizer constructors
function TranscriptionModel(optimizer_constructor;
                            kwargs...)::JuMP.Model
    model = JuMP.Model(optimizer_constructor; kwargs...)
    model.ext[:TransData] = TranscriptionData()
    return model
end

################################################################################
#                                BASIC QUERIES
################################################################################
"""
    is_transcription_model(model::JuMP.Model)::Bool

Return true if `model` is a `TranscriptionModel` or false otherwise.

**Example**
```julia-repl
julia> is_transcription_model(model)
true
```
"""
function is_transcription_model(model::JuMP.Model)::Bool
    return haskey(model.ext, :TransData)
end

"""
    transcription_data(model::JuMP.Model)::TranscriptionData

Return the `TranscriptionData` from a `TranscriptionModel`. Errors if it is not
a `TranscriptionModel`.
"""
function transcription_data(model::JuMP.Model)::TranscriptionData
    !is_transcription_model(model) && error("Model is not a transcription model.")
    return model.ext[:TransData]
end

"""
    has_internal_supports(model::JuMP.Model)::Bool

Return a `Bool` whether `model` has any internal supports that were collected.
"""
function has_internal_supports(model::JuMP.Model)::Bool
    return transcription_data(model).has_internal_supports
end

################################################################################
#                              VARIABLE QUERIES
################################################################################
# Define method for checking if the label needs to be accounted for
function _ignore_label(model::JuMP.Model, label::Type{<:InfiniteOpt.AbstractSupportLabel})::Bool
    return label == InfiniteOpt.All || (!has_internal_supports(model) && label == InfiniteOpt.PublicLabel)
end

"""
    transcription_variable(model::JuMP.Model,
        vref::InfiniteOpt.GeneralVariableRef;
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
         ndarray::Bool = false])

Return the transcribed variable reference(s) corresponding to `vref`. Errors
if no transcription variable is found. Also can query via the syntax:
```julia
transcription_variable(vref::InfiniteOpt.GeneralVariableRef;
    [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
     ndarray::Bool = false])
```
If the infinite model contains a built transcription model. By default, this
method returns only transcribed variables associated with public supports. All the
variables can be returned by setting `label = All`.

If `vref` is infinite and `ndarray = true` then an n-dimensional array will be
returned in accordance with the infinite parameters that have unique object
numbers. In this case, `label` will be used to search the intersection of variable
supports that use the label. This is defers from the default behavior which
considers the union.

**Example**
```julia-repl
julia> transcription_variable(trans_model, infvar)
2-element Array{VariableRef,1}:
 infvar(support: 1)
 infvar(support: 2)

julia> transcription_variable(trans_model, hdvar)
hdvar

julia> transcription_variable(infvar)
2-element Array{VariableRef,1}:
 infvar(support: 1)
 infvar(support: 2)

julia> transcription_variable(hdvar)
hdvar
```
"""
function transcription_variable(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef;
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    return transcription_variable(model, vref, InfiniteOpt._index_type(vref), label, ndarray)
end

# define convenient aliases
const InfVarIndex = Union{InfiniteOpt.InfiniteVariableIndex,
                          InfiniteOpt.ReducedVariableIndex,
                          InfiniteOpt.DerivativeIndex}
const FinVarIndex = Union{InfiniteOpt.HoldVariableIndex,
                          InfiniteOpt.PointVariableIndex}

## Define the variable mapping functions
# FinVarIndex
function transcription_variable(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{V},
    label::Type{<:InfiniteOpt.AbstractSupportLabel},
    ndarray::Bool
    )::JuMP.VariableRef where {V <: FinVarIndex}
    var = get(transcription_data(model).finvar_mappings, vref, nothing)
    if var === nothing
        error("Variable reference $vref not used in transcription model.")
    end
    return var
end

# InfVarIndex
function transcription_variable(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{V},
    label::Type{<:InfiniteOpt.AbstractSupportLabel},
    ndarray::Bool
    )::Array{JuMP.VariableRef} where {V <: InfVarIndex}
    vars = get(transcription_data(model).infvar_mappings, vref, nothing)
    if vars === nothing
        error("Variable reference $vref not used in transcription model.")
    end
    if ndarray
        return make_ndarray(model, vref, vars, label)
    elseif _ignore_label(model, label)
        return vars
    else
        labels = transcription_data(model).infvar_support_labels[vref]
        inds = map(s -> any(l -> l <: label, s), labels)
        return vars[inds]
    end
end

# ParameterFunctionIndex
function transcription_variable(model::JuMP.Model,
    fref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{InfiniteOpt.ParameterFunctionIndex},
    label::Type{<:InfiniteOpt.AbstractSupportLabel},
    ndarray::Bool
    )
    # get the object numbers of the expression and form the support iterator
    obj_nums = InfiniteOpt._object_numbers(fref)
    support_indices = support_index_iterator(model, obj_nums)
    vals = Vector{Float64}(undef, length(support_indices))
    check_labels = length(vals) > 1 && !_ignore_label(model, label)
    label_inds = ones(Bool, length(vals))
    # iterate over the indices and compute the values
    for (i, idx) in enumerate(support_indices)
        supp = index_to_support(model, idx)
        if check_labels && !any(l -> l <: label, index_to_labels(model, idx))
            @inbounds label_inds[i] = false
        end
        @inbounds vals[i] = transcription_expression(model, fref, supp)
    end
    # return the values
    if ndarray
        return make_ndarray(model, fref, vals, label)
    else
        return vals[label_inds]
    end
end

# Fallback
function transcription_variable(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type,
    label,
    ndarray)
    error("`transcription_variable` not defined for variables with indices of " *
          "type $(index_type) and/or is not defined for labels of type $(label).")
end

# Dispatch for internal models
function transcription_variable(vref::InfiniteOpt.GeneralVariableRef;
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    trans_model = InfiniteOpt.optimizer_model(JuMP.owner_model(vref))
    return transcription_variable(trans_model, vref, label = label,
                                  ndarray = ndarray)
end

"""
    InfiniteOpt.optimizer_model_variable(vref::InfiniteOpt.GeneralVariableRef,
        ::Val{:TransData};
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
        ndarray::Bool = false])

Proper extension of [`InfiniteOpt.optimizer_model_variable`](@ref) for
`TranscriptionModel`s. This simply dispatches to [`transcription_variable`](@ref).
"""
function InfiniteOpt.optimizer_model_variable(vref::InfiniteOpt.GeneralVariableRef,
    ::Val{:TransData};
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false)
    return transcription_variable(vref, label = label, ndarray = ndarray)
end

"""
    InfiniteOpt.variable_supports(model::JuMP.Model,
        vref::InfiniteOpt.DecisionVariableRef,
        key::Val{:TransData} = Val(:TransData);
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
        ndarray::Bool = false])

Return the support alias mapping associated with `vref` in the transcription model.
Errors if `vref` does not have transcripted variables. See `transcription_variable`
for an explanation of `ndarray`.
"""
function InfiniteOpt.variable_supports(model::JuMP.Model,
    dvref::Union{InfiniteOpt.InfiniteVariableRef, InfiniteOpt.ReducedVariableRef,
                 InfiniteOpt.DerivativeRef},
    key::Val{:TransData} = Val(:TransData);
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )::Array
    vref = InfiniteOpt._make_variable_ref(JuMP.owner_model(dvref), JuMP.index(dvref))
    if !haskey(transcription_data(model).infvar_mappings, vref)
        error("Variable reference $vref not used in transcription model.")
    elseif !haskey(transcription_data(model).infvar_supports, vref)
        prefs = InfiniteOpt.raw_parameter_refs(dvref)
        lookups = transcription_data(model).infvar_lookup[vref]
        type = typeof(Tuple(first(keys(lookups)), prefs))
        supps = Vector{type}(undef, length(lookups))
        for (s, i) in lookups
            supps[i] = Tuple(s, prefs)
        end
        transcription_data(model).infvar_supports[vref] = supps
    end
    supps = transcription_data(model).infvar_supports[vref]
    if ndarray
        return make_ndarray(model, dvref, supps, label)
    elseif _ignore_label(model, label)
        return supps
    else
        labels = transcription_data(model).infvar_support_labels[vref]
        inds = map(s -> any(l -> l <: label, s), labels)
        return supps[inds]
    end
end

# ParameterFunctionRef
function InfiniteOpt.variable_supports(model::JuMP.Model,
    dvref::InfiniteOpt.ParameterFunctionRef,
    key::Val{:TransData} = Val(:TransData);
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )::Array
    # get the object numbers of the expression and form the support iterator
    obj_nums = sort(InfiniteOpt._object_numbers(dvref))
    support_indices = support_index_iterator(model, obj_nums)
    supps = Vector{Tuple}(undef, length(support_indices))
    check_labels = length(supps) > 1 && !_ignore_label(model, label)
    param_supps = parameter_supports(model)
    label_inds = ones(Bool, length(supps))
    # iterate over the indices and compute the values
    for (i, idx) in enumerate(support_indices)
        if check_labels && !any(l -> l <: label, index_to_labels(model, idx))
            @inbounds label_inds[i] = false
        end
        @inbounds supps[i] = Tuple(param_supps[j][idx[j]] for j in obj_nums)
    end
    # return the supports
    if ndarray
        return make_ndarray(model, dvref, supps, label)
    else
        return supps[label_inds]
    end
end

"""
    lookup_by_support(model::JuMP.Model,
                      vref::InfiniteOpt.GeneralVariableRef,
                      support::Vector)

Return the transcription expression of `vref` defined at its `support`. This is
intended as a helper method for automated transcription.
"""
function lookup_by_support(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    support::Vector)
    return lookup_by_support(model, vref, InfiniteOpt._index_type(vref), support)
end

# define error function for not being able to find a variable by its support
_supp_error() = error("Unable to locate transcription variable by support, consider " *
                      "rebuilding the infinite model with less significant digits. " *
                      "Note this might be due to partially evaluating dependent parameters " *
                      "which is not supported by TranscriptionOpt. Such is the case with " *
                      "derivatives/measures that dependent on single dependent parameters.")

# InfiniteIndex
function lookup_by_support(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{V},
    support::Vector
    )::JuMP.VariableRef where {V <: InfVarIndex}
    if !haskey(transcription_data(model).infvar_lookup, vref)
        error("Variable reference $vref not used in transcription model.")
    end
    idx = get(_supp_error, transcription_data(model).infvar_lookup[vref], support)
    return transcription_data(model).infvar_mappings[vref][idx]
end

# ParameterFunctionIndex
function lookup_by_support(model::JuMP.Model,
    fref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{InfiniteOpt.ParameterFunctionIndex},
    support::Vector
    )
    prefs = InfiniteOpt.raw_parameter_refs(fref)
    func = InfiniteOpt.raw_function(fref)
    return func(Tuple(support, prefs)...)
end

# FiniteVariableIndex
function lookup_by_support(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{V},
    support::Vector
    )::JuMP.VariableRef where {V <: FinVarIndex}
    if !haskey(transcription_data(model).finvar_mappings, vref)
        error("Variable reference $vref not used in transcription model.")
    end
    return transcription_data(model).finvar_mappings[vref]
end

"""
    InfiniteOpt.internal_reduced_variable(
        vref::InfiniteOpt.ReducedVariableRef,
        ::Val{:TransData}
        )::InfiniteOpt.ReducedVariable{InfiniteOpt.GeneralVariableRef}

Return the internal reduced variable associated with `vref`, assuming it was
added internally during measure expansion at the transcription step. This
extends [`InfiniteOpt.internal_reduced_variable`](@ref) as described in its
docstring. Errors, if no such variable can be found.
"""
function InfiniteOpt.internal_reduced_variable(
    vref::InfiniteOpt.ReducedVariableRef,
    ::Val{:TransData}
    )::InfiniteOpt.ReducedVariable{InfiniteOpt.GeneralVariableRef}
    trans_model = InfiniteOpt.optimizer_model(JuMP.owner_model(vref))
    reduced_vars = transcription_data(trans_model).reduced_vars
    idx = -1 * JuMP.index(vref).value
    if idx in keys(reduced_vars)
        return reduced_vars[idx]
    else
        error("Invalid reduced variable reference, this likely is attributed " *
              "to its being deleted.")
    end
end

################################################################################
#                              MEASURE QUERIES
################################################################################
# MeasureIndex
function transcription_variable(model::JuMP.Model,
    mref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{InfiniteOpt.MeasureIndex},
    label::Type{<:InfiniteOpt.AbstractSupportLabel},
    ndarray::Bool = false
    )
    exprs = get(transcription_data(model).measure_mappings, mref, nothing)
    if exprs === nothing
        error("Measure reference $mref not used in transcription model.")
    end
    if ndarray
        return make_ndarray(model, mref, exprs, label)
    elseif length(exprs) > 1 && _ignore_label(model, label)
        return exprs
    elseif length(exprs) > 1
        labels = transcription_data(model).measure_support_labels[mref]
        inds = map(s -> any(l -> l <: label, s), labels)
        return exprs[inds]
    else
        return first(exprs)
    end
end

# Extend transcription_expression
function lookup_by_support(model::JuMP.Model,
    mref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{InfiniteOpt.MeasureIndex},
    support::Vector
    )::JuMP.AbstractJuMPScalar
    if !haskey(transcription_data(model).measure_lookup, mref)
        error("Measure reference $mref not used in transcription model.")
    end
    idx = get(_supp_error, transcription_data(model).measure_lookup[mref], support)
    return transcription_data(model).measure_mappings[mref][idx]
end

# Extend variable_supports
function InfiniteOpt.variable_supports(model::JuMP.Model,
    dmref::InfiniteOpt.MeasureRef,
    key::Val{:TransData} = Val(:TransData);
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    mref = InfiniteOpt._make_variable_ref(JuMP.owner_model(dmref), JuMP.index(dmref))
    if !haskey(transcription_data(model).measure_mappings, mref)
        error("Measure reference $mref not used in transcription model.")
    elseif !haskey(transcription_data(model).measure_supports, mref)
        lookups = transcription_data(model).measure_lookup[mref]
        prefs = InfiniteOpt.parameter_refs(dmref)
        vt_prefs = InfiniteOpt.Collections.VectorTuple(prefs)
        type = typeof(Tuple(first(keys(lookups)), vt_prefs))
        supps = Vector{type}(undef, length(lookups))
        for (supp, i) in lookups
            supps[i] = Tuple(supp, vt_prefs)
        end
        transcription_data(model).measure_supports[mref] = supps
    end
    supps = transcription_data(model).measure_supports[mref]
    if ndarray
        return make_ndarray(model, dmref, supps, label)
    elseif length(supps) > 1 && _ignore_label(model, label)
        return supps
    elseif length(supps) > 1
        labels = transcription_data(model).measure_support_labels[mref]
        inds = map(s -> any(l -> l <: label, s), labels)
        return supps[inds]
    else
        return first(supps)
    end
end

################################################################################
#                             EXPRESSION QUERIES
################################################################################
"""
    transcription_expression(model::JuMP.Model,
        expr::JuMP.AbstractJuMPScalar;
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
        ndarray::Bool = false])

Return the transcribed expression(s) corresponding to `expr`. Errors
if `expr` cannot be transcribed. Also can query via the syntax:
```julia
transcription_expression(expr::JuMP.AbstractJuMPScalar;
                         [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
                         ndarray::Bool = false])
```
If the infinite model contains a built transcription model. By default, this
method returns only transcribed expressions associated with public supports. All the
expressions can be returned by setting `label = All`.

If `expr` is infinite and `ndarray = true` then an n-dimensional array will be
returned in accordance with the infinite parameters that have unique object
numbers. In this case, `label` will be used to search the intersection of the
supports that use the label. This is defers from the default behavior which
considers the union.

**Example**
```julia-repl
julia> transcription_expression(trans_model, my_expr)
x(support: 1) - y

julia> transcription_expression(my_expr)
x(support: 1) - y
```
"""
function transcription_expression(model::JuMP.Model,
    expr::Union{JuMP.GenericAffExpr, JuMP.GenericQuadExpr};
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    # get the object numbers of the expression and form the support iterator
    obj_nums = InfiniteOpt._object_numbers(expr)
    support_indices = support_index_iterator(model, obj_nums)
    exprs = Vector{JuMP.AbstractJuMPScalar}(undef, length(support_indices))
    check_labels = length(exprs) > 1 && !_ignore_label(model, label)
    label_inds = ones(Bool, length(exprs))
    # iterate over the indices and compute the values
    for (i, idx) in enumerate(support_indices)
        supp = index_to_support(model, idx)
        if check_labels && !any(l -> l <: label, index_to_labels(model, idx))
            @inbounds label_inds[i] = false
        end
        @inbounds exprs[i] = transcription_expression(model, expr, supp)
    end
    # return the expressions
    if ndarray
        return make_ndarray(model, expr, exprs, label)
    else
        exprs = exprs[label_inds]
        return length(support_indices) > 1 ? exprs : first(exprs)
    end
end

# Define for variables
function transcription_expression(model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef;
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false)
    return transcription_variable(model, vref, label = label, ndarray = ndarray)
end

# Dispatch for internal models
function transcription_expression(expr::JuMP.AbstractJuMPScalar;
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    model = InfiniteOpt._model_from_expr(expr)
    if model === nothing
        return zero(JuMP.AffExpr) + JuMP.constant(expr)
    else
        trans_model = InfiniteOpt.optimizer_model(model)
    end
    return transcription_expression(trans_model, expr, label = label,
                                    ndarray = ndarray)
end

"""
    InfiniteOpt.optimizer_model_expression(expr::JuMP.AbstractJuMPScalar,
        ::Val{:TransData};
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
        ndarray::Bool = false])

Proper extension of [`InfiniteOpt.optimizer_model_expression`](@ref) for
`TranscriptionModel`s. This simply dispatches to [`transcription_expression`](@ref).
"""
function InfiniteOpt.optimizer_model_expression(
    expr::Union{JuMP.GenericAffExpr, JuMP.GenericQuadExpr},
    ::Val{:TransData};
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false)
    return transcription_expression(expr, label = label, ndarray = ndarray)
end

"""
    InfiniteOpt.expression_supports(model::JuMP.Model,
        expr::JuMP.AbstractJuMPScalar,
        key::Val{:TransData} = Val(:TransData);
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
        ndarray::Bool = false])

Return the support alias mappings associated with `expr`. Errors if `expr` cannot
be transcribed.
"""
function InfiniteOpt.expression_supports(model::JuMP.Model,
    expr::Union{JuMP.GenericAffExpr, JuMP.GenericQuadExpr},
    key::Val{:TransData} = Val(:TransData);
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    # get the object numbers of the expression and form the support iterator
    obj_nums = sort(InfiniteOpt._object_numbers(expr))
    support_indices = support_index_iterator(model, obj_nums)
    supps = Vector{Tuple}(undef, length(support_indices))
    check_labels = length(supps) > 1 && !_ignore_label(model, label)
    param_supps = parameter_supports(model)
    label_inds = ones(Bool, length(supps))
    # iterate over the indices and compute the values
    for (i, idx) in enumerate(support_indices)
        if check_labels && !any(l -> l <: label, index_to_labels(model, idx))
            @inbounds label_inds[i] = false
        end
        @inbounds supps[i] = Tuple(param_supps[j][idx[j]] for j in obj_nums)
    end
    # return the supports
    if ndarray
        return make_ndarray(model, expr, supps, label)
    else
        supps = supps[label_inds]
        return length(support_indices) > 1 ? supps : first(supps)
    end
end

################################################################################
#                             CONSTRAINT QUERIES
################################################################################
"""
    transcription_constraint(model::JuMP.Model,
        cref::InfiniteOpt.InfOptConstraintRef;
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
        ndarray::Bool = false])

Return the transcribed constraint reference(s) corresponding to `cref`. Errors
if `cref` has not been transcribed. Also can query via the syntax:
```julia
transcription_constraint(cref::InfiniteOpt.InfOptConstraintRef;
    [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false])
```
If the infinite model contains a built transcription model. By default, this
method returns only transcribed constraints associated with public supports. All the
constraints can be returned by setting `label = All`.

If `cref` is infinite and `ndarray = true` then an n-dimensional array will be
returned in accordance with the infinite parameters that have unique object
numbers. In this case, `label` will be used to search the intersection of the
supports that use the label. This is defers from the default behavior which
considers the union.

**Example**
```julia-repl
julia> transcription_constraint(trans_model, fin_con)
fin_con : x(support: 1) - y <= 3.0

julia> transcription_constraint(fin_con)
fin_con : x(support: 1) - y <= 3.0
```
"""
function transcription_constraint(model::JuMP.Model,
    cref::InfiniteOpt.InfOptConstraintRef;
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    constr = get(transcription_data(model).constr_mappings, cref, nothing)
    if constr === nothing
      error("Constraint reference $cref not used in transcription model.")
    end
    if ndarray
        return make_ndarray(model, cref, constr, label)
    elseif length(constr) > 1 && _ignore_label(model, label)
        return constr
    elseif length(constr) > 1
        labels = transcription_data(model).constr_support_labels[cref]
        inds = map(s -> any(l -> l <: label, s), labels)
        return constr[inds]
    else
        return first(constr)
    end
end

# Dispatch for internal models
function transcription_constraint(cref::InfiniteOpt.InfOptConstraintRef;
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    trans_model = InfiniteOpt.optimizer_model(JuMP.owner_model(cref))
    return transcription_constraint(trans_model, cref, label = label,
                                    ndarray = ndarray)
end

"""
    InfiniteOpt.optimizer_model_constraint(cref::InfiniteOpt.InfOptConstraintRef,
        ::Val{:TransData};
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel
        ndarray::Bool = false])

Proper extension of [`InfiniteOpt.optimizer_model_constraint`](@ref) for
`TranscriptionModel`s. This simply dispatches to [`transcription_constraint`](@ref).
"""
function InfiniteOpt.optimizer_model_constraint(
    cref::InfiniteOpt.InfOptConstraintRef,
    ::Val{:TransData};
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    return transcription_constraint(cref, label = label, ndarray = ndarray)
end

"""
    InfiniteOpt.constraint_supports(model::JuMP.Model,
        cref::InfiniteOpt.InfOptConstraintRef,
        key::Val{:TransData} = Val(:TransData);
        [label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
        ndarray::Bool = false])

Return the support alias mappings associated with `cref`. Errors if `cref` is
not transcribed.
"""
function InfiniteOpt.constraint_supports(model::JuMP.Model,
    cref::InfiniteOpt.InfOptConstraintRef,
    key::Val{:TransData} = Val(:TransData);
    label::Type{<:InfiniteOpt.AbstractSupportLabel} = InfiniteOpt.PublicLabel,
    ndarray::Bool = false
    )
    supps = get(transcription_data(model).constr_supports, cref, nothing)
    if supps === nothing
        error("Constraint reference $cref not used in transcription model.")
    end
    if ndarray
        return make_ndarray(model, cref, supps, label)
    elseif length(supps) > 1 && _ignore_label(model, label)
        return supps
    elseif length(supps) > 1
        labels = transcription_data(model).constr_support_labels[cref]
        inds = map(s -> any(l -> l <: label, s), labels)
        return supps[inds]
    else
        return first(supps)
    end
end

################################################################################
#                             OTHER QUERIES
################################################################################
"""
    parameter_supports(model::JuMP.Model)::Tuple

Return the collected parameter support tuple that is stored in
`TranscriptionData.supports`.
"""
function parameter_supports(model::JuMP.Model)::Tuple
    return transcription_data(model).supports
end

"""
    support_index_iterator(model::JuMP.Model, [obj_nums::Vector{Int}])::CartesianIndices

Return the `CartesianIndices` that determine the indices of the unique combinations
of `TranscriptionData.supports` stored in `model`. If `obj_nums` is specified,
then the indices will only include the tuple elements uses indices are included
in the object numbers `obj_nums` and all others will be assigned the last index
which should correspond to an appropriately sized placeholder comprised of `NaN`s.
Note this method assumes that [`set_parameter_supports`](@ref) has already been
called and that the last elements of each support vector contains a placeholder
value.
"""
function support_index_iterator(model::JuMP.Model)::CartesianIndices
    raw_supps = parameter_supports(model)
    return CartesianIndices(ntuple(i -> 1:length(raw_supps[i])-1, length(raw_supps)))
end

# Generate for a subset of object numbers (use last index as placeholder --> support with NaNs)
function support_index_iterator(model::JuMP.Model,
                                obj_nums::Vector{Int})::CartesianIndices
    raw_supps = parameter_supports(model)
    lens = map(i -> length(i), raw_supps)
    # prepare the indices of each support combo
    # note that the actual supports are afrom 1:length-1 and the placeholders are at the ends
    return CartesianIndices(ntuple(i -> i in obj_nums ? (1:lens[i]-1) : (lens[i]:lens[i]),
                                   length(raw_supps)))
end

"""
    index_to_support(model::JuMP.Model, index::CartesianIndex)::Vector{Float64}

Given a particular support `index` generated via [`support_index_iterator`](@ref)
using `model`, return the corresponding support from `TranscriptionData.supports`
using placeholder `NaN`s as appropriate for tuple elements that are unneeded.
"""
function index_to_support(model::JuMP.Model,
                          index::CartesianIndex)::Vector{Float64}
    raw_supps = parameter_supports(model)
    return [j for i in eachindex(index.I) for j in raw_supps[i][index[i]]]
end

"""
    index_to_labels(model::JuMP.Model, index::CartesianIndex)::Set{DataType}

Given a particular support `index` generated via [`support_index_iterator`](@ref)
using `model`, return the corresponding support label set from `TranscriptionData.support_labels`.
"""
function index_to_labels(model::JuMP.Model,
                         index::CartesianIndex)::Set{DataType}
    raw_labels = transcription_data(model).support_labels
    labels = Set{DataType}()
    for (i, j) in enumerate(index.I)
        union!(labels, raw_labels[i][j])
    end
    return labels
end

################################################################################
#                               QUERY FORMATERS
################################################################################
# Helper function for getting the array type T
function _get_array_type(array::Array{T, N}) where {T, N}
    return T
end

## Helper functions to consistently get object numbers
# Fallback
function _get_object_numbers(ref)
    return InfiniteOpt._object_numbers(ref)
end

# Expressions
function _get_object_numbers(expr::Union{JuMP.GenericAffExpr, JuMP.GenericQuadExpr})
    return sort(InfiniteOpt._object_numbers(expr))
end

"""
    make_narray(model::JuMP.Model,
                ref::Union{JuMP.AbstractJuMPScalar, InfiniteOpt.InfOptConstraintRef},
                info::Vector,
                label::Type{<:InfiniteOpt.AbstractSupportLabel})::Array

Take the results`info` associated with `ref` and rearrange them into an
n-dimensional array where the axes correspond to the infinite parameter dependencies
in accordance with their creation. Note that this works by querying the object
numbers. Thus, independent infinite parameters will each get their own dimension
(even if they are defined at the same time in an array) and each dependent infinite
parameter group will have its own dimension.
"""
function make_ndarray(model::JuMP.Model, ref, info::Vector, label::DataType)
    # get the object numbers
    obj_nums = _get_object_numbers(ref)
    # return result if it is from a finite object
    if isempty(obj_nums)
        return info
    end
    # determine the dimensions of the new array
    raw_supps = parameter_supports(model)
    dims = Tuple(length(raw_supps[i]) - 1 for i in eachindex(raw_supps) if i in obj_nums)
    # check that the lengths match (otherwise we'll have some sparse set)
    # TODO add capability to avoid this problem (make reduced array by looking at the supports)
    if length(info) != prod(dims)
        error("Unable to make `ndarray`. This is likely due to the object being " *
              "over a portion of the infinite-domain (e.g., bounded constraints and " *
              "certain reduced infinite variables.")
    end
    # make and populate the array
    narray = Array{_get_array_type(info)}(undef, dims)
    for (i, idx) in enumerate(eachindex(narray))
        narray[idx] = info[i]
    end
    # rearrange the array as needed to match the object number order
    sorted_array = issorted(obj_nums) ? narray : permutedims(narray, sortperm(obj_nums))
    # consider the label specified (this will enforce the intersection of labels)
    if _ignore_label(model, label)
        return sorted_array
    else
        labels = transcription_data(model).support_labels[obj_nums]
        inds = map(sets -> findall(s -> any(l -> l <: label, s), sets), labels)
        return sorted_array[inds...]
    end
end


using MacroTools
using Match
using Parameters
using Parameters
using AxisArrays
using Unitful

export cohere, component, mask, ncomponents, components, component_means,
  windowing, map_windowing

abstract type CoherenceMethod end
struct CParams{M,P}
  cort::P
  ncomponents::Int
  skipframes::Int
  window::typeof(1.0s)
  delta::typeof(1.0s)
  method::M
end

const Coherence = MetaArray{<:AxisArray,<:CParams}
const CParamLike = Union{CParams,Coherence}

CorticalSpecrtralTemporalResponses.Δt(as::CParamLike) = as.delta

function Base.show(io::IO,::MIME"text/plain",x::Coherence)
  if !get(io, :compact, false)
    println(io,"Coherence Components")
    describe_axes(io,x)
  else
    println(io,string(duration(x))," Coherence Components")
  end
end

ncomponents(x::CParams) = x.ncomponents
ncomponents(x::AbstractArray) = size(x,axisdim(x,Axis{:component}))
components(x::CParams) = 1:ncomponents(x)
components(x::AbstractArray) = axisvalues(AxisArrays.axes(x,Axis{:component}))[1]
component(x::AbstractArray,n) = x[Axis{:component}(n)]

CorticalSpecrtralTemporalResponses.hastimes(x::Coherence) = HasTimes()

function component_means(C)
  mdims = filter(x -> x != axisdim(C,Axis{:component}),1:ndims(C))
  vec(mean(C,dims=mdims))
end

CorticalSpecrtralTemporalResponses.frame_length(params::CParams,x) =
  max(1,floor(Int,params.delta / Δt(x)))

function CParams(x;ncomponents=1,window=1000ms,
                 delta=10ms,method=:nmf,skipframes=0,method_kwds...)
  method = CoherenceMethod(Val{method},method_kwds)

  CParams(getmeta(x), ncomponents, skipframes,
          convert(typeof(1.0s),asseconds(window)), convert(typeof(1.0s),asseconds(delta)), method)
end

windowing(x,params::CParams) =
  windowing(x,length=params.window,step=params.delta)
windowlen(params::CParams,x) = round(Int,params.window/Δt(x))

function nunits(params::CParams,x)
  mapreduce(*,AxisArrays.axes(x)) do ax
    isa(ax,Axis{:time}) || isa(ax,Axis{:rate}) ? 1 : length(ax)
  end
end

cohere(x::MetaUnion{AxisArray};progressbar=true,params...) =
  cohere(x,CParams(x;params...),progressbar)

function cohere_progress(progressbar,x,params)
  if progressbar
    windows = windowing(x,params)
    Progress(length(windows),desc="Temporal Coherence Analysis: ")
  end
end

function cohere(x::MetaUnion{AxisArray},params::CParams,
                progressbar=true,
                progress = cohere_progress(progressbar,x,params))
  @assert axisdim(x,Axis{:time}) == 1
  @assert axisdim(x,Axis{:rate}) == 2
  @assert axisdim(x,Axis{:scale}) == 3
  @assert axisdim(x,Axis{:freq}) == 4
  @assert ndims(x) == 4

  # if we already have components, just wrap up the values with
  # parameters (since we've already computed components)
  if :component in axisnames(x)
    return MetaArray(params,x)
  end

  windows = windowing(x,params)

  K = ncomponents(params)
  C_data = zeros(eltype(params.method,x),length(windows),size(x)[3:end]...,K)
  C = AxisArray(C_data,
                AxisArrays.axes(windows,Axis{:time}),
                AxisArrays.axes(x)[3:end]...,
                Axis{:component}(1:K))

  with_method(params.method,K) do extract
    for (i,w_inds) in enumerate(windows)
      skipped = w_inds[1:1+params.skipframes:end]
      # axis array can't handle skipped indices, so we assume
      # the right dimensionality
      components = extract(x[skipped,:,:,:])
      C[i,Base.axes(components)...] = components

      next!(progress)
    end
  end

  MetaArray(params,C)
end


# Fast sincos function for Julia 0.5
# (see https://github.com/JuliaLang/julia/pull/21589)
@inline function sincos(v::Float64)
    return Base.llvmcall("""
        %f = bitcast i8 *%1 to void (double, double *, double *)*
        %ps = alloca double
        %pc = alloca double
        call void %f(double %0, double *%ps, double *%pc)
        %s = load double, double* %ps
        %c = load double, double* %pc
        %res0 = insertvalue [2 x double] undef, double %s, 0
        %res = insertvalue [2 x double] %res0, double %c, 1
        ret [2 x double] %res
        """, Tuple{Float64,Float64}, Tuple{Float64,Ptr{Void}}, v,
        cglobal((:sincos, Base.Math.libm)))
end

sincos(v::Real) = ( (r, i) = reim(cis(v)); (i, r) )


# Routines for interaction with pymatgen, e.g. converting to
# its structures from the DFTK equivalents

function pymatgen_lattice(lattice::AbstractArray)
    # Notice: Pymatgen uses rows as lattice vectors, so we unpeel
    # our lattice column by column. The default unit in pymatgen is Ǎngström
    mg = pyimport("pymatgen")
    bohr_to_A = 1 / units.Ǎ
    mg.Lattice([Array(bohr_to_A .* lattice[:, i]) for i in 1:3])
end
pymatgen_lattice(model::Model) = pymatgen_lattice(model.lattice)

function pymatgen_structure(model_or_lattice, atoms)
    mg = pyimport("pymatgen")
    pylattice = pymatgen_lattice(model_or_lattice)

    n_species = sum(length(pos) for (spec, pos) in atoms)
    pyspecies = Vector{Int}(undef, n_species)
    pypositions = Array{Vector{Float64}}(undef, n_species)
    ispec = 1
    for (spec, pos) in atoms
        for coord in pos
            pyspecies[ispec] = spec.Z
            pypositions[ispec] = Vector{Float64}(coord)
            ispec = ispec + 1
        end
    end
    @assert ispec == n_species + 1

    mg.Structure(pylattice, pyspecies, pypositions)
end
pymatgen_structure(model::Model) = pymatgen_structure(model, model.atoms)

function pymatgen_bandstructure(basis, λ, εF, klabels)
    Spin = pyimport("pymatgen.electronic_structure.core").Spin
    bandstructure = pyimport("pymatgen.electronic_structure.bandstructure")

    # This assumes no spin polarization
    @assert basis.model.spin_polarization in (:none, :spinless)

    eigenvals_spin_up = Matrix{eltype(λ[1])}(undef, length(λ[1]), length(basis.kpoints))
    for (ik, λs) in enumerate(λ)
        eigenvals_spin_up[:, ik] = λs
    end
    eigenvals = Dict(Spin.up => eigenvals_spin_up)

    kcoords = [kpt.coordinate for kpt in basis.kpoints]
    pylattice = pymatgen_lattice(basis.model.lattice)
    bandstructure.BandStructureSymmLine(
        kcoords, eigenvals, pylattice.reciprocal_lattice, εF,
        labels_dict=klabels, coords_are_cartesian=true
    )
end

function load_lattice_pymatgen(T, pyobj::PyObject)
    mg = pyimport("pymatgen")

    if pyisinstance(pyobj, mg.Structure)
        load_lattice_pymatgen(T, pyobj.lattice)
    elseif pyisinstance(pyobj, mg.Lattice)
        lattice = Matrix{T}(undef, 3, 3)
        for i in 1:3, j in 1:3
            lattice[i, j] = units.Ǎ * get(get(pyobj.matrix, j-1), i-1)
        end
        Mat3{T}(lattice)
    else
        error("load_lattice_pymatgen not implemented for python type $pyobj")
    end
end

"""
Load a DFTK-compatible atoms representation from a supported pymatgen object.
All atoms are using a Coulomb model.
"""
function load_atoms_pymatgen(T, pyobj::PyObject)
    @assert pyisinstance(pyobj, pyimport("pymatgen").Structure)
    map(unique(pyobj.species)) do spec
        coords = [s.frac_coords for s in pyobj.sites if s.specie == spec]
        ElementCoulomb(spec.number) => coords
    end
end


export RawBLAS

module RawBLAS
using Base.LinAlg # force built-in BLAS library initialization
using ..Mocha.blasfunc
using Compat

for (gemm, elty) in ((:dgemm_, Float64), (:sgemm_, Float32))
  @eval begin
    function gemm!(transA::Char, transB::Char, M::Int, N::Int, K::Int, alpha::$elty,
        A, lda, B, ldb, beta::$elty, C, ldc)
      ccall(($(blasfunc(gemm)), Base.libblas_name), Void,
          (Ptr{UInt8}, Ptr{UInt8}, Ptr{Base.LinAlg.BlasInt}, Ptr{Base.LinAlg.BlasInt},
          Ptr{Base.LinAlg.BlasInt}, Ptr{$elty}, Ptr{$elty}, Ptr{Base.LinAlg.BlasInt},
          Ptr{$elty}, Ptr{Base.LinAlg.BlasInt}, Ptr{$elty}, Ptr{$elty},
          Ptr{Base.LinAlg.BlasInt}),
          &transA, &transB, &M, &N, &K, &alpha, A, &lda, B, &ldb, &beta, C, &ldc)
    end

    function gemm!(transA::Char, transB::Char, M::Int, N::Int, K::Int, alpha::$elty,
        A, B, beta::$elty, C)
      lda = (transA == 'N') ? M : K;
      ldb = (transB == 'N') ? K : N;
      ldc = M;
      gemm!(transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc)
    end
  end # eval
end

for (gemv, elty) in ((:dgemv_, Float64), (:sgemv_, Float32))
  @eval begin
    function gemv!(trans::Char, M::Int, N::Int, alpha::$elty, A, lda, x, incx, beta::$elty, y, incy)
      ccall(($(blasfunc(gemv)), Base.libblas_name), Void,
          (Ptr{UInt8}, Ptr{Base.LinAlg.BlasInt}, Ptr{Base.LinAlg.BlasInt}, Ptr{$elty},
          Ptr{$elty}, Ptr{Base.LinAlg.BlasInt}, Ptr{$elty}, Ptr{Base.LinAlg.BlasInt},
          Ptr{$elty}, Ptr{$elty}, Ptr{Base.LinAlg.BlasInt}),
          &trans, &M, &N, &alpha, A, &lda, x, &incx, &beta, y, &incy)
    end
    function gemv!(trans::Char, M::Int, N::Int, alpha::$elty, A, x, beta::$elty, y)
      lda = M
      incx = 1
      incy = 1
      gemv!(trans, M, N, alpha, A, lda, x, incx, beta, y, incy)
    end
  end
end

end # module RawBLAS


sun = Star("Kerbol", 261600000., 1.1723328e18, (254, 198, 20));
kerbin =    CelestialBody("Kerbin", 600000.,    3.5316000e12,       21549.425,          0.,             Orbit(13599840256.,     0.,             0.,                     0.,                     0.,             3.14,   0.,     sun),       (6,   154, 244))
mun =       CelestialBody("Mun",    200000.,    6.5138398e10,       138984.38,          deg2rad(230.),  Orbit(12000000.,        0.,             0.,                     0.,                     0.,             1.7,    0.,     kerbin),    (146, 150, 146))
minmus =    CelestialBody("Minmus", 60000.,     1765800026.,        40400.,             deg2rad(230.),  Orbit(47000000.,        0.,             deg2rad(6.),            deg2rad(78.),           deg2rad(38.),   0.9,    0.,     kerbin),    (160, 254, 176))
moho =      CelestialBody("Moho",   250000.,    168.609378655e9,    1210000.,           deg2rad(190.),  Orbit(5263138304.,      0.200000003,    deg2rad(7.),            deg2rad(70.),           deg2rad(15.),   3.14,   0.,     sun),       (210, 178, 110))
eve =       CelestialBody("Eve",    700000.,    8171.730229211e9,   80500.,             0.,             Orbit(9832684544.,      0.01,           deg2rad(2.099999905),   deg2rad(15.000000000),  deg2rad(0.),    3.14,   0.,     sun),       (200, 160, 240))
duna =      CelestialBody("Duna",   320000.,    301.363211975e9,    65517.859375,       deg2rad(90.),   Orbit(20726155264.,     0.051,          deg2rad(0.06),          deg2rad(135.5),         0.,             3.14,   0.,     sun),       (254, 66,  30 ))
ike =       CelestialBody("Ike",    130000.,    18568368573.,       65517.862134808,    0.,             Orbit(3200000.,         0.03,           deg2rad(0.2),           0.,                     0.,             1.7,    0.,     duna),      (168, 164, 150))
jool =      CelestialBody("Jool",   6000000.,   282528004209995.,   36000.,             0.,             Orbit(68773560320.,     0.05,           deg2rad(1.304),         deg2rad(52.),           deg2rad(0.),    0.1,    0.,     sun),       (126, 190, 2  ))
laythe =    CelestialBody("Laythe", 500000.,    1962000029236.,     52980.87905938,     deg2rad(90.),   Orbit(27184000.,        0.,             0.,                     0.,                     0.,             3.14,   0.,     jool),      (90,  126, 154))
vall =      CelestialBody("Vall",   300000.,    207481499474.,      105962.088893924,   0.,             Orbit(43152000.,        0.,             0.,                     0.,                     0.,             0.9,    0.,     jool),      (196, 254, 248))
bop =       CelestialBody("Bop",    65000.,     2486834944.,        544507.428516654,   deg2rad(230.),  Orbit(128500000.,       0.235,          deg2rad(15.),           deg2rad(10.),           deg2rad(25.),   0.9,    0.,     jool),      (226, 202, 118))
tylo =      CelestialBody("Tylo",   600000.,    2825280042100.,     211926.35802123,    0.,             Orbit(68500000.,        0.,             deg2rad(0.025),         deg2rad(0.),            deg2rad(0.),    3.14,   0.,     jool),      (216, 220, 214))
gilly =     CelestialBody("Gilly",  13000.,     8289450.,           28255.,             deg2rad(5.),    Orbit(31500000.,        0.55,           deg2rad(12.),           deg2rad(80.),           deg2rad(10.),   0.9,    0.,     eve),       (230, 218, 166))
pol =       CelestialBody("Pol",    44000.,     721702080.,         901902.623531173,   deg2rad(25.),   Orbit(179890000.,       0.17085,        deg2rad(4.25),          deg2rad(2.0),           deg2rad(15.),   0.9,    0.,     jool),      (214, 182, 10 ))
dres =      CelestialBody("Dres",   138000.,    21484488600.,       34800.,             deg2rad(25.),   Orbit(40839348203.,     0.145,          deg2rad(5.),            deg2rad(280.),          deg2rad(90.),   3.14,   0.,     sun),       (198, 202, 200))
eeloo =     CelestialBody("Eeloo",  210000.,    74410814527.,       19460.,             deg2rad(25.),   Orbit(90118820000.,     0.26,           deg2rad(6.15),          deg2rad(50.),           deg2rad(260.),  3.14,   0.,     sun),       (244, 244, 218))


using ModernGL, GLWindow, GLAbstraction, GLFW, GeometryTypes

const window = GLWindow.create_glcontext("Example", resolution=(512, 512), debugging=true)

const vsh = vert"""
{{GLSL_VERSION}}
in vec2 position;

void main(){
	gl_Position = vec4(position, 0, 1.0);
}
"""

const fsh = frag"""
{{GLSL_VERSION}}
out vec4 outColor;

void main() {
	outColor = vec4(1.0, 1.0, 1.0, 1.0);
}
"""

const triangle = std_renderobject(
	Dict{Symbol, Any}(
        :position => GLBuffer(Point2f0[(0.0, 0.5), (0.5, -0.5), (-0.5,-0.5)]),
    ),
	LazyShader(vsh, fsh)
)

glClearColor(0, 0, 0, 1)

while isopen(window)
  	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)

    render(triangle)

  	swapbuffers(window)
  	pollevents()
end


### A Pluto.jl notebook ###
# v0.17.1

using Markdown
using InteractiveUtils

# ╔═╡ dc3c2030-144f-11eb-08ab-0b68dc2698ac
begin
	using Pkg
	cd(joinpath(dirname(@__FILE__),".."))
    Pkg.activate(pwd())
	using BenchmarkTools
end

# ╔═╡ 615d2b78-9a47-41bf-bae4-f0f7fb875956
# Make cells wider
html"""<style>
/*              screen size more than:                     and  less than:                     */
@media screen and (max-width: 699px) { /* Tablet */
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1200px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/

}

@media screen and (min-width: 700px) and (max-width: 1199px) { /* Laptop*/
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1200px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/
}

@media screen and (min-width:1000px) and (max-width: 1920px) { /* Desktop */
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1000px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/
}

@media screen and (min-width:1921px) { /* Stadium */
  /* Nest everything into here */
    main { /* Same as before */
        max-width: 1200px !important; /* Same as before */
        margin-right: 100px !important; /* Same as before */
    } /* Same as before*/
}
</style>
"""

# ╔═╡ 1c49d8ae-144e-11eb-024a-137460f78b15
md"""
# Performance
When building more complex programs, performance can be an issue. The [performance tips](https://docs.julialang.org/en/v1/manual/performance-tips/) website already contains a lot tips. We will illustrate potential performance gains for a function by providing one or more alternatives. In order to make sure the the functions produce the same result, the results are verified to be identical before starting the benchmarks.

The actual performance is measured using [BenchmarkTools](https://github.com/JuliaCI/BenchmarkTools.jl). More advanced methods (that we cannot cover due to time constraints) included profiling your code. In Visual Studio Code you can get a detailed overview with the [`@profview`](https://discourse.julialang.org/t/julia-vs-code-extension-version-v0-17-released/42865) macro. For a REPL based method you could use [Traceur](https://github.com/JunoLab/Traceur.jl).

## Tools
Below you can find a set of very generic functions that we will be using to evaluate the functions that we will be benchmarking.

### Notes
At some point we try to make use of multithreading. You can see the number of threads that are currently being used with
```julia
Threads.nthreads()
```

Should this be only 1, you can set the number of threads to use by setting the environment variable `JULIA_NUM_THREADS` to the required before starting julia.

In a windows terminal:
```
$env:JULIA_NUM_THREADS = 4
julia
```
In a Linux/Mac OS terminal:
```
export JULIA_NUM_THREADS=4
julia
```

both examples above suppose "julia" is known in your terminal. If this is not the case, simply replace it by the path to the julia executable.

*Remark*: starting from julia 1.5 you can use command line arguments to specify the number of threads e.g. `julia --threads 4`
"""

# ╔═╡ a4ed00b8-144f-11eb-0c66-932242a1767c
begin
	"""
    testfuns(F, args...; kwargs...)

Benchmark the functions in F using `args` and `kwargs`.
"""
function testfuns(F::Union{Array{Function,1}, Array{Type,1}}, args...; kwargs...)
    # set up benchmark suite
    suite = BenchmarkGroup()
    for f in F
        suite[string(f)] = @benchmarkable $(f)($(args)...; $(kwargs)...)
    end
    # tune and run suite
    tune!(suite)
    results = run(suite, verbose=get(kwargs, :verbose, false))
    # show results
    for f in F
        println("\nResult for $(f):\n$("-"^length("Result for $(f):"))")
        display(results[string(f)])
    end
    medians = sort!([(fun, median(trial.times)) for (fun, trial) in results], by=x->x[2])
    speedup = round(Int,medians[end][2] / medians[1][2])
    println("\n$(speedup)x speedup (fastest: $(medians[1][1]), slowest: $(medians[end][1]))\n")
    return

end
	nothing
end

# ╔═╡ e32569b0-144f-11eb-33e9-8fd8755c208f
begin
	"""
		equality(F::Array{Function,1}, args...; kwargs...)

	Verify the result of an array of functions F is the same using `args` and `kwargs`.

	### Notes
	The first function in the array is used as the reference
	"""
	function equality(F::Array{Function,1}, args...; kwargs...)
		return all([equality(F[1], F[i], args...; kwargs...) for i in 2:length(F)])
	end

	"""
		equality(f::Function, g::Function, args...; kwargs...)

	Verify the result of function f and g is the same (element-wise) using `args` and `kwargs`.
	"""
	function equality(f::Function, g::Function, args...; kwargs...)
		return all(isequal.(map(f->f(args...; kwargs...), [f, g])...))
	end

	nothing
end

# ╔═╡ ff480c88-144f-11eb-2839-598b8452a945
md"""
## Hands-on
"""

# ╔═╡ 1e0ef46a-1450-11eb-2676-d94d5cc1f073
begin

	"""
		renew(char="", n::Int=0)

	Slow implementation of string joining
	"""
	function renew(char="", n::Int=0; kwargs...)
		res = ""
		for i in 1:n
			res *= char
		end

		return res
	end

	"""
		buffer(char="", n::Int=0)

	Fast implementation of string joining
	"""
	function buffer(char="", n::Int=0; kwargs...)
		res = IOBuffer()
		for i in 1:n
			print(res, char)
		end

		return String(take!(res))
	end

	"""
		demo1

	- Illustration of implementation issues

	speedup ∝ length
	"""
	function demo1()
		F = [renew, buffer];
		args=("a", 10000)
		# test validity
		@info "Testing demo 1"
		@assert equality(F, args...)
		# Run benchmark
		@info "Benching demo 1"
		testfuns(F, args...)
	end

	demo1()

end

# ╔═╡ 41ef249a-1450-11eb-2c52-0930531fa46e
begin
	function f_alloc(x::Vector{T}, y::Array{T,2}; props::Vector{T}, kwargs...) where T
		# read properties
		prop_A = props[1]
		prop_B = props[2]
		prop_C = props[3]
		prop_D = props[4]
		# initialise
		result = zeros(T, size(x))
		result .+= x
		# update result
		result = result .+ prop_A .* y[:,1]
		result = result .+ prop_B .* y[:,2]
		result = result .+ prop_C .* y[:,3]
		result = result .+ prop_D .* y[:,4]
		return result
	end

	struct Props{T<:AbstractFloat}
		prop_A::T
		prop_B::T
		prop_C::T
		prop_D::T
	end

	@views function f_noalloc(x::Vector{T}, y::Array{T,2}; p::Props, kwargs...) where T
		return x .+ p.prop_A .* y[:,1] .+ p.prop_B .* y[:,2] + p.prop_C .* y[:,3] + p.prop_D .* y[:,4]
	end

	"""
		demo2()

	- Avoiding unnecessary allocations
	- Using views
	- Avoiding fields with abstract types
	"""
	function demo2()
		F = [f_alloc, f_noalloc]
		N = 10000
		x = rand(N)
		y = rand(N,4)
		args = [x, y]
		props = Float64[1,2,3,4]
		p = Props(props...)
		kwargs = Dict(:props => props, :p => p)
		# test validity
		@info "Testing demo 2"
		@assert equality(F, args...; kwargs...)
		# run benchmark
		@info "Benching demo 2"
		testfuns(F, args...; kwargs...)
	end

	demo2()

end

# ╔═╡ 6332ef7e-1450-11eb-0a36-41818207b78f
begin
	# avoid changing type of a variable
	function f_typechange(x)
		res = 0
		for val in x
			res += val
		end
		return res
	end

	function f_notypechange(x::Array{T,1}) where T <: Number
		res = zero(T)
		for val in x
			res += val
		end
		return res
	end

	"""
		demo3()

	- avoid changing type of a variable
	"""
	function demo3()
		F = [f_typechange, f_notypechange]
		N = 2
		args = [rand(N)]
		# test validity
		@info "Testing demo 3"
		@assert equality(F, args...)
		# run benchmark
		@info "Benching demo 3"
		testfuns(F, args...)
	end

	demo3()
end

# ╔═╡ 75948388-1450-11eb-0a0e-431ad5929504
begin
	f_norm(x) = 3x.^2 + 4x + 7x.^3;
	f_vec(x) = @. 3x.^2 + 4x + 7x.^3;

	"""
		demo 4

	- use vectorisation where possible
	"""
	function demo4()
		F = [f_norm, f_vec]
		N::Int = 1e6
		args = [rand(N)]
		# test validity
		@info "Testing demo 4"
		@assert equality(F, args...)
		# run benchmark
		@info "Benching demo 4"
		testfuns(F, args...)
	end

	demo4()

end

# ╔═╡ 98e356f4-1450-11eb-0229-eb57f262ea4d
begin
	function f_nopre(x::Vector; kwargs...)
		y = similar(x)
		for i in eachindex(x)
			y[i] = sum(x[1:i])
		end
		return y
	end

	function f_pre!(x::Vector{T}; y::Vector{T}, kwargs...) where {T}
		for i in eachindex(x)
			y[i] = sum(x[1:i])
		end
		return y
	end

	function f_preboost!(x::Vector{T}; y::Vector{T}, kwargs...) where {T}
		Threads.@threads for i in eachindex(x)
			@inbounds y[i] = sum(x[1:i])
		end
		return y
	end

	@views function f_preturboboost!(x::Vector{T}; y::Vector{T}, kwargs...) where {T}
		Threads.@threads for i in eachindex(x)
			@inbounds y[i] = sum(x[1:i])
		end
		return y
	end

	"""
		demo5()

	- preallocate outputs if possible
	- make use of threads if possible
	- make use of views if possible
	- skip index check when sure of dimensions
	"""
	function demo5()
		F = [f_nopre, f_pre!, f_preboost!, f_preturboboost!]
		@info "Currently running on $(Threads.nthreads()) threads"
		N::Int = 1e4
		x::Vector = rand(N)
		y_pre = similar(x)
		args = [x]
		kwargs = Dict( :y => y_pre)
		# test validity
		@info "Testing demo 5"
		@assert equality(F, args...; kwargs...)
		# run benchmark
		@info "Benching demo 5"
		testfuns(F, args...; kwargs...)
	end

	demo5()
end

# ╔═╡ b341b4d2-1450-11eb-0549-7db5bd6756ba
begin
	function grow(N::Int; kwargs...)
		res = []
		for _ in 1:N
			push!(res, 1)
		end
		return res
	end

	function nogrow(N::Int; x::Array{T,1}) where{T}
		res = similar(x)
		for i in 1:N
			res[i] = 1
		end
		return res
	end

	"""
		demo6()

	- growing vs. allocating for known dimensions
	"""
	function demo6()
		N = 10000
		x = rand(N)
		F = [grow, nogrow]
		args = [N]
		kwargs = Dict(:x => x)
		# test validity
		@info "Testing demo 6"
		@assert equality(F, args...; kwargs...)
		# run benchmark
		@info "Benching demo 6"
		testfuns(F, args...; kwargs...)
	end

	demo6()
end

# ╔═╡ c6702048-1450-11eb-1f1d-a54089545949
begin
	struct mystruct
		a
		b
		c
	end

	struct mytypedstruct{T}
		a::T
		b::T
		c::T
	end

	mutable struct mymutablestruct
		a
		b
		c
	end

	"""
		demo7

	- creation time of mutable vs unmutable structs and the parameter version
	"""
	function demo7()
		structs = [mystruct, mytypedstruct, mymutablestruct]
		@info "Benching demo 7"
		testfuns(structs, [1;2;3]...)
	end

	demo7()

end

# ╔═╡ df4563da-1450-11eb-08a7-25ed365f578c
begin
	function reader(s::mystruct)
		return (s.a, s.b, s.c)
	end

	function reader(s::mytypedstruct)
		return (s.a, s.b, s.c)
	end

	function reader(sm::mymutablestruct)
		return (sm.a, sm.b, sm.c)
	end

	"""
		demo8

	- read times of fields of mutable vs unmutable structs and the parameter version
	"""
	function demo8()
		vals = [1;2;3]
		@info "Benching demo8"
		for arg in [mystruct, mytypedstruct, mymutablestruct]
			println("\nResult for $(arg):\n$("-"^length("Result for $(arg):"))")
			x = arg(vals...)
			display(@benchmark reader($(x)))
		end
	end

	demo8()
end

# ╔═╡ Cell order:
# ╟─615d2b78-9a47-41bf-bae4-f0f7fb875956
# ╟─1c49d8ae-144e-11eb-024a-137460f78b15
# ╠═dc3c2030-144f-11eb-08ab-0b68dc2698ac
# ╠═a4ed00b8-144f-11eb-0c66-932242a1767c
# ╠═e32569b0-144f-11eb-33e9-8fd8755c208f
# ╟─ff480c88-144f-11eb-2839-598b8452a945
# ╠═1e0ef46a-1450-11eb-2676-d94d5cc1f073
# ╠═41ef249a-1450-11eb-2c52-0930531fa46e
# ╠═6332ef7e-1450-11eb-0a36-41818207b78f
# ╠═75948388-1450-11eb-0a0e-431ad5929504
# ╠═98e356f4-1450-11eb-0229-eb57f262ea4d
# ╠═b341b4d2-1450-11eb-0549-7db5bd6756ba
# ╠═c6702048-1450-11eb-1f1d-a54089545949
# ╠═df4563da-1450-11eb-08a7-25ed365f578c


function _extract_tsfresh_feats(column, shift; ka...)
    @from tsfresh imports extract_relevant_features
    @from tsfresh.utilities.dataframe_functions imports roll_time_series
    dst = @sprintf("tsfresh/%s_%s.h5", column, shift)
    isfile(dst) && return dst
    df = pd.read_parquet("df.parquet", columns = ["代码", column])
    if df[column].dtype == "uint8"
        df[column] = df[column] / 128 - 1
    end
    df[column] = df[column].replace([Inf, -Inf], NaN).fillna(0)
    y = pd.read_pickle("y.pkl")["y"]
    if length(y) == length(df)
        df = roll_time_series(df, "代码", nothing, nothing, 1, shift)
        df.drop(columns = "代码", inplace = true)
        y.index = df.groupby("id")["sort"].last().sort_values().index
        df_erf = extract_relevant_features(df, y; column_id = "id", column_sort = "sort", ka...)
    else
        df_erf = extract_relevant_features(df, y; column_id = "代码", ka...)
        df_erf.drop(columns = "代码", inplace = true)
    end
    df_erf = df_erf.astype("float32")
    df_erf.columns = ["$c-$shift" for c in df_erf.columns]
    to_hdf5(df_erf, dst)
    @eval GC.gc()
    return dst
end

function extract_tsfresh_feats(df, y = nothing; shifts = ["20T"], horizon = "20T", remove = true, ka...)
    Δt = df["时间戳"].groupby(df["代码"]).diff().median()
    shifts = @. ceil(Int, parsefreq(shifts) / Δt)
    if isnothing(y)
        horizon = ceil(Int, parsefreq(horizon) / Δt)
        y = df["涨幅"].groupby(df["代码"]).rolling(horizon).sum().groupby("代码").shift(-horizon).fillna(0)
        y = Series(y.reset_index(level = "代码", drop = true).sort_index().astype("float32"), name = "y")
    end
    df_meta, df_fea = split_metafeat(df)
    df_ts = pdhcat(df_fea, df_meta[["代码"]])
    df_ts.to_parquet("df.parquet")
    y.to_frame().to_pickle("y.pkl")
    !isdir("tsfresh") && mkdir("tsfresh")
    h5s = pmap(Iterators.product(df_fea.columns, shifts)) do (column, shift)
        _extract_tsfresh_feats(column, shift; ka...)
    end
    if length(df) != length(y)
        df = df.drop_duplicates(subset = "代码", keep = "last")
        df.reset_index(inplace = true)
        @assert length(df) == length(y)
    end
    df = pdhcat(df, read_hdf5.(h5s)...)
    remove && foreach(rm, ["df.parquet", "y.pkl", h5s...])
    return df
end

function extract_talib_feats(df, bycol = "code"; keep_columns = [])
    ohlc_cols = ["open", "high", "low", "close", "volume"]
    df_ohlc = df[ohlc_cols].astype("double")
    df_talib = df[df.columns.drop(ohlc_cols)]
    @from talib imports get_function_groups
    normclose = [get_function_groups()["Overlap Studies"];
                get_function_groups()["Price Transform"];
                ["APO", "MACD", "MACDEXT", "MACDFIX", "MINUS_DM", "MOM", "PLUS_DM"];
                ["TRANGE", "ATR", "LINEARREG", "LINEARREG_INTERCEPT", "STDDEV", "TSF"]]
    groupfuncs = [(g, f) for (g, fs) in get_function_groups() for f in fs]
    dfs = @showprogress 10 pmap(groupfuncs) do (g, f)
        g = split(g, ' ')[1]
        g == "Math" && return
        f == "MAVP" && return
        f == "VAR" && return
        c = "talib:" * g * "_" * f
        !isempty(keep_columns) && c ∉ keep_columns && return
        println(c)
        py"""
import gc
import pandas as pd
from talib.abstract import Function
def apptalib(x):
    df = Function($f)(x)
    if len(df.shape) == 1:
        df = df.to_frame()
    return df
df = $df_ohlc.groupby($df_talib[$bycol]).apply(apptalib)
if $f in $$normclose:
    for c in df.columns:
        df[c] = df[c] / $df_ohlc["close"]
df = df.astype('float32')
df.columns = [$c if c == 0 else $c + '_' + c  for c in df]
gc.collect()
"""
        return py"df"
    end
    return pdhcat(df_talib, filter(!isnothing, dfs)...)
end

function TradingFCParameters(tscal = 1)
    Dict(
        "abs_energy" => nothing,
        "count_above_mean" => nothing,
        "first_location_of_maximum" => nothing,
        "first_location_of_minimum" => nothing,
        "kurtosis" => nothing,
        "last_location_of_maximum" => nothing,
        "last_location_of_minimum" => nothing,
        "longest_strike_above_mean" => nothing,
        "longest_strike_below_mean" => nothing,
        "maximum" => nothing,
        "mean" => nothing,
        "mean_abs_change" => nothing,
        "mean_change" => nothing,
        "mean_second_derivative_central" => nothing,
        "median" => nothing,
        "minimum" => nothing,
        # "sample_entropy" => nothing,
        "skewness" => nothing,
        "standard_deviation" => nothing,
        "variance" => nothing,
        # "time_reversal_asymmetry_statistic" => [Dict("lag" => tscal * lag) for lag in [1, 5]],
        # "c3" => [Dict("lag" => tscal * lag) for lag in [1, 5]],
        # "symmetry_looking" => [Dict("r" => r * 0.05) for r in [1, 5, 10]],
        # "large_standard_deviation" => [Dict("r" => r * 0.05) for r in [1, 5, 10]],
        # "quantile" => [Dict("q" => q) for q in [0.25, 0.75]],
        # "autocorrelation" => [Dict("lag" => tscal * lag) for lag in [1, 5, 10]],
        # "agg_autocorrelation" => [Dict("f_agg" => s, "maxlag" => 40 * tscal) for s in ["mean", "median", "var"]],
        # "partial_autocorrelation" => [Dict("lag" => tscal * lag) for lag in [1, 5, 10]],
        # !"number_peaks" => [Dict("n" => tscal * n) for n in [1, 5, 10]],
        "binned_entropy" => [Dict("max_bins" => 10)],
        # !"fft_aggregated" => [Dict("aggtype" => s) for s in ["centroid", "variance", "skew", "kurtosis"]],
        # "max_langevin_fixed_point" => [Dict("m" => 3, "r" => 30)],
        # !"linear_trend" => [Dict("attr" => "pvalue"), Dict("attr" => "rvalue"), Dict("attr" => "slope"), Dict("attr" => "stderr")],
        # "agg_linear_trend" => [Dict("attr" => attr, "chunk_len" => tscal * i, "f_agg" => f)
        #                      for attr in ["rvalue", "slope", "stderr"]
        #                      for i in [5, 10, 50]
        #                      for f in ["max", "min", "mean", "var"]],
        # "augmented_dickey_fuller" => [Dict("attr" => "teststat"), Dict("attr" => "pvalue"), Dict("attr" => "usedlag")],
        # "ratio_beyond_r_sigma" => [Dict("r" => x) for x in [1, 5]],
        # "ar_coefficient" => [Dict("coeff" => coeff, "k" => k) for coeff in 0:4 for k in [10]],
    )
end


### ODEProblem from AbstractReactionNetwork ###
function DiffEqBase.ODEProblem(rn::DiffEqBase.AbstractReactionNetwork, u0::Union{AbstractArray, Number}, args...; kwargs...)
    isa(rn.odefun, ODEFunction) || error("Call addodes! before constructing ODEProblems")
    ODEProblem(rn.odefun, u0::Union{AbstractArray, Number}, args...; kwargs...)
end

### SDEProblem from AbstractReactionNetwork ###
function DiffEqBase.SDEProblem(rn::DiffEqBase.AbstractReactionNetwork, u0::Union{AbstractArray, Number}, args...; kwargs...)
    isa(rn.sdefun, SDEFunction) || error("Call addsdes! before constructing SDEProblems")
    SDEProblem(rn.sdefun, rn.g::Function, u0, args...;noise_rate_prototype=rn.p_matrix, kwargs...)
end

### DiscreteProblem, passing through syms
function DiffEqBase.DiscreteProblem(rn::DiffEqBase.AbstractReactionNetwork, u0, tspan::Tuple, p=nothing; kwargs...)
    f = (du,u,p,t) -> du.=u    # identity function to make syms works
    df = DiscreteFunction(f,syms=rn.syms)
    DiscreteProblem(df, u0, tspan, p; kwargs...)
end

### JumpProblem ###
function build_jump_problem(prob, aggregator, rn, kwargs...)
    if typeof(prob)<:DiscreteProblem && any(x->typeof(x) <: VariableRateJump, rn.jumps)
        error("When using time dependant reaction rates a DiscreteProblem should not be used (try an ODEProblem). Also, use a continious solver.")
    end

    # get a JumpSet of the possible jumps
    jset = network_to_jumpset(rn, prob.p)

    # construct map from species index to indices of reactions that depend on it
    if needs_vartojumps_map(aggregator) || needs_depgraph(aggregator)
        rxidxs_to_jidxs   = rxidxs_to_jidxs_map(rn, get_num_majumps(jset))
        spec_to_jumps_set = spec_to_dep_jumps_map(rn, rxidxs_to_jidxs)
        spec_to_jumps_vec = [sort!(collect(specset)) for specset in spec_to_jumps_set]
        jump_to_specs_vec = jump_to_dep_specs_map(rn, rxidxs_to_jidxs)
    else
        rxidxs_to_jidxs   = nothing
        spec_to_jumps_set = nothing
        spec_to_jumps_vec = nothing
        jump_to_specs_vec = nothing
    end

    # construct reaction dependency graph
    if needs_depgraph(aggregator)
        dep_graph = depgraph_from_network(rn, jset, rxidxs_to_jidxs, spec_to_jumps_set)
    else
        dep_graph = nothing
    end

    JumpProblem(prob, aggregator, jset; dep_graph=dep_graph,
                                        vartojumps_map=spec_to_jumps_vec,
                                        jumptovars_map=jump_to_specs_vec,
                                        kwargs...)
end

### JumpProblem from AbstractReactionNetwork ###
function DiffEqJump.JumpProblem(prob, aggregator, rn::DiffEqBase.AbstractReactionNetwork; kwargs...)
    (rn.jumps !== nothing) || error("Call addjumps! before constructing JumpProblems")
    build_jump_problem(prob, aggregator, rn, kwargs...)
end

### SteadyStateProblems from AbstractReactionNetwork ###
function DiffEqBase.SteadyStateProblem(rn::DiffEqBase.AbstractReactionNetwork, args...; kwargs...)
    isa(rn.odefun, ODEFunction) || error("Call addodes! before constructing SteadyStateProblems")
    SteadyStateProblem(rn.odefun, args...; kwargs...)
end

function DiffEqBase.SteadyStateProblem{isinplace}(rn::DiffEqBase.AbstractReactionNetwork, args...; kwargs...) where isinplace
    isa(rn.odefun, ODEFunction) || error("Call addodes! before constructing SteadyStateProblems")
    SteadyStateProblem{isinplace}(rn.odefun, args...; kwargs...)
end