# create a new "build_series_args" which converts all inputs into xs = Any[xitems], ys = Any[yitems].
# Special handling for: no args, xmin/xmax, parametric, dataframes
# Then once inputs have been converted, build the series args, map functions, etc.
# This should cut down on boilerplate code and allow more focused dispatch on type
# note: returns meta information... mainly for use with automatic labeling from DataFrames for now

typealias FuncOrFuncs @compat(Union{Function, AVec{Function}})

all3D(d::KW) = trueOrAllTrue(lt -> lt in (:contour, :heatmap, :surface, :wireframe, :contour3d), get(d, :linetype, :none))

# missing
convertToAnyVector(v::@compat(Void), d::KW) = Any[nothing], nothing

# fixed number of blank series
convertToAnyVector(n::Integer, d::KW) = Any[zeros(0) for i in 1:n], nothing

# numeric vector
convertToAnyVector{T<:Number}(v::AVec{T}, d::KW) = Any[v], nothing

# string vector
convertToAnyVector{T<:@compat(AbstractString)}(v::AVec{T}, d::KW) = Any[v], nothing

# numeric matrix
function convertToAnyVector{T<:Number}(v::AMat{T}, d::KW)
    if all3D(d)
        Any[Surface(v)]
    else
        Any[v[:,i] for i in 1:size(v,2)]
    end, nothing
end

# other matrix... vector of columns
function convertToAnyVector(m::AMat, d::KW)
    Any[begin
        v = vec(m[:,i])
        length(v) == 1 ? v[1] : v
    end for i=1:size(m,2)], nothing
end

# function
convertToAnyVector(f::Function, d::KW) = Any[f], nothing

# surface
convertToAnyVector(s::Surface, d::KW) = Any[s], nothing

# vector of OHLC
convertToAnyVector(v::AVec{OHLC}, d::KW) = Any[v], nothing

# dates
convertToAnyVector{D<:Union{Date,DateTime}}(dts::AVec{D}, d::KW) = Any[dts], nothing

# list of things (maybe other vectors, functions, or something else)
function convertToAnyVector(v::AVec, d::KW)
    if all(x -> typeof(x) <: Number, v)
        # all real numbers wrap the whole vector as one item
        Any[convert(Vector{Float64}, v)], nothing
    else
        # something else... treat each element as an item
        vcat(Any[convertToAnyVector(vi, d)[1] for vi in v]...), nothing
        # Any[vi for vi in v], nothing
    end
end

# --------------------------------------------------------------------

# TODO: can we avoid the copy here?  one error that crops up is that mapping functions over the same array
#       result in that array being shared.  push!, etc will add too many items to that array

compute_x(x::Void, y::Void, z)      = 1:size(z,1)
compute_x(x::Void, y, z)            = 1:size(y,1)
compute_x(x::Function, y, z)        = map(x, y)
compute_x(x, y, z)                  = copy(x)

# compute_y(x::Void, y::Function, z)  = error()
compute_y(x::Void, y::Void, z)      = 1:size(z,2)
compute_y(x, y::Function, z)        = map(y, x)
compute_y(x, y, z)                  = copy(y)

compute_z(x, y, z::Function)        = map(z, x, y)
compute_z(x, y, z::AbstractMatrix)  = Surface(z)
compute_z(x, y, z::Void)            = nothing
compute_z(x, y, z)                  = copy(z)

@noinline function compute_xyz(x, y, z)
    x = compute_x(x,y,z)
    y = compute_y(x,y,z)
    z = compute_z(x,y,z)
    x, y, z
end

# not allowed
compute_xyz(x::Void, y::FuncOrFuncs, z)       = error("If you want to plot the function `$y`, you need to define the x values!")
compute_xyz(x::Void, y::Void, z::FuncOrFuncs) = error("If you want to plot the function `$z`, you need to define x and y values!")
compute_xyz(x::Void, y::Void, z::Void)        = error("x/y/z are all nothing!")

# --------------------------------------------------------------------

# create n=max(mx,my) series arguments. the shorter list is cycled through
# note: everything should flow through this
function build_series_args(plt::AbstractPlot, kw::KW) #, idxfilter)
    x, y, z = map(sym -> pop!(kw, sym, nothing), (:x, :y, :z))
    if nothing == x == y == z
        return [], nothing, nothing
    end

    xs, xmeta = convertToAnyVector(x, kw)
    ys, ymeta = convertToAnyVector(y, kw)
    zs, zmeta = convertToAnyVector(z, kw)

    fr = pop!(kw, :fillrange, nothing)
    fillranges, _ = if typeof(fr) <: Number
        ([fr],nothing)
    else
        convertToAnyVector(fr, kw)
    end

    mx = length(xs)
    my = length(ys)
    mz = length(zs)
    ret = Any[]
    for i in 1:max(mx, my, mz)

        # try to set labels using ymeta
        d = copy(kw)
        if !haskey(d, :label) && ymeta != nothing
            if isa(ymeta, Symbol)
                d[:label] = string(ymeta)
            elseif isa(ymeta, AVec{Symbol})
                d[:label] = string(ymeta[mod1(i,length(ymeta))])
            end
        end

        # build the series arg dict
        numUncounted = pop!(d, :numUncounted, 0)
        commandIndex = i + numUncounted
        n = plt.n + i

        dumpdict(d, "before getSeriesArgs")
        d = getSeriesArgs(plt.backend, getplotargs(plt, n), d, commandIndex, convertSeriesIndex(plt, n), n)
        dumpdict(d, "after getSeriesArgs")

        d[:x], d[:y], d[:z] = compute_xyz(xs[mod1(i,mx)], ys[mod1(i,my)], zs[mod1(i,mz)])
        lt = d[:linetype]

        # for linetype `line`, need to sort by x values
        if lt == :line
            # order by x
            indices = sortperm(d[:x])
            d[:x] = d[:x][indices]
            d[:y] = d[:y][indices]
            d[:linetype] = :path
        end

        # special handling for missing x in box plot... all the same category
        if lt == :box && xs[mod1(i,mx)] == nothing
            d[:x] = ones(Int, length(d[:y]))
        end

        # map functions to vectors
        if isa(d[:marker_z], Function)
            d[:marker_z] = map(d[:marker_z], d[:x])
        end

        # @show fillranges
        d[:fillrange] = fillranges[mod1(i,length(fillranges))]
        if isa(d[:fillrange], Function)
            d[:fillrange] = map(d[:fillrange], d[:x])
        end

        # handle error bars
        for esym in (:xerror, :yerror)
            if get(d, esym, nothing) != nothing
                # we make a copy of the KW and apply an errorbar recipe
                append!(ret, apply_series_recipe(copy(d), Val{esym}))
            end
        end

        # handle ribbons
        if get(d, :ribbon, nothing) != nothing
            rib = d[:ribbon]
            d[:fillrange] = (d[:y] - rib, d[:y] + rib)
        end

        # handle quiver plots
        if lt == :quiver
            d[:linetype] = lt = :path
            d[:linewidth] = 0
        end
        if get(d, :quiver, nothing) != nothing
            append!(ret, apply_series_recipe(copy(d), Val{:quiver}))
        end

        # now that we've processed a given series... optionally split into
        # multiple dicts through a recipe (for example, a box plot is split into component
        # parts... polygons, lines, and scatters)
        # note: we pass in a Val type (i.e. Val{:box}) so that we can dispatch on the linetype
        kwlist = apply_series_recipe(d, Val{lt})
        append!(ret, kwlist)

        # # add it to our series list
        # push!(ret, d)
    end

    ret, xmeta, ymeta
end

# --------------------------------------------------------------------
# process_inputs
# --------------------------------------------------------------------

# These methods take a plot and the keyword arguments, and processes the input
# arguments (x/y/z, group, etc), populating the KW dict with appropriate values.

# --------------------------------------------------------------------
# 0 arguments
# --------------------------------------------------------------------

# don't do anything
function process_inputs(plt::AbstractPlot, d::KW)
end

# --------------------------------------------------------------------
# 1 argument
# --------------------------------------------------------------------

function process_inputs(plt::AbstractPlot, d::KW, n::Integer)
    # d[:x], d[:y], d[:z] = zeros(0), zeros(0), zeros(0)
    d[:x] = d[:y] = d[:z] = n
end

# no special handling... assume x and z are nothing
function process_inputs(plt::AbstractPlot, d::KW, y)
    d[:y] = y
end

# matrix... is it z or y?
function process_inputs{T<:Number}(plt::AbstractPlot, d::KW, mat::AMat{T})
    if all3D(d)
        n,m = size(mat)
        d[:x], d[:y], d[:z] = 1:n, 1:m, mat
    else
        d[:y] = mat
    end
end

# images - grays
function process_inputs{T<:Gray}(plt::AbstractPlot, d::KW, mat::AMat{T})
    d[:linetype] = :image
    n,m = size(mat)
    d[:x], d[:y], d[:z] = 1:n, 1:m, Surface(mat)
    # handle images... when not supported natively, do a hack to use heatmap machinery
    if !nativeImagesSupported()
        d[:linetype] = :heatmap
        d[:yflip] = true
        d[:z] = Surface(convert(Matrix{Float64}, mat.surf))
        d[:fillcolor] = ColorGradient([:black, :white])
    end
end

# images - colors
function process_inputs{T<:Colorant}(plt::AbstractPlot, d::KW, mat::AMat{T})
    d[:linetype] = :image
    n,m = size(mat)
    d[:x], d[:y], d[:z] = 1:n, 1:m, Surface(mat)
    # handle images... when not supported natively, do a hack to use heatmap machinery
    if !nativeImagesSupported()
        d[:yflip] = true
        imageHack(d)
    end
end

# plotting arbitrary shapes/polygons
function process_inputs(plt::AbstractPlot, d::KW, shape::Shape)
    d[:x], d[:y] = shape_coords(shape)
    d[:linetype] = :shape
end
function process_inputs(plt::AbstractPlot, d::KW, shapes::AVec{Shape})
    d[:x], d[:y] = shape_coords(shapes)
    d[:linetype] = :shape
end
function process_inputs(plt::AbstractPlot, d::KW, shapes::AMat{Shape})
    x, y = [], []
    for j in 1:size(shapes, 2)
        tmpx, tmpy = shape_coords(vec(shapes[:,j]))
        push!(x, tmpx)
        push!(y, tmpy)
    end
    d[:x], d[:y] = x, y
    d[:linetype] = :shape
end

# function without range... use the current range of the x-axis
function process_inputs(plt::AbstractPlot, d::KW, f::FuncOrFuncs)
    process_inputs(plt, d, f, xmin(plt), xmax(plt))
end

# --------------------------------------------------------------------
# 2 arguments
# --------------------------------------------------------------------

function process_inputs(plt::AbstractPlot, d::KW, x, y)
    d[:x], d[:y] = x, y
end

# if functions come first, just swap the order (not to be confused with parametric functions...
# as there would be more than one function passed in)
function process_inputs(plt::AbstractPlot, d::KW, f::FuncOrFuncs, x)
    @assert !(typeof(x) <: FuncOrFuncs)  # otherwise we'd hit infinite recursion here
    process_inputs(plt, d, x, f)
end

# --------------------------------------------------------------------
# 3 arguments
# --------------------------------------------------------------------

# no special handling... just pass them through
function process_inputs(plt::AbstractPlot, d::KW, x, y, z)
    d[:x], d[:y], d[:z] = x, y, z
end

# 3d line or scatter
function process_inputs(plt::AbstractPlot, d::KW, x::AVec, y::AVec, zvec::AVec)
    # default to path3d if we haven't set a 3d linetype
    lt = get(d, :linetype, :none)
    if lt == :scatter
        d[:linetype] = :scatter3d
    elseif !(lt in _3dTypes)
        d[:linetype] = :path3d
    end
    d[:x], d[:y], d[:z] = x, y, zvec
end

# surface-like... function
function process_inputs{TX,TY}(plt::AbstractPlot, d::KW, x::AVec{TX}, y::AVec{TY}, zf::Function)
    x = TX <: Number ? sort(x) : x
    y = TY <: Number ? sort(y) : y
    # x, y = sort(x), sort(y)
    d[:z] = Surface(zf, x, y)  # TODO: replace with SurfaceFunction when supported
    d[:x], d[:y] = x, y
end

# surface-like... matrix grid
function process_inputs{TX,TY,TZ}(plt::AbstractPlot, d::KW, x::AVec{TX}, y::AVec{TY}, zmat::AMat{TZ})
    @assert size(zmat) == (length(x), length(y))
    if TX <: Number && !issorted(x)
        idx = sortperm(x)
        x, zmat = x[idx], zmat[idx, :]
    end
    if TY <: Number && !issorted(y)
        idx = sortperm(y)
        y, zmat = y[idx], zmat[:, idx]
    end
    d[:x], d[:y], d[:z] = x, y, Surface{Matrix{TZ}}(zmat)
    if !like_surface(get(d, :linetype, :none))
        d[:linetype] = :contour
    end
end

# surfaces-like... general x, y grid
function process_inputs{T<:Number}(plt::AbstractPlot, d::KW, x::AMat{T}, y::AMat{T}, zmat::AMat{T})
    @assert size(zmat) == size(x) == size(y)
    # d[:x], d[:y], d[:z] = Any[x], Any[y], Surface{Matrix{Float64}}(zmat)
    d[:x], d[:y], d[:z] = map(Surface{Matrix{Float64}}, (x, y, zmat))
    if !like_surface(get(d, :linetype, :none))
        d[:linetype] = :contour
    end
end

# --------------------------------------------------------------------
# Parametric functions
# --------------------------------------------------------------------

# special handling... xmin/xmax with function(s)
function process_inputs(plt::AbstractPlot, d::KW, f::FuncOrFuncs, xmin::Number, xmax::Number)
    width = get(plt.plotargs, :size, (100,))[1]
    x = linspace(xmin, xmax, width)
    process_inputs(plt, d, x, f)
end

# special handling... xmin/xmax with parametric function(s)
process_inputs{T<:Number}(plt::AbstractPlot, d::KW, fx::FuncOrFuncs, fy::FuncOrFuncs, u::AVec{T}) = process_inputs(plt, d, mapFuncOrFuncs(fx, u), mapFuncOrFuncs(fy, u))
process_inputs{T<:Number}(plt::AbstractPlot, d::KW, u::AVec{T}, fx::FuncOrFuncs, fy::FuncOrFuncs) = process_inputs(plt, d, mapFuncOrFuncs(fx, u), mapFuncOrFuncs(fy, u))
process_inputs(plt::AbstractPlot, d::KW, fx::FuncOrFuncs, fy::FuncOrFuncs, umin::Number, umax::Number, numPoints::Int = 1000) = process_inputs(plt, d, fx, fy, linspace(umin, umax, numPoints))

# special handling... 3D parametric function(s)
process_inputs{T<:Number}(plt::AbstractPlot, d::KW, fx::FuncOrFuncs, fy::FuncOrFuncs, fz::FuncOrFuncs, u::AVec{T}) = process_inputs(plt, d, mapFuncOrFuncs(fx, u), mapFuncOrFuncs(fy, u), mapFuncOrFuncs(fz, u))
process_inputs{T<:Number}(plt::AbstractPlot, d::KW, u::AVec{T}, fx::FuncOrFuncs, fy::FuncOrFuncs, fz::FuncOrFuncs) = process_inputs(plt, d, mapFuncOrFuncs(fx, u), mapFuncOrFuncs(fy, u), mapFuncOrFuncs(fz, u))
process_inputs(plt::AbstractPlot, d::KW, fx::FuncOrFuncs, fy::FuncOrFuncs, fz::FuncOrFuncs, umin::Number, umax::Number, numPoints::Int = 1000) = process_inputs(plt, d, fx, fy, fz, linspace(umin, umax, numPoints))

# --------------------------------------------------------------------
# Lists of tuples and FixedSizeArrays
# --------------------------------------------------------------------

# if we get an unhandled tuple, just splat it in
function process_inputs(plt::AbstractPlot, d::KW, tup::Tuple)
    process_inputs(plt, d, tup...)
end

# (x,y) tuples
function process_inputs{R1<:Number,R2<:Number}(plt::AbstractPlot, d::KW, xy::AVec{Tuple{R1,R2}})
    process_inputs(plt, d, unzip(xy)...)
end
function process_inputs{R1<:Number,R2<:Number}(plt::AbstractPlot, d::KW, xy::Tuple{R1,R2})
    process_inputs(plt, d, [xy[1]], [xy[2]])
end

# (x,y,z) tuples
function process_inputs{R1<:Number,R2<:Number,R3<:Number}(plt::AbstractPlot, d::KW, xyz::AVec{Tuple{R1,R2,R3}})
    process_inputs(plt, d, unzip(xyz)...)
end
function process_inputs{R1<:Number,R2<:Number,R3<:Number}(plt::AbstractPlot, d::KW, xyz::Tuple{R1,R2,R3})
    process_inputs(plt, d, [xyz[1]], [xyz[2]], [xyz[3]])
end

# 2D FixedSizeArrays
function process_inputs{T<:Number}(plt::AbstractPlot, d::KW, xy::AVec{FixedSizeArrays.Vec{2,T}})
    process_inputs(plt, d, unzip(xy)...)
end
function process_inputs{T<:Number}(plt::AbstractPlot, d::KW, xy::FixedSizeArrays.Vec{2,T})
    process_inputs(plt, d, [xy[1]], [xy[2]])
end

# 3D FixedSizeArrays
function process_inputs{T<:Number}(plt::AbstractPlot, d::KW, xyz::AVec{FixedSizeArrays.Vec{3,T}})
    process_inputs(plt, d, unzip(xyz)...)
end
function process_inputs{T<:Number}(plt::AbstractPlot, d::KW, xyz::FixedSizeArrays.Vec{3,T})
    process_inputs(plt, d, [xyz[1]], [xyz[2]], [xyz[3]])
end

# --------------------------------------------------------------------
# handle grouping
# --------------------------------------------------------------------

# function process_inputs(plt::AbstractPlot, d::KW, groupby::GroupBy, args...)
#     ret = Any[]
#     error("unfinished after series reorg")
#     for (i,glab) in enumerate(groupby.groupLabels)
#         # TODO: don't automatically overwrite labels
#         kwlist, xmeta, ymeta = process_inputs(plt, d, args...,
#                                             idxfilter = groupby.groupIds[i],
#                                             label = string(glab),
#                                             numUncounted = length(ret))  # we count the idx from plt.n + numUncounted + i
#         append!(ret, kwlist)
#     end
#     ret, nothing, nothing # TODO: handle passing meta through
# end

# --------------------------------------------------------------------
# For DataFrame support.  Imports DataFrames and defines the necessary methods which support them.
# --------------------------------------------------------------------

function setup_dataframes()
    @require DataFrames begin
    # @eval begin
        # import DataFrames

        DFS = Union{Symbol, AbstractArray{Symbol}}

        function handle_dfs(df::DataFrames.AbstractDataFrame, d::KW, letter, dfs::DFS)
            if isa(dfs, Symbol)
                get!(d, symbol(letter * "label"), string(dfs))
                collect(df[dfs])
            else
                get!(d, :label, reshape(dfs, 1, length(dfs)))
                Any[collect(df[s]) for s in dfs]
            end
        end

        function handle_group(df::DataFrames.AbstractDataFrame, d::KW)
            if haskey(d, :group)
                g = d[:group]
                if isa(g, Symbol)
                    d[:group] = collect(df[g])
                end
            end
        end

        @recipe function plot(df::DataFrames.AbstractDataFrame, sy::DFS)
            handle_group(df, d)
            handle_dfs(df, d, "y", sy)
        end

        @recipe function plot(df::DataFrames.AbstractDataFrame, sx::DFS, sy::DFS)
            handle_group(df, d)
            x = handle_dfs(df, d, "x", sx)
            y = handle_dfs(df, d, "y", sy)
            x, y
        end

        @recipe function plot(df::DataFrames.AbstractDataFrame, sx::DFS, sy::DFS, sz::DFS)
            handle_group(df, d)
            x = handle_dfs(df, d, "x", sx)
            y = handle_dfs(df, d, "y", sy)
            z = handle_dfs(df, d, "z", sz)
            x, y, z
        end

        # get_data(df::DataFrames.AbstractDataFrame, arg::Symbol) = df[arg]
        # get_data(df::DataFrames.AbstractDataFrame, arg) = arg
        #
        # function process_inputs(plt::AbstractPlot, d::KW, df::DataFrames.AbstractDataFrame, args...)
        #     # d[:dataframe] = df
        #     process_inputs(plt, d, map(arg -> get_data(df, arg), args)...)
        # end
        #
        # # expecting the column name of a dataframe that was passed in... anything else should error
        # function extractGroupArgs(s::Symbol, df::DataFrames.AbstractDataFrame, args...)
        #     if haskey(df, s)
        #         return extractGroupArgs(df[s])
        #     else
        #         error("Got a symbol, and expected that to be a key in d[:dataframe]. s=$s d=$d")
        #     end
        # end

        # function getDataFrameFromKW(d::KW)
        #     get(d, :dataframe) do
        #         error("Missing dataframe argument!")
        #     end
        # end

        # # the conversion functions for when we pass symbols or vectors of symbols to reference dataframes
        # convertToAnyVector(s::Symbol, d::KW) = Any[getDataFrameFromKW(d)[s]], s
        # convertToAnyVector(v::AVec{Symbol}, d::KW) = (df = getDataFrameFromKW(d); Any[df[s] for s in v]), v

    end
end


"""
`Bins{TS, TW, TBI, TT}`: A bin structure designed for WE

### Fields

* `Ω` - structure containing information for uniquely identifying each bin
* `n` - number of particles in each bin
* `target` - target number of particles in each bin
* `ν` - weight of each bin
* `d` - auxiliary bin data
"""
struct Bins{TS, TW<:AbstractFloat, TBI<:Integer, TT<:Real, TD} <: AbstractBins
   # structure which identifies the bins
   Ω::Vector{TS}
   # number of walkers in each bin
   n::Vector{TBI}
   # target number of walkers in each bin - specify if real/int with TT
   target::Vector{TT}
   # weight associated with each bin
   ν::Vector{TW}
   # auxiliary data structure for each bin
   d::Vector{TD}
end

function Base.push!(B::TB, Ω, n, target, ν, d) where {TB<:Bins}
   push!(B.Ω, Ω);
   push!(B.n, n);
   push!(B.target, target);
   push!(B.ν, ν);
   push!(B.d, d)
end

function Base.pop!(B::TB) where {TB<:Bins}
   Ω = pop!(B.Ω);
   n = pop!(B.n);
   target = pop!(B.target);
   ν = pop!(B.ν);
   d = pop!(B.d)
   #particles = pop!(B.particles)
   return Ω, n, target, ν, d
end

function Base.popfirst!(B::TB) where {TB<:Bins}
   Ω = popfirst!(B.Ω);
   n = popfirst!(B.n);
   target = popfirst!(B.target);
   ν = popfirst!(B.ν);
   d = popfirst!(B.d);
   #particles = pop!(B.particles)
   return Ω, n, target, ν, d
end

function Base.length(B::TB) where {TB<:Bins}
   return length(B.Ω)
end

function Base.eltype(B::TB) where {TB<:Bins}
   return typeof((B.Ω, B.n, B.target, B.ν, B.d))
end

function Base.iterate(B::TB, state = 1) where {TB<:Bins}

   if state > length(B)
      return nothing
   end
   return (B.Ω[state], B.n[state], B.target[state], B.ν[state], B.d[state]), state+1
end


type CallSpecifiedMultiProduct{M <: MarketModelMultiProduct, E <: ExerciseStrategy, M2 <: MarketModelMultiProduct} <: MarketModelMultiProduct
  underlying::M
  strategy::E
  rebate::M2
  evolution::EvolutionDescription
  isPresent::Vector{BitArray{1}}
  cashFlowTimes::Vector{Float64}
  rebateOffset::Int
  wasCalled::Bool
  dummyCashFlowsThisStep::Vector{Int}
  dummyCashFlowsGenerated::Vector{Vector{MarketModelCashFlow}}
  currentIndex::Int
  callable::Bool
end

function CallSpecifiedMultiProduct(underlying::MarketModelMultiProduct,
                                  strategy::ExerciseStrategy,
                                  rebate::MarketModelMultiProduct)

  underlying = clone(underlying)
  strategy = clone(strategy)
  rebate = clone(rebate)

  products = number_of_products(underlying)
  d1 = get_evolution(underlying)
  rateTimes1 = d1.rateTimes
  evolutionTimes1 = d1.evolutionTimes
  exerciseTimes = strategy.exerciseTimes

  # TODO check if rebate isn't there
  d2 = get_evolution(rebate)
  rateTimes2 = d2.rateTimes
  rateTimes1 == rateTimes2 || error("incompatible rate times")

  allEvolutionTimes = Vector{Vector{Float64}}(4)
  allEvolutionTimes[1] = evolutionTimes1
  allEvolutionTimes[2] = exerciseTimes
  allEvolutionTimes[3] = d2.evolutionTimes
  allEvolutionTimes[4] = relevant_times(strategy)

  mergedEvolutionTimes, isPresent = merge_times(allEvolutionTimes)

  evolution = EvolutionDescription(rateTimes1, mergedEvolutionTimes)
  cashFlowTimes = possible_cash_flow_times(underlying)
  rebateOffset = length(cashFlowTimes)
  rebateTimes = possible_cash_flow_times(rebate)
  cashFlowTimes = vcat(cashFlowTimes, rebateTimes)

  dummyCashFlowsThisStep = zeros(Int, products)
  n = max_number_of_cashflows_per_product_per_step(rebate)

  dummyCashFlowsGenerated = fill(Vector{MarketModelCashFlow}(n), products)

  return CallSpecifiedMultiProduct(underlying, strategy, rebate, evolution, isPresent, cashFlowTimes, rebateOffset, false, dummyCashFlowsThisStep,
                                  dummyCashFlowsGenerated, 1, true)
end

get_evolution(cs::CallSpecifiedMultiProduct) = cs.evolution
number_of_products(cs::CallSpecifiedMultiProduct) = number_of_products(cs.underlying)
possible_cash_flow_times(cs::CallSpecifiedMultiProduct) = cs.cashFlowTimes
function reset!(cs::CallSpecifiedMultiProduct)
  reset!(cs.underlying)
  reset!(cs.rebate)
  reset!(cs.strategy)
  cs.currentIndex = 1
  cs.wasCalled = false

  return cs
end

max_number_of_cashflows_per_product_per_step(cs::CallSpecifiedMultiProduct) =
              max(max_number_of_cashflows_per_product_per_step(cs.underlying), max_number_of_cashflows_per_product_per_step(cs.rebate))

function next_time_step!(cs::CallSpecifiedMultiProduct, currentState::CurveState, numberCashFlowsThisStep::Vector{Int}, genCashFlows::Vector{Vector{MarketModelCashFlow}})
  isUnderlyingTime = cs.isPresent[1][cs.currentIndex]
  isExerciseTime = cs.isPresent[2][cs.currentIndex]
  isRebateTime = cs.isPresent[3][cs.currentIndex]
  isStrategyRelevantTime = cs.isPresent[4][cs.currentIndex]

  isDone = false

  if ~cs.wasCalled && isStrategyRelevantTime
    next_step!(cs.strategy, currentState)
  end

  if ~cs.wasCalled && isExerciseTime && cs.callable
    cs.wasCalled = get_exercise!(cs.strategy, currentState)
  end

  if cs.wasCalled
    if isRebateTime
      isDone = next_time_step!(cs.rebate, currentState, numberCashFlowsThisStep, genCashFlows)
      for i in eachindex(numberCashFlowsThisStep)
        for j = 1:numberCashFlowsThisStep[i]
          genCashFlows[i][j].timeIndex += cs.rebateOffset
        end
      end
    end
  else
    if isRebateTime
      next_time_step!(cs.rebate, currentState, cs.dummyCashFlowsThisStep, cs.dummyCashFlowsGenerated)
    end

    if isUnderlyingTime
      isDone = next_time_step!(cs.underlying, currentState, numberCashFlowsThisStep, genCashFlows)
    end
  end

  cs.currentIndex += 1
  return isDone || cs.currentIndex == length(cs.evolution.evolutionTimes) + 1
end

clone(cs::CallSpecifiedMultiProduct) = CallSpecifiedMultiProduct(clone(cs.underlying), clone(cs.strategy), clone(cs.rebate), clone(cs.evolution), deepcopy(cs.isPresent),
                                        copy(cs.cashFlowTimes), cs.rebateOffset, cs.wasCalled, copy(cs.dummyCashFlowsThisStep), deepcopy(cs.dummyCashFlowsGenerated),
                                        cs.currentIndex, cs.callable)


using HDF5

function read_APS_file(filename, startIdx=0)

	#APS bit masks
	START_MINILL_MASK = 2^15
	END_MINILL_MASK = 2^14
	TA_PAIR_MASK = 2^12
	REPEAT_MASK = 2^10 - 1
	MAX_WAVEFORM_VALUE = 2^13 - 1
	ADDRESS_UNIT = 4

	chanStrs = ["chan_1", "chan_2", "chan_3", "chan_4"]
	mrkStrs = ["ch1m1", "ch2m1", "ch3m1", "ch4m1"]
	AWGData = [chan => Vector{Float64}[] for chan in chanStrs]

	h5open(filename, "r") do f
		for (ct, chanStr) in enumerate(chanStrs)
			if read(attrs(f[chanStr])["isIQMode"]) != 0
				curLLData = f[chanStrs[2*fld(ct-1, 2) + 1]]["linkListData"]
			else
				curLLData = f[chanStr]["linkListData"]
			end

			# pull out LL data
			addr = read(curLLData["addr"]) * ADDRESS_UNIT
			count = (1 .+ read(curLLData["count"])) * ADDRESS_UNIT
			repeat = read(curLLData["repeat"])
			trigger1 = read(curLLData["trigger1"]) * ADDRESS_UNIT
			trigger2 = read(curLLData["trigger2"]) * ADDRESS_UNIT
			numEntries = read(attrs(curLLData)["length"])[1]

			# pull out and scale waveform data
			wfLib = (1./MAX_WAVEFORM_VALUE)*read(f[chanStr]["waveformLib"])

			# initialize storage
			AWGData[mrkStrs[ct]] = Vector{Float64}[]

			# loop over LL entries
			for entryct = 1:numEntries
				# at the start of a miniLL, create an empty array
				if repeat[entryct] & START_MINILL_MASK != 0
					curWF = Float64[]
					triggerDelays = Int[]
				end

				# record the trigger delays
				if ct % 2 == 1
					if trigger1[entryct] > 0
						push!(triggerDelays, length(curWF) + trigger1[entryct])
					end
				else
					if trigger2[entryct] > 0
						push!(triggerDelays, length(curWF) + trigger2[entryct])
					end
				end

				# if it is a TA pair or a regular pulse
				curRepeat = (repeat[entryct] & REPEAT_MASK) + 1
				if repeat[entryct] & TA_PAIR_MASK != 0
					curWF = [curWF, wfLib[1+addr[entryct]]*ones(count[entryct] * curRepeat)]
				else
					try
						curWF = [curWF, repmat(wfLib[1+addr[entryct]:addr[entryct]+count[entryct]], curRepeat, 1)]
					catch
						print("addr: ")
						println(int(addr[entryct]))
						print("count: ")
						println(int(count[entryct]))
					end
				end

				# add the trigger pulses and push on the curWF
				if repeat[entryct] & END_MINILL_MASK != 0
					# println("Size of curWF: $(size(curWF))")
					push!(AWGData[chanStr], vec(curWF))
					triggerSeq = zeros(Uint8, length(curWF))
					triggerSeq[triggerDelays] = 1
					push!(AWGData[mrkStrs[ct]], triggerSeq)
				end
			end
		end
	end

	#Strip-off the starting zero points
	if startIdx > 0
		for seqs in values(AWGData)
			for seq in seqs
				deleteat!(seq, 1:startIdx)
			end
		end
	end

	return AWGData
end


function collect_commom_DAT!(DAT, Ed, iJR)

    # maps
    # ----------------------------------------------------------------------------------

    # ----------------------------------------------------------------------------------
    max_model = iJR.load_model("max_model"; uncompress = false)
    objider = iJR.BIOMASS_IDER

    # ----------------------------------------------------------------------------------
    for exp in DAT[:EXPS]
        # exp dat
        Ed_biom = Ed.val("D", exp)
        DAT[:exp, :flx, "D", exp] = Ed_biom
        DAT[:exp, :err, "D", exp] = 0.0

        # bounds
        fva_model = iJR.load_model("fva_models", exp; uncompress = false)
        max_lb, max_ub = ChU.bounds(max_model, objider)
        fva_lb, fva_ub = ChU.bounds(fva_model, objider)
        lb = max(max_lb, fva_lb)
        ub = min(max_ub, fva_ub)
        DAT[:bounds, "D", exp] = (lb, ub)

        # exchanges
        for Ed_ider in DAT[:EXCH_FLX_IDERS]

            # exp dat
            Ed_flx = Ed.uval(Ed_ider, exp)
            Ed_err = try; Ed.uerr(Ed_ider, exp) catch; 0.0 end
            DAT[:exp, :flx, Ed_ider, exp] = Ed_flx
            DAT[:exp, :err, Ed_ider, exp] = Ed_err

        end

        # inner
        ider_map = iJR.load_rxns_map()
        for model_ider in DAT[:INNER_FLX_IDERS]

            Ed_ider = ider_map[model_ider]

            # exp dat
            Ed_flx = Ed.val(Ed_ider, exp)
            Ed_err = try; Ed.err(Ed_ider, exp) catch; 0.0 end
            DAT[:exp, :flx, Ed_ider, exp] = Ed_flx
            DAT[:exp, :err, Ed_ider, exp] = Ed_err

        end
    end

    return DAT
end


"""
GenX: An Configurable Capacity Expansion Model
Copyright (C) 2021,  Massachusetts Institute of Technology
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
A complete copy of the GNU General Public License v2 (GPLv2) is available
in LICENSE.txt.  Users uncompressing this from an archive may not have
received this license file.  If not, see <http://www.gnu.org/licenses/>.
"""

@doc raw"""
	write_status(path::AbstractString, inputs::Dict, EP::Model)

Function for writing the final solve status of the optimization problem solved.
"""
function write_status(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)

	# https://jump.dev/MathOptInterface.jl/v0.9.10/apireference/#MathOptInterface.TerminationStatusCode
	status = termination_status(EP)

	# Note: Gurobi excludes constants from solver reported objective function value - MIPGap calculated may be erroneous
	if (setup["UCommit"] == 0 || setup["UCommit"] == 2)
		dfStatus = DataFrame(Status = status, Solve = inputs["solve_time"],
			Objval = objective_value(EP))
	else
		dfStatus = DataFrame(Status = status, Solve = inputs["solve_time"],
			Objval = objective_value(EP), Objbound= objective_bound(EP),FinalMIPGap =(objective_value(EP) -objective_bound(EP))/objective_value(EP) )
	end
	CSV.write(joinpath(path, "status.csv"),dfStatus)
end


module ScaledArrays

    export ScaledArray, unscale, rescale,
            LinQuant16Array, LinQuant24Array

    include("scaledarray.jl")
    include("binarytree.jl")
    include("quantarray.jl")
end


using KernelAbstractions
using Oceananigans.Utils: work_layout
using Oceananigans.Fields
using Oceananigans.Diagnostics: dims_to_result_size
import Oceananigans.Diagnostics: run_diagnostic!

import Oceananigans.Fields: location, total_size, compute!

"""
    Computation{T, R, O, G}

Represents an operation performed over the elements of a field.
"""
struct Computation{T, R, O, G}
      operation :: O
         result :: R
           grid :: G
    return_type :: T
end

"""
    Computation(operation, result; return_type=Array)

Returns a `Computation` representing an `operation` performed over the elements of
`operation.grid` and stored in `result`. `return_type` specifies the output type when the
`Computation` instances is called as a function.
"""
Computation(operation, result; return_type=Array) =
    Computation(operation, result, operation.grid, return_type)

# Utilities for limited field-like behavior
architecture(comp::Computation) = architecture(comp.result)
Base.parent(comp::Computation) = comp # this enables taking a "horizontal average" of a computation
location(comp::Computation) = location(comp.operation)
total_size(comp::Computation) = total_size(comp.result)

"""
    compute!(computation::Computation)

Perform a `computation`. The result is stored in `computation.result`.
"""
function compute!(computation::Computation)
    arch = architecture(computation.result)
    result_data = data(computation.result)
    loc = location(computation.result)

    workgroup, worksize = work_layout(computation.grid, :xyz, include_right_boundaries=true, location=loc)

    compute_kernel! = _compute!(device(arch), workgroup, worksize)

    event = compute_kernel!(result_data, computation.grid, computation.operation; dependencies=Event(device(arch)))

    wait(device(arch), event)

    return nothing
end

"""Compute an `operation` over `grid` and store in `result`."""
@kernel function _compute!(result, grid, operation)
    i, j, k = @index(Global, NTuple)
    @inbounds result[i, j, k] = operation[i, j, k]
end

"""
    (computation::Computation)(args...)

Performs the `compute(computation)` and returns the result if `isnothing(return_type)`,
or the result after being converted to `return_type`.
"""
function (computation::Computation)(args...)
    compute!(computation)
    return computation.return_type(interior(computation.result))
end

function (computation::Computation{<:Nothing})(args...)
    compute!(computation)
    return computation.result
end

#####
##### Functionality for using computations with Average
#####

"""
    Average(op::AbstractOperation, result; dims, kwargs...)

Returns the representation of an `Average` over the operation `op`, using `result` as
a temporary array to store the result of `operation` computed on `op.grid`.
"""
function Average(op::AbstractOperation, result; dims, kwargs...)
    computation = Computation(op, result)
    return Average(computation; dims=dims, kwargs...)
end

"""
    Average(op::AbstractOperation, model; dims, kwargs...)

Returns the representation of an `Average` over the operation `op`, using
`model.pressures.pHY′` as a temporary array to store the result of `operation` computed on
`op.grid`.
"""
Average(op::AbstractOperation, model::AbstractModel; dims, kwargs...) =
    Average(op, model.pressures.pHY′; dims=dims, kwargs...)

"""
    Average(field::Field; dims, iteration_interval=nothing, time_interval=nothing, return_type=Array)

Construct an `Average` of `field` along the dimensions specified by the tuple `dims`.

After the average is computed it will be stored in the `result` property.

The `Average` can be used as a callable object that computes and returns the average.

An `iteration_interval` or `time_interval` (or both) can be passed to indicate how often to
run this diagnostic if it is part of `simulation.diagnostics`. `iteration_interval` is a
number of iterations while `time_interval` is a time interval in units of `model.clock.time`.

A `return_type` can be used to specify the type returned when the `Average` is
used as a callable object. The default `return_type=Array` is useful when running a GPU
model and you want to save the output to disk by passing it to an output writer.
"""
function Average(field::Computation; dims, iteration_interval=nothing, time_interval=nothing, return_type=Array,
                 with_halos=true)

    dims isa Union{Int, Tuple} || error("Average dims must be an integer or tuple!")
    dims isa Int && (dims = tuple(dims))

    length(dims) == 0 && error("dims is empty! Must average over at least one dimension.")
    length(dims) > 3  && error("Models are 3-dimensional. Cannot average over 4+ dimensions.")
    all(1 <= d <= 3 for d in dims) || error("Dimensions must be one of 1, 2, 3.")

    arch = architecture(field)
    result_size = dims_to_result_size(field, dims, field.grid)
    result = zeros(arch, field.grid, result_size...)

    return Average(field, dims, result, iteration_interval, time_interval, 0.0,
                   return_type, with_halos)
end

"""Compute the average of a computation."""
function run_diagnostic!(avg::Average{<:Computation}, model)
    compute!(avg.field)
    zero_halo_regions!(parent(avg.field.result), (Cell, Cell, Cell), model.grid)
    sum!(avg.result, parent(avg.field.result))
    normalize_sum!(avg)
    return nothing
end


module KiteUtils

#= MIT License

Copyright (c) 2020, 2021 Uwe Fechner

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. =#

# data structures for the flight state and the flight log
# functions for creating a demo flight state, demo flight log, loading and saving flight logs
# function se() for reading the settings
# the parameter P is the number of points of the tether, equal to segments+1
# in addition helper functions for working with rotations

using Rotations, StaticArrays, StructArrays, RecursiveArrayTools, Arrow, YAML, LinearAlgebra, DocStringExtensions, Parameters
export Settings, SysState, SysLog, MyFloat

export demo_state, demo_syslog, demo_log, load_log, save_log, export_log # functions for logging
export demo_state_4p, initial_kite_ref_frame                             # functions for four point kite model
export rot, rot3d, ground_dist, calc_elevation, azimuth_east, acos2      # geometric functions
export set_data_path, load_settings, copy_settings, se                   # functions for reading and copying parameters

"""
    const MyFloat = Float32

Type used for position components and scalar SysState members.
"""
const MyFloat = Float32             # type to use for position components and scalar SysState members
const DATA_PATH = ["data"]          # path for log files and other data

include("settings.jl")
include("transformations.jl")

"""
    SysState{P}

Basic system state. One of these is saved per time step. P is the number
of tether particles.

$(TYPEDFIELDS)
"""
struct SysState{P}
    "time since start of simulation in seconds"
    time::Float64
    "orientation of the kite (quaternion, order w,x,y,z)"
    orient::MVector{4, Float32}
    "elevation angle in radians"
    elevation::MyFloat
    "azimuth angle in radians"
    azimuth::MyFloat
    "tether length [m]"
    l_tether::MyFloat
    "reel out velocity [m/s]"
    v_reelout::MyFloat
    "tether force [N]"
    force::MyFloat
    "depower settings"
    depower::MyFloat
    "norm of apparent wind speed [m/s]"
    v_app::MyFloat
    "vector of particle positions in x"
    X::MVector{P, MyFloat}
    "vector of particle positions in y"
    Y::MVector{P, MyFloat}
    "vector of particle positions in z"
    Z::MVector{P, MyFloat}
end

function Base.getproperty(st::SysState, sym::Symbol)
    if sym == :pos
        X = getfield(st, :X)
        Y = getfield(st, :Y)
        Z = getfield(st, :Z)
        pos = zeros(SVector{length(X), MVector{3, MyFloat}})
        for i in 1:length(X)
            pos[i] .= MVector(X[i], Y[i], Z[i])
        end
        pos
    else
        getfield(st, sym)
    end
end

function Base.getproperty(st::StructVector{SysState}, sym::Symbol)
    if sym == :x
        last.getfield(st, :X)
    elseif sym == :y
        last.getfield(st, :Y)
    elseif sym == :z
        last.getfield(st, :Z) # last.(st.Z)
    else
        getfield(st, sym)
    end
end

function Base.show(io::IO, st::SysState)
    println(io, "time      [s]:       ", st.time)
    println(io, "orient    [w,x,y,z]: ", st.orient)
    println(io, "elevation [rad]:     ", st.elevation)
    println(io, "azimuth   [rad]:     ", st.azimuth)
    println(io, "l_tether  [m]:       ", st.l_tether)
    println(io, "v_reelout [m/s]:     ", st.v_reelout)
    println(io, "force     [N]:       ", st.force)
    println(io, "depower   [-]:       ", st.depower)
    println(io, "v_app     [m/s]:     ", st.v_app)
    println(io, "X         [m]:       ", st.X)
    println(io, "Y         [m]:       ", st.Y)
    println(io, "Z         [m]:       ", st.Z)
end

"""
    SysLog{P}

Flight log, containing the basic data as struct of vectors which can be accessed as if it would
be an array structs.
In addition an extended view on the data that includes derived/ calculated values for plotting.
Finally it contains meta data like the name of the log file.

$(TYPEDFIELDS)
"""
struct SysLog{P}
    "name of the flight log"
    name::String
    "struct of vectors that can also be accessed like a vector of structs"
    syslog::StructArray{SysState{P}}
end

function Base.getproperty(log::SysLog, sym::Symbol)
    if sym == :x
        last.(getproperty(log.syslog, :X))
    elseif sym == :y
        last.(getproperty(log.syslog, :Y))
    elseif sym == :z
        last.(getproperty(log.syslog, :Z))
    else
        getfield(log, sym)
    end
end

# functions
function __init__()
    SETTINGS.segments=0 # force loading of settings.yaml
end

"""
    demo_state(P, height=6.0, time=0.0)

Create a demo state with a given height and time. P is the number of tether particles.

Returns a SysState instance.
"""
function demo_state(P, height=6.0, time=0.0)
    a = 10
    X = range(0, stop=10, length=P)
    Y = zeros(length(X))
    Z = (a .* cosh.(X./a) .- a) * height/ 5.430806
    r_xyz = RotXYZ(pi/2, -pi/2, 0)
    q = QuatRotation(r_xyz)
    orient = MVector{4, Float32}(Rotations.params(q))
    elevation = calc_elevation([X[end], 0.0, Z[end]])
    return SysState{P}(time, orient, elevation,0.,0.,0.,0.,0.,0.,X, Y, Z)
end

"""
    initial_kite_ref_frame(vec_c, v_app)

Calculate the initial orientation of the kite based on the last tether segment and
the apparent wind speed.

Parameters:
- `vec_c`: (pos_n-2) - (pos_n-1) n: number of particles without the three kite particles
                                    that do not belong to the main thether (P1, P2 and P3).
- v_app: vector of the apparent wind speed

Returns:
x, y, z:  the unit vectors of the kite reference frame in the ENU reference frame
"""
function initial_kite_ref_frame(vec_c, v_app)
    z = normalize(vec_c)
    y = normalize(cross(v_app, vec_c))
    x = normalize(cross(y, vec_c))
    return (x, y, z)
end

"""
    get_particles(height_k, height_b, width, m_k, pos_pod= [ 75., 0., 129.90381057], vec_c=[-15., 0., -25.98076211], v_app=[10.4855, 0, -3.08324])

Calculate the initial positions of the particels representing
a 4-point kite, connected to a kite control unit (KCU).

Parameters:
- height_k: height of the kite itself, not above ground [m]
- height_b: height of the bridle [m]
- width: width of the kite [m]
- mk: relative nose distance
- pos_pod: position of the control pod
- vec_c: vector of the last tether segment
"""
function get_particles(height_k, height_b, width, m_k, pos_pod= [ 75., 0., 129.90381057], vec_c=[-15., 0., -25.98076211], v_app=[10.4855, 0, -3.08324])
    # inclination angle of the kite; beta = atan(-pos_kite[2], pos_kite[1]) ???
    beta = pi/2.0
    x, y, z = initial_kite_ref_frame(vec_c, v_app)

    h_kx = height_k * cos(beta); # print 'h_kx: ', h_kx
    h_kz = height_k * sin(beta); # print 'h_kz: ', h_kz
    h_bx = height_b * cos(beta)
    h_bz = height_b * sin(beta)
    pos_kite = pos_pod - (h_kz + h_bz) * z + (h_kx + h_bx) * x   # top,        poing B in diagram
    pos_C = pos_kite + h_kz * z + 0.5 * width * y + h_kx * x     # side point, point C in diagram
    pos_A = pos_kite + h_kz * z + (h_kx + width * m_k) * x       # nose,       point A in diagram
    pos_D = pos_kite + h_kz * z - 0.5 * width * y + h_kx * x     # side point, point D in diagram
    pos0 = pos_kite + (h_kz + h_bz) * z + (h_kx + h_bx) * x      # equal to pos_pod, P_KCU in diagram
    [zeros(3), pos0, pos_A, pos_kite, pos_C, pos_D] # 0, p7, p8, p9, p10, p11
end

"""
    demo_state_4p(P, height=6.0, time=0.0)

Create a demo state, using the 4 point kite model with a given height and time. P is the number of tether particles.

Returns a SysState instance.
"""
function demo_state_4p(P, height=6.0, time=0.0)
    a = 10
    X = collect(range(0, stop=10, length=P))
    Y = zeros(length(X))
    Z = (a .* cosh.(X./a) .- a) * height/ 5.430806
    # append the kite particles to X, Y and z
    pod_pos = [X[end], Y[end], Z[end]]
    particles = get_particles(se().height_k, se().h_bridle, se().width, se().m_k, pod_pos)[3:end]
    for i in 1:4
        particle=particles[i]
        x, y, z = particle[1], particle[2], particle[3]
        push!(X, x)
        push!(Y, y)
        push!(Z, z)
    end
    r_xyz = RotXYZ(pi/2, -pi/2, 0)
    q = QuatRotation(r_xyz)
    orient = MVector{4, Float32}(Rotations.params(q))
    elevation = calc_elevation([X[end], 0.0, Z[end]])
    return SysState{P+4}(time, orient, elevation,0.,0.,0.,0.,0.,0.,X, Y, Z)
end

"""
    demo_syslog(P, name="Test flight"; duration=10)

Create a demo flight log  with given name [String] and duration [s] as StructArray. P is the number of tether
particles.
"""
function demo_syslog(P, name="Test flight"; duration=10)
    max_height = 6.03
    steps   = Int(duration * se().sample_freq) + 1
    time_vec = Vector{Float64}(undef, steps)
    myzeros = zeros(MyFloat, steps)
    elevation = Vector{Float64}(undef, steps)
    orient_vec = Vector{MVector{4, Float32}}(undef, steps)
    X_vec = Vector{MVector{P, MyFloat}}(undef, steps)
    Y_vec = Vector{MVector{P, MyFloat}}(undef, steps)
    Z_vec = Vector{MVector{P, MyFloat}}(undef, steps)
    for i in range(0, length=steps)
        state = demo_state(P, max_height * i/steps, i/se().sample_freq)
        time_vec[i+1] = state.time
        orient_vec[i+1] = state.orient
        elevation[i+1] = asin(state.Z[end]/state.X[end])
        X_vec[i+1] = state.X
        Y_vec[i+1] = state.Y
        Z_vec[i+1] = state.Z
    end
    return StructArray{SysState{P}}((time_vec, orient_vec, elevation, myzeros,myzeros,myzeros,myzeros,myzeros,myzeros, X_vec, Y_vec, Z_vec))
end

"""
    demo_log(P, name="Test_flight"; duration=10)

Create an artifical SysLog struct for demonstration purposes. P is the number of tether
particles.
"""
function demo_log(P, name="Test_flight"; duration=10)
    syslog = demo_syslog(P, name, duration=duration)
    return SysLog{P}(name, syslog)
end

"""
    save_log(flight_log, compress=true)

Save a fligh log of type SysLog as .arrow file. By default lz4 compression is used,
if you use **false** as second parameter no compression is used.
"""
function save_log(flight_log, compress=true)
    filename = joinpath(DATA_PATH[1], flight_log.name) * ".arrow"
    if compress
        Arrow.write(filename, flight_log.syslog, compress=:lz4)
    else
        Arrow.write(filename, flight_log.syslog)
    end
end

"""
    export_log(flight_log)

Save a fligh log of type SysLog as .csv file.
"""
function export_log(flight_log)
    @eval using CSV
    filename = joinpath(DATA_PATH[1], flight_log.name) * ".csv"
    Base.invokelatest(CSV.write, filename, flight_log.syslog)
end

"""
    load_log(P, filename::String)

Read a log file that was saved as .arrow file.  P is the number of tether
particles.
"""
function load_log(P, filename::String)
    if isnothing(findlast(isequal('.'), filename))
        fullname = joinpath(DATA_PATH[1], filename) * ".arrow"
    else
        fullname = joinpath(DATA_PATH[1], filename)
    end
    table = Arrow.Table(fullname)
    syslog = StructArray{SysState{P}}((table.time, table.orient, table.elevation, table.azimuth, table.l_tether, table.v_reelout, table.force, table.depower, table.v_app, table.X, table.Y, table.Z))
    return SysLog{P}(basename(fullname[1:end-6]), syslog)
end

function test(save=false)
    if save
        log_to_save=demo_log(7)
        save_log(log_to_save)
    end
    return(load_log(7, "Test_flight.arrow"))
end

precompile(load_log, (Int64, String,))

end


@testset "Small model join" begin
    m1 = load_model(model_paths["e_coli_core.json"])
    m2 = load_model(CoreModel, model_paths["e_coli_core.json"])

    exchange_rxn_ids = filter(looks_like_exchange_reaction, reactions(m2))
    exchange_met_ids =
        [first(keys(reaction_stoichiometry(m2, ex_rxn))) for ex_rxn in exchange_rxn_ids]

    biomass_ids = ["BIOMASS_Ecoli_core_w_GAM", "BIOMASS_Ecoli_core_w_GAM"]

    community = join_with_exchanges(
        [m1, m2],
        exchange_rxn_ids,
        exchange_met_ids;
        add_biomass_objective = true,
        biomass_ids = biomass_ids,
    )

    env_ex_inds = indexin(exchange_rxn_ids, reactions(community))
    m2_ex_inds = indexin(exchange_rxn_ids, reactions(m2))
    community.xl[env_ex_inds] .= m2.xl[m2_ex_inds]
    community.xu[env_ex_inds] .= m2.xu[m2_ex_inds]

    biomass_ids =
        ["species_1_BIOMASS_Ecoli_core_w_GAM", "species_2_BIOMASS_Ecoli_core_w_GAM"]
    add_objective!(
        community,
        biomass_ids;
        objective_column_index = first(
            indexin(["community_biomass"], reactions(community)),
        ),
    )

    d = flux_balance_analysis_dict(community, Tulip.Optimizer)
    @test size(stoichiometry(community)) == (166, 211)
    @test isapprox(d["community_biomass"], 0.41559777495618294, atol = TEST_TOLERANCE)
end

@testset "Heterogenous model join" begin
    m1 = load_model(CoreModel, model_paths["e_coli_core.json"])
    m2 = load_model(CoreModel, model_paths["iJO1366.mat"])

    exchange_rxn_ids = filter(looks_like_exchange_reaction, reactions(m2))
    exchange_met_ids =
        [first(keys(reaction_stoichiometry(m2, ex_rxn))) for ex_rxn in exchange_rxn_ids]

    biomass_ids = ["BIOMASS_Ecoli_core_w_GAM", "BIOMASS_Ec_iJO1366_core_53p95M"]

    community = join_with_exchanges(
        [m1, m2],
        exchange_rxn_ids,
        exchange_met_ids;
        add_biomass_objective = true,
        biomass_ids = biomass_ids,
    )

    env_ex_inds = indexin(exchange_rxn_ids, reactions(community))
    m2_ex_inds = indexin(exchange_rxn_ids, reactions(m2))
    m1_ex_inds = indexin(exchange_rxn_ids, reactions(m1))

    for (env_ex, m2_ex, m1_ex) in zip(env_ex_inds, m2_ex_inds, m1_ex_inds)
        m2lb = isnothing(m2_ex) ? 0.0 : m2.xl[m2_ex]
        m2ub = isnothing(m2_ex) ? 0.0 : m2.xu[m2_ex]

        m1lb = isnothing(m1_ex) ? 0.0 : m1.xl[m1_ex]
        m1ub = isnothing(m1_ex) ? 0.0 : m1.xu[m1_ex]

        community.xl[env_ex] = m1lb + m2lb
        community.xu[env_ex] = m1ub + m2ub
    end

    biomass_metabolite_inds = indexin(
        ["species_1_BIOMASS_Ecoli_core_w_GAM", "species_2_BIOMASS_Ec_iJO1366_core_53p95M"],
        metabolites(community),
    )

    community.S[biomass_metabolite_inds, end] .= -1.0
    community.c[end] = 1.0
    community.xl[end] = 0.0
    community.xu[end] = 1000.0

    d = flux_balance_analysis_dict(
        community,
        Tulip.Optimizer;
        modifications = [change_optimizer_attribute("IPM_IterationsLimit", 1000)],
    )

    @test size(stoichiometry(community)) == (2203, 3003)
    @test isapprox(d["community_biomass"], 0.8739215069675402, atol = TEST_TOLERANCE)
end

@testset "Community model modifications" begin
    m1 = load_model(CoreModel, model_paths["e_coli_core.json"])

    exchange_rxn_ids = filter(looks_like_exchange_reaction, reactions(m1))
    exchange_met_ids =
        [first(keys(reaction_stoichiometry(m1, ex_rxn))) for ex_rxn in exchange_rxn_ids]

    biomass_ids = ["BIOMASS_Ecoli_core_w_GAM"]

    community = join_with_exchanges(
        [m1],
        exchange_rxn_ids,
        exchange_met_ids;
        add_biomass_objective = true,
        biomass_ids = biomass_ids,
    )

    env_ex_inds = indexin(exchange_rxn_ids, reactions(community))
    m1_ex_inds = indexin(exchange_rxn_ids, reactions(m1))
    community.xl[env_ex_inds] .= m1.xl[m1_ex_inds]
    community.xu[env_ex_inds] .= m1.xu[m1_ex_inds]

    m2 = load_model(CoreModel, model_paths["e_coli_core.json"])

    community = add_model_with_exchanges(
        community,
        m2,
        exchange_rxn_ids,
        exchange_met_ids;
        model_name = "species_2",
        biomass_id = "BIOMASS_Ecoli_core_w_GAM",
    )

    biomass_ids =
        ["species_1_BIOMASS_Ecoli_core_w_GAM", "species_2_BIOMASS_Ecoli_core_w_GAM"]
    add_objective!(
        community,
        biomass_ids;
        objective_column_index = first(
            indexin(["community_biomass"], reactions(community)),
        ),
    )

    d = flux_balance_analysis_dict(community, Tulip.Optimizer)

    @test size(stoichiometry(community)) == (166, 211)
    @test isapprox(d["community_biomass"], 0.41559777495618294, atol = TEST_TOLERANCE)
end


struct RandomEffectsTerm <: AbstractTerm
    lhs::StatsModels.TermOrTerms
    rhs::StatsModels.TermOrTerms
    function RandomEffectsTerm(lhs,rhs)
        if isempty(intersect(StatsModels.termvars(lhs), StatsModels.termvars(rhs)))
            if !isa(rhs, Union{CategoricalTerm,InteractionTerm{<:NTuple{N,CategoricalTerm} where N}})
                throw(ArgumentError("blocking variables (those behind |) must be Categorical ($(rhs) is not)"))
            end
            new(lhs, rhs)
        else
            throw(ArgumentError("Same variable appears on both sides of |"))
        end
    end
end

function StatsModels.apply_schema(t::FunctionTerm{typeof(/)},
                                  sch::StatsModels.FullRank,
                                  Mod::Type{<:MixedModel})
    length(t.args_parsed) == 2 ||
        throw(ArgumentError("malformed nesting term: $t " *
                            "(Exactly two arguments are supported)"))

    first, second = apply_schema.(t.args_parsed, Ref(sch.schema), Mod)
    return first + first & second
end

RandomEffectsTerm(lhs, rhs::NTuple{2, AbstractTerm}) =
    (RandomEffectsTerm(lhs, rhs[1]), RandomEffectsTerm(lhs, rhs[2]))

Base.show(io::IO, t::RandomEffectsTerm) = print(io, "($(t.lhs) | $(t.rhs))")
StatsModels.is_matrix_term(::Type{RandomEffectsTerm}) = false

function StatsModels.termvars(t::RandomEffectsTerm)
    vcat(StatsModels.termvars(t.lhs), StatsModels.termvars(t.rhs))
end

function StatsModels.apply_schema(t::FunctionTerm{typeof(|)},
                                  schema::StatsModels.FullRank,
                                  Mod::Type{<:MixedModel})
    schema = StatsModels.FullRank(schema.schema)
    lhs, rhs = t.args_parsed
    if !StatsModels.hasintercept(lhs) && !StatsModels.omitsintercept(lhs)
        lhs = InterceptTerm{true}() + lhs
    end
    lhs, rhs = apply_schema.((lhs, rhs), Ref(schema), Mod)
    RandomEffectsTerm(MatrixTerm(lhs), rhs)
end

function StatsModels.modelcols(t::RandomEffectsTerm, d::NamedTuple)
    lhs = t.lhs
    z = Matrix(transpose(modelcols(lhs, d)))
    cnames = coefnames(lhs)
    T = eltype(z)
    S = size(z, 1)
    grp = t.rhs
    m = reshape(1:abs2(S), (S, S))
    inds = sizehint!(Int[], (S * (S + 1)) >> 1)
    for j in 1:S, i in j:S
        push!(inds, m[i,j])
    end
    refs, levels = _ranef_refs(grp, d)

    ReMat{T,S}(
        grp, refs, levels, isa(cnames, String) ? [cnames] : collect(cnames),
        z, z, LowerTriangular(Matrix{T}(I, S, S)), inds,
        adjA(refs, z), Matrix{T}(undef, (S, length(levels))))
end

# extract vector of refs from ranef grouping term and data
function _ranef_refs(grp::CategoricalTerm, d::NamedTuple)
    invindex = grp.contrasts.invindex
    refs = convert(Vector{Int32}, getindex.(Ref(invindex), d[grp.sym]))
    refs, grp.contrasts.levels
end

function _ranef_refs(grp::InteractionTerm{<:NTuple{N,CategoricalTerm}},
                     d::NamedTuple) where {N}
    combos = zip(getproperty.(Ref(d), [g.sym for g in grp.terms])...)
    uniques = unique(combos)
    invindex = Dict(x => i for (i,x) in enumerate(uniques))
    refs = convert(Vector{Int32}, getindex.(Ref(invindex), combos))
    refs, uniques
end

# add some syntax to manually promote to full dummy coding
fulldummy(t::AbstractTerm) =
    throw(ArgumentError("can't promote $t (of type $(typeof(t))) to full dummy " *
                        "coding (only CategoricalTerms)"))

function fulldummy(t::CategoricalTerm)
    new_contrasts = StatsModels.ContrastsMatrix(StatsModels.FullDummyCoding(),
                                                t.contrasts.levels)
    t = CategoricalTerm(t.sym, new_contrasts)
end

fulldummy(x) =
    throw(ArgumentError("fulldummy isn't supported outside of a MixedModel formula"))

function StatsModels.apply_schema(t::FunctionTerm{typeof(fulldummy)},
                                  sch::StatsModels.FullRank,
                                  Mod::Type{<:MixedModel})
    fulldummy(apply_schema.(t.args_parsed, Ref(sch), Mod)...)
end

# specify zero correlation
struct ZeroCorr <: AbstractTerm
    term::RandomEffectsTerm
end
StatsModels.is_matrix_term(::Type{ZeroCorr}) = false

"""
    zerocorr(term::RandomEffectsTerm)

Remove correlations between random effects in `term`.
"""
zerocorr(x) = ZeroCorr(x)

function StatsModels.apply_schema(t::FunctionTerm{typeof(zerocorr)},
                                  sch::StatsModels.FullRank,
                                  Mod::Type{<:MixedModel})
    ZeroCorr(apply_schema(t.args_parsed..., sch, Mod))
end

StatsModels.modelcols(t::ZeroCorr, d::NamedTuple) =
    zerocorr!(modelcols(t.term, d))


function set_vector_attribute(element::XMLElement, attr::AbstractString, vec::AbstractVector)
    set_attribute(element, attr, join(vec, ' '))
end

function to_urdf(body::RigidBody)
    xml_link = new_element("link")
    set_attribute(xml_link, "name", string(body))
    isroot = !has_defined_inertia(body)
    if !isroot
        xml_inertial = new_child(xml_link, "inertial")
        xml_origin = new_child(xml_inertial, "origin")
        xml_mass = new_child(xml_inertial, "mass")
        xml_inertia = new_child(xml_inertial, "inertia")

        inertia = spatial_inertia(body)
        if inertia.mass > 0
            origin = center_of_mass(inertia)
            centroidal = CartesianFrame3D()
            to_centroidal_frame = Transform3D(inertia.frame, centroidal, -origin.v)
            inertia = transform(inertia, to_centroidal_frame)
            @assert center_of_mass(inertia) ≈ Point3D(centroidal, zero(typeof(origin.v)))
        else
            origin = zero(center_of_mass(inertia))
        end

        set_vector_attribute(xml_origin, "xyz", origin.v)
        set_vector_attribute(xml_origin, "rpy", zero(origin.v))
        set_attribute(xml_mass, "value", inertia.mass)
        set_attribute(xml_inertia, "ixx", inertia.moment[1, 1])
        set_attribute(xml_inertia, "ixy", inertia.moment[1, 2])
        set_attribute(xml_inertia, "ixz", inertia.moment[1, 3])
        set_attribute(xml_inertia, "iyy", inertia.moment[2, 2])
        set_attribute(xml_inertia, "iyz", inertia.moment[2, 3])
        set_attribute(xml_inertia, "izz", inertia.moment[3, 3])
    end
    xml_link
end

function process_joint_type!(xml_joint::XMLElement, joint::Joint)
    if isfloating(joint)
        # handle this here instead of using dispatch to support user-defined
        # floating joint types out of the box
        set_attribute(xml_joint, "type", "floating")
    else
        throw(ArgumentError("Joint type $(typeof(joint_type(joint))) not handled."))
    end
    xml_joint
end

function process_joint_type!(xml_joint::XMLElement, joint::Joint{<:Any, <:Fixed})
    set_attribute(xml_joint, "type", "fixed")
    xml_joint
end

function process_joint_type!(xml_joint::XMLElement, joint::Joint{<:Any, <:Planar})
    set_attribute(xml_joint, "type", "planar")
    jtype = joint_type(joint)
    xml_axis = new_child(xml_joint, "axis")
    set_vector_attribute(xml_axis, "xyz", jtype.x_axis × jtype.y_axis)
    xml_joint
end

function process_joint_type!(xml_joint::XMLElement, joint::Joint{T, JT}) where {T, JT<:Union{Revolute, Prismatic}}
    jtype = joint_type(joint)
    xml_axis = new_child(xml_joint, "axis")
    set_vector_attribute(xml_axis, "xyz", jtype.axis)

    qbounds, vbounds, τbounds = position_bounds(joint), velocity_bounds(joint), effort_bounds(joint)
    @assert length(qbounds) == length(vbounds) == length(τbounds) == 1
    qbound, vbound, τbound = qbounds[1], vbounds[1], τbounds[1]

    @assert upper(vbound) == -lower(vbound)
    @assert upper(τbound) == -lower(τbound)

    realline = Bounds(-typemax(T), typemax(T))
    xml_limit = new_child(xml_joint, "limit")
    set_position_limits = true
    if JT <: Revolute
        if qbound == realline
            set_attribute(xml_joint, "type", "continuous")
            set_position_limits = false # continuous joints don't allow `lower` and `upper`
        else
            set_attribute(xml_joint, "type", "revolute")
        end
    else # Prismatic
        set_attribute(xml_joint, "type", "prismatic")
    end

    if set_position_limits
        set_attribute(xml_limit, "lower", lower(qbound))
        set_attribute(xml_limit, "upper", upper(qbound))
    end

    set_attribute(xml_limit, "effort", upper(τbound))
    set_attribute(xml_limit, "velocity", upper(vbound))

    xml_joint
end

function to_urdf(joint::Joint, mechanism::Mechanism)
    parent = predecessor(joint, mechanism)
    child = successor(joint, mechanism)
    to_parent = joint_to_predecessor(joint)
    xyz = translation(to_parent)
    rpy = RotZYX(rotation(to_parent))

    xml_joint = new_element("joint")
    set_attribute(xml_joint, "name", string(joint))
    xml_parent = new_child(xml_joint, "parent")
    set_attribute(xml_parent, "link", string(parent))
    xml_child = new_child(xml_joint, "child")
    set_attribute(xml_child, "link", string(child))
    xml_origin = new_child(xml_joint, "origin")
    set_vector_attribute(xml_origin, "xyz", xyz)
    set_vector_attribute(xml_origin, "rpy", [rpy.theta3, rpy.theta2, rpy.theta1])

    process_joint_type!(xml_joint, joint)
    xml_joint
end

function to_urdf(mechanism::Mechanism; robot_name::Union{Nothing, AbstractString}=nothing, include_root::Bool=true)
    @assert !has_loops(mechanism)
    xdoc = XMLDocument()
    xroot = create_root(xdoc, "robot")
    if robot_name !== nothing
        set_attribute(xroot, "name", robot_name)
    end
    bodies_to_include = include_root ? bodies(mechanism) : non_root_bodies(mechanism)
    for body in bodies(mechanism)
        !include_root && isroot(body, mechanism) && continue
        add_child(xroot, to_urdf(body))
    end
    for joint in tree_joints(mechanism)
        !include_root && isroot(predecessor(joint, mechanism), mechanism) && continue
        add_child(xroot, to_urdf(joint, mechanism))
    end
    xdoc
end

"""
Serialize a `Mechanism` to the [URDF](https://wiki.ros.org/urdf/XML/model) file format.

Limitations:

* for `<link>` tags, only the `<inertial>` tag is written; there is no support for `<visual>` and `<collision>` tags.
* for `<joint>` tags, only the `<origin>`, `<parent>`, `<child>`, and `<limit>` tags are written. There is no support for the `<calibration>` and `<safety_controller>` tags.

These limitations are simply due to the fact that `Mechanism`s do not store the required information to write these tags.

Keyword arguments:

* `robot_name`: used to set the `name` attribute of the root `<robot>` tag in the URDF. Default: `nothing` (name attribute will not be set).
* `include_root`: whether to include `root_body(mechanism)` in the URDF. If `false`, joints with `root_body(mechanism)` as their predecessor will also be omitted. Default: `true`.
"""
function write_urdf end

const write_urdf_name_kwarg_doc = "Optionally, the `robot_name` keyword argument can be used to specify the robot's name."

"""
$(SIGNATURES)

Write a URDF representation of `mechanism` to the stream `io` (a `Base.IO`).

$write_urdf_name_kwarg_doc
"""
function write_urdf(io::IO, mechanism::Mechanism; robot_name=nothing, include_root=true)
    show(io, to_urdf(mechanism; robot_name=robot_name, include_root=include_root))
end

"""
$(SIGNATURES)

Write a URDF representation of `mechanism` to a file.

$write_urdf_name_kwarg_doc
"""
function write_urdf(filename::AbstractString, mechanism::Mechanism; robot_name=nothing, include_root=true)
    open(filename, "w") do io
        write_urdf(io, mechanism, robot_name=robot_name, include_root=include_root)
    end
end


using VectorizationBase, SIMDPirates, StructuredMatrices, PaddedMatrices

"""
Creates a quite that already assumes L (and S if S != L) are VectorizationBase.vpointers

"""
function cholesky_quote(
    M::Int, ::Type{T} = Float64,
    base_offset = M, base_stride = M - 1,
    Lsym = :L, Ssym = :S,
    first_diag_expr =  :( $(Symbol(Lsym,:scalar_,1,:_,1)) = Base.FastMath.sqrt_fast(VectorizationBase.load($Ssym)) ),
    mask_first_load = true
) where {T}
    W, Wshift = VectorizationBase.pick_vector_width_shift(M, T)
    Wm1 = W - 1
    Mm1 = M - 1
    Mrep = Mm1 >> Wshift
    Mrem = Mm1 & Wm1
    V = Vec{W,T}
    Maxrowrep = (Mm1 + Wm1) >>> Wshift
    MT = VectorizationBase.mask_type(W)
    mask_bits = 8sizeof(MT)
    q = quote
        $first_diag_expr
        VectorizationBase.store!($Lsym, $(Symbol(Lsym,:scalar_, 1, :_, 1)))
    end
    # Load S
    # Load diagonal
    dind = 0
    if Mrem > 0
        # Offset for first diaognal vector
        initial_offset = 1 + Mrem - W
        if mask_first_load && initial_offset < 0
            mask = typemax(MT) << (-initial_offset)
            push!(q.args, Expr(:(=), Symbol(Lsym, :d_, dind), :(SIMDPirates.vload($V, $Ssym + $initial_offset, $mask) ) ) )
        else
            push!(q.args, Expr(:(=), Symbol(Lsym, :d_, dind), :(SIMDPirates.vload($V, $Ssym + $initial_offset) ) ) )
        end
        offset = initial_offset + W
        dind += 1
    else
        initial_offset = 1
        offset = 1
    end
    for m ∈ 1:Mrep # Should really be 0 or 1 (or 2 with Mrem == 0); greater leads to register spills
        push!(q.args, Expr(:(=), Symbol(Lsym, :d_, dind), :(SIMDPirates.vload($V, $Ssym + $offset) ) ) )
        offset += W
        dind += 1
    end
    # load off diagonal
    col_offset = Mrem == 0 ? base_offset : base_offset + Mrem - W
    rroffset = 0
    for c ∈ 1:Mm1
        rows = M - c
        rowreps = (rows + Wm1) >> Wshift
#        rowrem = rows & Wm1
        for rr ∈ 0:rowreps-1
            push!(q.args, Expr(:(=), Symbol(Lsym, :_, rr + rroffset, :_, c), :(SIMDPirates.vload($V, $Ssym + $(col_offset + rr * W)))))
        end
        col_offset += base_stride - c
        if ((rows - 1) & Wm1) == 0
            col_offset += W
            rroffset += 1
        end
    end
    # S has been loaded
    # Now, start the factorization
    rroffsetcoffset = Mrem == 0 ? -1 : Wm1 - Mrem
    col_offset = Mrem == 0 ? base_offset : base_offset + Mrem - W
    for couter ∈ 1:Mm1
        rroffset_outer = (couter + rroffsetcoffset) >> Wshift
        rows = M - couter
        rowreps = (rows + Wm1) >> Wshift
        rowrem = rows & Wm1
        # Optimizer will handle choice between Ldinv = 1 / Ld; Lcol * Ldinv vs Lcol / Ld
        vdiag = Symbol(:v, Lsym, :_, couter, :_, couter)
        push!(q.args, :( $vdiag = SIMDPirates.vbroadcast($V, $(Symbol(Lsym, :scalar_, couter, :_, couter)))))
        for rr ∈ rroffset_outer:rowreps-1 + rroffset_outer
            L_rr_couter = Symbol(Lsym, :_, rr, :_, couter)
            push!(q.args, Expr(:(=), L_rr_couter, :(SIMDPirates.vfdiv($L_rr_couter, $vdiag))))
            diagsym = Symbol(Lsym, :d_, rr)# + Maxrowrep - rowreps)
            push!(q.args, Expr(:(=), diagsym, :(SIMDPirates.vfnmadd($L_rr_couter,$L_rr_couter,$diagsym))))
            if rr == rroffset_outer # we sqrt and store diagonal element
                Lds = Symbol(Lsym, :scalar_, couter + 1, :_, couter + 1)
                doff = rowrem == 0 ? 1 : W + 1 - rowrem
                push!(q.args, Expr(:(=), Lds, :(@inbounds Base.FastMath.sqrt_fast($diagsym[ $doff ].value))))
                push!(q.args, :(VectorizationBase.store!($Lsym + $couter, $Lds)))
            end
            if rr == rroffset_outer && rowrem > 0 # mask store
                push!(q.args, :(SIMDPirates.vstore!($Lsym + $(col_offset), $L_rr_couter, $(typemax(MT) << (W - rowrem)))))
            else
                push!(q.args, :(SIMDPirates.vstore!($Lsym + $(col_offset + (rr-rroffset_outer)*W), $L_rr_couter)))
            end
        end
        scalar_base = col_offset - couter - 1 + ((W - rowrem) & Wm1)
        for cinner ∈ couter+1:Mm1
            colsadjusted = cinner + rroffsetcoffset
            rroffset_inner = (colsadjusted) >> Wshift
            rows = M - cinner
            rowreps = (rows + Wm1) >> Wshift
            Ls = Symbol(:Ls_,cinner,:_,couter)
            push!(q.args, Expr(:(=), Ls, :( SIMDPirates.vbroadcast($V,VectorizationBase.load($Lsym + $(cinner + scalar_base))  ))))
            for rr ∈ rroffset_inner:rowreps - 1 + rroffset_inner
                L_rr_cinner = Symbol(Lsym, :_, rr, :_, cinner)
                push!(q.args, :($L_rr_cinner = SIMDPirates.vfnmadd($(Symbol(Lsym, :_, rr, :_, couter)), $Ls, $L_rr_cinner)))
            end
        end
        col_offset += base_stride - couter
        if ((M - couter - 1) & Wm1) == 0
            col_offset += W
        end
    end
    q
end

@generated function choltest!(
    Lm::AbstractMutableLowerTriangularMatrix{M,T},
    Sm::AbstractSymmetricMatrixL{M,T}
#) where {T,M}
) where {M,T}
    chol_quote = cholesky_quote(M, T)
    quote
        L = VectorizationBase.vectorizable(Lm)
        S = VectorizationBase.vectorizable(Sm)
        $chol_quote
        Lm
    end
end

"""
Calculates and stores L, inv(diag(L)), and ∂(-logdet(L))/∂S
"""
function cholesky_∂det_quote(
    M::Int, ::Type{T} = Float64,
    base_offset = M, base_stride = M - 1,
    Lsym = :L, Ssym = :S,
    first_diag_expr =  :( $(Symbol(Lsym,:scalar_,1,:_,1)) = Base.FastMath.sqrt_fast(VectorizationBase.load($Ssym)) ),
    mask_first_load = true
) where {T}
    W, Wshift = VectorizationBase.pick_vector_width_shift(M, T)
    Wm1 = W - 1
    Mm1 = M - 1
    Mrep = Mm1 >> Wshift
    Mrem = Mm1 & Wm1
    V = Vec{W,T}
    Maxrowrep = (Mm1 + Wm1) >>> Wshift
    MT = VectorizationBase.mask_type(W)
    mask_bits = 8sizeof(MT)
    q = quote
        $first_diag_expr
        VectorizationBase.store!($Lsym, $(Symbol(Lsym,:scalar_, 1, :_, 1)))
    end
    # Load S
    # Load diagonal
    dind = 0
    if Mrem > 0
        # Offset for first diaognal vector
        initial_offset = 1 + Mrem - W
        if mask_first_load && initial_offset < 0
            mask = typemax(MT) << (-initial_offset)
            push!(q.args, Expr(:(=), Symbol(Lsym, :d_, dind), :(SIMDPirates.vload($V, $Ssym + $initial_offset, $mask) ) ) )
        else
            push!(q.args, Expr(:(=), Symbol(Lsym, :d_, dind), :(SIMDPirates.vload($V, $Ssym + $initial_offset) ) ) )
        end
        offset = initial_offset + W
        dind += 1
    else
        initial_offset = 1
        offset = 1
    end
    for m ∈ 1:Mrep # Should really be 0 or 1 (or 2 with Mrem == 0); greater leads to register spills
        push!(q.args, Expr(:(=), Symbol(Lsym, :d_, dind), :(SIMDPirates.vload($V, $Ssym + $offset) ) ) )
        offset += W
        dind += 1
    end
    # load off diagonal
    col_offset = Mrem == 0 ? base_offset : base_offset + Mrem - W
    rroffset = 0
    for c ∈ 1:Mm1
        rows = M - c
        rowreps = (rows + Wm1) >> Wshift
#        rowrem = rows & Wm1
        for rr ∈ 0:rowreps-1
            push!(q.args, Expr(:(=), Symbol(Lsym, :_, rr + rroffset, :_, c), :(SIMDPirates.vload($V, $Ssym + $(col_offset + rr * W)))))
        end
        col_offset += base_stride - c
        if ((rows - 1) & Wm1) == 0
            col_offset += W
            rroffset += 1
        end
    end
    # S has been loaded
    # Now, start the factorization
    rroffsetcoffset = Mrem == 0 ? -1 : Wm1 - Mrem
    col_offset = Mrem == 0 ? base_offset : base_offset + Mrem - W
    for couter ∈ 1:Mm1
        rroffset_outer = (couter + rroffsetcoffset) >> Wshift
        rows = M - couter
        rowreps = (rows + Wm1) >> Wshift
        rowrem = rows & Wm1
        # Optimizer will handle choice between Ldinv = 1 / Ld; Lcol * Ldinv vs Lcol / Ld
        vdiag = Symbol(:v, Lsym, :_, couter, :_, couter)
        push!(q.args, :( $vdiag = SIMDPirates.vbroadcast($V, $(Symbol(Lsym, :scalar_, couter, :_, couter)))))
        for rr ∈ rroffset_outer:rowreps-1 + rroffset_outer
            L_rr_couter = Symbol(Lsym, :_, rr, :_, couter)
            push!(q.args, Expr(:(=), L_rr_couter, :(SIMDPirates.vfdiv($L_rr_couter, $vdiag))))
            diagsym = Symbol(Lsym, :d_, rr)# + Maxrowrep - rowreps)
            push!(q.args, Expr(:(=), diagsym, :(SIMDPirates.vfnmadd($L_rr_couter,$L_rr_couter,$diagsym))))
            if rr == rroffset_outer # we sqrt and store diagonal element
                Lds = Symbol(Lsym, :scalar_, couter + 1, :_, couter + 1)
                doff = rowrem == 0 ? 1 : W + 1 - rowrem
                push!(q.args, Expr(:(=), Lds, :(@inbounds Base.FastMath.sqrt_fast($diagsym[ $doff ].value))))
                push!(q.args, :(VectorizationBase.store!($Lsym + $couter, $Lds)))
            end
            if rr == rroffset_outer && rowrem > 0 # mask store
                push!(q.args, :(SIMDPirates.vstore!($Lsym + $(col_offset), $L_rr_couter, $(typemax(MT) << (W - rowrem)))))
            else
                push!(q.args, :(SIMDPirates.vstore!($Lsym + $(col_offset + (rr-rroffset_outer)*W), $L_rr_couter)))
            end
        end
        scalar_base = col_offset - couter - 1 + ((W - rowrem) & Wm1)
        for cinner ∈ couter+1:Mm1
            colsadjusted = cinner + rroffsetcoffset
            rroffset_inner = (colsadjusted) >> Wshift
            rows = M - cinner
            rowreps = (rows + Wm1) >> Wshift
            Ls = Symbol(:Ls_,cinner,:_,couter)
            push!(q.args, Expr(:(=), Ls, :( SIMDPirates.vbroadcast($V,VectorizationBase.load($Lsym + $(cinner + scalar_base))  ))))
            for rr ∈ rroffset_inner:rowreps - 1 + rroffset_inner
                L_rr_cinner = Symbol(Lsym, :_, rr, :_, cinner)
                push!(q.args, :($L_rr_cinner = SIMDPirates.vfnmadd($(Symbol(Lsym, :_, rr, :_, couter)), $Ls, $L_rr_cinner)))
            end
        end
        col_offset += base_stride - couter
        if ((M - couter - 1) & Wm1) == 0
            col_offset += W
        end
    end
    q
end

#@generated function choltest!(
#    Lm::AbstractMutableLowerTriangularMatrix{M,T},
#    Sm::AbstractSymmetricMatrixL{M,T}
#) where {T,M}
##) where {M,T}
#    chol_quote = cholesky_quote(M, T)
#    quote
#        L = VectorizationBase.vectorizable(Lm)
#        S = VectorizationBase.vectorizable(Sm)
#        $chol_quote
#        Lm
#    end
#end


###
### Sampler states
###

struct Emcee{space, E<:AMH.Ensemble} <: InferenceAlgorithm
    ensemble::E
end

function Emcee(n_walkers::Int, stretch_length=2.0)
    # Note that the proposal distribution here is just a Normal(0,1)
    # because we do not need AdvancedMH to know the proposal for
    # ensemble sampling.
    prop = AMH.StretchProposal(nothing, stretch_length)
    ensemble = AMH.Ensemble(n_walkers, prop)
    return Emcee{(), typeof(ensemble)}(ensemble)
end

function Sampler(
    alg::Emcee,
    model::Model,
    s::Selector=Selector()
)
    # Set up info dict.
    info = Dict{Symbol, Any}()

    # Set up state struct.
    state = SamplerState(VarInfo(model))

    # Generate a sampler.
    return Sampler(alg, info, s, state)
end

function AbstractMCMC.sample_init!(
    rng::AbstractRNG,
    model::Model,
    spl::Sampler{<:Emcee},
    N::Integer;
    verbose::Bool=true,
    resume_from=nothing,
    kwargs...
)
    # Resume the sampler.
    set_resume!(spl; resume_from=resume_from, kwargs...)

    # Get `init_theta`
    initialize_parameters!(spl; verbose=verbose, kwargs...)

    # Link everything before sampling.
    link!(spl.state.vi, spl)
end

function AbstractMCMC.sample_end!(
    rng::AbstractRNG,
    model::Model,
    spl::Sampler{<:Emcee},
    N::Integer,
    transitions;
    kwargs...
)
    # Invlink everything after sampling.
    invlink!(spl.state.vi, spl)
end

function _typed_draw(model, vi::VarInfo)
    spl = SampleFromPrior()
    empty!(vi)
    model(vi, spl)
    DynamicPPL.link!(vi, spl)
    params = vi[spl]
    return AMH.Transition(params, getlogp(vi))
end

function AbstractMCMC.step!(
    rng::AbstractRNG,
    model::Model,
    spl::Sampler{<:Emcee},
    N::Integer,
    transition::Nothing;
    kwargs...
)
    # Generate a log joint function.
    densitymodel = AMH.DensityModel(gen_logπ(spl.state.vi, DynamicPPL.SampleFromPrior(), model))

    # Make the first transition.
    walkers = map(x -> _typed_draw(model, spl.state.vi), 1:spl.alg.ensemble.n_walkers)

    return walkers
end

function AbstractMCMC.step!(
    rng::AbstractRNG,
    model::Model,
    spl::Sampler{<:Emcee},
    N::Integer,
    transition;
    kwargs...
)
    # Generate a log joint function.
    densitymodel = AMH.DensityModel(gen_logπ(spl.state.vi, DynamicPPL.SampleFromPrior(), model))

    # Make the first transition.
    new_transitions = AbstractMCMC.step!(rng, densitymodel, spl.alg.ensemble, 1, transition)
    return new_transitions
end

function transform_transition(spl::Sampler{<:Emcee}, ts::Vector{<:Vector}, w::Int, i::Int; linked=true)
    trans = ts[i][w]
    linked && DynamicPPL.link!(spl.state.vi, spl)
    spl.state.vi[spl] = trans.params
    linked && DynamicPPL.invlink!(spl.state.vi, spl)
    setlogp!(spl.state.vi, trans.lp)

    return Transition(spl)
end

function AbstractMCMC.bundle_samples(
    rng::AbstractRNG,
    model::AbstractModel,
    spl::Sampler{<:Emcee},
    N::Integer,
    ts::Vector{<:Vector},
    chain_type::Type{MCMCChains.Chains};
    save_state = false,
    kwargs...
)
    # Transform the transitions by linking them to the constrained space.
    ts_transform = [map(i -> transform_transition(spl, ts, w, i), 1:N) for w in 1:spl.alg.ensemble.n_walkers]

     # Convert transitions to array format.
    # Also retrieve the variable names.
    params_vec = map(_params_to_array, ts_transform)

    # Extract names and values separately.
    nms = params_vec[1][1]
    vals_vec = [p[2] for p in params_vec]

    # Get the values of the extra parameters in each transition.
    extra_vec = map(get_transition_extras, ts_transform)

    # Get the extra parameter names & values.
    extra_params = extra_vec[1][1]
    extra_values_vec = [e[2] for e in extra_vec]

    # Extract names & construct param array.
    nms = [nms; extra_params]
    parray = map(x -> hcat(x[1], x[2]), zip(vals_vec, extra_values_vec))
    parray = cat(parray..., dims=3)

    # Get the average or final log evidence, if it exists.
    le = getlogevidence(spl)

    # Set up the info tuple.
    if save_state
        info = (range = rng, model = model, spl = spl)
    else
        info = NamedTuple()
    end

    # Concretize the array before giving it to MCMCChains.
    parray = MCMCChains.concretize(parray)

    # Chain construction.
    return MCMCChains.Chains(
        parray,
        nms,
        deepcopy(TURING_INTERNAL_VARS);
        evidence=le,
        info=info,
    ) |> sort
end


function check_error(err, error_msg)
    if err != 0
        error(error_msg * " with error: $(err)")
    end
end

const FREE_FONT_LIBRARY = FT_Library[C_NULL]

function ft_init()
    FREE_FONT_LIBRARY[1] != C_NULL && error("Freetype already initalized. init() called two times?")
    err = FT_Init_FreeType(FREE_FONT_LIBRARY)
    return err == 0
end

function ft_done()
    FREE_FONT_LIBRARY[1] == C_NULL && error("Library == CNULL. FreeTypeAbstraction.done() called before init(), or done called two times?")
    err = FT_Done_FreeType(FREE_FONT_LIBRARY[1])
    FREE_FONT_LIBRARY[1] = C_NULL
    return err == 0
end

function newface(facename, faceindex::Real=0, ftlib=FREE_FONT_LIBRARY)
    face = Ref{FT_Face}()
    err = FT_New_Face(ftlib[1], facename, Int32(faceindex), face)
    check_error(err, "Couldn't load font $facename")
    return face[]
end

struct FontExtent{T}
    vertical_bearing::Vec{2, T}
    horizontal_bearing::Vec{2, T}

    advance::Vec{2, T}
    scale::Vec{2, T}
end

hadvance(ext::FontExtent) = ext.advance[1]
inkwidth(ext::FontExtent) = ext.scale[1]
inkheight(ext::FontExtent) = ext.scale[2]
hbearing_ori_to_left(ext::FontExtent) = ext.horizontal_bearing[1]
hbearing_ori_to_top(ext::FontExtent) = ext.horizontal_bearing[2]
leftinkbound(ext::FontExtent) = hbearing_ori_to_left(ext)
rightinkbound(ext::FontExtent) = leftinkbound(ext) + inkwidth(ext)
bottominkbound(ext::FontExtent) = hbearing_ori_to_top(ext) - inkheight(ext)
topinkbound(ext::FontExtent) = hbearing_ori_to_top(ext)

BroadcastStyle(::Type{<: FontExtent}) = Style{FontExtent}()
BroadcastStyle(::Style{FontExtent}, x) = Style{FontExtent}()
BroadcastStyle(x, ::Style{FontExtent}) = Style{FontExtent}()

function broadcasted(op::Function, f::FontExtent, scaling::StaticVector)
    return FontExtent(
        op.(f.vertical_bearing, scaling[1]),
        op.(f.horizontal_bearing, scaling[2]),
        op.(f.advance, scaling),
        op.(f.scale, scaling),
    )
end

function broadcasted(op::Function, f::FontExtent)
    return FontExtent(
        op.(f.vertical_bearing),
        op.(f.horizontal_bearing),
        op.(f.advance),
        op.(f.scale),
    )
end

function broadcasted(op::Function, ::Type{T}, f::FontExtent) where T
    return FontExtent(
        map(x-> op(T, x), f.vertical_bearing),
        map(x-> op(T, x), f.horizontal_bearing),
        map(x-> op(T, x), f.advance),
        map(x-> op(T, x), f.scale),
    )
end

function FontExtent(fontmetric::FreeType.FT_Glyph_Metrics, scale::T = 64.0) where T <: AbstractFloat
    return FontExtent(
        Vec{2, T}(fontmetric.vertBearingX, fontmetric.vertBearingY) ./ scale,
        Vec{2, T}(fontmetric.horiBearingX, fontmetric.horiBearingY) ./ scale,
        Vec{2, T}(fontmetric.horiAdvance, fontmetric.vertAdvance) ./ scale,
        Vec{2, T}(fontmetric.width, fontmetric.height) ./ scale
    )
end

function ==(x::FontExtent, y::FontExtent)
    return (x.vertical_bearing == y.vertical_bearing &&
            x.horizontal_bearing == y.horizontal_bearing &&
            x.advance == y.advance &&
            x.scale == y.scale)
end

function FontExtent(fontmetric::FreeType.FT_Glyph_Metrics, scale::Integer)
    return FontExtent(
        div.(Vec{2, Int}(fontmetric.vertBearingX, fontmetric.vertBearingY), scale),
        div.(Vec{2, Int}(fontmetric.horiBearingX, fontmetric.horiBearingY), scale),
        div.(Vec{2, Int}(fontmetric.horiAdvance, fontmetric.vertAdvance), scale),
        div.(Vec{2, Int}(fontmetric.width, fontmetric.height), scale)
    )
end

function bearing(extent::FontExtent)
    return Vec2f0(extent.horizontal_bearing[1],
                  -(extent.scale[2] - extent.horizontal_bearing[2]))
end

function safe_free(face)
    ptr = getfield(face, :ft_ptr)
    if ptr != C_NULL && FREE_FONT_LIBRARY[1] != C_NULL
        FT_Done_Face(face)
    end
end

function boundingbox(extent::FontExtent)
    mini = bearing(extent)
    return Rect2D(mini, Vec2f0(extent.scale))
end

mutable struct FTFont
    ft_ptr::FreeType.FT_Face
    use_cache::Bool
    extent_cache::Dict{Char, FontExtent{Float32}}
    function FTFont(ft_ptr::FreeType.FT_Face, use_cache::Bool=true)
        extent_cache = Dict{Tuple{Int, Char}, FontExtent{Float32}}()
        face = new(ft_ptr, use_cache, extent_cache)
        finalizer(safe_free, face)
        return face
    end
end

use_cache(face::FTFont) = getfield(face, :use_cache)
get_cache(face::FTFont) = getfield(face, :extent_cache)

function FTFont(path::String)
    return FTFont(newface(path))
end

# C interop
function Base.cconvert(::Type{FreeType.FT_Face}, font::FTFont)
    return font
end

function Base.unsafe_convert(::Type{FreeType.FT_Face}, font::FTFont)
    return getfield(font, :ft_ptr)
end

function Base.propertynames(font::FTFont)
    return fieldnames(FreeType.FT_FaceRec)
end

function Base.getproperty(font::FTFont, fieldname::Symbol)
    fontrect = unsafe_load(getfield(font, :ft_ptr))
    field = getfield(fontrect, fieldname)
    if field isa Ptr{FT_String}
        field == C_NULL && return ""
        return unsafe_string(field)
    # Some fields segfault with unsafe_load...Lets find out which another day :D
    elseif field isa Ptr{FreeType.LibFreeType.FT_GlyphSlotRec}
        return unsafe_load(field)
    else
        return field
    end
end

# Allow broadcasting over fonts
Base.Broadcast.broadcastable(ft::FTFont) = Ref(ft)

function set_pixelsize(face::FTFont, size::Integer)
    err = FT_Set_Pixel_Sizes(face, size, size)
    check_error(err, "Couldn't set pixelsize")
    return size
end

function kerning(c1::Char, c2::Char, face::FTFont)
    i1 = FT_Get_Char_Index(face, c1)
    i2 = FT_Get_Char_Index(face, c2)
    kerning2d = Ref{FreeType.FT_Vector}()
    err = FT_Get_Kerning(face, i1, i2, FreeType.FT_KERNING_DEFAULT, kerning2d)
    # Can error if font has no kerning! Since that's somewhat expected, we just return 0
    err != 0 && return Vec2f0(0)
    # 64 since metrics are in 1/64 units (units to 26.6 fractional pixels)
    divisor = 64
    return Vec2f0(kerning2d[].x / divisor, kerning2d[].y / divisor)
end

function get_extent(face::FTFont, char::Char)
    if use_cache(face)
        get!(get_cache(face), char) do
            return internal_get_extent(face, char)
        end
    else
        return internal_get_extent(face, char)
    end
end

function internal_get_extent(face::FTFont, char::Char)
    #=
    Load chars without scaling. This leaves all glyph metrics that can be
    retrieved in font units, which can be normalized by dividing with the
    font's units_per_EM. This is more robust than relying on extents
    that are only valid with a specific pixelsize, because a font's
    pixelsize can be silently changed by third parties, such as Cairo.
    If that happens, all glyph metrics are incorrect. We avoid this by using the normalized space.
    =#
    err = FT_Load_Char(face, char, FT_LOAD_NO_SCALE)
    check_error(err, "Could not load char to get extent.")
    # This gives us the font metrics in normalized units (0, 1), with negative
    # numbers interpreted as an offset
    return FontExtent(face.glyph.metrics, Float32(face.units_per_EM))
end

descender(font) = font.descender / font.units_per_EM
ascender(font) = font.ascender / font.units_per_EM


# Sane behavior when run from the REPL
source_dir = typeof(Base.source_dir()) == Void ? joinpath(Pkg.dir("NeuralNet"), "src") : Base.source_dir()

"""
align(algorithm::Compat.String, gap_penalty::Float64, gap_cont_penalty::Float64,
            seq1::Compat.String, seq2::Compat.String, smat::Compat.String, outputfile::Compat.String)
Align SEQ1 to SEQ2 using the scoring matrix SMAT and paramters GAP_PENALTY and GAP_CONT_PENALTY
for the ALGORITHM defined by user.
"""
function simple_encoder(N::Int, alpha::Float64, outputfile::Compat.String)

    include(joinpath(source_dir, "algorithms", "nn.jl"));
    # run
    Base.invokelatest(encode_8x3x8, N, alpha, outputfile);

end

function train_model(N::Int, alpha::Float64, hl_size::Int, write_output::Bool,
            balance::Int, C::Int, pos_seqs::Compat.String, total_seq::Compat.String, outputfile::Compat.String)

    include(joinpath(source_dir, "algorithms", "nn.jl"));
    include(joinpath(source_dir, "utils", "parse.jl"));

    tdata, labels = Base.invokelatest(parse_training, pos_seqs, total_seq, balance);

    Base.invokelatest(train_nn, tdata, labels, hl_size, N, alpha,
                        outputfile, write_output, C);

end

function predict(N::Int, alpha::Float64, hl_size::Int, write_output::Bool,
            balance::Int, C::Int, pos_seqs::Compat.String, total_seq::Compat.String,
            predict_seq::Compat.String, outputfile::Compat.String)

    include(joinpath(source_dir, "algorithms", "nn.jl"));
    include(joinpath(source_dir, "utils", "parse.jl"));

    tdata, labels = Base.invokelatest(parse_training, pos_seqs, total_seq, balance);
    test_data = Base.invokelatest(parse_testing, predict_seq)

    Base.invokelatest(nn_predict_on_data, tdata, labels, test_data, hl_size, N, alpha,
                        outputfile, write_output, C);

end