using NeuralQuantum
using LinearAlgebra, SparseArrays
using Test

function test_cache(sc_1)
    @testset "test zero" begin
        zero!(sc_1)
        test_zero(sc_1)
    end

    @testset "test addition" begin
        sc_2 = deepcopy(sc_1)
        random_data!(sc_1)
        random_data!(sc_2)
        sc_3 = deepcopy(sc_1)

        NeuralQuantum.add!(sc_3, sc_2)
        test_sum!(sc_1, sc_2, sc_3)
    end
end

Nsites = 4
lind = quantum_ising_lind(SquareLattice([Nsites],PBC=true), g=1.3, V=2.0, γ=1.0)
net  = cached(NDM(Nsites, 2, 3))

@testset "Test sampling caches" begin
    include("MCMC_SR_cache.jl")
    @testset "Test sampling cache MCMCSREvaluationCache" begin
        prob = SteadyStateProblem(lind, operators=false, variance=false)
        sc   = SamplingCache(SR(), prob, net)
        @test sc isa MCMCSREvaluationCache
        test_cache(sc)
    end

    include("MCMC_SRL_cache.jl")
    @testset "Test sampling cache MCMCSRLEvaluationCache" begin
        prob = SteadyStateProblem(lind, operators=false, variance=true)
        sc   = SamplingCache(SR(), prob, net)
        @test sc isa MCMCSRLEvaluationCache
        test_cache(sc)
    end

    include("MCMC_Grad_cache.jl")
    @testset "Test sampling cache MCMCGradientEvaluationCache" begin
        prob = SteadyStateProblem(lind, operators=false, variance=false)
        sc   = SamplingCache(Gradient(), prob, net)
        @test sc isa MCMCGradientEvaluationCache
        test_cache(sc)
    end

    include("MCMC_GradL_cache.jl")
    @testset "Test sampling cache MCMCGradientLEvaluationCache" begin
        prob = SteadyStateProblem(lind, operators=false, variance=true)
        sc   = SamplingCache(Gradient(), prob, net)
        @test sc isa MCMCGradientLEvaluationCache
        test_cache(sc)
    end
end


"""
# Description

A logical node that represents a projection operation. It has two fields:

1. `source`: A node describing a relation that will be projected.
2. `expressions`: A tuple of `Expression` objects that define the projection.
"""
struct Projection <: LogicalNode
    source::Any
    expressions::Tuple
end

function Base.print(io::IO, node::Projection)
    println(io, "Projection LogicalNode")
    println(io, type(node.source))
    for e in node.expressions
        println(io, e)
    end
end


@testmodule DataReaderTest begin
  using MLLibrary

  function test_DataSetMemory()

    resources=["abalone.data","3percent-miscategorization.csv","10percent-miscatergorization.csv","Seperable.csv"]
    for res in resources
      delimiter=","
      path = joinpath(Pkg.dir("MLLibrary"),"resources","datasets",res)
      count = MLLibrary.DataParser.DataStream.getDataSetCount(path)

      featureCount = MLLibrary.DataParser.DataStream.getFeatureCount(path)
      threshold=.1
      srand(123)
      trainingsetindices = MLLibrary.DataSampling.ThresholdSampling.getThresholdTrainingSetIndicies(path,count,threshold)
      srand(123)
      testsetindices = MLLibrary.DataSampling.ThresholdSampling.getThresholdTestSetIndices(path,count,threshold)

      trainingset = MLLibrary.DataSampling.DataReader.getDataSetInMemoryFromIndices(path,delimiter,trainingsetindices)

      testset = MLLibrary.DataSampling.DataReader.getDataSetInMemoryFromIndices(path,delimiter,testsetindices)
      @test size(trainingset)==(convert(Int,round(count*(1-threshold))),featureCount)
      @test size(testset)==(convert(Int,round(count*threshold)),featureCount)

    end

  end

end


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
    SerialExecution

Functor object for using serial execution in a progressive-hedging algorithm. Create by supplying a [`Serial`](@ref) object through `execution` in the `ProgressiveHedgingSolver` factory function and then pass to a `StochasticPrograms.jl` model.

"""
struct SerialExecution{T <: AbstractFloat,
                       A <: AbstractVector,
                       PT <: AbstractPenaltyTerm} <: AbstractProgressiveHedgingExecution
    subproblems::Vector{SubProblem{T,A,PT}}

    function SerialExecution(::Type{T}, ::Type{A}, ::Type{PT}) where {T <: AbstractFloat, A <: AbstractVector, PT <: AbstractPenaltyTerm}
        return new{T,A,PT}(Vector{SubProblem{T,A,PT}}())
    end
end

function initialize_subproblems!(ph::AbstractProgressiveHedging,
                                 execution::SerialExecution{T},
                                 scenarioproblems::ScenarioProblems,
                                 penaltyterm::AbstractPenaltyTerm) where T <: AbstractFloat
    # Create subproblems
    for i = 1:num_subproblems(scenarioproblems)
        push!(execution.subproblems, SubProblem(
            subproblem(scenarioproblems, i),
            i,
            T(probability(scenarioproblems, i)),
            copy(penaltyterm)))
    end
    # Initial reductions
    update_iterate!(ph)
    update_dual_gap!(ph)
    return nothing
end

function finish_initilization!(execution::SerialExecution, penalty::AbstractFloat)
    for subproblem in execution.subproblems
        initialize!(subproblem, penalty)
    end
    return nothing
end

function restore_subproblems!(ph::AbstractProgressiveHedging, execution::SerialExecution)
    for subproblem in execution.subproblems
        restore_subproblem!(subproblem)
    end
    return nothing
end

function resolve_subproblems!(ph::AbstractProgressiveHedging, execution::SerialExecution)
    Qs = Vector{SubproblemSolution}(undef, length(execution.subproblems))
    # Reformulate and solve sub problems
    for (i, subproblem) in enumerate(execution.subproblems)
        reformulate_subproblem!(subproblem, ph.ξ, penalty(ph))
        Qs[i] = subproblem(ph.ξ)
    end
    # Return current objective value
    return sum(Qs)
end

function update_iterate!(ph::AbstractProgressiveHedging, execution::SerialExecution)
    # Update the estimate
    ξ_prev = copy(ph.ξ)
    ph.ξ .= mapreduce(+, execution.subproblems, init = zero(ph.ξ)) do subproblem
        π = subproblem.probability
        x = subproblem.x
        π * x
    end
    # Update δ₁
    ph.data.δ₁ = norm(ph.ξ - ξ_prev, 2) ^ 2
    return nothing
end

function update_subproblems!(ph::AbstractProgressiveHedging, execution::SerialExecution)
    # Update dual prices
    update_subproblems!(execution.subproblems, ph.ξ, penalty(ph))
    return nothing
end

function update_dual_gap!(ph::AbstractProgressiveHedging, execution::SerialExecution{T}) where T <: AbstractFloat
    # Update δ₂
    ph.data.δ₂ = mapreduce(+, execution.subproblems, init = zero(T)) do subproblem
        π = subproblem.probability
        x = subproblem.x
        π * norm(x - ph.ξ, 2) ^ 2
    end
    return nothing
end

function calculate_objective_value(ph::AbstractProgressiveHedging, execution::SerialExecution{T}) where T <: AbstractFloat
    return mapreduce(+, execution.subproblems, init = zero(T)) do subproblem
        _objective_value(subproblem)
    end
end

function scalar_subproblem_reduction(value::Function, execution::SerialExecution{T}) where T <: AbstractFloat
    return mapreduce(+, execution.subproblems, init = zero(T)) do subproblem
        π = subproblem.probability
        return π * value(subproblem)
    end
end

function vector_subproblem_reduction(value::Function, execution::SerialExecution{T}, n::Integer) where T <: AbstractFloat
    return mapreduce(+, execution.subproblems, init = zero(T, n)) do subproblem
        π = subproblem.probability
        return π * value(subproblem)
    end
end

# API
# ------------------------------------------------------------
function (execution::Serial)(::Type{T}, ::Type{A}, ::Type{PT}) where {T <: AbstractFloat, A <: AbstractVector, PT <: AbstractPenaltyTerm}
    return SerialExecution(T, A, PT)
end

function str(::Serial)
    return ""
end


const NODE_ID = Int
const SCENARIO_ID = Int
const STAGE_ID = Int
const INDEX = Int

"""
Unique identifier for a `ScenarioNode` in a `ScenarioTree`.
"""
struct NodeID
    value::NODE_ID
end
"""
    value(nid::NodeID)

Return the raw (non-type safe) value of a [`NodeID`](@ref)
"""
value(nid::NodeID)::NODE_ID = nid.value
Base.isless(a::NodeID, b::NodeID) = value(a) < value(b)

"""
Unique, type-safe identifier for a scenario.
"""
struct ScenarioID
    value::SCENARIO_ID
end

"""
    value(scid::ScenarioID)

Return the raw (non-type safe) value of a [`ScenarioID`](@ref)
"""
value(scid::ScenarioID)::SCENARIO_ID = scid.value
Base.isless(a::ScenarioID, b::ScenarioID) = value(a) < value(b)

"""
Unique, type-safe identifier for a stage.
"""
struct StageID
    value::STAGE_ID
end

"""
    value(stid::StageID)

Return the raw (non-type safe) value of a [`StageID`](@ref)
"""
value(stid::StageID)::STAGE_ID = stid.value
_increment(sid::StageID)::StageID = StageID(value(sid) + one(STAGE_ID))
Base.isless(a::StageID, b::StageID) = value(a) < value(b)

"""
Unique, type-safe identifier for variables associated with the same scenario tree node.
"""
struct Index
    value::INDEX
end

"""
    value(idx::Index)

Return the raw (non-type safe) value of an [`Index`](@ref)
"""
value(idx::Index)::INDEX = idx.value
_increment(index::Index)::Index = Index(value(index) + one(INDEX))
Base.isless(a::Index, b::Index) = value(a) < value(b)

"""
Unique, type-safe identifier for any variable in a (multi-stage) stochastic programming problem.  Composed of a `ScenarioID`, a `StageID` and an `Index`.
"""
struct VariableID
    scenario::ScenarioID # scenario to which this variable belongs
    stage::StageID # stage to which this variable belongs
    index::Index # coordinate in vector
end

"""
    scenario(vid::VariableID)::ScenarioID

Returns the [`ScenarioID`](@ref) of the specified [`VariableID`](@ref).
"""
function scenario(vid::VariableID)::ScenarioID
    return vid.scenario
end

"""
    stage(vid::VariableID)::StageID

Returns the [`StageID`](@ref) of the specified [`VariableID`](@ref).
"""
function stage(vid::VariableID)::StageID
    return vid.stage
end

"""
    index(vid::VariableID)::Index

Returns the [`Index`](@ref) of the specified [`VariableID`](@ref).
"""
function index(vid::VariableID)::Index
    return vid.index
end

function Base.isless(a::VariableID, b::VariableID)
    return (a.stage < b.stage ||
            (a.stage == b.stage &&
             (a.scenario < b.scenario ||
              (a.scenario == b.scenario && a.index < b.index))))
end

"""
    scid(n::$(SCENARIO_ID))::ScenarioID

Create `ScenarioID` from `n`.
"""
function scid(n::SCENARIO_ID)::ScenarioID
    return ScenarioID(SCENARIO_ID(n))
end

"""
    stid(n::$(STAGE_ID))::StageID

Create `StageID` from `n`.
"""
function stid(n::STAGE_ID)::StageID
    return StageID(STAGE_ID(n))
end

"""
    index(n::$(INDEX))::Index

Create `Index` from `n`.
"""
function index(n::INDEX)::Index
    return Index(INDEX(n))
end

"""
Unique identifier for consensus variables. The scenario variables being driven to consensus with this variable is given by `convert_to_variable_ids`.
"""
struct XhatID
    node::NodeID
    index::Index
end

function Base.isless(a::XhatID, b::XhatID)
    return (a.node < b.node ||
            (a.node == b.node && a.index < b.index))
end


module BenchmarkTools

using JSON
using Base.Iterators

using Logging: @logmsg, LogLevel
using Statistics
using UUIDs: uuid4
using Printf

const BENCHMARKTOOLS_VERSION = v"0.7.0"

##############
# Parameters #
##############

include("parameters.jl")

export loadparams!

##############
# Trial Data #
##############

include("trials.jl")

export gctime,
       memory,
       allocs,
       params,
       ratio,
       judge,
       isinvariant,
       isregression,
       isimprovement,
       median,
       mean,
       rmskew!,
       rmskew,
       trim

##################
# Benchmark Data #
##################

include("groups.jl")

export BenchmarkGroup,
       invariants,
       regressions,
       improvements,
       @tagged,
       addgroup!,
       leaves

######################
# Execution Strategy #
######################

include("execution.jl")

export tune!,
       warmup,
       @ballocated,
       @benchmark,
       @benchmarkable,
       @belapsed,
       @btime

#################
# Serialization #
#################

include("serialization.jl")

end # module BenchmarkTools


@static if VERSION ≥ v"1.1"
    # In Julia < 1.1, zero(Half{T}) and one(Half{T}) do not work for
    # T ∈ [SafeInt8, SafeInt16, SafeUInt8, SafeUInt16] due to a bug in Julia, cf.
    # https://github.com/JuliaLang/julia/issues/32203

    using SaferIntegers

    safeinttypes  = (:SafeInt, :SafeInt8, :SafeInt16, :SafeInt32, :SafeInt64, :SafeInt128)
    safeuinttypes = (:SafeUInt, :SafeUInt8, :SafeUInt16, :SafeUInt32, :SafeUInt64, :SafeUInt128)

    @testset "SaferIntegers" begin
        @test_throws OverflowError Half{SafeInt8}(typemax(Int8))
        @test_throws OverflowError typemax(Half{SafeInt8}) + one(HalfInt8)
        @test_throws OverflowError typemax(Half{SafeInt8}) + one(Int8)
        @test_throws OverflowError typemin(Half{SafeInt8}) - one(HalfInt8)
        @test_throws OverflowError typemin(Half{SafeInt8}) - one(Int8)
        for T in safeinttypes
            @eval @test HalfInteger(one($T)) === one(Half{$T})
            @eval @test_throws OverflowError Half{$T}(typemax($T))
            @eval @test_throws OverflowError Half{$T}(typemin($T))
            @eval @test_throws OverflowError Half{$T}(typemax($T)//one($T))
            @eval @test_throws OverflowError Half{$T}(typemin($T)//one($T))
            @eval @test_throws OverflowError Half{$T}(complex(typemax($T)))
            @eval @test_throws OverflowError Half{$T}(complex(typemin($T)))
            @eval @test_throws OverflowError Half{$T}(complex(typemax($T)//one($T)))
            @eval @test_throws OverflowError Half{$T}(complex(typemin($T)//one($T)))
            @eval @test_throws OverflowError -typemin(Half{$T})
            @eval @test_throws OverflowError abs(typemin(Half{$T}))
            @eval @test_throws OverflowError flipsign(typemin(Half{$T}), -1)
            @eval @test_throws OverflowError typemax(Half{$T}) + onehalf(Half{$T})
            @eval @test_throws OverflowError typemin(Half{$T}) - onehalf(Half{$T})
            @eval @test_throws OverflowError twice(typemax($T))
            @eval @test_throws OverflowError twice(typemin($T))
            @eval @test_throws OverflowError round(typemax(Half{$T}))
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundNearest)
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundNearestTiesAway)
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundNearestTiesUp)
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundUp)
        end
        for T in safeuinttypes
            @eval @test HalfInteger(one($T)) === one(Half{$T})
            @eval @test_throws OverflowError Half{$T}(typemax($T))
            @eval @test_throws OverflowError Half{$T}(typemax($T)//one($T))
            @eval @test_throws OverflowError Half{$T}(complex(typemax($T)))
            @eval @test_throws OverflowError Half{$T}(complex(typemax($T)//one($T)))
            @eval @test_throws OverflowError -onehalf(Half{$T})
            @eval @test_throws OverflowError flipsign(onehalf(Half{$T}), -1)
            @eval @test_throws OverflowError typemax(Half{$T}) + onehalf(Half{$T})
            @eval @test_throws OverflowError typemin(Half{$T}) - onehalf(Half{$T})
            @eval @test_throws OverflowError twice(typemax($T))
            @eval @test_throws OverflowError round(typemax(Half{$T}))
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundNearest)
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundNearestTiesAway)
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundNearestTiesUp)
            @eval @test_throws OverflowError round(typemax(Half{$T}), RoundUp)
        end
    end
end


## Double Pendulum
import TrajectoryOptimization
traj_folder = joinpath(dirname(pathof(TrajectoryOptimization)),"..")
urdf_folder = joinpath(traj_folder, "dynamics/urdf")
urdf_doublependulum = joinpath(urdf_folder, "doublependulum.urdf")

doublependulum = Model(urdf_doublependulum)


# Global constant allowing to chose between ForwardDiff and Zygote
# for hyperparameter optimization
const ADBACKEND = Ref(:Zygote)

function setadbackend(ad_backend::Symbol)
    (ad_backend == :ForwardDiff || ad_backend == :Zygote) ||
        error("Wrong AD Backend symbol, options are :ForwardDiff or :Zygote")
    return ADBACKEND[] = ad_backend
end

# Type piracy but will be introduced in a future version
# See https://github.com/FluxML/Optimisers.jl/pull/24
Optimisers.init(::Any, ::Any) = nothing

## Generic fallback when gradient is nothing
update_kernel!(::Any, ::Any, ::Nothing, ::Any) = nothing
update_Z!(::Any, ::Any, ::Nothing, ::Any) = nothing

## Generic fallback when optimizer is nothing
update_kernel!(::Nothing, ::Kernel, ::NamedTuple, ::NamedTuple) = nothing
update_kernel!(::Nothing, ::AbstractVector, ::NamedTuple, ::NamedTuple) = nothing
update_Z!(::Nothing, ::AbstractVector, ::AbstractVector) = nothing
update_Z!(::Nothing, ::AbstractVector, ::NamedTuple) = nothing

## Updating prior mean parameters ##
# function update!(μ::PriorMean, g::AbstractVector, X::AbstractVector, state)
# return update!(μ, g, X, state)
# end

## Updating kernel parameters ##

## ForwardDiff.jl approach (with destructure())
function update!(opt, k::Kernel, Δ::AbstractVector)
    ps = params(k)
    i = 1
    for p in ps
        d = length(p)
        Δp = Δ[i:(i + d - 1)]
        Δlogp = Optimise.apply!(opt, p, p .* Δp)
        @. p = exp(log(p) + Δlogp)
        i += d
    end
end

## Zygote.jl approach with named tuple
function update_kernel!(opt, k::Union{Kernel,Transform}, g::NamedTuple, state::NamedTuple)
    return NamedTuple(
        map(keys(g)) do fieldname
            Pair(
                fieldname,
                update_kernel!(
                    opt,
                    getfield(k, fieldname),
                    getfield(g, fieldname),
                    getfield(state, fieldname),
                ),
            )
        end,
    )
end

function update_kernel!(opt, x::AbstractArray, g::AbstractArray, state)
    state, Δ = Optimisers.apply(opt, state, x, x .* g)
    @. x = exp(log(x) + Δ) # Always assume that parameters need to be positive
    return state
end

## Updating inducing points
function update_Z!(opt, Z::AbstractVector, Z_grads::AbstractVector, state)
    return map(Z, Z_grads, state) do z, zgrad, st
        st, ΔZ = Optimisers.apply(opt, st, z, zgrad)
        z .+= ΔZ
        return st
    end
end

function update_Z!(opt, Z::Union{ColVecs,RowVecs}, Z_grads::NamedTuple, state)
    st, Δ = Optimisers.apply(opt, state, Z.X, Z_grads.X)
    Z.X .+= Δ
    return st
end


# This package is based on the shrinkage estimator from
# https://cran.r-project.org/web/packages/entropy/

module ShrinkageEstimators

include("ExportedFunctions.jl")
include("Discretization.jl")
include("Shrinkage.jl")
include("Formulae.jl")

end # module


using DataStructures

"""
	type SimpleNetwork <: Network
		time::Int64
		layers::DataStructures.OrderedDict{Symbol, Layer}
		plasticconnections::Array{Connection, 1}
		layeriterator::Base.ValueIterator{DataStructures.OrderedDict{Symbol,Simulator.Layer}}
"""

type SimpleNetwork <: Network
	time::Int64
	layers::DataStructures.OrderedDict{Symbol, Layer}
	plasticconnections::Array{Connection, 1}
	# currently iterating a OrderedDict causes memory allocation, see
	# https://github.com/JuliaLang/DataStructures.jl/issues/93
	# shouldn't be a big deal for this application
	layeriterator::Base.ValueIterator{DataStructures.OrderedDict{Symbol,Layer}}
end

"""
	SimpleNetwork()

Create an empty network
"""
function SimpleNetwork()
	layers = DataStructures.OrderedDict{Symbol, Layer}()
	it = values(layers)
	SimpleNetwork(0, layers, Connection[], it)
end

export SimpleNetwork

import Base.reverse!

function reverse!(d::DataStructures.OrderedDict)
    reverse!(d.keys)
    reverse!(d.vals)
    DataStructures.rehash!(d)
end

"""
	reverselayerorder!(net::SimpleNetwork)

"""
function reverselayerorder!(net::SimpleNetwork)
	reverse!(net.layers)
	net.layeriterator = values(net.layers)
end
export reverselayerorder!

"""
    connect!(net::SimpleNetwork, from::Symbol, to::Symbol, conorkind; label = :default)
"""
function connect!(net::SimpleNetwork, from::Symbol, to::Symbol, conorkind;
				  label = :default)
	contype = connect!(net.layers[from], net.layers[to], conorkind, label = label)
	if ismatch(r"Plastic", contype)
		push!(net.plasticconnections,
		      net.layers[to].inputconnections[label][from])
	end
end

"""
	addlayer!(net::SimpleNetwork, layer::Layer)
"""
function addlayer!(net::SimpleNetwork, layer::Layer)
	net.layers[layer.name] = layer
	net.layeriterator = values(net.layers)
end
export addlayer!

"""
	updatenet!(net::SimpleNetwork)

Call updateneuron! for all layers in `net.layers`
and updateconnection! for all connections in `net.plasticconnections`
and increment `net.time`.
"""
function updatenet!(net::SimpleNetwork)
	net.time += 1
	for l in net.layeriterator
		collectmessages!(l.neurons, l.inputconnections)
	end
	for l in net.layeriterator
		updateneuron!(l.neurons)
	end
	for c in net.plasticconnections
		updateconnection!(c)
	end
end
export updatenet!

function updatenetasync!(net::SimpleNetwork)
	net.time += 1
	for l in net.layeriterator
		collectmessages!(l.neurons, l.inputconnections)
		updateneuron!(l.neurons)
	end
	for c in net.plasticconnections
		updateconnection!(c)
	end
end
export updatenetasync!


###
### Indexing
###

@inline function inbounds_getindex2(x::Kmer{A,K,N}, i::Integer) where {A,K,N}
    # Emulation of BitIndex type
    i′ = i + div(64N - 2K, 2)
    val = (i′ - 1) << 1
    idx = (val >> 6) + 1
    off = 62 - (val & (UInt8(64) - 0x01))
    @inbounds begin
        chunk = x.data[idx]
    end
    bits = (chunk >> off) & UInt64(3)
    return reinterpret(eltype(x), 0x01 << bits)
end

"""
Get the Vector or NTuple of unsigned integers that the encoded
"""
function packed_data end

@inline packed_data(seq::Kmer) = seq.data

@inline function inbounds_getindex(seq::Kmer, i::Integer)
    bidx = bitindex(seq, i)
    elem = extract_encoded_element(bidx, packed_data(seq))
    return decode(Alphabet(seq), elem)
end


module simulator

export matrixcore, fresnelltools, analyticalmaterials, cmtgratings, utilities, materials

include("utilities.jl")
include("analyticalmaterials.jl")
include("ema.jl")
include("materials.jl")
include("matrixcore.jl")
include("fresnelltools.jl")
include("targetfigures.jl")

end # simulator


function big_fraction_matrix_to_real_matrix(::Type{MatrixUtils}, arg0::FieldMatrix)
    return jcall(MatrixUtils, "bigFractionMatrixToRealMatrix", Array2DRowRealMatrix, (FieldMatrix,), arg0)
end

function block_inverse(::Type{MatrixUtils}, arg0::RealMatrix, arg1::jint)
    return jcall(MatrixUtils, "blockInverse", RealMatrix, (RealMatrix, jint), arg0, arg1)
end

function check_addition_compatible(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::AnyMatrix)
    return jcall(MatrixUtils, "checkAdditionCompatible", void, (AnyMatrix, AnyMatrix), arg0, arg1)
end

function check_column_index(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::jint)
    return jcall(MatrixUtils, "checkColumnIndex", void, (AnyMatrix, jint), arg0, arg1)
end

function check_matrix_index(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::jint, arg2::jint)
    return jcall(MatrixUtils, "checkMatrixIndex", void, (AnyMatrix, jint, jint), arg0, arg1, arg2)
end

function check_multiplication_compatible(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::AnyMatrix)
    return jcall(MatrixUtils, "checkMultiplicationCompatible", void, (AnyMatrix, AnyMatrix), arg0, arg1)
end

function check_row_index(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::jint)
    return jcall(MatrixUtils, "checkRowIndex", void, (AnyMatrix, jint), arg0, arg1)
end

function check_same_column_dimension(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::AnyMatrix)
    return jcall(MatrixUtils, "checkSameColumnDimension", void, (AnyMatrix, AnyMatrix), arg0, arg1)
end

function check_same_row_dimension(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::AnyMatrix)
    return jcall(MatrixUtils, "checkSameRowDimension", void, (AnyMatrix, AnyMatrix), arg0, arg1)
end

function check_sub_matrix_index(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::Vector{jint}, arg2::Vector{jint})
    return jcall(MatrixUtils, "checkSubMatrixIndex", void, (AnyMatrix, Vector{jint}, Vector{jint}), arg0, arg1, arg2)
end

function check_sub_matrix_index(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::jint, arg2::jint, arg3::jint, arg4::jint)
    return jcall(MatrixUtils, "checkSubMatrixIndex", void, (AnyMatrix, jint, jint, jint, jint), arg0, arg1, arg2, arg3, arg4)
end

function check_subtraction_compatible(::Type{MatrixUtils}, arg0::AnyMatrix, arg1::AnyMatrix)
    return jcall(MatrixUtils, "checkSubtractionCompatible", void, (AnyMatrix, AnyMatrix), arg0, arg1)
end

function check_symmetric(::Type{MatrixUtils}, arg0::RealMatrix, arg1::jdouble)
    return jcall(MatrixUtils, "checkSymmetric", void, (RealMatrix, jdouble), arg0, arg1)
end

function create_column_field_matrix(::Type{MatrixUtils}, arg0::Vector{FieldElement})
    return jcall(MatrixUtils, "createColumnFieldMatrix", FieldMatrix, (Vector{FieldElement},), arg0)
end

function create_column_real_matrix(::Type{MatrixUtils}, arg0::Vector{jdouble})
    return jcall(MatrixUtils, "createColumnRealMatrix", RealMatrix, (Vector{jdouble},), arg0)
end

function create_field_diagonal_matrix(::Type{MatrixUtils}, arg0::Vector{FieldElement})
    return jcall(MatrixUtils, "createFieldDiagonalMatrix", FieldMatrix, (Vector{FieldElement},), arg0)
end

function create_field_identity_matrix(::Type{MatrixUtils}, arg0::Field, arg1::jint)
    return jcall(MatrixUtils, "createFieldIdentityMatrix", FieldMatrix, (Field, jint), arg0, arg1)
end

function create_field_matrix(::Type{MatrixUtils}, arg0::Field, arg1::jint, arg2::jint)
    return jcall(MatrixUtils, "createFieldMatrix", FieldMatrix, (Field, jint, jint), arg0, arg1, arg2)
end

function create_field_matrix(::Type{MatrixUtils}, arg0::Vector{Vector{FieldElement}})
    return jcall(MatrixUtils, "createFieldMatrix", FieldMatrix, (Vector{Vector{FieldElement}},), arg0)
end

function create_field_vector(::Type{MatrixUtils}, arg0::Field, arg1::jint)
    return jcall(MatrixUtils, "createFieldVector", FieldVector, (Field, jint), arg0, arg1)
end

function create_field_vector(::Type{MatrixUtils}, arg0::Vector{FieldElement})
    return jcall(MatrixUtils, "createFieldVector", FieldVector, (Vector{FieldElement},), arg0)
end

function create_real_diagonal_matrix(::Type{MatrixUtils}, arg0::Vector{jdouble})
    return jcall(MatrixUtils, "createRealDiagonalMatrix", RealMatrix, (Vector{jdouble},), arg0)
end

function create_real_identity_matrix(::Type{MatrixUtils}, arg0::jint)
    return jcall(MatrixUtils, "createRealIdentityMatrix", RealMatrix, (jint,), arg0)
end

function create_real_matrix(::Type{MatrixUtils}, arg0::Vector{Vector{jdouble}})
    return jcall(MatrixUtils, "createRealMatrix", RealMatrix, (Vector{Vector{jdouble}},), arg0)
end

function create_real_matrix(::Type{MatrixUtils}, arg0::jint, arg1::jint)
    return jcall(MatrixUtils, "createRealMatrix", RealMatrix, (jint, jint), arg0, arg1)
end

function create_real_vector(::Type{MatrixUtils}, arg0::Vector{jdouble})
    return jcall(MatrixUtils, "createRealVector", RealVector, (Vector{jdouble},), arg0)
end

function create_real_vector(::Type{MatrixUtils}, arg0::jint)
    return jcall(MatrixUtils, "createRealVector", RealVector, (jint,), arg0)
end

function create_row_field_matrix(::Type{MatrixUtils}, arg0::Vector{FieldElement})
    return jcall(MatrixUtils, "createRowFieldMatrix", FieldMatrix, (Vector{FieldElement},), arg0)
end

function create_row_real_matrix(::Type{MatrixUtils}, arg0::Vector{jdouble})
    return jcall(MatrixUtils, "createRowRealMatrix", RealMatrix, (Vector{jdouble},), arg0)
end

function deserialize_real_matrix(::Type{MatrixUtils}, arg0::Object, arg1::JString, arg2::ObjectInputStream)
    return jcall(MatrixUtils, "deserializeRealMatrix", void, (Object, JString, ObjectInputStream), arg0, arg1, arg2)
end

function deserialize_real_vector(::Type{MatrixUtils}, arg0::Object, arg1::JString, arg2::ObjectInputStream)
    return jcall(MatrixUtils, "deserializeRealVector", void, (Object, JString, ObjectInputStream), arg0, arg1, arg2)
end

function fraction_matrix_to_real_matrix(::Type{MatrixUtils}, arg0::FieldMatrix)
    return jcall(MatrixUtils, "fractionMatrixToRealMatrix", Array2DRowRealMatrix, (FieldMatrix,), arg0)
end

function inverse(::Type{MatrixUtils}, arg0::RealMatrix)
    return jcall(MatrixUtils, "inverse", RealMatrix, (RealMatrix,), arg0)
end

function inverse(::Type{MatrixUtils}, arg0::RealMatrix, arg1::jdouble)
    return jcall(MatrixUtils, "inverse", RealMatrix, (RealMatrix, jdouble), arg0, arg1)
end

function is_symmetric(::Type{MatrixUtils}, arg0::RealMatrix, arg1::jdouble)
    return jcall(MatrixUtils, "isSymmetric", jboolean, (RealMatrix, jdouble), arg0, arg1)
end

function matrix_exponential(::Type{MatrixUtils}, arg0::RealMatrix)
    return jcall(MatrixUtils, "matrixExponential", RealMatrix, (RealMatrix,), arg0)
end

function serialize_real_matrix(::Type{MatrixUtils}, arg0::RealMatrix, arg1::ObjectOutputStream)
    return jcall(MatrixUtils, "serializeRealMatrix", void, (RealMatrix, ObjectOutputStream), arg0, arg1)
end

function serialize_real_vector(::Type{MatrixUtils}, arg0::RealVector, arg1::ObjectOutputStream)
    return jcall(MatrixUtils, "serializeRealVector", void, (RealVector, ObjectOutputStream), arg0, arg1)
end

function solve_lower_triangular_system(::Type{MatrixUtils}, arg0::RealMatrix, arg1::RealVector)
    return jcall(MatrixUtils, "solveLowerTriangularSystem", void, (RealMatrix, RealVector), arg0, arg1)
end

function solve_upper_triangular_system(::Type{MatrixUtils}, arg0::RealMatrix, arg1::RealVector)
    return jcall(MatrixUtils, "solveUpperTriangularSystem", void, (RealMatrix, RealVector), arg0, arg1)
end


export smooth_gaps, smooth_gaps!, zero_out, zero_out!

"""
  smooth_gaps(S)

Remove and taper around gaps in `SeisData` `S`.

Use `ungap!(S,tap=true,m=false)` before applying this function to fill gaps
with `nan` values.

# Arguments
- `S::SeisData`: 3-component `SeisData` for detection.

# Keywords
- `pad::Int`: Number of sample points to taper before and after gap.
"""
function smooth_gaps!(S::SeisData;pad::Real=100)

    # check pad input
    pad = max(2,pad)
    if !isa(pad,Int)
        pad = convert(Int,ceil(pad))
    end
    for ii = 1:S.n
        # find all nan / gaps
        ind = findall(isnan.(S.x[ii]))
        if length(ind) == 0
            continue
        end

        # set all nan equal to zero
        out = unique(ind)
        T = eltype(S.x[ii])
        S.x[ii][out] .= T(0)
        padarr = T(0.):T(pad-1)

        # right tapering of gaps
        rightind = findall(diff(ind) .!= 1)
        for jj in ind[rightind]
            maxind = min(jj+pad-1,length(S[ii].x))
            if length(jj:maxind) == length(padarr)
                S.x[ii][jj:maxind] .*= cos.(T(pi) ./ T(2) .+ T(pi) ./ T(2) .* padarr ./ pad).^2
            else
                S.x[ii][jj:maxind] .= T(0)
            end
        end

        # left tapering of gaps
        prepend!(ind,1)
        leftind = findall(diff(ind) .> 1)
        for jj in ind[leftind]
            minind = max(1,jj-pad+1)
            if length(minind:jj) == length(padarr)
                S.x[ii][minind:jj] .*= cos.(T(pi) ./ T(2) .* padarr ./ pad).^2
            else
                S.x[ii][minind:jj] .= T(0)
            end
        end
    end
    return nothing
end
smooth_gaps(S::SeisData;pad::Int=100) = (U = deepcopy(S);smooth_gaps!(U,pad=pad);return U)

"""
  zero_out!(S)

Set data in SeisData `S` == 0 where `S.x .> thresh`
"""
function zero_out!(S::SeisData;thresh::AbstractFloat=1e-7)
    for ii = 1:S.n
        T = eltype(S.x[ii])
        ind = findall(abs.(S.x[ii]) .< thresh)
        S.x[ii][ind] .= T(0)
    end
    return nothing
end
zero_out(S::SeisData; thresh::AbstractFloat=1e-7) = (U = deepcopy(S);zero_out!(U,thresh=thresh);return U)


# Author: Ritchie Lee, ritchie.lee@sv.cmu.edu
# Date: 8/26/2016

# Aircraft encounter model 90 degree "side-on" initial aircraft states
# DBN samples are generated at runtime at each step.

#TODO: This module is copied from StarDBNImpl
#Both modules need to be cleaned up and unified
#Performance is probably an issue as well
module SideOnDBNImpl

export
    AddObserver,

    getInitialState,
    initialize,
    update,
    get,

    SideOnDBNParams,
    SideOnDBN

import Compat.ASCIIString

using AbstractEncounterDBNImpl
using AbstractEncounterDBNInterfaces
using CommonInterfaces
using Util
using RLESUtils, Observers
using Encounter
import CorrAEMImpl: CorrAEMParameters, CorrAEMInitialState, CorrAEMCommand

import CommonInterfaces.addObserver
import CommonInterfaces.initialize
import CommonInterfaces.update
import AbstractEncounterDBNInterfaces.get
import AbstractEncounterDBNInterfaces.getInitialState

include(Pkg.dir("SISLES/src/Encounter/CorrAEMImpl/corr_aem_sample.jl"))
include(Pkg.dir("SISLES/src/Encounter/CorrAEMImpl/corr_aem_load_params.jl"))
include(Pkg.dir("SISLES/src/EncounterDBN/DBNShared/DBNShared.jl"))

const MAP_G2L = Dict(2 => 1, 9 => 2, 11 => 3, 13 => 4, 17 => 5,
    19 => 6) #global to local
const MAP_L2G = Dict(1 => 2, 2 => 9, 3 => 11, 4 => 13, 5=> 17,
    6 => 19) #local to global
const MAP_VAR2IND_L = Dict(:L => 1, :v_d => 2, :h_d0 => 3, :psi_d0 => 4,
    :h_d1 => 5, :psi_d1 => 6) #variable names to local
const MAP_IND2VAR_L = Dict(1 => :L, 2 => :v_d, 3 => :h_d0, 4 => :psi_d0,
    5 => :h_d1, 6 => :psi_d1) #local to variable names
const TEMPORAL_MAP = [3 5; 4 6] #[dynamic_variables0; dynamic_variables1]

immutable SideOnDBNParams
    tca::Float64 #time of closest approach in seconds
    v_min::Float64
    v_max::Float64
    vdot_min::Float64
    vdot_max::Float64
    dh_min::Float64 #feet
    dh_max::Float64 #feet
    hdot_min::Float64 #feet per second
    hdot_max::Float64 #feet per second
    psidot_min::Float64 #degrees
    psidot_max::Float64 #degrees
    l_min::Int64
    l_max::Int64
end

function SideOnDBNParams(;
    tca::Float64=40.0,
    v_min::Float64=400.0,
    v_max::Float64=600.0,
    vdot_min::Float64=-2.0,
    vdot_max::Float64=2.0,
    dh_min::Float64=-1000.0,
    dh_max::Float64=1000.0,
    hdot_min::Float64=-10.0,
    hdot_max::Float64=10.0,
    psidot_min::Float64=0.0,
    psidot_max::Float64=0.0,
    l_min::Int64=2,
    l_max::Int64=5)

    SideOnDBNParams(tca, v_min, v_max, vdot_min, vdot_max, dh_min, dh_max, hdot_min,
        hdot_max, psidot_min, psidot_max, l_min, l_max)
end

type SideOnDBN <: AbstractEncounterDBN
  number_of_aircraft::Int64

  parameters::SideOnDBNParams

  parameter_file::ASCIIString
  aem_parameters::CorrAEMParameters

  encounter_seed::UInt64

  dirichlet_transition

  t::Int64

  #initial state
  initial_states::Vector{CorrAEMInitialState}

  #initial command
  initial_commands_d::Vector{Vector{Int64}} #discrete current state
  initial_commands::Vector{Vector{Float64}} #continuous of current dynamic variables

  #current command
  commands_d::Vector{Vector{Int64}} #discrete current state
  commands::Vector{Vector{Float64}} #continuous of current dynamic variables

  #caching and reuse
  dynamic_variables0::Vector{Int64}
  dynamic_variables1::Vector{Int64}
  parents_cache::Dict{Int64, Vector{Bool}}
  weights_cache::Dict{Tuple{Int64,Int64}, Vector{Float64}}
  cumweights_cache::Dict{Tuple{Int64,Int64}, Vector{Float64}}

  #pre-allocated output to avoid repeating reallocations
  output_commands::Vector{CorrAEMCommand}

  logProb::Float64 #log probability of output

  function SideOnDBN(number_of_aircraft::Int,
                   parameter_file::AbstractString,
                   encounter_seed::UInt64,
                   p::SideOnDBNParams = SideOnDBNParams())

    dbn = new()

    dbn.number_of_aircraft = number_of_aircraft
    dbn.parameters = p
    dbn.parameter_file = parameter_file
    dbn.aem_parameters = CorrAEMParameters()
    em_read(dbn.aem_parameters, dbn.parameter_file)

    dbn.encounter_seed = encounter_seed

    dbn.dirichlet_transition = bn_dirichlet_prior(dbn.aem_parameters.N_transition)

    dbn.t = 0

    #compute initial states of variables
    dbn.dynamic_variables0 = TEMPORAL_MAP[:, 1]
    dbn.dynamic_variables1 = TEMPORAL_MAP[:, 2]

    #FIXME: don't use srand here
    srand(encounter_seed) #There's a rand inside generateEncounter
    generateEncounter(dbn) #sets initial_states, initial_commands_d, initial_commands

    dbn.commands_d = deepcopy(dbn.initial_commands_d)
    dbn.commands = deepcopy(dbn.initial_commands)

    #precompute and cache these quantities
    dbn.parents_cache = Dict{Int64, Vector{Bool}}()
    dbn.weights_cache = Dict{Tuple{Int64,Int64}, Vector{Float64}}()
    dbn.cumweights_cache = Dict{Tuple{Int64,Int64}, Vector{Float64}}()

    for i = 1:length(dbn.aem_parameters.N_transition)
      dbn.parents_cache[i] = dbn.aem_parameters.G_transition[:, i]

      for j = 1:1:size(dbn.dirichlet_transition[i], 2)
        dbn.weights_cache[(i, j)] = dbn.aem_parameters.N_transition[i][:, j] + dbn.dirichlet_transition[i][:, j]
        dbn.weights_cache[(i, j)] /= sum(dbn.weights_cache[(i, j)])
        dbn.cumweights_cache[(i, j)] = cumsum(dbn.weights_cache[(i, j)])
      end
    end

    dbn.output_commands = CorrAEMCommand[ CorrAEMCommand(0.0, 0.0, 0.0, 0.0) for i = 1:number_of_aircraft ]
    dbn.logProb = 0.0

    return dbn
  end

end

#this is where most of the initialization differences are
function generateEncounter(dbn::SideOnDBN)
    p = dbn.parameters
    L = rand(p.l_min:p.l_max) #randint
    h_min, h_max = alt_bounds(L)
    h_mid = (h_min + h_max) / 2

    #initial aircraft states - place in star pattern heading towards origin
    dbn.initial_states = Array(CorrAEMInitialState, dbn.number_of_aircraft)

    @assert dbn.number_of_aircraft == 2
    for i = 1:dbn.number_of_aircraft
        t = 0
        v = randfloat(p.v_min, p.v_max)
        h = i==1 ? h_mid : h_mid + randfloat(p.dh_min, p.dh_max)
        h_d = randfloat(p.hdot_min, p.hdot_max)
        psi = i == 1 ? 0.0 : 90.0
        x = v * p.tca * cosd(psi + 180)
        y = v * p.tca * sind(psi + 180)

        dbn.initial_states[i] = CorrAEMInitialState(t, x, y, h, v, psi, h_d)
    end

    #initial aircraft commands
    #TODO: Change the data structure here
    #[L,v_d,h_d,psi_d]
    dbn.initial_commands = Array(Vector{Float64}, dbn.number_of_aircraft)
    #[L,v_d,h_d,psi_d,[hd_tp1,psid_tp1]]
    dbn.initial_commands_d = Array(Vector{Int64}, dbn.number_of_aircraft)

    for i = 1:dbn.number_of_aircraft
        h_d = dbn.initial_states[i].h_d  #set commanded same as current h_d
        psi_d = randfloat(p.psidot_min, p.psidot_max)
        v_d = randfloat(p.vdot_min, p.vdot_max)

        #TODO: These should be typed rather than as Vectors...
        dbn.initial_commands[i] = Float64[L, v_d, h_d, psi_d]
        initial_commands_d = discretize(dbn.aem_parameters,
            unconvert_units(dbn.initial_commands[i]))

        dbn.initial_commands_d[i] = [initial_commands_d;
            zeros(Int64, length(dbn.dynamic_variables1))]
    end
end

addObserver(dbn::SideOnDBN, f::Function) = add_observer(aem.observer, f)
addObserver(dbn::SideOnDBN, tag::AbstractString, f::Function) = add_observer(aem.observer, tag, f)

function initialize(dbn::SideOnDBN)
  #reset to initial state
  for i = 1:dbn.number_of_aircraft
    copy!(dbn.commands_d[i], dbn.initial_commands_d[i])
    copy!(dbn.commands[i], dbn.initial_commands[i])
  end
  dbn.t = 0
end

getInitialState(dbn::SideOnDBN, index::Int) = dbn.initial_states[index]
get(dbn::SideOnDBN, aircraft_number::Int) = dbn.output_commands[aircraft_number]

function update(dbn::SideOnDBN)
  logProb = 0.0 #to accumulate over each aircraft

  for i = 1:dbn.number_of_aircraft
    logProb += step_dbn(dbn, dbn.commands_d[i], dbn.commands[i])

    dbn.output_commands[i].t = dbn.t
    dbn.output_commands[i].v_d = dbn.commands[i][MAP_VAR2IND_L[:v_d]]
    dbn.output_commands[i].h_d = dbn.commands[i][MAP_VAR2IND_L[:h_d0]]
    dbn.output_commands[i].psi_d = dbn.commands[i][MAP_VAR2IND_L[:psi_d0]]
  end
  dbn.t += 1

  return logProb
end

#TODO: This function needs a cleanup
function step_dbn(dbn::SideOnDBN, command_d::Vector{Int64}, command::Vector{Float64})
    #_G is global (to the file), _L is local (to this module)

    p = dbn.aem_parameters
    logProb = 0.0
    for (o,i_L) in enumerate(dbn.dynamic_variables1)
        i_G = MAP_L2G[i_L]
        j_G = 1
        if !isempty(find(dbn.parents_cache[i_G]))
            parents_L = map(i -> MAP_G2L[i], find(dbn.parents_cache[i_G]))
            dims = tuple(p.r_transition[dbn.parents_cache[i_G]]...)
            j_G = sub2ind(dims, command_d[parents_L]...)
        end

        command_d[i_L] = select_random_cumweights(dbn.cumweights_cache[(i_G, j_G)])
        logProb += log(dbn.weights_cache[(i_G, j_G)][command_d[i_L]]) #prob of bin

        #Resampling and dediscretizing process
        i0_L = dbn.dynamic_variables0[o]
        i0_G = MAP_L2G[i0_L]

        if command_d[i_L] != command_d[i0_L] #Changed bins, resample
            command[i0_L], prob = dediscretize(command_d[i_L], p.boundaries[i0_G],
                p.zero_bins[i0_G])
            command[i0_L] = convert_units(command[i0_L], MAP_IND2VAR_L[i0_L])
            logProb += log(prob)
        elseif rand() < p.resample_rates[i0_G] #same bin and meets rate
            logProb += log(p.resample_rates[i0_G]) #prob of meeting resample rate
            command[i0_L], prob = dediscretize(command_d[i_L],
            p.boundaries[i0_G], p.zero_bins[i0_G])
            command[i0_L] = convert_units(command[i0_L], MAP_IND2VAR_L[i0_L])
            logProb += log(prob)
        else #command_d[i_L] == command_d[i0_L] same bin, but does not meet rate
            logProb += log(1.0 - p.resample_rates[i0_G]) #prob of not meeting
                #resample rate
            #don't resample state here
        end
    end

    # update x(t) with x(t+1)
    command_d[dbn.dynamic_variables0] = command_d[dbn.dynamic_variables1]

    return logProb
end

end #module


########################################################################
# AutoGrad functions extended for CuArray (copied from karray.jl)

# using CUDA: CuArray, CuPtr
# using Knet: c2i, @knet8
using Base.Broadcast: Broadcasted
using AutoGrad: Value, recording

import AutoGrad: Sparse, matches, ungetindex, addto!, addtoindex!, zeroslike
import Base: copyto!
import LinearAlgebra: axpy!

Sparse(a::CuArray{T,N},v,i) where {T,N} = Sparse{T,N}(a,v,i)

axpy!(a, x::Sparse, y::CuArray) = addto!(y, a*x)

function copyto!(a::CuArray, bc::Broadcasted{S,A,F,X}) where
    {S, A, F <: Union{typeof(+),typeof(-)}, X <: Tuple{Any,Sparse}}
    (b,c) = bc.args
    if !(size(a) == size(b) == size(c.container))
        a .= bc.f.(b, full(c))
        return a
    end
    a === b || copyto!(a, b)
    F <: typeof(-) && (c = -c)
    addto!(a, c)
    return a
end

matches(a::CuArray,b::CuArray)=(size(a)==size(b))

function addto!(a::CuArray{T},b::CuArray{T}) where {T}
    if AutoGrad.recording(); a = copy(a); end  # support highorder gradients
    axpy!(1,b,a) # (a+b)
end

function addto!(a::CuArray,b::Sparse)
    @assert size(a) == size(b.container)
    if AutoGrad.recording(); a = copy(a); end  # support highorder gradients
    for (idx,val) in zip(b.indices, b.values)
        addtoindex!(a, val, idx...)
    end
    return a
end

addto!(a::Sparse, b::CuArray) = addto!(b, a)

import Base: +, -
+(a::CuArray, s::Sparse) = addto!(copy(a), s)
+(s::Sparse, a::CuArray) = addto!(copy(a), s)
-(a::CuArray, s::Sparse) = addto!(copy(a), -s)
-(s::Sparse, a::CuArray) = addto!(-a, s)

function ungetindex(x::CuArray{T},dxi,i) where T
    if isbitstype(T)
        if dxi isa Value
            forw(addto!, zeroslike(x), forw(ungetindex, x, dxi, i))
        elseif recording()
            addtoindex!(zero(x), dxi, i...)
        else
            Sparse(x,[dxi],[i])
        end
    else
        # Using addtoindex! instead of setindex! to handle repeated indices
        addtoindex!(Array{Union{T,Nothing}}(nothing, size(x)), dxi, i...)
    end
end

zeroslike(a::CuArray)=zero(a) # Still need this because zero(::Array{!isbits}) is not defined

# This only works when there are no repeated indices. This is true for index types:
# Real, (Real...), CartesianIndex, Colon, AbstractArray{Bool}, Range, EmptyArray
# and pairs of Union{Real,AbstractUnitRange,Colon} and (Colon,Range)
addtoindex!(A::CuArray, X, I::Union{Real,Colon,AbstractRange,AbstractArray{Bool}}...) =
    setindex!(A, addto!(getindex(A,I...), X), I...)
addtoindex!(A::CuArray, X, I...) =
    setindex!(A, getindex(A,I...) .+ X, I...)

# The following index types may have repeated indices:
# AbstractArray{Real}, AbstractArray{CartesianIndex}, (Colon,AbstractVector{Real}), (AbstractVector{Real},Colon)

addtoindex!(A::CuArray, X, I::AbstractArray{<:CartesianIndex})=addtoindex!(A,X,c2i(size(A),I))

for F in (32,64); T=Symbol("Float$F"); @eval begin

    function addtoindex!(A::CuArray{$T}, X, I::AbstractArray{<:Real})
        I = CuArray{Int32}(I)
        X = CuArray{$T}(X)
        @knet8($("addents_$F"),(Cint,CuPtr{Int},CuPtr{$T},CuPtr{$T}),
               length(I), I, A, X)
        return A
    end

    function addtoindex!(A::CuArray{$T}, X, ::Colon, I::AbstractArray{<:Real})
        I = CuArray{Int32}(I)
        X = CuArray{$T}(X)
        @knet8($("addcols_$F"),(Cint,Cint,Cint,CuPtr{Int},CuPtr{$T},CuPtr{$T}),
               size(A,1), size(A,2), length(I), I, A, X)
        return A
    end

    function addtoindex!(A::CuArray{$T}, X, I::AbstractArray{<:Real}, ::Colon)
        I = CuArray{Int32}(I)
        X = CuArray{$T}(X)
        @knet8($("addrows_$F"),(Cint,Cint,Cint,CuPtr{Int},CuPtr{$T},CuPtr{$T}),
               size(A,1), size(A,2), length(I), I, A, X)
        return A
    end

    addtoindex!(A::CuArray{$T}, X, I::AbstractArray{Bool})=addtoindex!(A,X,findall(vec(I)))
    addtoindex!(A::CuArray{$T}, X, c::Colon, I::AbstractArray{Bool})=addtoindex!(A,X,c,findall(vec(I)))
    addtoindex!(A::CuArray{$T}, X, I::AbstractArray{Bool}, c::Colon)=addtoindex!(A,X,findall(vec(I)),c)

    # Fix #578: addtoindex! ambiguity:
    addtoindex!(A::CuArray{$T}, X, I::UnitRange{<:Real})=addtoindex!(A,X,CuArray{Int32}(I))
    addtoindex!(A::CuArray{$T}, X, c::Colon, I::UnitRange{<:Real})=addtoindex!(A,X,c,CuArray{Int32}(I))
    addtoindex!(A::CuArray{$T}, X, I::UnitRange{<:Real}, c::Colon)=addtoindex!(A,X,CuArray{Int32}(I),c)

end; end


#=
    A data structure used for dealing with the oprations performed by samplingSolver in a cache friendly way.

    Inspired by an implementation for linked lists used for Incomplete Cholesky in the TAUCS package, this
    code works based on the following guidelines:
        - initial size of the data structure is given by `size` blocks
        - each block contains `blockSize` entries, all corresponding to a certain vertex (to offer locality)
        - when all the blocks are filled up, we increase the number of blocks by `sizeIncrease`. this quantity
        increases by 25% after each expansion
        - we hold a queue of free blocks to reuse memory
=#

immutable element{Tv,Ti}
    # values stored in the linked list relating to the samplingSolver algorithm
	edgeWeight::Tv
	edgeCount::Tv
	neighbor::Ti

    # next will point to the position of the next element in the linked list
    # If totalSize % blockSize == 0 for the current position, then next will be pointing to a new block
	next::Ti

	# usedBy is for debugging purposes
	usedBy::Ti
end

#=
    The linked list starts out with a small size, say close to 1000. At each step we increase its size
    by sizeIncrease. We hold blockSize consecutive elements in memory for every
=#
type LinkedListStorage{Tv,Ti}
	val::Array{element{Tv,Ti},1}		# a big block of memory storing the values from the linked lists

	free::Array{Ti,1}					# a circular array storing all the starting positions of blocks of free memory in [left, right)
	left::Ti 							# the first position in the array containing a free element
	right::Ti 							# the end of the sequence of free elements. val[right] is occupied in memory

	first::Array{Ti,1}					# the position of the first element in i's linked list
	last::Array{Ti,1}					# the position of the last element in i's linked list

	size::Ti 							# total size of the data structure (just counts the number of blocks)
	sizeIncrease::Ti                    # the size of each expansion
	blockSize::Ti 						# size of individual blocks of memory
end

function llsInit{Tv,Ti}(a::SparseMatrixCSC{Tv,Ti}; startingSize::Ti = 1000, blockSize::Ti = 20)

	n = a.n
	startingSize::Ti = max(blockSize, startingSize - startingSize % blockSize)
	sizeIncrease::Ti = startingSize

	first = -ones(Ti,n)
	last = -ones(Ti,n)

	val = Array{element{Tv,Ti}}(startingSize * blockSize)

	free = collect(Ti, 1:startingSize)
	left = startingSize
	right = -1

	return LinkedListStorage(val, free, left, right, first, last, startingSize, sizeIncrease, blockSize)

end

function llsAdd{Tv,Ti}(lls::LinkedListStorage{Tv,Ti}, v::Ti, t::Tuple{Tv,Tv,Ti})

	@assert(v < t[3], "Adding $(t[3]) to $v's linked list")

	if lls.last[v] == -1
		# nothing was added to the current element's linked list so far

		# if we are out of memory - increase the size of the linked list
		if moduloNext(lls.left, lls.size) == lls.right
			incSize(lls)
		end

		# get the position of a new free block
		lls.left = moduloNext(lls.left, lls.size)
		ind = (lls.free[lls.left] - 1) * lls.blockSize + 1;
		@assert(lls.left != lls.right, "Overwriting values in the linked list struct")

		# this new block will be the starting point for v's linked list
		lls.first[v] = lls.last[v] = ind
		lls.val[ind] = element(t[1], t[2], t[3], -1, v)

		# set lls.right on the first insertion
		if lls.right == -1
			lls.right = lls.left
		end
	else
		# we're adding an element to an already existing linked list

		# check if we still have space in the current block
		if lls.last[v] % lls.blockSize != 0
			ind = lls.last[v] + 1
		else
			# if we are out of memory - increase the size of the linked list
			if moduloNext(lls.left, lls.size) == lls.right
				incSize(lls)
			end

			lls.left = moduloNext(lls.left, lls.size)
			ind = (lls.free[lls.left] - 1) * lls.blockSize + 1;
			@assert(lls.left != lls.right, "Overwriting values in the linked list struct")
		end

		# update the 'next' field for the last element in v' linked list
		prev = lls.last[v]
		lls.val[prev] = element(lls.val[prev].edgeWeight, lls.val[prev].edgeCount, lls.val[prev].neighbor, ind, v)

		# add in the new element to the linked list
		lls.last[v] = ind
		lls.val[ind] = element(t[1], t[2], t[3], -1, v)
	end

end

function llsPurge{Tv,Ti}(lls::LinkedListStorage{Tv,Ti}, pos::Ti, auxVal::Array{Tv,1}, auxMult::Array{Tv,1},
	weight::Array{Tv,1}, mult::Array{Tv,1}, ind::Array{Ti,1}; capEdge::Bool=false, rho::Tv=0.0, xhat::Array{Tv,2}=zeros(1,1))

 	multSum::Tv = 0
 	diag::Tv = 0

	i = lls.first[pos]
	while i != -1
		@assert(pos == lls.val[i].usedBy, "accessing storage blocks for $(lls.val[i].usedBy) instead of $pos")

		neigh = lls.val[i].neighbor
		w = lls.val[i].edgeWeight
		e = lls.val[i].edgeCount

		auxVal[neigh] += w
		diag += w
		auxMult[neigh] += e

		i = lls.val[i].next
	end

    numPurged = 0

    i = lls.first[pos]
    while i != -1
        neigh = lls.val[i].neighbor

        if auxVal[neigh] != 0
            @assert(pos < neigh, "current element < neigh in purge")
            if neigh == pos
                @assert(false, "current element = neigh in purge")
            else
            	# this is an edge between pos and neigh. if capEdge is true, we will use the effective resistance estimates to cap this edge
                actualMult = min(auxMult[neigh], rho)
                # actualMult = auxMult[neigh]

                if capEdge
                	p = pos
                	q = neigh
                	w = auxVal[neigh]
                	lev = w * norm(xhat[p,:] - xhat[q,:])^2

                	actualMult = min(actualMult, rho * lev)
                end

                numPurged += 1
                weight[numPurged] = auxVal[neigh]
                mult[numPurged] = actualMult
                ind[numPurged] = neigh

                multSum = multSum + actualMult

                auxVal[neigh] = 0
                auxMult[neigh] = 0
            end
        end

        # i was just freed
        if i % lls.blockSize == 0
        	# if i == 20
        	# 	println("block ", i / lls.blockSize, " was freed by ", pos)
        	# end

        	lls.free[lls.right] = i / lls.blockSize
        	lls.right = moduloNext(lls.right, lls.size)
        end

        i = lls.val[i].next
    end

    return diag, multSum, numPurged

end

# increase the size of a lls structure
function incSize{Tv,Ti}(lls::LinkedListStorage{Tv,Ti})

	#=
    	we know that if we are increasing the size of the structure, we are out of free blocks
    	thus we can add aditional memory at the end of the structure
	=#
	lls.left = lls.size + 1
	lls.right = 1

	append!(lls.val, Array{element{Tv,Ti}}(lls.sizeIncrease * lls.blockSize))
	append!(lls.free, collect(Ti, (lls.size + 1):(lls.size + lls.sizeIncrease)))

	# update the number of free blocks and the number of blocks by which we increase the size every time
	lls.size += lls.sizeIncrease
	lls.sizeIncrease += ceil(Ti, 0.25 * lls.sizeIncrease)

    # this was to make sizeIncrease divisible by blockSize. probably an artifact
	# lls.sizeIncrease = max(lls.blockSize, lls.sizeIncrease - lls.sizeIncrease % lls.blockSize)

end

function moduloNext{Ti}(i::Ti, maxSize::Ti)
	return i < maxSize ? i + 1 : 1
end


THIS_DIR = dirname(@__FILE__)
CSV_PATH = joinpath(THIS_DIR, "../../data/hr/hr.csv")

TREASURER_DEPT = "TREASURER"
TREASURER_HEAD = "CITY TREASURER"
ACC_HEAD = "DIR OF ACCOUNTING"
ACCS = ["AUDITOR IV", "ACCOUNTANT IV", "ACCOUNTANT III", "STAFF ASST", "ACCOUNTANT I"]

csv = readcsv(CSV_PATH, header=true)[1]
N = size(csv, 1)

dept_name_set = Set{String}()

tr_head = nothing
trs = []
acc_head = nothing
accs = []

for eid = 1:N
    department_name = csv[eid, 1]
    push!(dept_name_set, department_name)
    if department_name == TREASURER_DEPT
        position = csv[eid, 4]
        if position == TREASURER_HEAD
            tr_head = eid
        elseif position == ACC_HEAD
            acc_head = eid
            push!(trs, eid)
        elseif position in ACCS
            push!(accs, eid)
        else
            push!(trs, eid)
        end
    end
end

M = length(dept_name_set)

dept_data = 1:M
emp_data = 1:N

dept_name_data = String[]
dept_employee_data = Vector{Int}[]

emp_name_data = String[]
emp_position_data = String[]
emp_salary_data = Int[]
emp_department_data = Int[]
emp_manager_data = Nullable{Int}[]
emp_subordinate_data = Vector{Int}[]

did_by_name = Dict{String, Int}()

for eid = 1:N
    department_name, name, surname, position, salary = csv[eid, :]
    if !(department_name in keys(did_by_name))
        did = length(did_by_name)+1
        push!(dept_name_data, department_name)
        push!(dept_employee_data, Int[])
        did_by_name[department_name] = did
    end
    did = did_by_name[department_name]
    push!(dept_employee_data[did], eid)
    push!(emp_name_data, name * " " * surname[1:1])
    push!(emp_position_data, position)
    push!(emp_salary_data, salary)
    push!(emp_department_data, did)
    if department_name == TREASURER_DEPT
        if position == TREASURER_HEAD
            push!(emp_manager_data, nothing)
            push!(emp_subordinate_data, trs)
        elseif position == ACC_HEAD
            push!(emp_manager_data, tr_head)
            push!(emp_subordinate_data, accs)
        elseif position in ACCS
            push!(emp_manager_data, acc_head)
            push!(emp_subordinate_data, Int[])
        else
            push!(emp_manager_data, tr_head)
            push!(emp_subordinate_data, Int[])
        end
    else
        push!(emp_manager_data, nothing)
        push!(emp_subordinate_data, Int[])
    end
end


module RealNeuralNetworks

include("Utils/include.jl");
include("SWCs.jl");
include("NodeNets.jl");
include("Manifests.jl");
include("Neurons.jl")
include("NeuralNets.jl")
include("NBLASTs.jl")

using .SWCs
using .NodeNets
using .Manifests
using .Neurons
using .NeuralNets
using .NBLASTs

end # end of module


using Blink
using CSV, DataFrames, Interact, Plots

mutable struct Viewer
    window::Window
    observables::Dict{String, Observable}
    widgets::Dict{String, Widget}
end

function makebuttons(view::Viewer)
    buttons = button.(string.(names(view.observables["data"])))
    for (btn, name) in zip(buttons, names(view.observables["data"]))
        map!(t -> histogram(view.observables["data"][name]), view.observables["plt"], btn)
    end
    dom"div"(hbox(buttons))
end

function makebuttons(df, view)
    buttons = button.(string.(names(df)))
    for (btn, name) in zip(buttons, names(df))
        map!(t -> histogram(df[name]), view.observables["plt"], btn)
    end
    dom"div"(hbox(buttons))
end

function read(dir::String)
    buttons = button.(string.(readdir(dir)))
    dom"div"(vbox(buttons))
end

function Viewer()

    view = Viewer(Window(), Dict{String, Observable}(), Dict{String, Widget}())
    window = view.window
    o = view.observables
    w = view.widgets

    w["loadbutton"] = filepicker()
    o["folder_list"] = Observable{Any}(dom"div"())

    map!(read, o["folder_list"], w["loadbutton"])
    # map!((df) -> makebuttons(df, view), o["columnbuttons"], o["data"])
    ui = dom"div"(w["loadbutton"], o["folder_list"])

    body!(window, ui)

    return view

end

Base.close(v::Viewer) = begin; close(v.window); end

function create_ui()

    # Make a Blink Window
    w = Window()
    ui = button()
    body!(w, ui)
    on(println, ui)

    return w
end

close_ui(w) = close(w)


# vim: set ft=julia sw=4 ts=4:
#
#
# A set of (micro-) benchmarks for the Julia programming language
#

module Benchmark

include("Fibonacci.jl")
#import Fibonacci

println("Fibonacci numbers")
println("-----------------")
@time println(lpad("fib_naive(35)", 16) * " = $(Fibonacci.fib_naive(35))")
@time println(lpad("fib(35)", 16) * " = $(Fibonacci.fib(35))")
@time println(lpad("fib(1000)", 16) * " = $(Fibonacci.fib(1000))")
println()

println("Perfect numbers")
println("---------------")
println(" todo...")
println()

println("Mandelbrot set")
println("--------------")
println(" todo...")
println()

println(" done.")

end # module


"""
`munsell_specs()`

Returns an array containing all 1269 munsell spectra, measured from 380 to 800 nm in 1 nm steps.
"""
function munsell_specs()
    filename = "munsell.dat"
    fp=@__DIR__
    cd(fp)
    cd("..\\spectra")
    sfile=readlines(filename)
    map(x->parse(Float64,x),sfile)
end

"""
`munsell_spec(idx::Int, speclist)`

Returns reflectance spectrum of the Munsell dataset at index `idx`.
Possible indices go from 1 to 1269

# Example

```jldoctest
julia> munsell_spec(437, speclist)
RSpec(Real[380, 381, 382, 383, 384, 385, 386, 387, 388, 389  …  791, 792, 793, 794, 795, 796, 797, 798, 799, 800], Real[0.1184, 0.1208, 0.124, 0.1301, 0.1349, 0.1431, 0.1498, 0.1579, 0.1631, 0.1731  …  0.3355, 0.3364, 0.3345, 0.3337, 0.3327, 0.3348, 0.3392, 0.3377, 0.335, 0.3351])
```
"""
function munsell_spec(idx::Int, speclist = munsell_specs())
    n=421
    0 < idx < 1270 ? (i=(idx-1)*n+1) : throw(DomainError(idx, "Index is outside the range of 1 ≤ idx ≤ 1269."))
    y=speclist[i:i+n-1]
    RSpec(collect(380:800),y)
end

function munsell_spec(munsellcolor::AbstractString)
    idx = get(munsell_dict(), munsellcolor, 9999)
    ifelse(idx == 9999, throw(DomainError(idx, "Munsell Color does not exist")), munsell_spec(idx, munsell_specs()))
end

function spec_color(col::AbstractString,colordict::AbstractString)
    ind = get(munsell_dict(), col, "Color key $(col) does not exist")
    if haskey(td,col)
        return spec_idx(ind, munsell_specs())
    else throw(DomainError(ind, "Color key does not exist"))
    end
end

function munsell_dict()
    d = Dict()
    n=length(munsell_keys)
    for i in 1:n
        push!(d, munsell_keys[i] => i)
    end
    d
end

function plotmunsell(index)
    λ=collect(380:800)
    n=421
    plot(λ, zeros(n))
    plot!(λ, spec_idx(index, munsell_specs())[i:i+n-1])
end

# The full set of 1269 munsell color names
const munsell_keys=[
    "2.5R 9/2";
    "2.5R 8/2";
    "2.5R 7/2";
    "2.5R 6/2";
    "2.5R 5/2";
    "2.5R 4/2";
    "2.5R 3/2";
    "2.5R 2.5/2";
    "2.5R 8/4";
    "2.5R 7/4";
    "2.5R 6/4";
    "2.5R 5/4";
    "2.5R 4/4";
    "2.5R 3/4";
    "2.5R 7/6";
    "2.5R 6/6";
    "2.5R 5/6";
    "2.5R 4/6";
    "2.5R 3/6";
    "2.5R 7/8";
    "2.5R 6/8";
    "2.5R 5/8";
    "2.5R 4/8";
    "2.5R 7/10";
    "2.5R 6/10";
    "2.5R 5/10";
    "2.5R 4/10";
    "2.5R 6/12";
    "2.5R 5/12";
    "2.5R 4/12";
    "5R 9/1";
    "5R 8/1";
    "5R 7/1";
    "5R 6/1";
    "5R 5/1";
    "5R 4/1";
    "5R 3/1";
    "5R 2.5/1";
    "5R 9/2";
    "5R 8/2";
    "5R 7/2";
    "5R 6/2";
    "5R 5/2";
    "5R 4/2";
    "5R 3/2";
    "5R 2.5/2";
    "5R 8/4";
    "5R 7/4";
    "5R 6/4";
    "5R 5/4";
    "5R 4/4";
    "5R 3/4";
    "5R 7/6";
    "5R 6/6";
    "5R 5/6";
    "5R 4/6";
    "5R 3/6";
    "5R 7/8";
    "5R 6/8";
    "5R 5/8";
    "5R 4/8";
    "5R 7/10";
    "5R 6/10";
    "5R 5/10";
    "5R 4/10";
    "5R 6/12";
    "5R 5/12";
    "5R 4/12";
    "5R 6/14";
    "5R 5/14";
    "5R 4/14";
    "7.5R 9/2";
    "7.5R 8/2";
    "7.5R 7/2";
    "7.5R 6/2";
    "7.5R 5/2";
    "7.5R 4/2";
    "7.5R 3/2";
    "7.5R 2.5/2";
    "7.5R 8/4";
    "7.5R 7/4";
    "7.5R 6/4";
    "7.5R 5/4";
    "7.5R 4/4";
    "7.5R 3/4";
    "7.5R 7/6";
    "7.5R 6/6";
    "7.5R 5/6";
    "7.5R 4/6";
    "7.5R 3/6";
    "7.5R 7/8";
    "7.5R 6/8";
    "7.5R 5/8";
    "7.5R 4/8";
    "7.5R 7/10";
    "7.5R 6/10";
    "7.5R 5/10";
    "7.5R 4/10";
    "7.5R 6/12";
    "7.5R 5/12";
    "7.5R 4/12";
    "10R 9/1";
    "10R 8/1";
    "10R 7/1";
    "10R 6/1";
    "10R 5/1";
    "10R 4/1";
    "10R 3/1";
    "10R 2.5/1";
    "10R 9/2";
    "10R 8/2";
    "10R 7/2";
    "10R 6/2";
    "10R 5/2";
    "10R 4/2";
    "10R 3/2";
    "10R 2.5/2";
    "10R 8/4";
    "10R 7/4";
    "10R 6/4";
    "10R 5/4";
    "10R 4/4";
    "10R 3/4";
    "10R 8/6";
    "10R 7/6";
    "10R 6/6";
    "10R 5/6";
    "10R 4/6";
    "10R 8/8";
    "10R 7/8";
    "10R 6/8";
    "10R 5/8";
    "10R 4/8";
    "10R 7/10";
    "10R 6/10";
    "10R 5/10";
    "10R 7/12";
    "10R 6/12";
    "10R 5/12";
    "2.5YR 9/2";
    "2.5YR 8/2";
    "2.5YR 7/2";
    "2.5YR 6/2";
    "2.5YR 5/2";
    "2.5YR 4/2";
    "2.5YR 3/2";
    "2.5YR 2.5/2";
    "2.5YR 8/4";
    "2.5YR 7/4";
    "2.5YR 6/4";
    "2.5YR 5/4";
    "2.5YR 4/4";
    "2.5YR 8/6";
    "2.5YR 7/6";
    "2.5YR 6/6";
    "2.5YR 5/6";
    "2.5YR 4/6";
    "2.5YR 8/8";
    "2.5YR 7/8";
    "2.5YR 6/8";
    "2.5YR 5/8";
    "2.5YR 4/8";
    "2.5YR 7/10";
    "2.5YR 6/10";
    "2.5YR 5/10";
    "2.5YR 7/12";
    "2.5YR 6/12";
    "2.5YR 6/14";
    "5YR 9/1";
    "5YR 8/1";
    "5YR 7/1";
    "5YR 6/1";
    "5YR 5/1";
    "5YR 4/1";
    "5YR 3/1";
    "5YR 2.5/1";
    "5YR 9/2";
    "5YR 8/2";
    "5YR 7/2";
    "5YR 6/2";
    "5YR 5/2";
    "5YR 4/2";
    "5YR 3/2";
    "5YR 9/4";
    "5YR 8/4";
    "5YR 7/4";
    "5YR 6/4";
    "5YR 5/4";
    "5YR 4/4";
    "5YR 8/6";
    "5YR 7/6";
    "5YR 6/6";
    "5YR 5/6";
    "5YR 4/6";
    "5YR 8/8";
    "5YR 7/8";
    "5YR 6/8";
    "5YR 5/8";
    "5YR 7/10";
    "5YR 6/10";
    "5YR 7/12";
    "5YR 6/12";
    "7.5YR 9/2";
    "7.5YR 8/2";
    "7.5YR 7/2";
    "7.5YR 6/2";
    "7.5YR 5/2";
    "7.5YR 4/2";
    "7.5YR 3/2";
    "7.5YR 9/4";
    "7.5YR 8/4";
    "7.5YR 7/4";
    "7.5YR 6/4";
    "7.5YR 5/4";
    "7.5YR 4/4";
    "7.5YR 8/6";
    "7.5YR 7/6";
    "7.5YR 6/6";
    "7.5YR 5/6";
    "7.5YR 4/6";
    "7.5YR 8/8";
    "7.5YR 7/8";
    "7.5YR 6/8";
    "7.5YR 5/8";
    "7.5YR 8/10";
    "7.5YR 7/10";
    "7.5YR 6/10";
    "7.5YR 7/12";
    "10YR 9/1";
    "10YR 8/1";
    "10YR 7/1";
    "10YR 6/1";
    "10YR 5/1";
    "10YR 4/1";
    "10YR 3/1";
    "10YR 2.5/1";
    "10YR 9/2";
    "10YR 8/2";
    "10YR 7/2";
    "10YR 6/2";
    "10YR 5/2";
    "10YR 4/2";
    "10YR 3/2";
    "10YR 9/4";
    "10YR 8/4";
    "10YR 7/4";
    "10YR 6/4";
    "10YR 5/4";
    "10YR 4/4";
    "10YR 8/6";
    "10YR 7/6";
    "10YR 6/6";
    "10YR 5/6";
    "10YR 8/8";
    "10YR 7/8";
    "10YR 6/8";
    "10YR 5/8";
    "10YR 8/10";
    "10YR 7/10";
    "10YR 6/10";
    "10YR 7/12";
    "2.5Y 9/2";
    "2.5Y 8.5/2";
    "2.5Y 8/2";
    "2.5Y 7/2";
    "2.5Y 6/2";
    "2.5Y 5/2";
    "2.5Y 4/2";
    "2.5Y 3/2";
    "2.5Y 9/4";
    "2.5Y 8.5/4";
    "2.5Y 8/4";
    "2.5Y 7/4";
    "2.5Y 6/4";
    "2.5Y 5/4";
    "2.5Y 4/4";
    "2.5Y 9/6";
    "2.5Y 8.5/6";
    "2.5Y 8/6";
    "2.5Y 7/6";
    "2.5Y 6/6";
    "2.5Y 5/6";
    "2.5Y 8.5/8";
    "2.5Y 8/8";
    "2.5Y 7/8";
    "2.5Y 6/8";
    "2.5Y 8.5/10";
    "2.5Y 8/10";
    "2.5Y 7/10";
    "2.5Y 8/12";
    "2.5Y 7/12";
    "5Y 9/1";
    "5Y 8.5/1";
    "5Y 8/1";
    "5Y 7/1";
    "5Y 6/1";
    "5Y 5/1";
    "5Y 4/1";
    "5Y 3/1";
    "5Y 2.5/1";
    "5Y 9/2";
    "5Y 8.5/2";
    "5Y 8/2";
    "5Y 7/2";
    "5Y 6/2";
    "5Y 5/2";
    "5Y 4/2";
    "5Y 3/2";
    "5Y 9/4";
    "5Y 8.5/4";
    "5Y 8/4";
    "5Y 7/4";
    "5Y 6/4";
    "5Y 5/4";
    "5Y 4/4";
    "5Y 9/6";
    "5Y 8.5/6";
    "5Y 8/6";
    "5Y 7/6";
    "5Y 6/6";
    "5Y 5/6";
    "5Y 9/8";
    "5Y 8.5/8";
    "5Y 8/8";
    "5Y 7/8";
    "5Y 6/8";
    "5Y 8.5/10";
    "5Y 8/10";
    "5Y 7/10";
    "5Y 8.5/12";
    "5Y 8/12";
    "5Y 7/12";
    "7.5Y 9/2";
    "7.5Y 8.5/2";
    "7.5Y 8/2";
    "7.5Y 7/2";
    "7.5Y 6/2";
    "7.5Y 5/2";
    "7.5Y 4/2";
    "7.5Y 3/2";
    "7.5Y 9/4";
    "7.5Y 8.5/4";
    "7.5Y 8/4";
    "7.5Y 7/4";
    "7.5Y 6/4";
    "7.5Y 5/4";
    "7.5Y 4/4";
    "7.5Y 9/6";
    "7.5Y 8.5/6";
    "7.5Y 8/6";
    "7.5Y 7/6";
    "7.5Y 6/6";
    "7.5Y 5/6";
    "7.5Y 9/8";
    "7.5Y 8.5/8";
    "7.5Y 8/8";
    "7.5Y 7/8";
    "7.5Y 6/8";
    "7.5Y 9/10";
    "7.5Y 8.5/10";
    "7.5Y 8/10";
    "7.5Y 7/10";
    "7.5Y 8.5/12";
    "7.5Y 8/12";
    "10Y 9/1";
    "10Y 8.5/1";
    "10Y 8/1";
    "10Y 7/1";
    "10Y 6/1";
    "10Y 5/1";
    "10Y 4/1";
    "10Y 3/1";
    "10Y 2.5/1";
    "10Y 9/2";
    "10Y 8.5/2";
    "10Y 8/2";
    "10Y 7/2";
    "10Y 6/2";
    "10Y 5/2";
    "10Y 4/2";
    "10Y 3/2";
    "10Y 9/4";
    "10Y 8.5/4";
    "10Y 8/4";
    "10Y 7/4";
    "10Y 6/4";
    "10Y 5/4";
    "10Y 4/4";
    "10Y 9/6";
    "10Y 8.5/6";
    "10Y 8/6";
    "10Y 7/6";
    "10Y 6/6";
    "10Y 5/6";
    "10Y 9/8";
    "10Y 8.5/8";
    "10Y 8/8";
    "10Y 7/8";
    "10Y 6/8";
    "10Y 9/10";
    "10Y 8.5/10";
    "10Y 8/10";
    "10Y 7/10";
    "10Y 8/12";
    "2.5GY 9/2";
    "2.5GY 8.5/2";
    "2.5GY 8/2";
    "2.5GY 7/2";
    "2.5GY 6/2";
    "2.5GY 5/2";
    "2.5GY 4/2";
    "2.5GY 3/2";
    "2.5GY 9/4";
    "2.5GY 8.5/4";
    "2.5GY 8/4";
    "2.5GY 7/4";
    "2.5GY 6/4";
    "2.5GY 5/4";
    "2.5GY 4/4";
    "2.5GY 9/6";
    "2.5GY 8.5/6";
    "2.5GY 8/6";
    "2.5GY 7/6";
    "2.5GY 6/6";
    "2.5GY 5/6";
    "2.5GY 9/8";
    "2.5GY 8.5/8";
    "2.5GY 8/8";
    "2.5GY 7/8";
    "2.5GY 6/8";
    "2.5GY 8.5/10";
    "2.5GY 8/10";
    "2.5GY 7/10";
    "5GY 9/1";
    "5GY 8.5/1";
    "5GY 8/1";
    "5GY 7/1";
    "5GY 6/1";
    "5GY 5/1";
    "5GY 4/1";
    "5GY 3/1";
    "5GY 2.5/1";
    "5GY 9/2";
    "5GY 8.5/2";
    "5GY 8/2";
    "5GY 7/2";
    "5GY 6/2";
    "5GY 5/2";
    "5GY 4/2";
    "5GY 3/2";
    "5GY 9/4";
    "5GY 8.5/4";
    "5GY 8/4";
    "5GY 7/4";
    "5GY 6/4";
    "5GY 5/4";
    "5GY 4/4";
    "5GY 9/6";
    "5GY 8.5/6";
    "5GY 8/6";
    "5GY 7/6";
    "5GY 6/6";
    "5GY 5/6";
    "5GY 8.5/8";
    "5GY 8/8";
    "5GY 7/8";
    "5GY 6/8";
    "5GY 5/8";
    "5GY 8.5/10";
    "5GY 8/10";
    "7.5GY 9/2";
    "7.5GY 8.5/2";
    "7.5GY 8/2";
    "7.5GY 7/2";
    "7.5GY 6/2";
    "7.5GY 5/2";
    "7.5GY 4/2";
    "7.5GY 3/2";
    "7.5GY 9/4";
    "7.5GY 8.5/4";
    "7.5GY 8/4";
    "7.5GY 7/4";
    "7.5GY 6/4";
    "7.5GY 5/4";
    "7.5GY 4/4";
    "7.5GY 8.5/6";
    "7.5GY 8/6";
    "7.5GY 7/6";
    "7.5GY 6/6";
    "7.5GY 5/6";
    "7.5GY 8/8";
    "7.5GY 7/8";
    "7.5GY 6/8";
    "7.5GY 5/8";
    "7.5GY 7/10";
    "7.5GY 6/10";
    "10GY 9/1";
    "10GY 8.5/1";
    "10GY 8/1";
    "10GY 7/1";
    "10GY 6/1";
    "10GY 5/1";
    "10GY 4/1";
    "10GY 3/1";
    "10GY 2.5/1";
    "10GY 9/2";
    "10GY 8.5/2";
    "10GY 8/2";
    "10GY 7/2";
    "10GY 6/2";
    "10GY 5/2";
    "10GY 4/2";
    "10GY 3/2";
    "10GY 2.5/2";
    "10GY 9/4";
    "10GY 8.5/4";
    "10GY 8/4";
    "10GY 7/4";
    "10GY 6/4";
    "10GY 5/4";
    "10GY 4/4";
    "10GY 8.5/6";
    "10GY 8/6";
    "10GY 7/6";
    "10GY 6/6";
    "10GY 5/6";
    "10GY 7/8";
    "10GY 6/8";
    "10GY 5/8";
    "10GY 7/10";
    "10GY 6/10";
    "2.5G 9/2";
    "2.5G 8/2";
    "2.5G 7/2";
    "2.5G 6/2";
    "2.5G 5/2";
    "2.5G 4/2";
    "2.5G 3/2";
    "2.5G 2.5/2";
    "2.5G 8/4";
    "2.5G 7/4";
    "2.5G 6/4";
    "2.5G 5/4";
    "2.5G 4/4";
    "2.5G 3/4";
    "2.5G 8/6";
    "2.5G 7/6";
    "2.5G 6/6";
    "2.5G 5/6";
    "2.5G 4/6";
    "2.5G 7/8";
    "2.5G 6/8";
    "2.5G 5/8";
    "2.5G 7/10";
    "2.5G 6/10";
    "2.5G 5/10";
    "2.5G 6/12";
    "5G 9/1";
    "5G 8/1";
    "5G 7/1";
    "5G 6/1";
    "5G 5/1";
    "5G 4/1";
    "5G 3/1";
    "5G 2.5/1";
    "5G 9/2";
    "5G 8/2";
    "5G 7/2";
    "5G 6/2";
    "5G 5/2";
    "5G 4/2";
    "5G 3/2";
    "5G 2.5/2";
    "5G 8/4";
    "5G 7/4";
    "5G 6/4";
    "5G 5/4";
    "5G 4/4";
    "5G 3/4";
    "5G 8/6";
    "5G 7/6";
    "5G 6/6";
    "5G 5/6";
    "5G 4/6";
    "5G 7/8";
    "5G 6/8";
    "5G 5/8";
    "5G 4/8";
    "5G 7/10";
    "5G 6/10";
    "7.5G 9/2";
    "7.5G 8/2";
    "7.5G 7/2";
    "7.5G 6/2";
    "7.5G 5/2";
    "7.5G 4/2";
    "7.5G 3/2";
    "7.5G 2.5/2";
    "7.5G 8/4";
    "7.5G 7/4";
    "7.5G 6/4";
    "7.5G 5/4";
    "7.5G 4/4";
    "7.5G 3/4";
    "7.5G 8/6";
    "7.5G 7/6";
    "7.5G 6/6";
    "7.5G 5/6";
    "7.5G 4/6";
    "7.5G 7/8";
    "7.5G 6/8";
    "7.5G 5/8";
    "7.5G 4/8";
    "7.5G 7/10";
    "7.5G 6/10";
    "10G 9/1";
    "10G 8/1";
    "10G 7/1";
    "10G 6/1";
    "10G 5/1";
    "10G 4/1";
    "10G 3/1";
    "10G 2.5/1";
    "10G 9/2";
    "10G 8/2";
    "10G 7/2";
    "10G 6/2";
    "10G 5/2";
    "10G 4/2";
    "10G 3/2";
    "10G 2.5/2";
    "10G 8/4";
    "10G 7/4";
    "10G 6/4";
    "10G 5/4";
    "10G 4/4";
    "10G 3/4";
    "10G 7/6";
    "10G 6/6";
    "10G 5/6";
    "10G 4/6";
    "10G 7/8";
    "10G 6/8";
    "10G 5/8";
    "10G 4/8";
    "10G 6/10";
    "2.5BG 9/2";
    "2.5BG 8/2";
    "2.5BG 7/2";
    "2.5BG 6/2";
    "2.5BG 5/2";
    "2.5BG 4/2";
    "2.5BG 3/2";
    "2.5BG 2.5/2";
    "2.5BG 8/4";
    "2.5BG 7/4";
    "2.5BG 6/4";
    "2.5BG 5/4";
    "2.5BG 4/4";
    "2.5BG 3/4";
    "2.5BG 7/6";
    "2.5BG 6/6";
    "2.5BG 5/6";
    "2.5BG 4/6";
    "2.5BG 3/6";
    "2.5BG 7/8";
    "2.5BG 6/8";
    "2.5BG 5/8";
    "2.5BG 4/8";
    "2.5BG 6/10";
    "5BG 9/1";
    "5BG 8/1";
    "5BG 7/1";
    "5BG 6/1";
    "5BG 5/1";
    "5BG 4/1";
    "5BG 3/1";
    "5BG 2.5/1";
    "5BG 9/2";
    "5BG 8/2";
    "5BG 7/2";
    "5BG 6/2";
    "5BG 5/2";
    "5BG 4/2";
    "5BG 3/2";
    "5BG 2.5/2";
    "5BG 8/4";
    "5BG 7/4";
    "5BG 6/4";
    "5BG 5/4";
    "5BG 4/4";
    "5BG 3/4";
    "5BG 7/6";
    "5BG 6/6";
    "5BG 5/6";
    "5BG 4/6";
    "5BG 7/8";
    "5BG 6/8";
    "5BG 5/8";
    "5BG 4/8";
    "7.5BG 9/2";
    "7.5BG 8/2";
    "7.5BG 7/2";
    "7.5BG 6/2";
    "7.5BG 5/2";
    "7.5BG 4/2";
    "7.5BG 3/2";
    "7.5BG 2.5/2";
    "7.5BG 8/4";
    "7.5BG 7/4";
    "7.5BG 6/4";
    "7.5BG 5/4";
    "7.5BG 4/4";
    "7.5BG 3/4";
    "7.5BG 7/6";
    "7.5BG 6/6";
    "7.5BG 5/6";
    "7.5BG 4/6";
    "7.5BG 7/8";
    "7.5BG 6/8";
    "7.5BG 5/8";
    "7.5BG 4/8";
    "10BG 9/1";
    "10BG 8/1";
    "10BG 7/1";
    "10BG 6/1";
    "10BG 5/1";
    "10BG 4/1";
    "10BG 3/1";
    "10BG 2.5/1";
    "10BG 9/2";
    "10BG 8/2";
    "10BG 7/2";
    "10BG 6/2";
    "10BG 5/2";
    "10BG 4/2";
    "10BG 3/2";
    "10BG 2.5/2";
    "10BG 8/4";
    "10BG 7/4";
    "10BG 6/4";
    "10BG 5/4";
    "10BG 4/4";
    "10BG 3/4";
    "10BG 7/6";
    "10BG 6/6";
    "10BG 5/6";
    "10BG 4/6";
    "10BG 7/8";
    "10BG 6/8";
    "10BG 5/8";
    "10BG 4/8";
    "2.5B 9/2";
    "2.5B 8/2";
    "2.5B 7/2";
    "2.5B 6/2";
    "2.5B 5/2";
    "2.5B 4/2";
    "2.5B 3/2";
    "2.5B 2.5/2";
    "2.5B 8/4";
    "2.5B 7/4";
    "2.5B 6/4";
    "2.5B 5/4";
    "2.5B 4/4";
    "2.5B 3/4";
    "2.5B 7/6";
    "2.5B 6/6";
    "2.5B 5/6";
    "2.5B 4/6";
    "2.5B 7/8";
    "2.5B 6/8";
    "2.5B 5/8";
    "2.5B 4/8";
    "5B 9/1";
    "5B 8/1";
    "5B 7/1";
    "5B 6/1";
    "5B 5/1";
    "5B 4/1";
    "5B 3/1";
    "5B 2.5/1";
    "5B 9/2";
    "5B 8/2";
    "5B 7/2";
    "5B 6/2";
    "5B 5/2";
    "5B 4/2";
    "5B 3/2";
    "5B 2.5/2";
    "5B 8/4";
    "5B 7/4";
    "5B 6/4";
    "5B 5/4";
    "5B 4/4";
    "5B 3/4";
    "5B 7/6";
    "5B 6/6";
    "5B 5/6";
    "5B 4/6";
    "5B 3/6";
    "5B 7/8";
    "5B 6/8";
    "5B 5/8";
    "5B 4/8";
    "7.5B 9/2";
    "7.5B 8/2";
    "7.5B 7/2";
    "7.5B 6/2";
    "7.5B 5/2";
    "7.5B 4/2";
    "7.5B 3/2";
    "7.5B 2.5/2";
    "7.5B 8/4";
    "7.5B 7/4";
    "7.5B 6/4";
    "7.5B 5/4";
    "7.5B 4/4";
    "7.5B 3/4";
    "7.5B 7/6";
    "7.5B 6/6";
    "7.5B 5/6";
    "7.5B 4/6";
    "7.5B 3/6";
    "7.5B 7/8";
    "7.5B 6/8";
    "7.5B 5/8";
    "7.5B 4/8";
    "7.5B 6/10";
    "7.5B 5/10";
    "10B 9/1";
    "10B 8/1";
    "10B 7/1";
    "10B 6/1";
    "10B 5/1";
    "10B 4/1";
    "10B 3/1";
    "10B 2.5/1";
    "10B 9/2";
    "10B 8/2";
    "10B 7/2";
    "10B 6/2";
    "10B 5/2";
    "10B 4/2";
    "10B 3/2";
    "10B 2.5/2";
    "10B 8/4";
    "10B 7/4";
    "10B 6/4";
    "10B 5/4";
    "10B 4/4";
    "10B 3/4";
    "10B 2.5/4";
    "10B 7/6";
    "10B 6/6";
    "10B 5/6";
    "10B 4/6";
    "10B 3/6";
    "10B 7/8";
    "10B 6/8";
    "10B 5/8";
    "10B 4/8";
    "10B 6/10";
    "10B 5/10";
    "2.5PB 9/2";
    "2.5PB 8/2";
    "2.5PB 7/2";
    "2.5PB 6/2";
    "2.5PB 5/2";
    "2.5PB 4/2";
    "2.5PB 3/2";
    "2.5PB 2.5/2";
    "2.5PB 8/4";
    "2.5PB 7/4";
    "2.5PB 6/4";
    "2.5PB 5/4";
    "2.5PB 4/4";
    "2.5PB 3/4";
    "2.5PB 2.5/4";
    "2.5PB 8/6";
    "2.5PB 7/6";
    "2.5PB 6/6";
    "2.5PB 5/6";
    "2.5PB 4/6";
    "2.5PB 3/6";
    "2.5PB 7/8";
    "2.5PB 6/8";
    "2.5PB 5/8";
    "2.5PB 4/8";
    "2.5PB 6/10";
    "2.5PB 5/10";
    "2.5PB 4/10";
    "5PB 9/1";
    "5PB 8/1";
    "5PB 7/1";
    "5PB 6/1";
    "5PB 5/1";
    "5PB 4/1";
    "5PB 3/1";
    "5PB 2.5/1";
    "5PB 9/2";
    "5PB 8/2";
    "5PB 7/2";
    "5PB 6/2";
    "5PB 5/2";
    "5PB 4/2";
    "5PB 3/2";
    "5PB 2.5/2";
    "5PB 8/4";
    "5PB 7/4";
    "5PB 6/4";
    "5PB 5/4";
    "5PB 4/4";
    "5PB 3/4";
    "5PB 2.5/4";
    "5PB 8/6";
    "5PB 7/6";
    "5PB 6/6";
    "5PB 5/6";
    "5PB 4/6";
    "5PB 3/6";
    "5PB 7/8";
    "5PB 6/8";
    "5PB 5/8";
    "5PB 4/8";
    "5PB 3/8";
    "5PB 6/10";
    "5PB 5/10";
    "5PB 4/10";
    "5PB 5/12";
    "5PB 4/12";
    "7.5PB 9/2";
    "7.5PB 8/2";
    "7.5PB 7/2";
    "7.5PB 6/2";
    "7.5PB 5/2";
    "7.5PB 4/2";
    "7.5PB 3/2";
    "7.5PB 2.5/2";
    "7.5PB 8/4";
    "7.5PB 7/4";
    "7.5PB 6/4";
    "7.5PB 5/4";
    "7.5PB 4/4";
    "7.5PB 3/4";
    "7.5PB 2.5/4";
    "7.5PB 8/6";
    "7.5PB 7/6";
    "7.5PB 6/6";
    "7.5PB 5/6";
    "7.5PB 4/6";
    "7.5PB 3/6";
    "7.5PB 2.5/6";
    "7.5PB 7/8";
    "7.5PB 6/8";
    "7.5PB 5/8";
    "7.5PB 4/8";
    "7.5PB 3/8";
    "7.5PB 6/10";
    "7.5PB 5/10";
    "7.5PB 4/10";
    "7.5PB 3/10";
    "7.5PB 5/12";
    "7.5PB 4/12";
    "10PB 9/1";
    "10PB 8/1";
    "10PB 7/1";
    "10PB 6/1";
    "10PB 5/1";
    "10PB 4/1";
    "10PB 3/1";
    "10PB 2.5/1";
    "10PB 9/2";
    "10PB 8/2";
    "10PB 7/2";
    "10PB 6/2";
    "10PB 5/2";
    "10PB 4/2";
    "10PB 3/2";
    "10PB 2.5/2";
    "10PB 8/4";
    "10PB 7/4";
    "10PB 6/4";
    "10PB 5/4";
    "10PB 4/4";
    "10PB 3/4";
    "10PB 2.5/4";
    "10PB 7/6";
    "10PB 6/6";
    "10PB 5/6";
    "10PB 4/6";
    "10PB 3/6";
    "10PB 2.5/6";
    "10PB 7/8";
    "10PB 6/8";
    "10PB 5/8";
    "10PB 4/8";
    "10PB 3/8";
    "10PB 6/10";
    "10PB 5/10";
    "10PB 4/10";
    "2.5P 9/2";
    "2.5P 8/2";
    "2.5P 7/2";
    "2.5P 6/2";
    "2.5P 5/2";
    "2.5P 4/2";
    "2.5P 3/2";
    "2.5P 2.5/2";
    "2.5P 8/4";
    "2.5P 7/4";
    "2.5P 6/4";
    "2.5P 5/4";
    "2.5P 4/4";
    "2.5P 3/4";
    "2.5P 2.5/4";
    "2.5P 7/6";
    "2.5P 6/6";
    "2.5P 5/6";
    "2.5P 4/6";
    "2.5P 3/6";
    "2.5P 7/8";
    "2.5P 6/8";
    "2.5P 5/8";
    "2.5P 4/8";
    "2.5P 3/8";
    "2.5P 6/10";
    "2.5P 5/10";
    "2.5P 4/10";
    "5P 9/1";
    "5P 8/1";
    "5P 7/1";
    "5P 6/1";
    "5P 5/1";
    "5P 4/1";
    "5P 3/1";
    "5P 2.5/1";
    "5P 9/2";
    "5P 8/2";
    "5P 7/2";
    "5P 6/2";
    "5P 5/2";
    "5P 4/2";
    "5P 3/2";
    "5P 2.5/2";
    "5P 8/4";
    "5P 7/4";
    "5P 6/4";
    "5P 5/4";
    "5P 4/4";
    "5P 3/4";
    "5P 2.5/4";
    "5P 7/6";
    "5P 6/6";
    "5P 5/6";
    "5P 4/6";
    "5P 3/6";
    "5P 2.5/6";
    "5P 7/8";
    "5P 6/8";
    "5P 5/8";
    "5P 4/8";
    "5P 3/8";
    "5P 5/10";
    "5P 4/10";
    "7.5P 9/2";
    "7.5P 8/2";
    "7.5P 7/2";
    "7.5P 6/2";
    "7.5P 5/2";
    "7.5P 4/2";
    "7.5P 3/2";
    "7.5P 2.5/2";
    "7.5P 8/4";
    "7.5P 7/4";
    "7.5P 6/4";
    "7.5P 5/4";
    "7.5P 4/4";
    "7.5P 3/4";
    "7.5P 2.5/4";
    "7.5P 8/6";
    "7.5P 7/6";
    "7.5P 6/6";
    "7.5P 5/6";
    "7.5P 4/6";
    "7.5P 3/6";
    "7.5P 7/8";
    "7.5P 6/8";
    "7.5P 5/8";
    "7.5P 4/8";
    "7.5P 3/8";
    "7.5P 6/10";
    "7.5P 5/10";
    "7.5P 4/10";
    "10P 9/1";
    "10P 8/1";
    "10P 7/1";
    "10P 6/1";
    "10P 5/1";
    "10P 4/1";
    "10P 3/1";
    "10P 2.5/1";
    "10P 9/2";
    "10P 8/2";
    "10P 7/2";
    "10P 6/2";
    "10P 5/2";
    "10P 4/2";
    "10P 3/2";
    "10P 2.5/2";
    "10P 8/4";
    "10P 7/4";
    "10P 6/4";
    "10P 5/4";
    "10P 4/4";
    "10P 3/4";
    "10P 2.5/4";
    "10P 8/6";
    "10P 7/6";
    "10P 6/6";
    "10P 5/6";
    "10P 4/6";
    "10P 3/6";
    "10P 7/8";
    "10P 6/8";
    "10P 5/8";
    "10P 4/8";
    "10P 3/8";
    "10P 6/10";
    "10P 5/10";
    "10P 4/10";
    "10P 5/12";
    "2.5RP 9/2";
    "2.5RP 8/2";
    "2.5RP 7/2";
    "2.5RP 6/2";
    "2.5RP 5/2";
    "2.5RP 4/2";
    "2.5RP 3/2";
    "2.5RP 2.5/2";
    "2.5RP 8/4";
    "2.5RP 7/4";
    "2.5RP 6/4";
    "2.5RP 5/4";
    "2.5RP 4/4";
    "2.5RP 3/4";
    "2.5RP 2.5/4";
    "2.5RP 8/6";
    "2.5RP 7/6";
    "2.5RP 6/6";
    "2.5RP 5/6";
    "2.5RP 4/6";
    "2.5RP 3/6";
    "2.5RP 7/8";
    "2.5RP 6/8";
    "2.5RP 5/8";
    "2.5RP 4/8";
    "2.5RP 7/10";
    "2.5RP 6/10";
    "2.5RP 5/10";
    "2.5RP 4/10";
    "2.5RP 6/12";
    "2.5RP 5/12";
    "5RP 9/1";
    "5RP 8/1";
    "5RP 7/1";
    "5RP 6/1";
    "5RP 5/1";
    "5RP 4/1";
    "5RP 3/1";
    "5RP 2.5/1";
    "5RP 9/2";
    "5RP 8/2";
    "5RP 7/2";
    "5RP 6/2";
    "5RP 5/2";
    "5RP 4/2";
    "5RP 3/2";
    "5RP 2.5/2";
    "5RP 8/4";
    "5RP 7/4";
    "5RP 6/4";
    "5RP 5/4";
    "5RP 4/4";
    "5RP 3/4";
    "5RP 2.5/4";
    "5RP 8/6";
    "5RP 7/6";
    "5RP 6/6";
    "5RP 5/6";
    "5RP 4/6";
    "5RP 3/6";
    "5RP 7/8";
    "5RP 6/8";
    "5RP 5/8";
    "5RP 4/8";
    "5RP 6/10";
    "5RP 5/10";
    "5RP 4/10";
    "5RP 5/12";
    "5RP 4/12";
    "7.5RP 9/2";
    "7.5RP 8/2";
    "7.5RP 7/2";
    "7.5RP 6/2";
    "7.5RP 5/2";
    "7.5RP 4/2";
    "7.5RP 3/2";
    "7.5RP 2.5/2";
    "7.5RP 8/4";
    "7.5RP 7/4";
    "7.5RP 6/4";
    "7.5RP 5/4";
    "7.5RP 4/4";
    "7.5RP 3/4";
    "7.5RP 2.5/4";
    "7.5RP 8/6";
    "7.5RP 7/6";
    "7.5RP 6/6";
    "7.5RP 5/6";
    "7.5RP 4/6";
    "7.5RP 3/6";
    "7.5RP 7/8";
    "7.5RP 6/8";
    "7.5RP 5/8";
    "7.5RP 4/8";
    "7.5RP 6/10";
    "7.5RP 5/10";
    "7.5RP 4/10";
    "7.5RP 5/12";
    "7.5RP 4/12";
    "10RP 9/1";
    "10RP 8/1";
    "10RP 7/1";
    "10RP 6/1";
    "10RP 5/1";
    "10RP 4/1";
    "10RP 3/1";
    "10RP 2.5/1";
    "10RP 9/2";
    "10RP 8/2";
    "10RP 7/2";
    "10RP 6/2";
    "10RP 5/2";
    "10RP 4/2";
    "10RP 3/2";
    "10RP 2.5/2";
    "10RP 8/4";
    "10RP 7/4";
    "10RP 6/4";
    "10RP 5/4";
    "10RP 4/4";
    "10RP 3/4";
    "10RP 2.5/4";
    "10RP 8/6";
    "10RP 7/6";
    "10RP 6/6";
    "10RP 5/6";
    "10RP 4/6";
    "10RP 3/6";
    "10RP 7/8";
    "10RP 6/8";
    "10RP 5/8";
    "10RP 4/8";
    "10RP 6/10";
    "10RP 5/10";
    "10RP 4/10";
    "10RP 5/12";
    "10RP 4/12"]


@testset "Partial MD" begin
    st = raw"""
        \newcommand{\com}{HH}
        \newcommand{\comb}[1]{HH#1HH}
        A list
        * \com and \comb{blah}
        * $f$ is a function
        * a last element
        """

    steps = explore_md_steps(st)
    lxdefs, tokens, braces, blocks, lxcoms = steps[:latex]

    @test length(braces) == 1
    @test F.content(braces[1]) == "blah"

    @test length(blocks) == 1
    @test blocks[1].name == :MATH_A
    @test F.content(blocks[1]) == "f"

    b2insert, = steps[:b2insert]

    inter_md, mblocks = F.form_inter_md(st, b2insert, lxdefs)
    @test inter_md // """
            A list
            *  ##FDINSERT## and  ##FDINSERT##
            *  ##FDINSERT## is a function
            * a last element"""
    inter_html = F.md2html(inter_md)
    @test inter_html // """

A list

##FDINSERT## and  ##FDINSERT##

##FDINSERT## is a function

a last element

        """
end

# index arithmetic over a string is a bit trickier when using all symbols
# we can use `prevind` and `nextind` to make sure it works properly
@testset "Inter Md 2" begin
    st = raw"""
        ~~~
        this⊙ then ⊙ ⊙ and
        ~~~
        finally ⊙⊙𝛴⊙ and
        ~~~
        escape ∀⊙∀
        ~~~
        done
        """

    inter_md, = explore_md_steps(st)[:inter_md]
    @test inter_md // """
                     ##FDINSERT##
                    finally ⊙⊙𝛴⊙ and
                     ##FDINSERT##
                    done
                    """
end

@testset "Latex eqa" begin
    st = raw"""
        a\newcommand{\eqa}[1]{\begin{eqnarray}#1\end{eqnarray}}b@@d .@@
        \eqa{\sin^2(x)+\cos^2(x) &=& 1}
        """

    steps = explore_md_steps(st)
    lxdefs, tokens, braces, blocks, lxcoms = steps[:latex]
    b2insert, = steps[:b2insert]
    inter_md, mblocks = steps[:inter_md]
    @test inter_md // """
                ab

                ##FDINSERT##

                 ##FDINSERT##
                """

    inter_html, = steps[:inter_html]

    @test F.convert_block(b2insert[1], lxdefs) == "
.
"
    @test isapproxstr(F.convert_block(b2insert[2], lxdefs), "\\[\\begin{array}{c} \\sin^2(x)+\\cos^2(x) &=& 1\\end{array}\\]")
    hstring = F.convert_inter_html(inter_html, b2insert, lxdefs)
    @test isapproxstr(hstring, raw"""

ab

.

                    \[\begin{array}{c} \sin^2(x)+\cos^2(x) &=& 1\end{array}\]""")
end

@testset "MD>HTML" begin
    st = raw"""
        text A1 \newcommand{\com}{blah}text A2 \com and
        ~~~
        escape B1
        ~~~
        \newcommand{\comb}[ 1]{\mathrm{#1}} text C1 $\comb{b}$ text C2
        \newcommand{\comc}[ 2]{part1:#1 and part2:#2} then \comc{AA}{BB}.
        """

    steps = explore_md_steps(st)
    lxdefs, tokens, braces, blocks, lxcoms = steps[:latex]
    b2insert, = steps[:b2insert]
    inter_md, mblocks = steps[:inter_md]
    inter_html, = steps[:inter_html]

    @test isapproxstr(inter_md, """
            text A1 text A2  ##FDINSERT## and
             ##FDINSERT##
             text C1  ##FDINSERT## text C2
             then  ##FDINSERT##.""")

    @test isapproxstr(inter_html, """
text A1 text A2  ##FDINSERT## and  ##FDINSERT##  text C1  ##FDINSERT## text C2  then  ##FDINSERT##.
""")

    hstring = F.convert_inter_html(inter_html, b2insert, lxdefs)
    @test isapproxstr(hstring, raw"""

text A1 text A2 blah and
                    escape B1
                    text C1 \(\mathrm{ b}\) text C2  then part1: AA and part2: BB.
""")
end

@testset "headers" begin
    set_curpath("index.md")
    h = """
        # Title
        and then
        ## Subtitle cool!
        done
        """ |> seval
    @test h // """

Title

and then

Subtitle cool!

done
"""
end


ds_small = GDAL.open("data/utmsmall.tif", GDAL.GA_ReadOnly)
ds_point = GDAL.openex("data/point.geojson", GDAL.GDAL_OF_VECTOR, C_NULL, C_NULL, C_NULL)

# GDALInfo
# infooptionsnew checks if the options are valid
@test_throws GDAL.GDALError GDAL.infooptionsnew(["-novalidoption"], C_NULL)
# check not only that a GDALError is thrown, but also its contents
try
    GDAL.infooptionsnew(["-novalidoption"], C_NULL)
catch err
    @test err.class === GDAL.CE_Failure
    @test err.code === Cint(6)
    @test err.msg == "Unknown option name '-novalidoption'"
end

options = GDAL.infooptionsnew(["-checksum"], C_NULL)
infostr = GDAL.info(ds_small, options)
@test contains(infostr, "Checksum=50054")
GDAL.infooptionsfree(options)

# GDALTranslate
# resample to a 5×5 ascii grid
optvec = ["-of","AAIGrid","-r","cubic","-tr","1200","1200"]
options = GDAL.translateoptionsnew(optvec, C_NULL)
ds_tiny_asc = GDAL.translate("data/utmtiny.asc", ds_small, options, C_NULL)
GDAL.translateoptionsfree(options)
GDAL.close(ds_tiny_asc)
@test replace(readstring("data/utmtiny.asc"), "\r", "") == """
    ncols        5
    nrows        5
    xllcorner    440720.000000000000
    yllcorner    3745320.000000000000
    cellsize     1200.000000000000
     128 126 161 185 193
     171 164 175 206 205
     127 148 177 205 209
     93 114 164 172 181
     83 101 140 128 122
    """
# also create a Byte GeoTIFF version for later use
optvec = ["-r","cubic","-tr","1200","1200","-ot","Byte"]
options = GDAL.translateoptionsnew(optvec, C_NULL)
ds_tiny = GDAL.translate("data/utmtiny.tif", ds_small, options, C_NULL)
GDAL.translateoptionsfree(options)

tinydata = UInt8[
    128 126 161 185 193;
    171 164 175 206 205;
    127 148 177 205 209;
     93 114 164 172 181;
     83 101 140 128 122]

band = GDAL.getrasterband(ds_tiny, 1)
read_data = zeros(tinydata)
GDAL.rasterio(band, GDAL.GF_Read, 0, 0, 5, 5,
              read_data, 5, 5, GDAL.GDT_Byte, 0, 0)
@test read_data' == tinydata

# GDALWarp

# cannot reproject file
# ERROR: LoadError: LoadError: GDALError (Failure, code 6):
#        Unable to load PROJ.4 library (libproj.so), creation of OGRCoordinateTransformation failed.
# when using
# options = GDAL.warpappoptionsnew(["-t_srs","EPSG:4326"], C_NULL)
# should we require PROJ.4 or not?
options = GDAL.warpappoptionsnew(["-of","MEM"], C_NULL)  # in memory
ds_warped = GDAL.warp("data/utmsmall.mem", Ptr{GDAL.GDALDatasetH}(C_NULL), 1, [ds_small], options, C_NULL)
GDAL.warpappoptionsfree(options)
GDAL.close(ds_warped)

# GDALDEMProcessing
# calculate hillshade
options = GDAL.demprocessingoptionsnew(["-of","AAIGrid"], C_NULL)
ds_dempr = GDAL.demprocessing("data/utmtiny-hillshade.asc", ds_tiny, "hillshade", C_NULL, options, C_NULL)
GDAL.demprocessingoptionsfree(options)
GDAL.close(ds_dempr)
@test replace(readstring("data/utmtiny-hillshade.asc"), "\r", "") == """
    ncols        5
    nrows        5
    xllcorner    440720.000000000000
    yllcorner    3745320.000000000000
    cellsize     1200.000000000000
    NODATA_value 0
     0 0 0 0 0
     0 183 184 183 0
     0 180 182 181 0
     0 181 181 177 0
     0 0 0 0 0
    """
rm("data/utmtiny-hillshade.asc")
rm("data/utmtiny-hillshade.prj")

# GDALNearblack
# not ascii because it doesn't support `create`
options = GDAL.nearblackoptionsnew(["-of","GTiff","-color","0"], C_NULL)
ds_nearblack = GDAL.nearblack("data/utmtiny-nearblack.tif", Ptr{GDAL.GDALDatasetH}(C_NULL), ds_tiny, options, C_NULL)
GDAL.nearblackoptionsfree(options)
# do rasterio to read result
band = GDAL.getrasterband(ds_nearblack, 1)
read_data = zeros(tinydata)
GDAL.rasterio(band, GDAL.GF_Read, 0, 0, 5, 5,
              read_data, 5, 5, GDAL.GDT_Byte, 0, 0)
nbdata = read_data'
# by default the outer two pixels are zeroed
nbexpected = zeros(tinydata)
nbexpected[3,3] = tinydata[3,3]
@test nbdata == nbexpected

GDAL.close(ds_nearblack)
rm("data/utmtiny-nearblack.tif")

# GDALGrid
options = GDAL.gridoptionsnew(["-of","MEM","-outsize","3","10","-txe","100","100.3","-tye","0","0.1"], C_NULL)
ds_grid = GDAL.grid("data/point-grid.mem", ds_point, options, C_NULL)
GDAL.gridoptionsfree(options)
geotransform = zeros(6)
GDAL.getgeotransform(ds_grid, geotransform)
@test geotransform ≈ [100.0,0.1,0.0,0.0,0.0,0.01]
GDAL.close(ds_grid)

# GDALRasterize
options = GDAL.rasterizeoptionsnew(["-of","MEM","-tr","0.05","0.05"], C_NULL)
ds_rasterize = GDAL.rasterize("data/point-rasterize.mem", Ptr{GDAL.GDALDatasetH}(C_NULL), ds_point, options, C_NULL)
GDAL.rasterizeoptionsfree(options)
geotransform = zeros(6)
GDAL.getgeotransform(ds_rasterize, geotransform)
@test geotransform ≈ [99.975,0.05,0.0,0.1143,0.0,-0.05]
GDAL.close(ds_rasterize)

# GDALBuildVRT
options = GDAL.buildvrtoptionsnew([], C_NULL)
ds_buildvrt = GDAL.buildvrt("data/utmtiny.vrt", 1, [ds_tiny], C_NULL, options, C_NULL)
GDAL.buildvrtoptionsfree(options)
GDAL.close(ds_buildvrt)
@test contains(readstring("data/utmtiny.vrt"), "<SourceFilename relativeToVRT=\"1\">utmtiny.tif</SourceFilename>")
rm("data/utmtiny.vrt")

# GDALVectorTranslate
# convert a GeoJSON to CSV with X and Y columns
options = GDAL.vectortranslateoptionsnew(["-f","CSV","-lco","GEOMETRY=AS_XY"], C_NULL)
ds_csv = GDAL.vectortranslate("data/point.csv", Ptr{GDAL.GDALDatasetH}(C_NULL), 1, [ds_point], options, C_NULL)
GDAL.vectortranslateoptionsfree(options)
GDAL.close(ds_csv)
dstcsv = """
    X,Y,FID,pointname
    100,0,2,point-a
    100.2785,0.0893,3,point-b
    100,0,0,a
    100.2785,0.0893,3,b
    """
@test replace(readstring("data/point.csv"), "\r", "") == dstcsv
rm("data/point.csv")

GDAL.close(ds_small)
GDAL.close(ds_tiny)
GDAL.close(ds_point)

rm("data/utmtiny.asc.aux.xml")
rm("data/utmtiny.prj")
rm("data/utmtiny.asc")
rm("data/utmtiny.tif")


"""
Represents everything needed to run and record a single simulation, including model, initial conditions, and metadata.

A vector of `Sim` objects can be executed with [`run`](@ref) or [`run_parallel`](@ref).

## Keyword Arguments
- `rng::AbstractRNG=Random.GLOBAL_RNG`
- `max_steps::Int=typemax(Int)`
- `simulator::Simulator=HistoryRecorder(rng=rng, max_steps=max_steps)`
- `metadata::NamedTuple a named tuple (or dictionary) of metadata for the sim that will be recorded, e.g. `(solver_iterations=500,)`.
"""
abstract type Sim end

struct POMDPSim <: Sim
    simulator::Simulator
    pomdp::POMDP
    policy::Policy
    updater::Updater
    initial_belief::Any
    initialstate::Any
    metadata::NamedTuple
end

problem(sim::POMDPSim) = sim.pomdp

struct MDPSim <: Sim
    simulator::Simulator
    mdp::MDP
    policy::Policy
    initialstate::Any
    metadata::NamedTuple
end

problem(sim::MDPSim) = sim.mdp

"""
    Sim(p::POMDP, policy::Policy, metadata=(note="a note",))
    Sim(p::POMDP, policy::Policy[, updater[, initial_belief[, initialstate]]]; kwargs...)

Create a `Sim` object that represents a POMDP simulation.
"""
function Sim(pomdp::POMDP,
             policy::Policy,
             up=updater(policy),
             initial_belief=initialstate_distribution(pomdp),
             initialstate=nothing;
             rng::AbstractRNG=Random.GLOBAL_RNG,
             max_steps::Int=typemax(Int),
             simulator::Simulator=HistoryRecorder(rng=rng, max_steps=max_steps),
             metadata = NamedTuple()
            )

    if initialstate == nothing && statetype(pomdp) != Nothing
        is = rand(rng, initial_belief)
    else
        is = initialstate
    end
    return POMDPSim(simulator, pomdp, policy, up, initial_belief, is, merge(NamedTuple(), metadata))
end

"""
    Sim(p::MDP, policy::Policy, metadata=(note="a note",))
    Sim(p::MDP, policy::Policy[, initialstate]; kwargs...)

Create a `Sim` object that represents a MDP simulation.
"""
function Sim(mdp::MDP,
             policy::Policy,
             initialstate=nothing;
             rng::AbstractRNG=Random.GLOBAL_RNG,
             max_steps::Int=typemax(Int),
             simulator::Simulator=HistoryRecorder(rng=rng, max_steps=max_steps),
             metadata = NamedTuple()
            )

    if initialstate == nothing && statetype(mdp) != Nothing
        is = POMDPs.initialstate(mdp, rng)
    else
        is = initialstate
    end
    return MDPSim(simulator, mdp, policy, is, merge(NamedTuple(), metadata))
end

POMDPs.simulate(s::POMDPSim) = simulate(s.simulator, s.pomdp, s.policy, s.updater, s.initial_belief, s.initialstate)
POMDPs.simulate(s::MDPSim) = simulate(s.simulator, s.mdp, s.policy, s.initialstate)

default_process(s::Sim, r::Float64) = (reward=r,)
default_process(s::Sim, hist::SimHistory) = default_process(s, discounted_reward(hist))

run_parallel(queue::AbstractVector; kwargs...) = run_parallel(default_process, queue; kwargs...)

"""
    run_parallel(queue::Vector{Sim})
    run_parallel(f::Function, queue::Vector{Sim})

Run `Sim` objects in `queue` in parallel and return results as a `DataFrame`.

By default, the `DataFrame` will contain the reward for each simulation and the metadata provided to the sim.

# Arguments
- `queue`: List of `Sim` objects to be executed
- `f`: Function to process the results of each simulation
This function should take two arguments, (1) the `Sim` that was executed and (2) the result of the simulation, by default a `SimHistory`. It should return a named tuple that will appear in the dataframe. See Examples below.

## Keyword Arguments
- `progress`: a `ProgressMeter.Progress` for showing progress through the simulations; `progress=false` will suppress the progress meter

# Examples

```julia
run_parallel(queue) do sim, hist
    return (n_steps=n_steps(hist), reward=discounted_reward(hist))
end
```
will return a dataframe with with the number of steps and the reward in it.
"""
function run_parallel(process::Function, queue::AbstractVector;
                      progress=Progress(length(queue), desc="Simulating..."),
                      proc_warn=true)

    #=
    frame_lines = pmap(progress, queue) do sim
        result = simulate(sim)
        return process(sim, result)
    end
    =#

    np = nprocs()
    if np == 1 && proc_warn
        @warn("""
             run_parallel(...) was started with only 1 process, so simulations will be run in serial.

             To supress this warning, use run_parallel(..., proc_warn=false).

             To use multiple processes, use addprocs() or the -p option (e.g. julia -p 4).
             """)
    end
    n = length(queue)
    i = 1
    prog = 0
    # based on the simple implementation of pmap here: https://docs.julialang.org/en/latest/manual/parallel-computing
    frame_lines = Vector{Any}(missing, n)
    nextidx() = (idx=i; i+=1; idx)
    prog_lock = ReentrantLock()
    @sync begin
        for p in 1:np
            if np == 1 || p != myid()
                @async begin
                    while true
                        idx = nextidx()
                        if idx > n
                            break
                        end
                        frame_lines[idx] = remotecall_fetch(p, queue[idx]) do sim
                            result = simulate(sim)
                            output = process(sim, result)
                            return merge(sim.metadata, output)
                        end
                        if progress isa Progress
                            lock(prog_lock)
                            update!(progress, prog+=1)
                            unlock(prog_lock)
                        end
                    end
                end
            end
        end
    end
    if progress isa Progress
        lock(prog_lock)
        finish!(progress)
        unlock(prog_lock)
    end

    return create_dataframe(frame_lines)
end

Base.run(queue::AbstractVector) = run(default_process, queue)

"""
    run(queue::Vector{Sim})
    run(f::Function, queue::Vector{Sim})

Run the `Sim` objects in `queue` on a single process and return the results as a dataframe.

See `run_parallel` for more information.
"""
function Base.run(process::Function, queue::AbstractVector; show_progress=true)
    lines = []
    if show_progress
        @showprogress for sim in queue
            result = simulate(sim)
            output = process(sim, result)
            line = merge(sim.metadata, output)
            push!(lines, line)
        end
    else
        for sim in queue
            result = simulate(sim)
            output = process(sim, result)
            line = merge(sim.metadata, output)
            push!(lines, line)
        end
    end
    return create_dataframe(lines)
end

function create_dataframe(lines::Vector)
    master = Dict{Symbol, AbstractVector}()
    for line in lines
        push_line!(master, line)
    end
    return DataFrame(master)
end

function push_line!(d::Dict{Symbol, AbstractVector}, line::NamedTuple)
    if isempty(d)
        len = 0
    else
        len = length(first(values(d)))
    end
    for (key, val) in pairs(line)
        T = typeof(val)
        if !haskey(d, key)
            d[key] = Vector{Union{T,Missing}}(missing, len)
        end
        data = d[key]
        if !isa(val,eltype(data))
            d[key] = convert(Array{promote_type(typeof(val), eltype(data)),1}, data)
        end
        push!(d[key], val)
    end
    for da in values(d)
        if length(da) < len + 1
            push!(da, missing)
        end
    end
    return d
end