export Spectrum, AlphaSpectrum
export AlphaSpectrum
export example_alpha_spectrum, to_histogram

abstract type Spectrum end

find_peak(s::Spectrum) = []
slice(s::Spectrum) = Spectrum()
to_histogram(s::Spectrum) = nothing

@static if @isdefined(DEBUG) || @isdefined(TEST) ALPHA_SPECTRUM_TEST_FILE = "Alpha2.csv" end

"""spectrum from a solid state sensor"""
struct AlphaSpectrum <: Spectrum
    channels::Vector{Int}
    energies::Vector{Float64}
    numchannels::Int32

    "constructor for default AlphaSpectrum"
    function AlphaSpectrum()
        chs = zeros(Int32,1024)
        e = zeros(1024)
        numchs = 512
        new(chs,e,512)
    end

    "alpha spectrum from a DataFrameRow instance"
    function AlphaSpectrum(row::DataFrameRow)
        numcolumns = length(row)
        # @show numcolumns
        indexfirstch = 4
        numchs = numcolumns - indexfirstch + 1
        chs = [row[i] for i=indexfirstch:numcolumns]
        RANGE_MEV = 10_000_000;
        b = convert(Float64,RANGE_MEV) / convert(Float64,numcolumns)
        es = [b * i for i=1:numcolumns-3]
        new(chs,es,numchs)
    end
end

# abstract type SpectrumSource end
# get_spectra(source::SpectrumSource) = Vector{Spectrum}
# alphaDataFrame = CSV.File("/home/b/Data/Spectra/Alpha2.csv"; header=false) |> DataFrame
# return array of alpha spectra
"""read and return an array of alpha spectra from a CSV file"""
function read_alpha_spectrum_file(file)
    frame = CSV.File(file; header=false) |> DataFrame
    num_rows = nrow(frame)
    ret_alpha_spects = [AlphaSpectrum(frame[i,:]) for i in 1:num_rows]
    return ret_alpha_spects
end

function to_histogram(as::AlphaSpectrum)
    len = as.numchannels;
    edges = collect(1:len+1)
    return hist = StatsBase.Histogram(edges,as.channels)
end

function example_alpha_spectrum()
    asdatafile = joinpath(DetectAlpha.TEST_DATA_FOLDER,ALPHA_SPECTRUM_TEST_FILE)
    return read_alpha_spectrum_file(asdatafile)
end


__precompile__()

module PersonalBanking

export main

using ExcelReaders, DataFrames, CSV, StringDistances

include(joinpath(Pkg.dir("PersonalBanking"), "src", "categorise.jl"))

# const rawdata = "/home/yakir/documents/money/rawdata"
const rawdata = "rawdata"

function cleandescription(X)
    x = lowercase(X)
    x = replace(x, r"ö", 'o')
    x = replace(x, r"ä", 'a')
    x = replace(x, r"å", 'a')
    x = replace(x, r"[^a-z0-9]", ' ')
    join(unique(split(x)), ' ')
end

function fixdescription(x)::String
    m = match(r"^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\s?(.*)", x)
    if m ≠ nothing
        strip(first(m.captures))
    else
        strip(x)
    end
end

function excelskandia(file)
    f = openxl(file)
    sheetname = "Kontoutdrag"
    sheet = f.workbook[:sheet_by_name](sheetname)
    startrow, startcol, endrow, endcol = ExcelReaders.convert_args_to_row_col(sheet)
    a = readxl(f, "$(sheetname)!A5:C$endrow")
    description = cleandescription.(fixdescription.(string.(a[:,2])))
    df = DataFrame(date = Date.(strip.(a[:,1])), description = description, amount = float.(a[:,3]))
    sort!(df, :date)
end

fixdate(x) = Date(strip(x)) - Date(0,1,1) + Date(2000,1,1)

function excelswedbank(file)
    f = openxl(file)
    sheetname = "Blad1"
    sheet = f.workbook[:sheet_by_name](sheetname)
    startrow, startcol, endrow, endcol = ExcelReaders.convert_args_to_row_col(sheet)
    date = readxl(f, "$(sheetname)!E6:E$endrow")
    description = readxl(f, "$(sheetname)!G6:G$endrow")
    amount = readxl(f, "$(sheetname)!I6:I$endrow")
    description = cleandescription.(string.(vec(description)))
    df = DataFrame(date = fixdate.(vec(date)), description = description, amount = float.(vec(amount)))
    sort!(df, :date)
end

fixsum(x) = parse(Float64, replace(filter(!isspace, x), ',', '.'))

function csvswedbank(file)
    a = CSV.read(file, delim = ';', header = ["description", "transaktionsdatum", "date", "amount", "saldo"], missingstring = "-")
    dropmissing!(a)
    disallowmissing!(a)
    a[:date] = Date.(a[:date])
    a[:amount] = fixsum.(a[:amount])
    a[:description] .= cleandescription.(a[:description])
    sort!(a[:, [:date, :description, :amount]], :date)
    # a = CSV.read(file, delim = ';', transforms = Dict{Int, Function}(1 => x -> strip(string(x)), 2 => x -> Date(x), 3 => identity, 4 => fixsum, 5 => identity), header = ["description", "a", "date", "amount", "b"])
end

function loadbank(person, ext, file)
    if person == "yakir"
        excelskandia(file)
    else
        if ext == ".csv"
            csvswedbank(file)
        else
            excelswedbank(file)
        end
    end
end

function overlap(t1, t2)
    date, description, amount = convert(Array, t1[end, :])
    date < t2[1, :date] && return 1
    i1 = findfirst(x -> x ≥ date - Dates.Day(1), t2[:date])
    i2 = findlast(x -> x ≤ date + Dates.Day(1), t2[:date])
    i1 == i2 && return i1 + 1
    candidates = filter(i -> amount - 1 ≤ t2[i, :amount] ≤ amount + 1, i1:i2)
    isempty(candidates) && return 1
    length(candidates) == 1 && return candidates[1] + 1
    _, i = findmax(compare(Hamming(), description, t2[candidate, :description]) for candidate in candidates)
    candidates[i] + 1
end

function concatenate!(t1, t2)
    t1, t2 = t1[end, :date] < t2[end, :date] ? (t1, t2) : (t2, t1)
    i = overlap(t1, t2)
    append!(t1, t2[i:end, :])
end

function getall(folder)
    all = Dict("yakir" => Dict{String, DataFrame}(), "ninna" => Dict{String, DataFrame}())
    for f in readdir(folder)
        if f[1] ≠ '.'
            file = joinpath(folder, f)
            person, account, rest = split(f, '_')
            _, ext = splitext(rest)
            bank = loadbank(person, ext, file)
            bank[:account] = Symbol(account)
            bank[:person] = Symbol(person)
            if haskey(all[person], account)
                concatenate!(all[person][account], bank)
            else
                all[person][account] = bank
            end
        end
    end
    all
end

function gather(a)
    table = DataFrame(date = Date(1), description = "", amount = 0.0, account = :a, person = :b)
    deleterows!(table, 1)
    for person in values(a), account in values(person)
        append!(table, account)
    end
    sort!(table, :date)
end

function main(folder)
    a = getall(joinpath(folder, rawdata))
    table = gather(a)
    categorise!(table, folder)
end

end # module

#=badwords = ["malmo", "lund", "till", "montreal", "koebenhavn", "haifa", "fran", "milano", "francisco", "san", "kastrup", "stockholm", "malm", "tel", "aviv", "triangeln", "durham", "the", "nya", "och", "eslov", "via", "brugge", "goteborg", "firenze", "molndal", "oslo", "ltd", "com", "www", "vaxjo", "ab"]

goodword(w) = length(w) > 1 && w ∉ badwords

function break2labels(Str)
    str = lowercase(Str)
    str = replace(str, r"ö", 'o')
    str = replace(str, r"ä", 'a')
    str = replace(str, r"å", 'a')
    str = replace(str, r"[^a-z0-9]", ' ')
    txts::Vector{String} = split(str)
    utxts = unique(strip.(txts))
    # filter!(goodword, utxts)
end

regexit(words) = isempty(words) ? r"" : Regex(string("[", join(words, "|"), "]"))

a = main()
d = a[:description]

uds = unique(break2labels(u) for u in d)

for i in d
    s = 0
    words = break2labels(i)
    for u in uds
        if
            s += 1
        end
    end
    println(s)
end

find(uds,
# a[:label] = join.(break2labels.(a[:description]), " ")

a = main()
d = a[:description]
uds = unique(break2labels(u) for u in d)
# sum(length, uds)
words = cat(1, uds...)
x = countmap(words)
w = collect(keys(x))
c = collect(values(x))
o = sortperm(c, rev=true)
y = OrderedDict(w[i] => c[i] for i in o)

x = SortedDict(x)

y = sort(x, by=last)

i = 105
d[i]
break2labels(d[i])

for j in 1:length(uds)
    u = uds[j]
    n = length(u)
    if n > 1
        for i in 1:n - 1
            if u[1:i] ∉ uds
                uds[j] = u[1:i]
            end
        end
    end
end

sum(length, uds)

a = main()

msg = """
Liability
    1) Transfer
    2) Other

Expense
    Food
        3) Cafe
        4) Groceries
        5) Restaurant

    Alcohol
        6) Store
        7) Bar

    Health
        8) Sport
        9) Other

    Transport
        10) Private
        11) Public

    FixedCost
        12) Rent
        13) Internet
        14) Electricity
        15) Phone
        16) Daycare
        17) CSN

    18) Shopping

    19) Fun

    20) Travel

    21) Other

Revenue
    22) ParentalSupport
    23) Salary
    24) ExternalTransfer
    25) Other

26) Undefined
"""

categories = [:Transfer, :Cafe, :Groceries, :Restaurant, :Systembolaget, :Bar, :Sport, :OtherHealth, :PrivateTransportation, :PublicTransportation, :Rent, :Internet, :Electricity, :Phone, :Daycare, :Electricity, :Phone, :PeriodsKort, :CSN, :Shopping, :Fun, :Travel, :OtherExpenses, :Föräldrarstöd, :Salary, :ExternalTransfer, :Undefined]
c = zeros(Int, nrow(a))
for i in 1:1#nrow(a)
    println(a[i,:])
    println(msg)
    x = readline()
    c[i] = parse(Int, x)
end

b = filter(x -> x[:category] <: Undefined, a)
open("tmp.txt", "w") do o
    println.(o, b[:description])
end

const days = Dates.Day.([0, -1, 1])
function findtransfer(a, i)::Union{Missing, Int}
    for d in days
        date = a[i, :date] + d
        i1 = findfirst(x -> x == date, a[:date])
        i1 == 0 && continue
        i2 = findlast(x -> x == date, a[:date])
        i2 == 0 && continue
        for ii in i1:i2
            if a[ii, :amount] == -a[i, :amount]
                return ii
            end
        end
    end
    return missing
end

tokill = Vector{Pair{Int, Int}}()
for i in 1:nrow(a)
    if a[i, :amount] > 0
        ii = findtransfer(a, i)
        if !ismissing(ii)
            push!(tokill, Pair(i, ii))
        end
    end
end

function findtransfer(r, a)::Union{Missing, Int}
    r[:amount] ≥ 0 && return missing
    i1 = findfirst(x -> x[:date] ≥ r[:date] - d, eachrow(a))
    i2 = findlast(x -> x[:date] ≤ r[:date] + d, eachrow(a))
    candidates = filter(i -> a[i, :amount] == -r[:amount], i1:i2)
    isempty(candidates) && return missing
    _, i = findmax(compare(Hamming(), r[:description], a[candidate, :description]) for candidate in candidates)
    candidates[i]
end

transfer(x) = r"(?:Insättning|plutt|SUGARMAMA|yakirhuvud2yakirbuffert|ninnalön2ninnahuvud|buffert|Övf|unknown|Överföring|swish|Insättning)"i(x)

i1 = findfirst(x -> x[:amount] > 0, eachrow(a))
i2 =
struct Transfer
    i1::Int
    i2::Int
    d::Dates.Day
end
ps = Dict{Float64, Vector{Transfer}}()
for i in i1:i2 - 1
    if a[i, :amount] > 0 && transfer((a[i, :description]))
        for j in i + 1:i2
            if a[i, :amount] == -a[j, :amount] && transfer(a[j, :description])
                Δ = abs(a[i, :date] - a[j, :date])
                if haskey(ps, a[j, :amount])
                    push!(ps[a[i, :amount]], Transfer(i, j, Δ))
                else
                    ps[a[i, :amount]] = [Transfer(i, j, Δ)]
                end
            end
        end
    end
end

for (k,v) in ps
    while !isempty(v)
        _, i = findmin(x.d for x in v)
        m = v[i]
        push!(tokill, Pair(m.i1, m.i2))
        filter!(x -> x.i1 ≠ m.i1 && x.i2 ≠ m.i2, v)
    end
end

tokill = Vector{Pair{Int, Int}}()
for i1 = 1:nrow(a)
    _i2 = findlast(j -> a[j, :date] ≤ a[i, :date] + d, rng)

    function distance(a, i, j)
        if a[i, :amount] == -a[j, :amount] && transfer(a[j, :description])
            abs(a[i, :date] - a[j, :date])
        else
            Dates.Day(10000)
        end
    end

    d = Dates.Day(3)
    tokill = Vector{Pair{Int, Int}}()
    for i in 1:nrow(a) - 1
        if a[i, :amount] < 0 && transfer((a[i, :description]))
            rng = i+1:nrow(a)
            _i2 = findlast(j -> a[j, :date] ≤ a[i, :date] + d, rng)
            if _i2 ≠ 0
                rng = rng[1:_i2]
                v, j = findmin(distance(a, i, j) for j in rng)
                if v ≠ Dates.Day(10000)
                    push!(tokill, i => rng[j])
                end
            end
        end
    end

    for i in 1:nrow(a)
        j
        d = [distance(a, i, j) for i = 1:nrow(a), j = 1:nrow(a)]

        for r1 in eachrow(a), r2 in eachrow(a)

            d = Dates.Day(1)
            tokill = Vector{Pair{Int, Int}}()
            for (i, r) in enumerate(eachrow(a))
                j = findtransfer(r, a)
                if !ismissing(j) && j ∉ last.(
                                              push!(tokill, i => j)
                                          end
                                      end

                                      open("tmp.txt", "w") do o
                                          for k in tokill
                                              i,j = k
                                              println(o, i, j, a[[i,j], :])
                                          end
                                      end

a = ["Comment" "Category";
     "2018-06-10 ICA SUPERMARKET SODER, MALMO" "Expense:Food:Groceries";
     "2018-06-05 SKANETRAFIKEN, MALMO C" "Expense:Transport:Public"]
=#


using DataStructures
using Base.Meta

import Base.ImmutableDict

const EClassId = Int64

# struct ENode{T, M}
mutable struct ENode{T}
    head::Any
    args::Vector{EClassId}
    # metadata::M
    # the nodes where it came from
    # proof_src::Vector{Tuple{Rule, ENode, Int}}
    # the enodes that were generated by this enode
    # proof_trg::Vector{Tuple{Rule, ENode, Int}}
    # age of the egraph when this enode was created
    # age::Int
    hash::Ref{UInt} # hash cache
end

# function ENode{T}(head, c_ids::AbstractVector{EClassId}, ps=[], pt=[], age=0) where {T}
function ENode{T}(head, c_ids::AbstractVector{EClassId}) where {T}
        # static_args = MVector{length(c_ids), Int64}(c_ids...)
    # m = metadata(e)
    # ENode{T}(head, c_ids, ps, pt, age, Ref{UInt}(0))
    ENode{T}(head, c_ids, Ref{UInt}(0))
end

ENode(a) = ENode{typeof(a)}(a, EClassId[])

ENode(a::ENode) =
    error("constructor of ENode called on enode. This should never happen")

function Base.:(==)(a::ENode, b::ENode)
    isequal(a.args, b.args) &&
    isequal(a.head, b.head)
end

# ===============================================================
# proof functions
# ===============================================================

# function setage!(n::ENode, age::Int)
#     n.age = age
# end

# function addproofsrc!(n::ENode{T}, rule::Rule, src::ENode, age::Int) where {T}
#     push!(n.proof_src, (rule, src, age))
# end

# function addprooftrg!(n::ENode{T}, rule::Rule, trg::ENode, age::Int) where {T}
#     push!(n.proof_trg, (rule, trg, age))
# end

# function hasproofdata(n::ENode)
#     !(isempty(n.proof_src) && isempty(n.proof_trg))
# end

# function mergeproof!(target::ENode, source::ENode)
#     append!(target.proof_src, source.proof_src)
#     append!(target.proof_trg, source.proof_trg)
# end

# This optimization comes from SymbolicUtils
# The hash of an enode is cached to avoid recomputing it.
# Shaves off a lot of time in accessing dictionaries with ENodes as keys.
function Base.hash(t::ENode{T}, salt::UInt) where {T}
    !iszero(salt) && return hash(hash(t, zero(UInt)), salt)
    h = t.hash[]
    !iszero(h) && return h
    # h′ = hash(t.args,  hash(t.metadata, hash(t.head, hash(T, salt))))
    h′ = hash(t.args,  hash(t.head, hash(T, salt)))
    t.hash[] = h′
    return h′
end

TermInterface.arity(n::ENode) = length(n.args)
# TermInterface.metadata(n::ENode) = n.metadata

termtype(x::ENode{T}) where T = T

function Base.show(io::IO, x::ENode{T}) where {T}
    print(io, "ENode{$T}(", x.head)
    n = arity(x)
    if n == 0
        print(io, ")")
        return
    else
        print(io, " ")
    end
    for i ∈ 1:n
        if i < n
            print(io, x.args[i], " ")
        else
            print(io, x.args[i], ")")
        end
    end
end


"""
printFreq: defaults to 0 => prints every 10%. turn off printing by
           setting to -1.
joint_update_r (Bool): If true, updates each `r_{ik}` sequentially.
                       If false, updates `r` via a metropolis step
                       (as a matrix), by flipping one random bit in `r`.
"""
function fit_fs!(init::StateFS, c::ConstantsFS, d::DataFS;
                 nmcmc::Int, nburn::Int,
                 tuners::Union{Nothing, TunersFS}=nothing,
                 monitors=[[:theta__Z, :theta__v, :theta__alpha,
                            :omega, :r, :theta__lam, :W_star, :theta__eta,
                            :theta__W, :theta__delta, :theta__sig2]],
                 fix::Vector{Symbol}=Symbol[],
                 thins::Vector{Int}=[1],
                 thin_dden::Int=1,
                 printFreq::Int=0, flushOutput::Bool=false,
                 computeDIC::Bool=false, computeLPML::Bool=false,
                 computedden::Bool=false,
                 sb_ibp::Bool=false,
                 use_repulsive::Bool=true, joint_update_Z::Bool=true,
                 joint_update_r::Bool=false,
                 _r_marg_lam_freq::Float64=1.0,
                 verbose::Int=1, time_updates::Bool=false, Z_thin::Int=0,
                 seed::Int=-1)

  # Set random seed if needed
  if seed >= 0
    Random.seed!(seed)
  end

  # We don't want to use noisy distribution.
  # Assert that all eps == 0
  if any(init.theta.eps .!= 0)
    msg = "WARNING: `init.theta.eps`: $(init.theta.eps) contains non-zeros values! "
    msg *= "Setting all eps to 0!"
    println(msg)

    # Set eps == 0
    init.theta.eps .= 0
  end

  @assert 0 <= _r_marg_lam_freq <= 1

  @assert 0 <= Z_thin
  if Z_thin == 0
    Z_thin = d.data.J
  end

  if verbose >= 1
    fixed_vars_str = join(fix, ", ")
    if fixed_vars_str == ""
      fixed_vars_str = "nothing"
    end
    println("fixing: $fixed_vars_str")
    println("Use stick-breaking IBP: $(sb_ibp)")
    println("joint_update_Z: $(joint_update_Z)")
    println("use_repulsive: $(use_repulsive)")
    println("Z_thin: $(Z_thin)")
    println("joint_update_r: $(joint_update_r)")
  end

  @assert printFreq >= -1
  if printFreq == 0
    numPrints = 10
    printFreq = Int(ceil((nburn + nmcmc) / numPrints))
  end

  # Initialize tuners if needed
  if tuners == nothing
    t = Tuners(d.data.y, c.constants.K)
    TunersFS(t, init.theta, d.data.X)
  end

  function printMsg(iter::Int, msg::String)
    if printFreq > 0 && iter % printFreq == 0
      print(msg)
    end
  end

  # Loglike
  loglike = Float64[]

  # Instantiate (but not initialize) CPO stream
  if computeLPML
    cpoStream = MCMC.CPOstream{Float64}()
  end

  # DIC
  if computeDIC
    local tmp = DICparam(p=deepcopy(d.data.y),
                         mu=deepcopy(d.data.y),
                         sig=[zeros(Float64, d.data.N[i]) for i in 1:d.data.I],
                         y=deepcopy(d.data.y))
    dicStream = MCMC.DICstream{DICparam}(tmp)

    function updateParams(d::MCMC.DICstream{DICparam}, param::DICparam)
      d.paramSum.p += param.p
      d.paramSum.mu += param.mu
      d.paramSum.sig += param.sig
      d.paramSum.y += param.y

      return
    end

    function paramMeanCompute(d::MCMC.DICstream{DICparam})::DICparam
      return DICparam(d.paramSum.p ./ d.counter,
                      d.paramSum.mu ./ d.counter,
                      d.paramSum.sig ./ d.counter,
                      d.paramSum.y ./ d.paramSum.y)
    end

    function loglikeDIC(param::DICparam)::Float64
      ll = 0.0

      for i in 1:d.data.I
        for j in 1:d.data.J
          for n in 1:d.data.N[i]
            y_inj_is_missing = (d.data.m[i][n, j] == 1)

            # NOTE: Refer to `../compute_loglike.jl` for reasoning.
            if y_inj_is_missing

              # Compute p(m_inj | y_inj, theta) term.
              ll += log(param.p[i][n, j])
            end

            # Compute p(y_inj | theta) term.
            ll += logpdf(Normal(param.mu[i][n, j], param.sig[i][n]),
                         param.y[i][n, j])
          end
        end
      end

      return ll
    end

    function convertStateToDicParam(s::State)::DICparam
      p = [[prob_miss(s.y_imputed[i][n, j], c.constants.beta[:, i])
            for n in 1:d.data.N[i], j in 1:d.data.J]
           for i in 1:d.data.I]

      mu = [[mus(i, n, j, s, c.constants, d.data)
             for n in 1:d.data.N[i], j in 1:d.data.J]
            for i in 1:d.data.I]

      sig = [fill(sqrt(s.sig2[i]), d.data.N[i]) for i in 1:d.data.I]

      y = deepcopy(s.y_imputed)

      return DICparam(p, mu, sig, y)
    end
  end

  dden = Matrix{Vector{Float64}}[]

  function update!(s::StateFS, iter::Int, out)
    update_state_feature_select!(s, c, d, tuners,
                                 ll=loglike, fix=fix,
                                 use_repulsive=use_repulsive,
                                 joint_update_Z=joint_update_Z,
                                 joint_update_r=joint_update_r,
                                 sb_ibp=sb_ibp, time_updates=time_updates,
                                 r_marg_lam_freq=_r_marg_lam_freq,
                                 Z_thin=Z_thin)

    if computedden && iter > nburn && (iter - nburn) % thin_dden == 0
      append!(dden,
              [[datadensity(i, j, s.theta, c.constants, d.data)
                for i in 1:d.data.I, j in 1:d.data.J]])
    end

    if computeLPML && iter > nburn
      # Inverse likelihood for each data point
      like = [[compute_like(i, n, s.theta, c.constants, d.data)
               for n in 1:d.data.N[i]] for i in 1:d.data.I]

      # Update (or initialize) CPO
      MCMC.updateCPO(cpoStream, vcat(like...))

      # Add to printMsg
      printMsg(iter, " -- LPML: $(MCMC.computeLPML(cpoStream))")
    end

    if computeDIC && iter > nburn
      # Update DIC
      MCMC.updateDIC(dicStream, s.theta, updateParams,
                     loglikeDIC, convertStateToDicParam)

      # Add to printMsg
      printMsg(iter, " -- DIC: $(MCMC.computeDIC(dicStream, loglikeDIC,
                                                 paramMeanCompute))")
    end

    printMsg(iter, "\n")
    if flushOutput
      flush(stdout)
    end
  end

  if isinf(compute_loglike(init.theta, c.constants, d.data, normalize=true))
    println("Warning: Initial state yields likelihood of zero.")
    msg = """
    It is likely the case that the initialization of missing values
    is not consistent with the provided missing mechanism. The MCMC
    will almost certainly reject the initial values and sample new
    ones in its place.
    """
    println(join(split(msg), " "))
  else
    println("")
  end

  out, lastState = MCMC.gibbs(init, update!, monitors=monitors,
                              thins=thins, nmcmc=nmcmc, nburn=nburn,
                              printFreq=printFreq,
                              loglike=loglike, flushOutput=flushOutput,
                              printlnAfterMsg=false)

  mega_out = Dict{Symbol, Any}(:samples => out,
                               :lastState => lastState,
                               :loglike => loglike,
                               :c => c,
                               :init => init)

  if computeDIC || computeLPML
    LPML = computeLPML ? MCMC.computeLPML(cpoStream) : NaN
    Dmean, pD = computeDIC ? MCMC.computeDIC(dicStream,
                                             loglikeDIC,
                                             paramMeanCompute,
                                             return_Dmean_pD=true) : (NaN, NaN)
    metrics = Dict(:LPML => LPML,
                   :DIC => Dmean + pD,
                   :Dmean => Dmean,
                   :pD => pD)
    println()
    println("metrics:")
    for (k, v) in metrics
      mega_out[k] = v
      println("$k => $v")
    end
  end

  if computedden
    mega_out[:dden] = dden
  end

  mega_out[:nburn] = nburn
  mega_out[:Z_thin] = Z_thin
  mega_out[:m] = Matrix{Bool}.(d.data.m)

  return mega_out
end


using GeometricEquations
using GeometricEquations: symplectic_matrix
using Test

include("functions.jl")
include("initial_conditions.jl")

@testset "$(rpad("Ordinary Differential Equations (ODE)",80))" begin

    ode  = ODE(ode_v, NullInvariants(), NullParameters(), NullPeriodicity())
    ode1 = ODE(ode_v)
    ode2 = ODE(ode_v; invariants=NullInvariants())
    ode3 = ODE(ode_v; parameters=NullParameters())
    ode4 = ODE(ode_v; periodicity=NullPeriodicity())

    @test ode == ode1
    @test ode == ode2
    @test ode == ode3
    @test ode == ode4

    @test hash(ode) == hash(ode1)
    @test hash(ode) == hash(ode2)
    @test hash(ode) == hash(ode3)
    @test hash(ode) == hash(ode4)

    @test functions(ode) == NamedTuple{(:v,)}((ode_v,))
    @test solutions(ode) == NamedTuple()

    @test parameters(ode) == NullParameters()
    @test invariants(ode) == NullInvariants()
    @test periodicity(ode) == NullPeriodicity()

    @test hasvectorfield(ode) == true
    @test hassolution(ode) == false
    @test hasprimary(ode) == false
    @test hassecondary(ode) == false

    @test hasinvariants(ode) == false
    @test hasparameters(ode) == false
    @test hasperiodicity(ode) == false

    @test hashamiltonian(ode) == false
    @test haslagrangian(ode) == false

    funcs = functions(ode)

    @test_nowarn funcs.v(t₀, x₀, zero(x₀), NullParameters())

    funcs = functions(ode, NullParameters())

    @test_nowarn funcs.v(t₀, x₀, zero(x₀))

end

@testset "$(rpad("Split Ordinary Differential Equations (SODE)",80))" begin

    sode  = SODE(sode_v, nothing, NullInvariants(), NullParameters(), NullPeriodicity())
    sode1 = SODE(sode_v)
    sode2 = SODE(sode_v; invariants=NullInvariants())
    sode3 = SODE(sode_v; parameters=NullParameters())
    sode4 = SODE(sode_v; periodicity=NullPeriodicity())

    @test sode == sode1
    @test sode == sode2
    @test sode == sode3
    @test sode == sode4

    @test hash(sode) == hash(sode1)
    @test hash(sode) == hash(sode2)
    @test hash(sode) == hash(sode3)
    @test hash(sode) == hash(sode4)

    @test functions(sode) == NamedTuple{(:v,)}((sode_v,))
    @test solutions(sode) == NamedTuple()

    @test parameters(sode) == NullParameters()
    @test invariants(sode) == NullInvariants()
    @test periodicity(sode) == NullPeriodicity()

    @test hasvectorfield(sode) == true
    @test hassolution(sode) == false
    @test hasprimary(sode) == false
    @test hassecondary(sode) == false

    @test hasinvariants(sode) == false
    @test hasparameters(sode) == false
    @test hasperiodicity(sode) == false

    @test hashamiltonian(sode) == false
    @test haslagrangian(sode) == false

    sode  = SODE(sode_v, sode_q, NullInvariants(), NullParameters(), NullPeriodicity())
    sode1 = SODE(sode_v, sode_q)
    sode2 = SODE(sode_v, sode_q; invariants=NullInvariants())
    sode3 = SODE(sode_v, sode_q; parameters=NullParameters())
    sode4 = SODE(sode_v, sode_q; periodicity=NullPeriodicity())

    @test sode == sode1
    @test sode == sode2
    @test sode == sode3
    @test sode == sode4

    @test hash(sode) == hash(sode1)
    @test hash(sode) == hash(sode2)
    @test hash(sode) == hash(sode3)
    @test hash(sode) == hash(sode4)

    @test functions(sode) == NamedTuple{(:v,)}((sode_v,))
    @test solutions(sode) == NamedTuple{(:q,)}((sode_q,))

    @test parameters(sode) == NullParameters()
    @test invariants(sode) == NullInvariants()
    @test periodicity(sode) == NullPeriodicity()

    @test hasvectorfield(sode) == true
    @test hassolution(sode) == true
    @test hasprimary(sode) == false
    @test hassecondary(sode) == false

    @test hasinvariants(sode) == false
    @test hasparameters(sode) == false
    @test hasperiodicity(sode) == false

    @test hashamiltonian(sode) == false
    @test haslagrangian(sode) == false

    sode  = SODE(nothing, sode_q, NullInvariants(), NullParameters(), NullPeriodicity())
    sode1 = SODE(nothing, sode_q)
    sode2 = SODE(nothing, sode_q; invariants=NullInvariants())
    sode3 = SODE(nothing, sode_q; parameters=NullParameters())
    sode4 = SODE(nothing, sode_q; periodicity=NullPeriodicity())

    @test sode == sode1
    @test sode == sode2
    @test sode == sode3
    @test sode == sode4

    @test hash(sode) == hash(sode1)
    @test hash(sode) == hash(sode2)
    @test hash(sode) == hash(sode3)
    @test hash(sode) == hash(sode4)

    @test functions(sode) == NamedTuple()
    @test solutions(sode) == NamedTuple{(:q,)}((sode_q,))

    @test parameters(sode) == NullParameters()
    @test invariants(sode) == NullInvariants()
    @test periodicity(sode) == NullPeriodicity()

    @test hasvectorfield(sode) == false
    @test hassolution(sode) == true
    @test hasprimary(sode) == false
    @test hassecondary(sode) == false

    @test hasinvariants(sode) == false
    @test hasparameters(sode) == false
    @test hasperiodicity(sode) == false

    @test hashamiltonian(sode) == false
    @test haslagrangian(sode) == false

end

@testset "$(rpad("Partitioned Ordinary Differential Equations (PODE)",80))" begin

    pode  = PODE(pode_eqs..., NullInvariants(), NullParameters(), NullPeriodicity())
    pode1 = PODE(pode_eqs...)
    pode2 = PODE(pode_eqs...; invariants=NullInvariants())
    pode3 = PODE(pode_eqs...; parameters=NullParameters())
    pode4 = PODE(pode_eqs...; periodicity=NullPeriodicity())

    @test pode == pode1
    @test pode == pode2
    @test pode == pode3
    @test pode == pode4

    @test hash(pode) == hash(pode1)
    @test hash(pode) == hash(pode2)
    @test hash(pode) == hash(pode3)
    @test hash(pode) == hash(pode4)

    @test functions(pode) == NamedTuple{(:v,:f)}(pode_eqs)
    @test solutions(pode) == NamedTuple()

    @test parameters(pode) == NullParameters()
    @test invariants(pode) == NullInvariants()
    @test periodicity(pode) == NullPeriodicity()

    @test hasvectorfield(pode) == true
    @test hassolution(pode) == false
    @test hasprimary(pode) == false
    @test hassecondary(pode) == false

    @test hasinvariants(pode) == false
    @test hasparameters(pode) == false
    @test hasperiodicity(pode) == false

    @test hashamiltonian(pode) == false
    @test haslagrangian(pode) == false

    funcs = functions(pode)

    @test_nowarn funcs.v(t₀, q₀, p₀, zero(q₀), NullParameters())
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(p₀), NullParameters())

    funcs = functions(pode, NullParameters())

    @test_nowarn funcs.v(t₀, q₀, p₀, zero(q₀))
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(p₀))

    # funcs = functions(pode)
    # @test funcs.v == pode_v == pode.v
    # @test funcs.f == pode_f == pode.f

    # rode = ODE(ode_v, t₀, [x₀])
    # code = convert(ODE, pode)
    # v₁ = zero(x₀)
    # v₂ = zero(x₀)
    # rode.v(rode.t₀, rode.q₀[begin], v₁)
    # code.v(code.t₀, code.q₀[begin], v₂)
    # @test v₁ == v₂

    # rode = SODE((v_sode_1, v_sode_2), t₀, [x₀])
    # code = convert(SODE, pode)
    # v₁ = zero(x₀)
    # v₂ = zero(x₀)
    # rode.v[1](rode.t₀, rode.q₀[begin], v₁)
    # code.v[1](code.t₀, code.q₀[begin], v₂)
    # @test v₁ == v₂
    # rode.v[2](rode.t₀, rode.q₀[begin], v₁)
    # code.v[2](code.t₀, code.q₀[begin], v₂)
    # @test v₁ == v₂

end

@testset "$(rpad("Implicit Ordinary Differential Equations (IODE)",80))" begin

    iode  = IODE(iode_eqs..., iode_v, iode_f, NullInvariants(), NullParameters(), NullPeriodicity())

    iode1 = IODE(iode_eqs...; v̄=iode_v)
    iode2 = IODE(iode_eqs...; v̄=iode_v, invariants=NullInvariants())
    iode3 = IODE(iode_eqs...; v̄=iode_v, parameters=NullParameters())
    iode4 = IODE(iode_eqs...; v̄=iode_v, periodicity=NullPeriodicity())
    iode5 = IODE(iode_eqs...; v̄=iode_v, f̄=iode_f)
    iode6 = IODE(iode_eqs...; v̄=iode_v, f̄=iode_f, invariants=NullInvariants())
    iode7 = IODE(iode_eqs...; v̄=iode_v, f̄=iode_f, parameters=NullParameters())
    iode8 = IODE(iode_eqs...; v̄=iode_v, f̄=iode_f, periodicity=NullPeriodicity())

    @test iode == iode1
    @test iode == iode2
    @test iode == iode3
    @test iode == iode4
    @test iode == iode5
    @test iode == iode6
    @test iode == iode7
    @test iode == iode8

    @test hash(iode) == hash(iode1)
    @test hash(iode) == hash(iode2)
    @test hash(iode) == hash(iode3)
    @test hash(iode) == hash(iode4)
    @test hash(iode) == hash(iode5)
    @test hash(iode) == hash(iode6)
    @test hash(iode) == hash(iode7)
    @test hash(iode) == hash(iode8)

    # @test functions(iode) == NamedTuple{(:ϑ,:f,:g,:v̄,:f̄)}(iode_eqs)
    @test solutions(iode) == NamedTuple()

    @test parameters(iode) == NullParameters()
    @test invariants(iode) == NullInvariants()
    @test periodicity(iode) == NullPeriodicity()

    @test hasvectorfield(iode) == true
    @test hassolution(iode) == false
    @test hasprimary(iode) == false
    @test hassecondary(iode) == false

    @test hasinvariants(iode) == false
    @test hasparameters(iode) == false
    @test hasperiodicity(iode) == false

    @test hashamiltonian(iode) == false
    @test haslagrangian(iode) == false

    funcs = functions(iode)

    @test_nowarn funcs.ϑ(t₀, q₀, p₀, zero(q₀), NullParameters())
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(q₀), NullParameters())
    @test_nowarn funcs.g(t₀, q₀, p₀, zero(q₀), NullParameters())

    funcs = functions(iode, NullParameters())

    @test_nowarn funcs.ϑ(t₀, q₀, p₀, zero(q₀))
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(q₀))
    @test_nowarn funcs.g(t₀, q₀, p₀, zero(q₀))

    # @test periodicity(iode) == zero(q₀)
    # @test initial_conditions(iode) == (t₀, [q₀], [p₀], [λ₀])

    # funcs = functions(iode)
    # @test funcs.ϑ == iode_ϑ == iode.ϑ
    # @test funcs.f == iode_f == iode.f
    # @test funcs.g == iode_g == iode.g
    # @test funcs.v̄ == iode_v == iode.v̄
    # @test funcs.f̄ == iode_f == iode.f̄

    # @test iode == similar(iode, t₀, q₀, p₀, λ₀)
    # @test iode == similar(iode, t₀, q₀, p₀)
    # @test iode == similar(iode, q₀, p₀)

end

@testset "$(rpad("Hamiltonian Ordinary Differential Equations (HODE)",80))" begin

    hode_eqs = (pode_v, pode_f, hode_ω, hode_h)

    hode  = HODE(pode_v, pode_f, hode_ω, hode_h, NullInvariants(), NullParameters(), NullPeriodicity())

    hode1 = HODE(hode_eqs...)
    hode2 = HODE(hode_eqs...; invariants=NullInvariants())
    hode3 = HODE(hode_eqs...; parameters=NullParameters())
    hode4 = HODE(hode_eqs...; periodicity=NullPeriodicity())

    @test hode == hode1
    @test hode == hode2
    @test hode == hode3
    @test hode == hode4

    @test hash(hode) == hash(hode1)
    @test hash(hode) == hash(hode2)
    @test hash(hode) == hash(hode3)
    @test hash(hode) == hash(hode4)

    @test functions(hode) == NamedTuple{(:v,:f,:poisson,:h)}((pode_v, pode_f, hode_ω, hode_h))
    @test solutions(hode) == NamedTuple()

    @test parameters(hode) == NullParameters()
    @test invariants(hode) == NullInvariants()
    @test periodicity(hode) == NullPeriodicity()

    @test hasvectorfield(hode) == true
    @test hassolution(hode) == false
    @test hasprimary(hode) == false
    @test hassecondary(hode) == false

    @test hasinvariants(hode) == false
    @test hasparameters(hode) == false
    @test hasperiodicity(hode) == false

    @test hashamiltonian(hode) == true
    @test haslagrangian(hode) == false

    funcs = functions(hode)

    @test_nowarn funcs.v(t₀, q₀, p₀, zero(q₀), NullParameters())
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(p₀), NullParameters())
    @test_nowarn funcs.h(t₀, q₀, p₀, NullParameters())
    @test_nowarn funcs.poisson(t₀, q₀, p₀, zeros(2,2), NullParameters())

    funcs = functions(hode, NullParameters())

    @test_nowarn funcs.v(t₀, q₀, p₀, zero(q₀))
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(p₀))
    @test_nowarn funcs.h(t₀, q₀, p₀)
    @test_nowarn funcs.poisson(t₀, q₀, p₀, zeros(2,2))

    # funcs = functions(hode)
    # @test funcs.v == pode_v == hode.v
    # @test funcs.f == pode_f == hode.f
    # @test funcs.h == pode_h == hode.hamiltonian

    # @test hode == similar(hode, t₀, q₀, p₀)
    # @test hode == similar(hode, q₀, p₀)

    # rode = ODE(ode_v, t₀, [x₀])
    # code = convert(ODE, hode)
    # v₁ = zero(x₀)
    # v₂ = zero(x₀)
    # rode.v(rode.t₀, rode.q₀[begin], v₁)
    # code.v(code.t₀, code.q₀[begin], v₂)
    # @test v₁ == v₂

    # pode = convert(PODE, hode)
    # v₁ = zero(q₀)
    # v₂ = zero(q₀)
    # f₁ = zero(p₀)
    # f₂ = zero(p₀)
    # hode.v(hode.t₀, hode.q₀[begin], hode.p₀[begin], v₁)
    # pode.v(pode.t₀, pode.q₀[begin], pode.p₀[begin], v₂)
    # hode.f(hode.t₀, hode.q₀[begin], hode.p₀[begin], f₁)
    # pode.f(pode.t₀, pode.q₀[begin], pode.p₀[begin], f₂)
    # @test v₁ == v₂
    # @test f₁ == f₂

    # rode = SODE((v_sode_1, v_sode_2), t₀, [x₀])
    # code = convert(SODE, hode)
    # v₁ = zero(x₀)
    # v₂ = zero(x₀)
    # rode.v[1](rode.t₀, rode.q₀[begin], v₁)
    # code.v[1](code.t₀, code.q₀[begin], v₂)
    # @test v₁ == v₂
    # rode.v[2](rode.t₀, rode.q₀[begin], v₁)
    # code.v[2](code.t₀, code.q₀[begin], v₂)
    # @test v₁ == v₂

end

@testset "$(rpad("Lagrangian Ordinary Differential Equations (LODE)",80))" begin

    lode_eqs = (iode_ϑ, iode_f, iode_g, lode_ω, lode_l)

    lode = LODE(iode_ϑ, iode_f, iode_g, lode_ω, iode_v, iode_f, lode_l, NullInvariants(), NullParameters(), NullPeriodicity())

    lode1 = LODE(lode_eqs..., v̄=iode_v)
    lode2 = LODE(lode_eqs..., v̄=iode_v, invariants=NullInvariants())
    lode3 = LODE(lode_eqs..., v̄=iode_v, parameters=NullParameters())
    lode4 = LODE(lode_eqs..., v̄=iode_v, periodicity=NullPeriodicity())

    @test lode == lode1
    @test lode == lode2
    @test lode == lode3
    @test lode == lode4

    @test hash(lode) == hash(lode1)
    @test hash(lode) == hash(lode2)
    @test hash(lode) == hash(lode3)
    @test hash(lode) == hash(lode4)

    @test functions(lode) == NamedTuple{(:ϑ,:f,:g,:ω,:v̄,:f̄,:l)}((iode_ϑ, iode_f, iode_g, lode_ω, iode_v, iode_f, lode_l))
    @test solutions(lode) == NamedTuple()

    @test parameters(lode) == NullParameters()
    @test invariants(lode) == NullInvariants()
    @test periodicity(lode) == NullPeriodicity()

    @test hasvectorfield(lode) == true
    @test hassolution(lode) == false
    @test hasprimary(lode) == false
    @test hassecondary(lode) == false

    @test hasinvariants(lode) == false
    @test hasparameters(lode) == false
    @test hasperiodicity(lode) == false

    @test hashamiltonian(lode) == false
    @test haslagrangian(lode) == true

    funcs = functions(lode)

    @test_nowarn funcs.ϑ(t₀, q₀, p₀, zero(q₀), NullParameters())
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(q₀), NullParameters())
    @test_nowarn funcs.g(t₀, q₀, p₀, zero(q₀), NullParameters())
    @test_nowarn funcs.l(t₀, q₀, p₀, NullParameters())
    @test_nowarn funcs.ω(t₀, q₀, p₀, zeros(2,2), NullParameters())

    funcs = functions(lode, NullParameters())

    @test_nowarn funcs.ϑ(t₀, q₀, p₀, zero(q₀))
    @test_nowarn funcs.f(t₀, q₀, p₀, zero(q₀))
    @test_nowarn funcs.g(t₀, q₀, p₀, zero(q₀))
    @test_nowarn funcs.l(t₀, q₀, p₀)
    @test_nowarn funcs.ω(t₀, q₀, p₀, zeros(2,2))

    # funcs = functions(lode)
    # @test funcs.ϑ == iode_ϑ == lode.ϑ
    # @test funcs.f == iode_f == lode.f
    # @test funcs.g == iode_g == lode.g
    # @test funcs.ω == lode_ω == lode.ω
    # @test funcs.v̄ == iode_v == lode.v̄
    # @test funcs.f̄ == iode_f == lode.f̄
    # @test funcs.l == lode_l == lode.lagrangian

    # @test lode == similar(lode, t₀, q₀, p₀, λ₀)
    # @test lode == similar(lode, t₀, q₀, p₀)
    # @test lode == similar(lode, q₀, p₀)

    # iode = convert(IODE, lode)
    # v₁ = zero(q₀)
    # v₂ = zero(q₀)
    # ϑ₁ = zero(q₀)
    # ϑ₂ = zero(q₀)
    # f₁ = zero(p₀)
    # f₂ = zero(p₀)
    # g₁ = zero(p₀)
    # g₂ = zero(p₀)
    # lode.v̄(lode.t₀, lode.q₀[begin], lode.p₀[begin], v₁)
    # iode.v̄(iode.t₀, iode.q₀[begin], iode.p₀[begin], v₂)
    # lode.ϑ(lode.t₀, lode.q₀[begin], v₁, ϑ₁)
    # iode.ϑ(iode.t₀, iode.q₀[begin], v₂, ϑ₂)
    # lode.f(lode.t₀, lode.q₀[begin], v₁, f₁)
    # iode.f(iode.t₀, iode.q₀[begin], v₂, f₂)
    # lode.g(lode.t₀, lode.q₀[begin], v₁, g₁)
    # iode.g(iode.t₀, iode.q₀[begin], v₂, g₂)
    # @test v₁ == v₂
    # @test ϑ₁ == ϑ₂
    # @test f₁ == f₂
    # @test g₁ == g₂

end


using Serialization
using .OSRM
using ProgressBars

struct TransitNetwork
    stops::Vector{Stop}
    stopidx_for_id::Dict{String,Int64}

    routes::Vector{Route}
    routeidx_for_id::Dict{String, Int64}

    services::Vector{Service}
    serviceidx_for_id::Dict{String, Int64}

    trips::Vector{Trip}
    tripidx_for_id::Dict{String, Int64}

    patterns::Vector{TripPattern}

    # todo could this have less indirection?
    transfers::Vector{Vector{Transfer}}

    # NB could also put this inside stop and pattern structs, but would prevent stop packing
    patterns_for_stop::Vector{Vector{Int64}}
    trips_for_pattern::Vector{Vector{Int64}}
end

TransitNetwork() = TransitNetwork(
    Vector{Stop}(),
    Dict{String,Int64}(),
    Vector{Route}(),
    Dict{String,Int64}(),
    Vector{Service}(),
    Dict{String,Int64}(),
    Vector{Trip}(),
    Dict{String,Int64}(),
    Vector{TripPattern}(),
    Vector{Vector{Transfer}}(),
    Vector{Vector{Int64}}(),
    Vector{Vector{Int64}}()
    )

# ensure that stop times are ordered by stop stop_sequence in each trip
function sort_stoptimes!(net::TransitNetwork)
    for trip in net.trips
        sort!(trip.stop_times, by=st -> st.stop_sequence)
    end
end

function hash_tp(stop_times::Vector{StopTime}, service::Int64)
    hash::Int64 = 0

    for (i, st) in enumerate(stop_times)
        hash += st.stop * primes[i % length(primes)]
    end

    hash += service * primes[length(primes)]

    return hash
end

# create trip patterns
function find_trip_patterns!(net::TransitNetwork)
    # maps from trip pattern hash to trip patterns with that hash
    tp_hashes = Dict{Int64, Vector{Tuple{Int64, TripPattern}}}()

    trips_with_patterns = Vector{Trip}()
    sizehint!(trips_with_patterns, length(net.trips))

    for trip in net.trips
        hash = hash_tp(trip.stop_times, trip.service)
        if !haskey(tp_hashes, hash)
            tp_hashes[hash] = Vector{Tuple{Int64, TripPattern}}()
        end

        stops::Vector{Int64} = map(st -> st.stop, trip.stop_times)

        # check for a matching trip pattern
        found_pattern = false
        for (tpidx, tp) in tp_hashes[hash]
            # short circuit and means vector equality won't be tested unless vectors are same length
            if ((tp.service == trip.service) && length(tp.stops) == length(stops) && all(tp.stops .== stops))
                found_pattern = true
                new_trip = Trip(
                    trip.stop_times,
                    trip.route,
                    trip.service,
                    tpidx
                )
                push!(trips_with_patterns, new_trip)
            end
        end

        if !found_pattern
            # create a new trip pattern
            tp = TripPattern(stops, trip.service)
            push!(net.patterns, tp)
            tpidx = length(net.patterns)
            push!(tp_hashes[hash], (tpidx, tp))
        end
    end

    # clear the entire trips array
    empty!(net.trips)
    append!(net.trips, trips_with_patterns)

    @info "created $(length(net.patterns)) trip patterns"
end

# find transfers based on crow-flies distance
function find_transfers_distance!(net::TransitNetwork, max_distance_meters::Real)
    empty!(net.transfers)
    total_transfers = 0
    sizehint!(net.transfers, length(net.stops))
    for stop in net.stops
        # find nearby stops
        # could use a spatial index if this is slow
        lat_diff = meters_to_degrees_lat(max_distance_meters)
        lon_diff = meters_to_degrees_lon(max_distance_meters, stop.stop_lat)

        # bbox query for nearby stops
        candidate_stops = filter(t -> (
            (t[2].stop_lat > stop.stop_lat - lat_diff) &&
            (t[2].stop_lat < stop.stop_lat + lat_diff) &&
            (t[2].stop_lon > stop.stop_lon - lon_diff) &&
            (t[2].stop_lon < stop.stop_lon + lon_diff) &&
            (t[2] !== stop)
            ), collect(enumerate(net.stops)))

        candidate_xfers = map(t -> Transfer(
            t[1],
            distance_meters(stop.stop_lat, stop.stop_lon, t[2].stop_lat, t[2].stop_lon)
        ), candidate_stops)

        # some overselection possible in corners of bbox
        xfers = filter(xfer -> xfer.distance_meters <= max_distance_meters, candidate_xfers)
        total_transfers += length(xfers)
        push!(net.transfers, xfers)
    end

    @assert length(net.transfers) == length(net.stops)

    @info "Created $total_transfers transfers from $(length(net.stops)) stops"
end

# Find transfers through the street network using OSRM
function find_transfers_osrm!(net::TransitNetwork, osrm::OSRMInstance, max_distance_meters::Real)
    empty!(net.transfers)
    total_transfers = 0
    sizehint!(net.transfers, length(net.stops))
    for stop in ProgressBar(net.stops)
        # find nearby stops
        # could use a spatial index if this is slow
        lat_diff = meters_to_degrees_lat(max_distance_meters)
        lon_diff = meters_to_degrees_lon(max_distance_meters, stop.stop_lat)

        # bbox query for nearby stops
        candidate_stops = collect(filter(t -> (
            (t[2].stop_lat > stop.stop_lat - lat_diff) &&
            (t[2].stop_lat < stop.stop_lat + lat_diff) &&
            (t[2].stop_lon > stop.stop_lon - lon_diff) &&
            (t[2].stop_lon < stop.stop_lon + lon_diff) &&
            (t[2] !== stop)
            ), collect(enumerate(net.stops))))

        # destinations has same order as candidate_stops
        destinations = collect(map(s -> Coordinate(s[2].stop_lat, s[2].stop_lon), candidate_stops))

        dmat = distance_matrix(osrm, [Coordinate(stop.stop_lat, stop.stop_lon)], destinations)

        xfers = Vector{Transfer}()

        for dest_idx in 1:length(destinations)
            # convert the index in destinations back to the index in net.stops
            stop_idx = candidate_stops[dest_idx][1]
            netdist = dmat.distances[1, dest_idx]
            if netdist < max_distance_meters
                push!(xfers, Transfer(stop_idx, netdist))
            end
        end

        total_transfers += length(xfers)
        push!(net.transfers, xfers)
    end

    @assert length(net.transfers) == length(net.stops)
    @info "Created $total_transfers transfers from $(length(net.stops)) stops"
end

function index_network!(net::TransitNetwork)
    # first, patterns for stops
    sizehint!(net.patterns_for_stop, length(net.stops))
    for stop in net.stops
        push!(net.patterns_for_stop, Vector{Int64}())
    end

    for (i, pattern) in enumerate(net.patterns)
        for stop in pattern.stops
            push!(net.patterns_for_stop[stop], i)
        end
    end

    # now, trips for pattern
    sizehint!(net.trips_for_pattern, length(net.patterns))
    for pattern in net.patterns
        push!(net.trips_for_pattern, Vector{Int64}())
    end

    for (i, trip) in enumerate(net.trips)
        push!(net.trips_for_pattern[trip.pattern], i)
    end
end

function _interpolate_segment_times(net::TransitNetwork, stop_times::Vector{StopTime})::Vector{StopTime}
    first_time = stop_times[1].departure_time
    last_time = stop_times[length(stop_times)].arrival_time

    distances = Vector{Float64}()
    sizehint!(distances, length(stop_times) - 1)

    for i in 1:(length(stop_times) - 1)
        fr = net.stops[stop_times[i].stop]
        to = net.stops[stop_times[i + 1].stop]
        push!(distances, distance_meters(
            fr.stop_lat,
            fr.stop_lon,
            to.stop_lat,
            to.stop_lon
        ))
    end

    distfrac = cumsum(distances) ./ sum(distances)

    timedelta = last_time - first_time

    new_stop_times = Vector{StopTime}()
    sizehint!(new_stop_times, length(stop_times))
    push!(new_stop_times, stop_times[1])

    for i in 2:(length(stop_times) - 1)
        interp_time::Int32 = first_time + round(distfrac[i - 1] * timedelta)
        push!(new_stop_times, StopTime(
            stop_times[i].stop,
            stop_times[i].stop_sequence,
            interp_time,
            interp_time
        ))
    end

    push!(new_stop_times, stop_times[length(stop_times)])

    @assert length(stop_times) == length(new_stop_times)

    return new_stop_times
end

function interpolate_stoptimes!(net::TransitNetwork)
    for trip in net.trips
        last_bona_fide_stidx = 0
        # collect should avoid concurrent modification issues
        for (i, st) in collect(enumerate(trip.stop_times))
            is_bona_fide_stidx = false
            if (st.arrival_time == INT_MISSING && st.departure_time != INT_MISSING)
                st.arrival_time = st.departure_time
                is_bona_fide_stidx = true
            elseif (st.departure_time == INT_MISSING && st.arrival_time != INT_MISSING)
                st.departure_time = st.arrival_time
                is_bona_fide_stidx = true
            elseif (st.departure_time != INT_MISSING && st.arrival_time != INT_MISSING)
                # do nothing, continue to accumulate
                is_bona_fide_stidx = true
            end

            if is_bona_fide_stidx
                if last_bona_fide_stidx != i - 1
                    trip.stop_times[last_bona_fide_stidx:i] = _interpolate_segment_times(net, trip.stop_times[last_bona_fide_stidx:i])
                end
                last_bona_fide_stidx = i
            end
        end

        # check our work
        @assert all(diff(map(st -> st.arrival_time, trip.stop_times)) .>= 0)
        @assert all(diff(map(st -> st.departure_time, trip.stop_times)) .>= 0)
    end
end

function save_network(network::TransitNetwork, filename::String)
    serialize(filename, network)
end

function load_network(filename::String)::TransitNetwork
    return deserialize(filename)
end


##############################
#
# obj-Files
#
##############################

FileIO.load(fn::File{format"OBJ"}, MeshType=GLNormalMesh) = open(fn) do s
    load(s, MeshType)
end

function FileIO.load{MT <: Mesh}(io::Stream{format"OBJ"}, MeshType::Type{MT}=GLNormalMesh)
    io           = stream(io)
    lineNumber   = 1
    mesh         = MeshType()
    last_command = ""
    attrib_type  = nothing
    for line in eachline(io)
        # read a line, remove newline and leading/trailing whitespaces
        line = strip(chomp(line))
        !isvalid(line) && error("non valid ascii in obj")

        if !startswith(line, "#") && !isempty(line) && !iscntrl(line) #ignore comments
            lines   = split(line)
            command = shift!(lines) #first is the command, rest the data

            if "v" == command # mesh always has vertices
                push!(vertices(mesh), Point{3, Float32}(lines)) # should automatically convert to the right type in vertices(mesh)
            elseif "vn" == command && has_normals(mesh)
                push!(normals(mesh), Normal{3, Float32}(lines))
            elseif "vt" == command && has_texturecoordinates(mesh)
                length(lines) == 2 && push!(lines, "0.0") # length can be two, but obj normaly does three coordinates with the third equals 0.
                push!(texturecoordinates(mesh), UVW{Float32}(lines))
            elseif "f" == command #mesh always has faces
                if any(x->contains(x, "/"), lines)
                    fs = process_face_uv_or_normal(lines)
                elseif any(x->contains(x, "//"), lines)
                    fs = process_face_normal(lines)
                else
                    return push!(faces(mesh), Triangle{Uint32}(lines))
                end
                push!(faces(mesh), Triangle{Uint32}(map(first, fs)))
            else
                #TODO
            end
        end
        # read next line
        lineNumber += 1
    end
    if isempty(mesh.normals) || length(mesh.normals) != length(mesh.vertices) # silly way of dealing with normals that are not per vertex.
        empty!(mesh.normals)
        append!(mesh.normals, normals(mesh.vertices, mesh.faces))
    end
    return mesh
end

# face indices are allowed to be negative, this methods handles this correctly
function handle_index{T <: Integer}(bufferlength::Integer, s::AbstractString, index_type::Type{T})
    i = parse(T, s)
    i < 0 && return convert(T, bufferlength) + i + one(T) # account for negative indexes
    return i
end
function push_index!{T}(buffer::Vector{T}, s::AbstractString)
    push!(buffer, handle_index(length(buffer), s, eltype(T)))
end

immutable SplitFunctor <: Base.Func{1}
    seperator::ASCIIString
end
call(s::SplitFunctor, array) = split(array, s.seperator)

# of form "f v1 v2 v3 ....""
process_face{S <: AbstractString}(lines::Vector{S}) = (lines,) # just put it in the same format as the others
# of form "f v1//vn1 v2//vn2 v3//vn3 ..."
process_face_normal{S <: AbstractString}(lines::Vector{S}) = map(SplitFunctor("//"), lines)
# of form "f v1/vt1 v2/vt2 v3/vt3 ..." or of form "f v1/vt1/vn1 v2/vt2/vn2 v3/vt3/vn3 ...."
process_face_uv_or_normal{S <: AbstractString}(lines::Vector{S}) = map(SplitFunctor("/"), lines)

function process{F <: Face, S <: AbstractString}(::Type{F}, lines::Vector{S}, mesh::Mesh, line::Int)
    if any(x->contains(x, "/"), lines)
        faces = process_face_uv_or_normal(lines)
    elseif any(x->contains(x, "//"), lines)
        faces = process_face_normal(lines)
    else
        return push!(mesh[F], F(Triangle{Uint32}(lines)))
    end
    push!(mesh[F], F(Triangle{Uint32}(map(first, faces))))
end


#The LLRBTree data structure, that allows us to acces keys in log2 time.
#Translated from code by Robert Sedgewick
module LLRBTrees

import Base: haskey, getindex, setindex!, delete!, minimum, maximum, push!, show, replace

export TreeNode, LLRBTree, getnodefrom_key, getmaxdepth, orderedpairs, levelorder,
         populate!, isleftleaf, isrightleaf, ischildleaf, isleaf, isred

type TreeNode{K,V}
    key:: Nullable{K}
    value:: Nullable{V}
    isRed:: Bool
    left:: TreeNode{K,V}
    right:: TreeNode{K,V}

    TreeNode() = method_exists(isless, (K,K) ) ? new(Nullable{K}(), Nullable{V}(), false) : error("keys must be comparable")
    TreeNode(key::K, value::V, isRed::Bool, left::TreeNode{K,V}, right::TreeNode{K,V}) =
    method_exists(isless, (K,K) ) ? new(key, value, isRed, left, right): error("keys must be comparable")
    TreeNode(key::K, value::V, isRed::Bool) = method_exists(isless, (K,K) ) ? new(key, value, isRed) : error("keys must be comparable")

end
# New nodes are always red so no method that explicitly assigns the color is neccessary
TreeNode{K,V}(key::K,value::V,left::TreeNode{K,V},right::TreeNode{K,V}) = TreeNode(key,value,true,left,right)
TreeNode{K,V}(key::K,value::V) = TreeNode{K,V}(key, value, true)
TreeNode{K,V}(key::K,value::V, isRed::Bool) = TreeNode{K,V}(key, value, isRed)

# To be able to have different instances of LLRB trees, we create a tree type
# that characterizes a tree by its root
type LLRBTree{K,V}
    root:: TreeNode{K,V}

    LLRBTree()=new()
    LLRBTree(node:: TreeNode{K,V}) = (tree = new(); tree.root=node; tree.root.isRed=false; tree)
    LLRBTree(key::K, value::V) = (tree=new(TreeNode(key,value)); tree.root.isRed=false; tree)
end

LLRBTree{K,V}(key::K,value::V) = LLRBTree{K,V}(key, value)
LLRBTree{K,V}(node::TreeNode{K,V}) = LLRBTree{K,V}(node)

function isleaf{K,V}(node::TreeNode{K,V})
    isnull( node.key )
end

function ischildleaf{K,V}(node::TreeNode{K,V}, left::Bool)
    if(left)
        if(!isdefined(node, :left))
            return true
        else
            return isnull(node.left.key)
        end
    else
        if(!isdefined(node, :right) )
            return true
        else
            return isnull(node.right.key)
        end
    end
end

isleftleaf{K,V}(node::TreeNode{K,V})= ischildleaf(node, true)
isrightleaf{K,V}(node::TreeNode{K,V}) = ischildleaf(node, false)

function isred{K,V}(node::TreeNode{K,V}, dirs::Symbol...)

    steps = length(dirs)
    isred = node.isRed
    foundleaf = 0
    foundundef = 0

    for i in 1:steps
        if(dirs[i]!= :left && dirs[i]!= :right)
            Base.error("Directions must be :left or :right")
        end

    end

    for i in 1:steps

        dirbool = dirs[i]== :left ? true : false

        if(isdefined(node, dirs[i]))
            node = dirbool ? node.left : node.right
            isred = node.isRed

            foundleaf = isnull(node.key) ? i : foundleaf
        else
            isred = false
            foundundef = i
            break
        end
    end

    overflow = (foundundef < steps && foundundef!=0)
    overflow |= (foundleaf < steps && foundleaf!=0)

    return  overflow ? Base.error("Directions move beyond leaves") : isred

end

function getmaxdepth{K,V}(node::TreeNode{K,V}, depth=1)
      #We do a caching up of the number of recursions with the use of parameters and returns
      depthleft=depthright=depth
    if !isleftleaf(node)
          #("left in at $depth")
          depthleft = getmaxdepth(node.left, depth+1)
      end
    if !isrightleaf(node)
          #("right in at $depth")
          depthright = getmaxdepth(node.right, depth+1)
      end
      #("out at $depth")
      #([depthleft,depthright])
      return maximum([depthleft,depthright])
  end
  getmaxdepth(tree::LLRBTree) = getmaxdepth(tree.root)

function replace(original::ASCIIString, start::Int64, fin::Int64, string::ASCIIString)
    left = original[1:start]
    right = original[fin:end]
    return left*string*right
end

function show{K,V}(io::IO, tree::LLRBTree{K,V})

    if(isdefined(tree, :root))
        totalheight = getmaxdepth(tree)
        height = getmaxdepth(tree) < 5 ? totalheight : 5
        width = 2^(height +2)

        len = width * height * 2 + 2 |> Int
        stringtree = ""
        for i = 1:len
            stringtree*= i < len - 2 && i % width == 0 ? "\n" : " "
        end

        if(!isnull(tree.root.key))
            stringtree = showR(stringtree, Int(width / 2), 1, Int(width / 4), width, tree.root, " ")
        else
            stringtree = typeof(tree)
        end
    else
            stringtree = typeof(tree)
    end

    println(io, stringtree)
end

function showR(stringtree::ASCIIString, center::Int, row::Int, indent::Int, width::Int, node::TreeNode,
            edge::ASCIIString)

    if(indent%2 == 0)

        if (!isleftleaf(node))
            stringtree = showR(stringtree, center - indent, row + 2, Int(indent/2), width, node.left, " /");
        end

        valuestring = string(node.value.value)
        valuestring = node.isRed ? valuestring*"_r" : valuestring

        idx1 = Int( row * width + center - div( (length(valuestring) + 1) , 2 ) )
        idx2 = idx1 + length(valuestring) |> Int
        idx3 = idx1 - width |> Int
        if (idx2 < length(stringtree))
            stringtree = replace(stringtree, idx1, idx2 +1, valuestring)
            stringtree = replace(stringtree, idx3, idx3 + 3, edge)
        end

        if (!isrightleaf(node))
            stringtree = showR(stringtree, center + indent, row + 2, Int(indent/2), width, node.right, "\\ ")
        end
    end

    return stringtree
end

# The function that returns the node that corresponds to a certain key, below a root-node
# it returns a TreeLeaf when the root node is empty
function getnodefrom_key{K,V}(part::TreeNode{K,V}, key::K)
  # The recursive search for the key
    if get(part.key) == key
        return part
    elseif key < get(part.key)
        if( !isleftleaf(part) )
            getnodefrom_key(part.left, key)
        else
            return TreeNode{K,V}()
        end
    else
        if(!isrightleaf(part))
            getnodefrom_key(part.right, key)
        else
          return TreeNode{K,V}()
        end
    end
end
getnodefrom_key{K,V}(tree::LLRBTree{K,V}, key::K) = getnodefrom_key(tree.root, key)

haskey{K,V}(tree::LLRBTree{K,V}, key::K) =  !isleaf( getnodefrom_key(tree.root, key) )
haskey{K,V}(node::TreeNode{K,V}, key::K) = !isleaf( getnodefrom_key(node, key) )

# A getindex only that the indexes are keys and we get the value of the first node with the given key
function getindex{K,V}(node::TreeNode{K,V}, key::K)
    part::TreeNode{K,V} = getnodefrom_key(node,key)
    # Throw a KeyException if the key is node.left present

    if isleaf( part )
        throw(KeyError(key))
    else
        return part.value
    end
end

getindex{K,V}(tree::LLRBTree{K,V}, key::K) = getindex(tree.root, key)

function setindex!{K,V}(tree::LLRBTree{K,V}, value::V, key::K)
    if(isdefined(tree, :root))
        part::TreeNode{K,V} = getnodefrom_key(tree.root,key)
    else
        part = TreeNode{K,V}()
    end

    if isleaf( part )
        push!(tree, key, value)
    else
        part.value= value
    end

    tree
end

function rotateleft{K,V}(node::TreeNode{K,V})

    if isrightleaf(node)
        error("The right part of the given node must be a node")
    end
    son=node.right

    if !isleftleaf(son)
        node.right=son.left
    else
        node.right=TreeNode{K,V}()
    end
    son.left=node

    son.isRed=node.isRed
    node.isRed=true
    return son
end

function rotateright{K,V}(node::TreeNode{K,V})

    if isleftleaf(node)
        error("The left part of the given node must be a node")
    end
    son=node.left

    if !isrightleaf(son)
        node.left=son.right
    else
        node.left=TreeNode{K,V}()
    end
    son.right=node

    son.isRed=node.isRed
    node.isRed=true
    return son
end

function flipcolor!{K,V}(node::TreeNode{K,V})

    node.isRed = !node.isRed
    if !isleftleaf(node)
        node.left.isRed = !node.left.isRed
    end
    if !isrightleaf(node)
        node.right.isRed = !node.right.isRed
    end

end

function add_node{K,V}(node::TreeNode{K,V}, key::K, value::V)

    if isleaf(node)
        #("Add the node only when a leaf is found  ",tree)
        return TreeNode(key, value)
    end

    #put flipcolor here to get 2-3-4 trees

    if get(node.key) < key
        #("went right")
        node.right = !isrightleaf(node) ? node.right: TreeNode{K,V}()
        node.right = add_node(node.right, key, value)
    elseif get(node.key) > key
        #("went left")
        node.left = !isleftleaf(node) ? node.left: TreeNode{K,V}()
        node.left = add_node(node.left, key, value)
    else
        #("node was same")
        node.value = value
    end

    if ( isred(node, :right) && !isred(node, :left))
        #("Rotate left to prevent red node on right ", node)
        node=rotateleft(node)
    end

    if ( isred(node, :left) && isred(node, :left, :left) )
        #("Rotate right to prevent consecutive red nodes ", node)
        node=rotateright(node)
    end
    if isred(node, :left) && isred(node, :right)
        #("Split node with two red children")
        flipcolor!(node)
    end

    return node
end
# For trees and nodes as parameters; different name so the other one doesn't get used
push!{K,V}(tree::LLRBTree{K,V}, key::K, value::V) = (tree.root = isdefined(tree, :root)
? add_node(tree.root, key, value): TreeNode(key, value); tree.root.isRed=false; tree)

push!{K,V}(tree::LLRBTree{K,V}, node::TreeNode{K,V}) = push!(tree, node.key.value, node.value.value)

function populate!{K,V}(tree::LLRBTree{K,V}, N::Int64)
    keys = []

    for i in 1:N
        r::Int64 = rand(1:N*2+100)
        push!(tree, r, r )

        #rep_index = indexin([r], keys)
        #while rep_index != [0]
        #    deleteat!(keys, rep_index)
        #    rep_index = indexin([r], keys)
        #end

        push!(keys, r)
    end
    return keys
end

function moveredleft{K,V}(node::TreeNode{K,V})
    flipcolor!(node)

    if !isrightleaf(node)
        if ( isred(node, :right, :left) )
            node.right = rotateright(node.right)
            node = rotateleft(node)
            flipcolor!(node)
        end
    end
    return node
end

function moveredright{K,V}(node::TreeNode{K,V})
    flipcolor!(node)

    if ( isred(node, :left, :left) )
        node= rotateright(node)
        flipcolor!(node)
    end
    return node
end

function minimum{K,V}(node::TreeNode{K,V})
    while !isleftleaf(node)
        node=node.left
    end
    return node
end
minimum(tree::LLRBTree) = minimum(tree.root)

function maximum{K,V}(node::TreeNode{K,V})
    while !isrightleaf(node)
        node=node.right
    end
    return node
end
maximum(tree::LLRBTree) = maximum(tree.root)

function fixup{K,V}(node::TreeNode{K,V})

    if isred(node, :right)
        node=rotateleft(node)
    end

    if(!isred(node, :left) && isred(node, :right) )
        node=rotateleft(node)
    end

    if(isred(node, :left) && isred(node, :left, :left) )
        node=rotateright(node)
    end

    if (isred(node, :left) && isred(node, :right) )
        flipcolor!(node)
    end

    return node
end

function deletemin{K,V}(node::TreeNode{K,V})

    if(!isleftleaf(node))

        if(!isred(node, :left) && !isred(node, :left, :left) )
            moveredleft(node)
        end

        node.left = deletemin(node.left)

        return fixup(node)
    else

        return TreeNode{K,V}()
    end
end

function delete_node{K,V}(node::TreeNode{K,V}, key::K)
    if key<node.key.value

        if !isleftleaf(node)

            if(!isred(node, :left) && !isred(node, :left, :left) )
                node=moveredleft(node)
            end

            node.left=delete_node(node.left, key)

        else
            Base.warn_once("Key not found:", key)
            return fixup(node)
        end

    else
        if(isred(node, :left))
            node=rotateright(node)
        end

        if (key==node.key.value && isrightleaf(node))
            return TreeNode{K,V}()
        end

        if !isrightleaf(node)

            if(!isred(node, :right) && !isred(node, :right, :left) )
                node=moveredright(node)
            end

            if key==node.key.value
                min = minimum(node.right)
                node.key=min.key
                node.value=min.value
                node.right=deletemin(node.right)
            else
                node.right=delete_node(node.right,key)
            end

        else
            Base.warn_once("Key not found:", key)
            return fixup(node)
        end
    end
    return fixup(node)
end

function delete!{K}(tree::LLRBTree, key::K)

    if( isdefined(tree, :root))
        if(! isleaf(tree.root) )
            tree.root = delete_node(tree.root, key)

            tree.root.isRed = false
        else
            Base.warn_once("Root cannot be leaf")
        end
    else
        Base.warn_once("Root cannot be leaf")
    end

    tree
end
delete!{K,V}(tree::LLRBTree, node::TreeNode{K,V}) = delete!(tree, node.key.value)

function levelorder{K,V}(tree::LLRBTree{K,V})
    queue = []
    push!(queue, tree.root)
    println("levelorder:  ")
    count = 20
    last20 = ones(1:20)

    while (!isempty(queue))
        node = shift!(queue)

        #Only print the first 20 values
        if(count>0)
            print(node.value.value,"  ")
        elseif count == 0
            print("...  ")
        end

        #Keep the last 20 entries to print later
        shift!(last20)
        #pop!(last20)
        push!(last20, node.value.value)
        #println(size(last20,1))

        if( !isleftleaf(node) )
            push!(queue, node.left)
        end
        if( !isrightleaf(node) )
            push!(queue, node.right)
        end

        count -= 1
    end

    if count < -19
        for i in 0:19
            print(last20[end-i]," ")
        end
    end
end

#Return an ordered list with recursion
function inorder{K,V}(node::TreeNode{K,V}, list=Any[])
  if !isleaf(node)

      if !isleftleaf(node)
          #("Go down to the left first")
          list = inorder(node.left, list)[2]
      end

      #("Push to the list when leftmost")
      push!(list, [node.key.value, node.value.value])

      if !isrightleaf(node)
          #("Go right if no left")
          list = inorder(node.right, list)[2]
      end
  end
  #("Go up;  ",node,";  ",list)
  return node, list
end
orderedpairs{K,V}(tree::LLRBTree{K,V}) = inorder(tree.root)[2]

end


import Base.==
import Base.convert

export Goal, PackedGoal, BasicPackedGoal, InterleavedPackedGoal, print_goal, compose, convert, ==, read_plu

# A goal describes a boolean function with one or more inputs and one or more
# outputs. It defines the number of inputs and the values of each output for
# all possible input combinations.
immutable Goal{N}
    num_inputs::Integer
    truth_table::NTuple{N, BitString}
end

Goal{N}(n::Integer, t::NTuple{N, Integer}) = Goal(n, convert(NTuple{N, BitString}, t))
Goal(n::Integer, b::Vector{BitString}) = Goal(n, ntuple(i -> b[i], length(b)))
Goal(c::Chromosome, b::Vector{BitString}) = Goal(c.num_inputs, b)

# The equality operator for Goals is true if the number of inputs, number of
# outputs, and the output truth tables are identical and false otherwise.
=={N}(g::Goal{N}, h::Goal{N}) = (g.num_inputs == h.num_inputs) && (g.truth_table == h.truth_table)
=={G, H}(g::Goal{G}, h::Goal{H}) = false

# Prints a goal in hex format. The truth table entries are printed from highest
# to lowest index. In other words: `print_goal(Goal(1, (0x8, 0x6)))` will
# print "(1, 2, [0x6, 0x8])".
function print_goal{N}(g::Goal{N})
    @printf("(%d, %d, [", g.num_inputs, N)
    for i in N:-1:2
        @printf("%#x ",g.truth_table[i])
    end
    @printf("%#x])\n",g.truth_table[1])
end

# Goals can have multiple outputs.
# There are three formats for goals:
# unpacked:  each output is stored in a separate BitString (this is the output format of execute chromosome)
# packed (also called non-interleved):  All outputs are stored in one BitString.
#      All bits of an output are consecutive.
# interleaved:  All outputs are stored in one BitString.
#      The bits of outputs are interleaved.  This format is used by the g_compose f function.
# Packed goals combine all outputs into a single bitstring and, consequently,
# can only handle functions with an appropriately small number of outputs.
abstract PackedGoal

immutable BasicPackedGoal <: PackedGoal
    num_inputs::Integer
    num_outputs::Integer
    truth_table::BitString
end

immutable InterleavedPackedGoal <: PackedGoal
    num_inputs::Integer
    num_outputs::Integer
    truth_table::BitString
end

function =={T <: PackedGoal}(g::T, h::T)
    (g.num_inputs == h.num_inputs) &&
    (g.num_outputs == h.num_outputs) &&
    (g.truth_table == h.truth_table)
end
==(g::BasicPackedGoal, h::InterleavedPackedGoal) = error("Cannot compare basic and interleaved goals")
==(g::InterleavedPackedGoal, h::BasicPackedGoal) = h == g
=={T <: PackedGoal}(g::Goal, h::T) = error("Cannot compare unpacked and packed goals")
=={T <: PackedGoal}(g::T, h::Goal) = h == g

# Prints a packed goal in hex by converting it to an unpacked goal.
function print_goal{T <: PackedGoal}(g::T)
    print_goal(convert(Goal, g))
end

function convert(::Type{Goal}, g::BasicPackedGoal)
    temp_ttable = g.truth_table
    ttable = Array(BitString, g.num_outputs)
    mask = output_mask(g.num_inputs)
    for i = 1:g.num_outputs
        ttable[i] = mask & temp_ttable
        temp_ttable >>= (2 ^ g.num_inputs)
    end
    return Goal(g.num_inputs, ttable)
end

function convert{N}(::Type{BasicPackedGoal}, g::Goal{N})
    if (2 ^ g.num_inputs) * N > 8*sizeof(BitString)  # 8 is the number of bits in a byte
        error("The goal is too large to be packed.")
    end

    ttable = g.truth_table[N]
    for i = (N-1):-1:1
        ttable <<= (2 ^ g.num_inputs)
        ttable $= g.truth_table[i]
    end
    return BasicPackedGoal(g.num_inputs, N, ttable)
end

function convert(::Type{InterleavedPackedGoal}, g::BasicPackedGoal)
    ttable = convert(BitString, 0)
    one = convert(BitString, 1)
    p = 2 ^ g.num_inputs
    for j = (p-1):-1:0
        for k = (g.num_outputs-1):-1:0
            r = g.truth_table >> (k * (p - 1) + j + k)
            ttable = (ttable << 1) $ (r & one)
        end
    end
    return InterleavedPackedGoal(g.num_inputs, g.num_outputs, ttable)
end

function convert(::Type{BasicPackedGoal}, g::InterleavedPackedGoal)
    ttable = convert(BitString, 0)
    one = convert(BitString, 1)
    t_g = g.truth_table
    p = 2 ^ g.num_inputs
    for j = (g.num_outputs-1):-1:0
        for k = p-1:-1:0
            r = t_g >> (k * g.num_outputs + j)
            ttable = (ttable << 1) $ (r & one)
        end
    end
    return BasicPackedGoal(g.num_inputs, g.num_outputs, ttable)
end

function convert(::Type{Goal}, g::InterleavedPackedGoal)
    return convert(Goal, convert(BasicPackedGoal, g))
end

function convert{N}(::Type{InterleavedPackedGoal}, g::Goal{N})
    return convert(InterleavedPackedGoal, convert(BasicPackedGoal, g))
end

# Compose two goals g, and h. The number of outputs from h must be
# equal to the number of inputs to g.
function compose(g::InterleavedPackedGoal, h::InterleavedPackedGoal)
    if g.num_inputs != h.num_outputs
        error("Cannot compose g and h, input-output mismatch.")
    end

    ones = typemax(BitString)

    mask_g = (ones << g.num_outputs) $ ones
    mask_h = (ones << h.num_outputs) $ ones

    ttable = convert(BitString, 0)

    t_g = g.truth_table
    t_h = h.truth_table

    for i = 0:(2 ^ h.num_outputs - 1)
        g_i = t_g & mask_g
        h_g_i = (t_h >> (h.num_outputs * g_i)) & mask_h
        ttable = ttable $ h_g_i << (i * h.num_outputs)
        t_g = t_g >> g.num_outputs
    end
    return InterleavedPackedGoal(g.num_inputs, h.num_outputs, ttable)
end

# Reads the ".plu" file whose name is fname.
# "*.plu" files define goals in Julian Miller's version 1.1 of GGP.
# This version is limited to goals with at most 4 inputs and at most 4 outputs.
function read_plu(fname)
    num_inputs = 0
    num_outputs = 0
    outputs = BitString[]
    f = open(fname,"r")
    for line in eachline(f)
        fields = split(line,' ')
        if fields[1][1] == '.'
            if fields[1][2] == 'i'
                num_inputs = parse(Int, fields[2])
                if num_inputs > 4
                    error("the number of inputs in this version of read_plu is limited to 4")
                end
            elseif fields[1][2] == 'o'
                num_outputs = parse(Int, fields[2])
                if num_outputs > 4
                    error("the number of outputs in this version of read_plu is limited to 4")
                end
            end
        else
            input_flag = true
            for f in fields
                if length(f) == 0
                    input_flag = false
                    #println()
                elseif f != "\r\n"
                    #@printf(" %#X",int(f))
                    if !input_flag
                        push!(outputs,convert(BitString, parse(Int, f)))
                    end
                end
            end
            #println()
        end
    end
    word_size = 2^num_inputs
    truth_table = convert(BitString,0)
    for o in outputs
        #@printf(" %#X",o)
        truth_table = truth_table << word_size
        truth_table = truth_table $ o
    end
    return BasicPackedGoal(num_inputs, num_outputs, truth_table)
end


module Abm

# TODO: make this cleaner, see https://github.com/sbromberger/LightGraphs.jl/blob/master/src/LightGraphs.jl
using Agents:
    @agent, ABM, AbstractAgent, DiscreteSpace, GridSpace, add_agent!,
    add_agent_pos!, allids, dummystep, genocide!, ids_in_position, kill_agent!,
    move_agent!, nagents, nearby_ids, nearby_positions, positions, random_position, run!

using InteractiveDynamics: abmexploration
using GLMakie
using Random: Xoshiro, rand

include("impl.jl")
include("parameters.jl")
include("step.jl")
include("utils.jl")

function initialize(; M=33, seed=125)
    space = GridSpace((M, M))
    properties = Parameters()
    rng = Xoshiro(seed)

    model = ABM(
        Organism, space;
        properties, rng
    )

    for n ∈ 1:properties.bacteria_count
        agent = Organism(n, (1, 1), rand(model.rng) < 0.5 ? :a : :b, Vector{Int}(), :nothing, :nothing, -1, :bacterium)
        add_bacterium_single!(agent, model)
    end
    for _ ∈ 1:properties.phages_count
        roll = rand(model.rng)
        if roll < 0.45
            kind = :temperate
        elseif roll < 0.85
            kind = :virulent
        else
            kind = :deficient
        end
        add_agent!(Organism, model, roll < 0.5 ? :a : :b, Vector{Int}(), kind, :free, 0, :phage)
    end

    return model
end

function plot(model::ABM)
    params = Dict(
        :a => 0.0:0.1:1.0,
        :b => 0.0:0.1:1.0,
        :m => 0.0:1.0:40.0,
        :α => 0.0:1.0:100.0,
        :κ => 0.01:0.01:0.5,
        :moi_proxy_radius => 1:3,
        :infection_distance => 1:3,
        :latent_period => 1:10,
        :burst_size => 3:8,
        :growth_rate => 0.2:0.1:1.0,
        :decay_factor => 0.01:0.01:0.2,
        :p_burst => 0.4:0.1:0.9
    )

    ac(a::A) where {A<:AbstractAgent} = a.type === :bacterium ? :cyan : :magenta
    as = 10
    am(a::A) where {A<:AbstractAgent} = a.type === :bacterium ? :circle : :diamond
    f(model::ABM{<:DiscreteSpace,A}) where {A<:AbstractAgent} = map(v -> length(v), model.space.s)

    heatarray = f
    heatkwargs = (colorrange=(0, 7), colormap=:thermal)
    plotkwargs = (;
        ac, as, am,
        scatterkwargs=(strokewidth=1.0,),
        heatarray, heatkwargs
    )

    mdata = [:bacteria_count, :phages_count]
    fig, p = abmexploration(model; (model_step!)=complex_step!, params, plotkwargs..., mdata, figure = (resolution = (600,600),))
    return fig
end

function run(model, n)
    run!(model, dummystep, complex_step!, n;
        mdata=[:bacteria_count, :phages_count])
end

end


using Keras
using Pyall
@pyimport numpy

model = Keras.load("structure.json", "weights.h5")


# This file is part of Kpax3. License is MIT.

# A1 = [1 0 0 0 0 0;
#       1 0 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 0 0 1]
#
# A2 = [1 0 0 0 0 0;
#       0 1 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 1 0 0;
#       0 0 0 0 1 0;
#       0 0 0 0 1 0]
#
# Each cutpoint greater than 1 has probability 1 / 5 of being selected
#
# COLUMN 2
# If cutpoint == 7, Kpax3.crossover breaks the second child:
#
# B1 = [1 0 0 0 0 0;
#       0 1 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 1 0 0;
#       0 0 0 0 1 0;
#       0 0 0 0 1 0]
#
# B2 = [1 0 0 0 0 0;
#       0 0 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 0 0 1]
#
# Pr(B1 = [1; 2; 3; 4; 5; 5] | cutpoint = 7) = 1
#
# Pr(B2 = [1; 1; 3; 3; 3; 6] | cutpoint = 7) = 1 / 2
# Pr(B2 = [1; 2; 3; 3; 3; 6] | cutpoint = 7) = 1 / 2
#
# COLUMN 3
# If cutpoint == 14, Kpax3.crossover does not break anything
#
# B1 = [1 0 0 0 0 0;
#       1 0 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 1 0 0;
#       0 0 0 0 1 0;
#       0 0 0 0 1 0]
#
# B2 = [1 0 0 0 0 0;
#       0 1 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 0 0 1]
#
# Pr(B1 = [1; 1; 3; 4; 5; 5] | cutpoint = 14) = 1
#
# Pr(B2 = [1; 2; 3; 3; 3; 6] | cutpoint = 14) = 1
#
# COLUMN 4
# If cutpoint == 21, Kpax3.crossover breaks the second child:
#
# B1 = [1 0 0 0 0 0;
#       1 0 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 1 0 0;
#       0 0 0 0 1 0;
#       0 0 0 0 1 0]
#
# B2 = [1 0 0 0 0 0;
#       0 1 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 0 0 0;
#       0 0 0 0 0 0;
#       0 0 0 0 0 1]
#
# Pr(B1 = [1; 1; 3; 4; 5; 5] | cutpoint = 21) = 1
#
# Pr(B2 = [1; 2; 3; 1; 1; 6] | cutpoint = 21) = (1 / 4) * (2 / 5) = 1 / 10
# Pr(B2 = [1; 2; 3; 2; 1; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 3; 1; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 4; 1; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
#
# Pr(B2 = [1; 2; 3; 1; 2; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 2; 2; 6] | cutpoint = 21) = (1 / 4) * (2 / 5) = 1 / 10
# Pr(B2 = [1; 2; 3; 3; 2; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 4; 2; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
#
# Pr(B2 = [1; 2; 3; 1; 3; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 2; 3; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 3; 3; 6] | cutpoint = 21) = (1 / 4) * (2 / 5) = 1 / 10
# Pr(B2 = [1; 2; 3; 4; 3; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
#
# Pr(B2 = [1; 2; 3; 4; 4; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
#
# Pr(B2 = [1; 2; 3; 1; 5; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 2; 5; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 3; 5; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 4; 5; 6] | cutpoint = 21) = (1 / 4) * (1 / 5) = 1 / 20
#
# COLUMN 5
# If cutpoint == 28, Kpax3.crossover breaks the second child:
#
# B1 = [1 0 0 0 0 0;
#       1 0 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 0 1 0;
#       0 0 0 0 1 0]
#
# B2 = [1 0 0 0 0 0;
#       0 1 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 1 0 0;
#       0 0 0 0 0 0;
#       0 0 0 0 0 1]
#
# Pr(B1 = [1; 1; 3; 3; 5; 5] | cutpoint = 28) = 1
#
# Pr(B2 = [1; 2; 3; 4; 1; 6] | cutpoint = 28) = 1 / 5
# Pr(B2 = [1; 2; 3; 4; 2; 6] | cutpoint = 28) = 1 / 5
# Pr(B2 = [1; 2; 3; 4; 3; 6] | cutpoint = 28) = 1 / 5
# Pr(B2 = [1; 2; 3; 4; 4; 6] | cutpoint = 28) = 1 / 5
# Pr(B2 = [1; 2; 3; 4; 5; 6] | cutpoint = 28) = 1 / 5
#
# COLUMN 6
# If cutpoint == 35, Kpax3.crossover breaks the first child:
#
# A1 = [1 0 0 0 0 0;
#       1 0 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 0 0 0]
#
# A2 = [1 0 0 0 0 0;
#       0 1 0 0 0 0;
#       0 0 1 0 0 0;
#       0 0 0 1 0 0;
#       0 0 0 0 1 0;
#       0 0 0 0 0 1]
#
# Pr(B1 = [1; 1; 3; 3; 3; 1] | cutpoint = 35) = 2 / 6
# Pr(B1 = [1; 1; 3; 3; 3; 3] | cutpoint = 35) = 3 / 6
# Pr(B1 = [1; 1; 3; 3; 3; 6] | cutpoint = 35) = 1 / 6
#
# Pr(B2 = [1; 2; 3; 4; 5; 6] | cutpoint = 35) = 1
#
# SUMMARY
# Unique partitions that are possible to be obtained from this Kpax3.crossover:
#
# Pr(B1 = [1; 2; 3; 4; 5; 5]) = Pr(B1 = R2) = 1 / 5
# Pr(B1 = [1; 1; 3; 4; 5; 5]) = 2 / 5
# Pr(B1 = [1; 1; 3; 3; 5; 5]) = 1 / 5
# Pr(B1 = [1; 1; 3; 3; 3; 1]) = (1 / 5) * (2 / 6) = 1 / 15
# Pr(B1 = [1; 1; 3; 3; 3; 3]) = (1 / 5) * (3 / 6) = 1 / 10
# Pr(B1 = [1; 1; 3; 3; 3; 6]) = (1 / 5) * (1 / 6) = 1 / 30
#
# Pr(B2 = [1; 1; 3; 3; 3; 6]) = (1 / 5) * (1 / 2) = 1 / 10
# Pr(B2 = [1; 2; 3; 3; 3; 6]) = (1 / 5) * (1 / 2) + (1 / 5) * 1 + (1 / 5) * (1 / 10) = 8 / 25
# Pr(B2 = [1; 2; 3; 1; 1; 6]) = (1 / 5) * (1 / 10) = 1 / 50
# Pr(B2 = [1; 2; 3; 2; 1; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 3; 1; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 4; 1; 6]) = (1 / 5) * (1 / 20) + (1 / 5) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 1; 2; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 2; 2; 6]) = (1 / 5) * (1 / 10) = 1 / 50
# Pr(B2 = [1; 2; 3; 3; 2; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 4; 2; 6]) = (1 / 5) * (1 / 20) + (1 / 5) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 1; 3; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 2; 3; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 4; 3; 6]) = (1 / 5) * (1 / 20) + (1 / 5) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 4; 4; 6]) = (1 / 5) * (1 / 20) + (1 / 5) * (1 / 5) = 1 / 20
# Pr(B2 = [1; 2; 3; 1; 5; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 2; 5; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 3; 5; 6]) = (1 / 5) * (1 / 20) = 1 / 100
# Pr(B2 = [1; 2; 3; 4; 5; 6]) = (1 / 5) * (1 / 20) + (1 / 5) * (1 / 5) + (1 / 5) * 1 = 1 / 4

function test_crossover()
  support = Kpax3.GASupport(10, 6)

  R1 = [1; 1; 2; 2; 2; 3]
  R2 = [1; 2; 3; 4; 5; 5]

  N = 1000000

  pr = [1 / 5; 2 / 5; 1 / 5; 1 / 15; 1 / 10; 1 / 30; 1 / 10; 8 / 25; 1 / 50; 1 / 100; 1 / 100; 1 / 20; 1 / 100; 1 / 50; 1 / 100; 1 / 20; 1 / 100; 1 / 100; 1 / 20; 1 / 20; 1 / 100; 1 / 100; 1 / 100; 1 / 4]

  tmp = zeros(Float64, length(pr))

  for i in 1:N
    Kpax3.crossover!(R1, R2, support)

    if support.oi.R == [1; 2; 3; 4; 5; 5]
      tmp[ 1] += 1
    elseif support.oi.R == [1; 1; 3; 4; 5; 5]
      tmp[ 2] += 1
    elseif support.oi.R == [1; 1; 3; 3; 5; 5]
      tmp[ 3] += 1
    elseif support.oi.R == [1; 1; 3; 3; 3; 1]
      tmp[ 4] += 1
    elseif support.oi.R == [1; 1; 3; 3; 3; 3]
      tmp[ 5] += 1
    elseif support.oi.R == [1; 1; 3; 3; 3; 6]
      tmp[ 6] += 1
    end

    if support.oj.R == [1; 1; 3; 3; 3; 6]
      tmp[ 7] += 1
    elseif support.oj.R == [1; 2; 3; 3; 3; 6]
      tmp[ 8] += 1
    elseif support.oj.R == [1; 2; 3; 1; 1; 6]
      tmp[ 9] += 1
    elseif support.oj.R == [1; 2; 3; 2; 1; 6]
      tmp[10] += 1
    elseif support.oj.R == [1; 2; 3; 3; 1; 6]
      tmp[11] += 1
    elseif support.oj.R == [1; 2; 3; 4; 1; 6]
      tmp[12] += 1
    elseif support.oj.R == [1; 2; 3; 1; 2; 6]
      tmp[13] += 1
    elseif support.oj.R == [1; 2; 3; 2; 2; 6]
      tmp[14] += 1
    elseif support.oj.R == [1; 2; 3; 3; 2; 6]
      tmp[15] += 1
    elseif support.oj.R == [1; 2; 3; 4; 2; 6]
      tmp[16] += 1
    elseif support.oj.R == [1; 2; 3; 1; 3; 6]
      tmp[17] += 1
    elseif support.oj.R == [1; 2; 3; 2; 3; 6]
      tmp[18] += 1
    elseif support.oj.R == [1; 2; 3; 4; 3; 6]
      tmp[19] += 1
    elseif support.oj.R == [1; 2; 3; 4; 4; 6]
      tmp[20] += 1
    elseif support.oj.R == [1; 2; 3; 1; 5; 6]
      tmp[21] += 1
    elseif support.oj.R == [1; 2; 3; 2; 5; 6]
      tmp[22] += 1
    elseif support.oj.R == [1; 2; 3; 3; 5; 6]
      tmp[23] += 1
    elseif support.oj.R == [1; 2; 3; 4; 5; 6]
      tmp[24] += 1
    end
  end

  tmp /= N

  @test isapprox(tmp, pr, rtol=0.01)

  nothing
end

test_crossover()


# This file defines the Attrition type. This type represents an attrition scheme
#   to make it possible for personnel members to leave service before their
#   mandatory retirement.

# The attrition type requires no additional types. [ XXX true? ]
requiredTypes = [ ]

for reqType in requiredTypes
    if !isdefined( Symbol( uppercase( string( reqType[ 1 ] ) ) * reqType[ 2:end ] ) )
        include( joinpath( typePath, reqType * ".jl" ) )
    end  # if !isdefined( Symbol( ...
end  # for reqType in requiredTypes

export Attrition
type Attrition
    # The name of the attrition scheme.
    name::String

    # The attrition period.
    attrPeriod::Float64

    # The time points of the attrition curve. This curve is always sorted, and
    #   the first element is 0.0.
    attrCurvePoints::Vector{Float64}

    # The attrition rates at the points of the attrition curve.
    attrRates::Vector{Float64}

    # The lambdas of each part of the attrition time distribution.
    lambdas::Vector{Float64}

    # An auxiliary vector of parameters.
    betas::Vector{Float64}

    # The exceeding probabilities for each time node of the attrition time
    #   distribution.
    gammas::Vector{Float64}

    function Attrition( name::String, rate::T1 = 0.0, period::T2 = 1.0 ) where T1 <: Real where T2 <: Real

        if ( rate < 0.0 ) || ( rate >= 1.0 )
            error( "Attrition rate must be between 0.0 and 1.0." )
        end  # if ( rate < 0.0 ) || ( rate > 100.0 )

        if period <= 0.0
            error( "Attrition period must be > 0.0." )
        end  # if period <= 0.0

        newAttr = new()
        newAttr.name = name
        # If there's no attrition, the period doesn't matter.
        newAttr.attrPeriod = rate == 0.0 ? 1.0 : period
        newAttr.attrCurvePoints = [ 0.0 ]
        newAttr.attrRates = [ rate ]
        computeDistPars( newAttr )
        return newAttr

    end  # Attrition( name, rate, period )

    function Attrition( rate::T1 = 0.0, period::T2 = 1.0 ) where T1 <: Real where T2 <: Real

        newAttr = Attrition( "default", rate, period )
        return newAttr

    end  # Attrition( rate, period )

    function Attrition( name::String, curve::T1, period::T2 = 1.0 ) where T1 <: Union{Dict{Float64, Float64}, Array{Float64, 2}} where T2 <: Real

        if period <= 0.0
            error( "Attrition period must be > 0.0." )
        end  # if period <= 0.0

        newAttr = Attrition( name )
        setAttritionCurve( newAttr, curve )
        setAttritionPeriod( newAttr, period )
        return newAttr

    end  # Attrition( name, curve, period )

    function Attrition( curve::T1, period::T2 = 1.0 ) where T1 <: Union{Dict{Float64, Float64}, Array{Float64, 2}} where T2 <: Real

        if period <= 0.0
            error( "Attrition period must be > 0.0." )
        end  # if period <= 0.0

        newAttr = Attrition( "default" )
        setAttritionCurve( newAttr, curve )
        setAttritionPeriod( newAttr, period )
        return newAttr

    end  # Attrition( curve, period )

end  # type Attrition


# automation script for generating READEME gif
# Runs example script for direct problem
# Generates gif
# Saves it in "assets/" folder
# Later, you should manually upload it via Git-LFS

#package prefix
prefix =
dirname(dirname(pathof(NonLinearReactionAdvectionDiffusionWithFrontData)));
assets_path =  joinpath(prefix, "assets/");
gif_path = joinpath(assets_path, "readme_direct.gif");

include( joinpath(prefix, "examples/example_direct_nonuniform_dparams.jl"))

ENV["GKS_ENCODING"] = "utf-8"
@info "Writing README gif" gif_path
make_gif(u, XX, Tₘ, ϕl, ϕr, ϕ, f1_data, f2_data;
         name=gif_path, title = "Прямая задача и экспериментальные данные")


using MixedModelsPermutations
using Test

@testset "nonparametricbootstrap" begin
    include("nonparametricbootstrap.jl")
end

@testset "permutation" begin
    include("permutation.jl")
end

@testset "ols" begin
    include("ols.jl")
end


# zt: should Linear Omega be default?
"""
LinearΩ: Stores data in single `AbstractVector{V}`

`lω[i] = lω.ωvec[lω.ids[i]]`

- No overhead for linearization / delinearization
- Good for optimization / inference methods which require linear interface
- If heterogenous elements are used, must convert (which can be slow)
"""
struct LinearΩ{I, SEG, V <: AbstractArray} <: ΩBase{I}
  ids::Dict{I, SEG} # zt: too precise, any associable will do. Map from Ω index I to segment of omvec
  ωvec::V
end

"`emp(::Type{T})` Empty container of type T"
function emp end

emp(a::Type{<:Array}) = a()

"$(SIGNATURES) Empty `LinearΩ`"
LinearΩ{I, SEG, V}() where {I, SEG, V} = LinearΩ{I, SEG, V}(Dict{I, SEG}(), emp(V))

linearize(lω::LinearΩ) = lω.ωvec
unlinearize(ωvec, lω::LinearΩ) = LinearΩ(lω.ids, ωvec)

# zt: fixme, dont use T, determine it by the input
flat(rv, ω::T) where T <: LinearΩ = floatvec -> rv(T(ω.ids, floatvec))

# flat2(rv, ω::LinearΩ) = LinearΩ(floatvec)

# zt: renamed this? Wanted to distinguish getting ith Omega value from
# projection. Maybe projection shouldnt overload getindex?
getdim(lω, i) = lω.ωvec[i]

# Update #

"Apply `kernel` to ith component"
function update(lω::LinearΩ, i::Int, kernel::Function) # zt: use name other than kernel
  # zt: `kernel` Might be a callable
  # zt: isn't there a more efficient version which just copies ωvec?
  lω_ = deepcopy(lω)
  lω_.ωvec[i] = kernel(lω_.ωvec[i])
  lω_
end

function update(lω::LinearΩ, i::Int, val)
  # This update != previous update, dont pun!
  lω_ = deepcopy(lω)
  lω_.ωvec[i] = val
  lω_
end

# randtype #

# Defualt behaviour: return type is T
randrtype(lω::LinearΩ{I, SEG, V}, ::Type{T}) where {T, I, SEG, V} = T

# Handle ForwardDiff: If `lω.ωvec` is a Vector of Duals then return type will be a DUal
randrtype(lω::LinearΩ{I, SEG, DT}, ::Type{T}) where {Q<:ForwardDiff.Dual, T, I, SEG, DT <: AbstractArray{Q}} = Q

# If Dims specified, return array of this type
randrtype(ω, T, ::Dims{N}) where N = Array{randrtype(ω, T), N}

# Specialization for when T is Type
randrtype(ω, ::Type{T}, ::Dims{N}) where {T, N} = Array{randrtype(ω, T), N}

# WHAT ABOUT Static Array
# sol? If V is a static vector then return a StaticArray
#    prob: reshaping it will return a reshape vector
#    prob: what if we want an actual staticarray instead
#  How to distingusih between staticarray rt and reshapedarray
# 1. specify in the model code
# 2. one issue is that Dims values aren't included in type information\: so there's no enough information in the model to specify the shape
# 3. It's might be possible with constant propagation that you get static types
# 4.
# WHAT ABOUT Float32
# I think that's handled fine from the type of T
# Is there ever a case when say we want values to be tracked (duals) and some not?
# Maybe if we have some discrete parameters and dont want them to continuized
# But what if we do want them to be.
#

nelem(lω) = length(lω.ωvec)
Base.values(lω::LinearΩ) = [lω.ωvec]
Base.isempty(lω::LinearΩ) = isempty(lω.ωvec)

# memrand #

# Flux-specific #

emp(a::Type{Tracker.TrackedArray{A, B, C}}) where {A, B, C} = C()

# zt: This is type piracy: use a different method name
Base.append!(ta::Tracker.TrackedArray, a::Array) =
  (append!(ta.data, a); append!(ta.grad, zero(a)))

Base.append!(ta::Tracker.TrackedArray, tb::Tracker.TrackedArray) =
  (append!(ta.data, tb.data); append!(ta.grad, tb.grad))

# using ZenUtils

function randrtype(lω::LinearΩ{I, SEG, V}, ::Type{T}, dims::Dims{N}) where {I, SEG, T, V <: Tracker.TrackedArray{T}, N}
  Tracker.TrackedArray{T, N, Array{T, N}}
end

# function randrtype(lω::LinearΩ{I, SEG, V}, ::Type{T}) where {I, SEG, V<: Tracker.TrackedArray, T <: AbstractFloat}
#   Tracker.TrackedReal{T}
# end

# coerce(::Type{T}, x) where T = convert(T, x)
# coerse(::Type{T}, x) where T

function memrand(lω::LinearΩ, id, X, dims::Dims; rng)
  RT = randrtype(lω,  X,  dims)
  seg = get(lω.ids, id, 0)
  if seg == 0
    res = rand(rng, X, dims)
    startidx = length(lω.ωvec) + 1
    lω.ids[id] = startidx:startidx + prod(dims) - 1
    res_::RT = res  # zt: issue si that might need to convert into correct format
    typeof(res_)
    typeof(lω.ωvec)
    # @grab res_
    p = lω.ωvec
    # @grab p
    append!(lω.ωvec, res_)
    res_
  else
    seg = lω.ids[id]
    subωvec = lω.ωvec[seg]
    res::RT = reshape(subωvec, dims) # reshaping a thing might produce a vector
    res
  end
end

memrand(lω::LinearΩ{I, SEG, V}, id, X; rng) where {I, SEG, V<: Tracker.TrackedArray} =
  first(memrand(lω, id, X, (1,); rng = rng))

randrtype(lω, x::Distributions.Distribution) = eltype(x)

function memrand(lω::LinearΩ, id, X; rng)
  RT = randrtype(lω, X)
  seg = get(lω.ids, id, 0)
  if seg == 0
    res::RT = rand(rng, X)
    res_::RT = res
    push!(lω.ωvec, res_)
    startidx = length(lω.ωvec)
    lω.ids[id] = startidx:startidx
    res_
  else
    # @show seg = lω.ids[id]
    # @show id
    subωvec = lω.ωvec[first(seg)]::RT
  end
end


methods_implemented = Dict{String,Vector{String}}()
methods_implemented["GaussianLikelihood"] = []
methods_implemented["StudentTLikelihood"] = ["AnalyticVI","AnalyticSVI","QuadratureVI","QuadratureSVI"] # ["QuadratureVI","QuadratureSVI"]
methods_implemented["LaplaceLikelihood"] = ["AnalyticVI","AnalyticSVI","QuadratureVI","QuadratureSVI"]
methods_implemented["HeteroscedasticLikelihood"] = []
methods_implemented["LogisticLikelihood"] = ["AnalyticVI","AnalyticSVI","QuadratureVI","QuadratureSVI"]
methods_implemented["BayesianSVM"] = ["AnalyticVI","AnalyticSVI"]
methods_implemented["LogisticSoftMaxLikelihood"] = ["AnalyticVI","AnalyticSVI"]# "NumericalVI","NumericalSVI"]
methods_implemented["SoftMaxLikelihood"] = ["MCIntegrationVI","MCIntegrationSVI"]
methods_implemented["PoissonLikelihood"] = ["AnalyticVI","AnalyticSVI"]

methods_implemented_VGP = deepcopy(methods_implemented)
push!(methods_implemented_VGP["StudentTLikelihood"],"GibbsSampling")
push!(methods_implemented_VGP["LaplaceLikelihood"],"GibbsSampling")
push!(methods_implemented_VGP["LogisticLikelihood"],"GibbsSampling")
push!(methods_implemented_VGP["LogisticSoftMaxLikelihood"],"GibbsSampling")
push!(methods_implemented_VGP["HeteroscedasticLikelihood"],"AnalyticVI")
methods_implemented_SVGP = deepcopy(methods_implemented)
methods_implemented_SVGP["GaussianLikelihood"] = ["AnalyticVI","AnalyticSVI"]

stoch(s::Bool,inference::String) = inference != "GibbsSampling" ? (s ? inference[1:end-2]*"SVI" : inference) : inference
addiargument(s::Bool,inference::String) = inference == "GibbsSampling" ? "nBurnin=0" : (s ? "b" : "")
addlargument(likelihood::String) = begin
    if (likelihood == "StudentTLikelihood")
        return "ν"
    else
        return ""
    end
end

function testconv(model::AbstractGP,problem_type::String,X::AbstractArray,y::AbstractArray)
    μ,Σ = predict_f(model,X,covf=true)
    y_pred = predict_y(model,X)
    py_pred = proba_y(model,X)
    if problem_type == "Regression"
        err=mean(abs.(y_pred-y))
        @info "Regression Error" err
        return err < 1.5
    elseif problem_type == "Classification"
        err=mean(y_pred.!=y)
        @info "Classification Error" err
        return err < 0.5
    elseif problem_type == "MultiClass"
        err = mean(y_pred.!=y)
        @info "Multiclass Error" err
        return err < 0.5
    elseif problem_type == "Event"
        err = mean(abs.(y_pred-y))
        @info "Event Error" err
        return err < 1.0
    end
end