function place_first_grid_order!(current, ticker, config, open_orders, bot_path, client, usr)
    @info("NEW order found!")
    id_str = open_orders["NEW"]["trigger"] * "--" * string(now())
    quant = ceil(config["order_volume"] / current, digits=config["precision"])
    if config["precision"] < 1
        quant = string(Int(quant))
    end
    order = try client.order_limit_buy(
        symbol=config["symbol"],
        quantity=quant,
        price=ticker);
    catch
        @info("order_limit_buy failed")
        nothing
    end
    if order !== nothing
        open_orders["BUY"]["EXTRA-0_" * id_str] = order
        open_orders["NEW"] = Dict()
        open(joinpath(bot_path, "open_orders.json"), "w") do f
            JSON.print(f, open_orders, 4)
        end
        p = Dict()
        p["ticker"] = ticker
        create_notification("new cycle", config, usr, params = p)
    end
    return open_orders
end
export place_first_grid_order!

function place_buy_order_command(k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    quant = round(v["origQty"], digits=config["precision"])
    new_id = k
    if v["price"] != "market"
        new_price = Pinguu.price2string(parse(Float64, v["price"]), config["sell_precision"])
        order = try client.order_limit_buy(
            symbol= config["symbol"],
            quantity=quant,
            price=new_price);
            catch
            nothing
        end
    else
        order = try client.order_market_buy(
            symbol= config["symbol"],
            quantity=quant);
            catch
            nothing
        end
    end
    if order !== nothing
        order["time"] = string(now())
        open_orders["BUY"][new_id] = order
    else
        open_orders["BUY"][new_id] = Dict()
        open_orders["BUY"][new_id]["origQty"] = quant
        open_orders["BUY"][new_id]["price"]   = new_price
        open_orders["BUY"][new_id]["status"]  = "place order"
        open_orders["BUY"][new_id]["time"]    = string(now())
    end
    open(joinpath(bot_path, "open_orders.json"), "w") do f
        JSON.print(f, open_orders, 4)
    end
    open(joinpath(bot_path, "trigger.json"), "w") do f
        JSON.print(f, Dict(), 4)
    end
end
export place_buy_order_command

function buy_executed(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    #
    ## Cancel order, if not done yet
    if tmp["status"] != client.ORDER_STATUS_FILLED && tmp["status"] != client.ORDER_STATUS_CANCELED
        result = try client.cancel_order(
            symbol=config["symbol"],
            orderId=tmp["orderId"])
        catch
            nothing
        end
        if result == nothing
            return nothing
        end
    end
    extra_iteration = parse(Int64, split(split(k, "_")[1], "-")[2])
    #
    ## Move to closed_orders
    !(id_str in keys(closed_orders)) ? closed_orders[id_str] = Dict() : ""
    tmp["time"] = string(now())
    closed_orders[id_str][split(k, "_")[1]] = deepcopy(tmp);
    open(joinpath(bot_path, "closed_orders.json"), "w") do f
        JSON.print(f, closed_orders, 4)
    end
    #
    ## Remove from open_orders
    delete!(open_orders["BUY"], k)
    open(joinpath(bot_path, "open_orders.json"), "w") do f
        JSON.print(f, open_orders, 4)
    end
    #
    ## Place EXTRA
    new_id = "EXTRA-" * string(extra_iteration + 1) * "_" * id_str

    last_price = parse(Float64, tmp["price"]) == 0.0 ? v["price"] : tmp["price"]

    new_price = Pinguu.price2string(parse(Float64, last_price) * (1 - config["extra_order_step"]), config["sell_precision"], round_up=true)
    quant = floor(config["order_volume"] * (config["extra_order_martingale"])^(extra_iteration + 1) / parse(Float64, new_price), digits=config["precision"])
    if extra_iteration < parse(Float64, config["extra_order_count"])
        open_orders["BUY"][new_id] = Dict()
        open_orders["BUY"][new_id]["origQty"] = quant
        open_orders["BUY"][new_id]["price"]   = new_price
        open_orders["BUY"][new_id]["status"]  = "place order"
        open_orders["BUY"][new_id]["time"]    = string(now())
        open(joinpath(bot_path, "open_orders.json"), "w") do f
            JSON.print(f, open_orders, 4)
        end
        open(joinpath(bot_path, "trigger.json"), "w") do f
            JSON.print(f, Dict(), 4)
        end
    end
    #
    ## Cancel current take profit orders
    if length(open_orders["SELL"]) > 0
        for (k2, v2) in open_orders["SELL"]
            if split(k2, "_")[end] == id_str
                tmp_sell = try client.get_order(
                    symbol=config["symbol"],
                    orderId=v2["orderId"])
                    catch
                    try sleep(0.5)
                        client.get_order(
                            symbol=config["symbol"],
                            orderId=v2["orderId"])
                        catch nothing
                    end
                end
                if !(tmp_sell["status"] in [client.ORDER_STATUS_CANCELED, client.ORDER_STATUS_FILLED]) || tmp_sell == nothing
                    sleep(0.5)
                    tmp_sell = try client.cancel_order(
                        symbol=config["symbol"],
                        orderId=v2["orderId"])
                    catch
                        nothing
                    end
                end
                if tmp_sell !== nothing
                    if parse(Float64, v2["executedQty"]) > 0
                        random_id_str = "SELL-" * lpad(rand(1:1:10000), 5, "0")
                        while haskey(closed_orders[id_str], random_id_str)
                            random_id_str = "SELL-" * lpad(rand(1:1:10000), 5, "0")
                        end
                        closed_orders[id_str][random_id_str] = result
                        open(joinpath(bot_path, "closed_orders.json"), "w") do f
                            JSON.print(f, closed_orders, 4)
                        end
                    end
                    delete!(open_orders["SELL"], k2)
                else
                    #
                    ## Cancel failed
                    delete!(open_orders["SELL"], k2)
                    v2["status"] = "cancel order"
                    #
                    ## Give different ID to be able to create a new "main" order later
                    random_id_str = "SELL-" * lpad(rand(1:1:10000), 5, "0")
                    open_orders[random_id_str] = v2
                end
            end
        end
    end

    #
    ## Place command for take profit order
    open_orders["SELL"]["SELL_" * id_str] = Dict()
    open_orders["SELL"]["SELL_" * id_str]["status"] = "place order"
    open_orders["SELL"]["SELL_" * id_str]["id_str"] = id_str

    open(joinpath(bot_path, "open_orders.json"), "w") do f
        JSON.print(f, open_orders, 4)
    end
    open(joinpath(bot_path, "trigger.json"), "w") do f
        JSON.print(f, Dict(), 4)
    end

    #
    ## Create notification
    p = Dict()
    p["extra_iteration"] = extra_iteration
    p["executedQty"] = tmp["executedQty"]
    create_notification("buy exec", config, usr, params = p)
    if extra_iteration >= parse(Float64, config["extra_order_count"])
        create_notification("no volume", config, usr)
    end
end
export buy_executed

function cancel_buy(k, v, id_str, config, open_orders, closed_orders, bot_path, client)
    @info("Cancel Order")
    if haskey(v, "orderId")
        result = try client.cancel_order(
            symbol=config["symbol"],
            orderId=v["orderId"])
        catch
            nothing
        end
        if result !== nothing
            delete!(open_orders["BUY"], k)
            open(joinpath(bot_path, "open_orders.json"), "w") do f
                JSON.print(f, open_orders, 4)
            end
            if parse(Float64, result["executedQty"]) > 0
                random_id_str = "EXTRA-" * lpad(rand(100:1:10000), 5, "0")
                while haskey(closed_orders[id_str], random_id_str)
                    random_id_str = "EXTRA-" * lpad(rand(100:1:10000), 5, "0")
                end
                closed_orders[id_str][random_id_str] = result
                open(joinpath(bot_path, "closed_orders.json"), "w") do f
                    JSON.print(f, closed_orders, 4)
                end
            end
        end
    else
        @info("Order has not been placed")
        #
        ## Order not placed, yet
        delete!(open_orders["BUY"], k)
        open(joinpath(bot_path, "open_orders.json"), "w") do f
            JSON.print(f, open_orders, 4)
        end

    end
    open(joinpath(bot_path, "trigger.json"), "w") do f
        JSON.print(f, Dict(), 4)
    end
end
export cancel_buy

function buy_now(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees)
    extra_iteration = parse(Int64, split(split(k, "_")[1], "-")[2])
    if extra_iteration < parse(Float64, config["extra_order_count"])
        #
        # Cancel current BUY order
        if tmp["status"] != client.ORDER_STATUS_CANCELED
            result = try client.cancel_order(
                symbol=config["symbol"],
                orderId=v["orderId"])
            catch
                nothing
            end
            if result == nothing
                return nothing
            end
        else
            result = tmp
        end

        #
        # Check if that order was partially filled -> if so: move cancelled order to closed_orders
        if parse(Float64, result["executedQty"]) > 0
            result["time"] = string(now())
            !haskey(closed_orders, id_str) ? closed_orders[id_str] = Dict() : ""
            closed_orders[id_str][split(k, "_")[1]] = result
            extra_iteration += 1
            open(joinpath(bot_path, "closed_orders.json"), "w") do f
                JSON.print(f, closed_orders, 4)
            end
        end
        #
        # Remove from open_orders
        delete!(open_orders["BUY"], k)
        #
        # Get new order details
        ticker = client.get_ticker(symbol=config["symbol"])["lastPrice"]
        quant = floor(parse(Float64, v["origQty"]), digits=config["precision"])
        new_id = "EXTRA-" * string(extra_iteration) * "_" * id_str
        #
        # Place new order
        order = try client.order_limit_buy(
            symbol=config["symbol"],
            quantity=quant,
            price=ticker)
        catch
            nothing
        end

        if order !== nothing
            order["time"] = string(now())
#                 if parse(Float64, order["price"]) == 0
#                     order["price"] = Pinguu.price2string(parse(Float64, order["cummulativeQuoteQty"]) / parse(Float64, order["origQty"]), config["sell_precision"])
#                 end
            open_orders["BUY"][new_id] = order
        else
            open_orders["BUY"][new_id] = Dict()
            open_orders["BUY"][new_id]["origQty"] = quant
            open_orders["BUY"][new_id]["price"]   = ticker
            open_orders["BUY"][new_id]["status"]  = "place order"
            open_orders["BUY"][new_id]["time"]    = string(now())
        end
        open(joinpath(bot_path, "open_orders.json"), "w") do f
            JSON.print(f, open_orders, 4)
        end
    end
    open(joinpath(bot_path, "trigger.json"), "w") do f
        JSON.print(f, Dict(), 4)
    end
end
export buy_now

function partially_order(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees, minutes)
    if v["partial_time"] < string(now() - Minute(minutes))
        result = try client.cancel_order(
            symbol=config["symbol"],
            orderId=tmp["orderId"])
        catch
            nothing
        end
        if result !== nothing
            tmp["partial_time"] = v["partial_time"]
            v = deepcopy(tmp)
            v["status"] = "continue"
            open_orders["BUY"][k] = v
            open(joinpath(bot_path, "open_orders.json"), "w") do f
                JSON.print(f, open_orders, 4)
            end
            p = Dict()
            p["extra_iteration"] = extra_iteration
            p["executedQty"] = tmp["executedQty"]
            create_notification("partial buy exec", config, usr, params = p)
        end
        if tmp["status"] == client.ORDER_STATUS_CANCELED
            v["status"] = "continue"
            open_orders["BUY"][k] = v
            open(joinpath(bot_path, "open_orders.json"), "w") do f
                JSON.print(f, open_orders, 4)
            end
        end
    end
end
export partially_order

function first_order_takes_too_long(tmp, k, v, id_str, config, open_orders, closed_orders, bot_path, client, usr, fees, minutes)
    t = DateTime(split(id_str, "--")[end])
    if now() - t > Minute(minutes)
        result = try client.cancel_order(
            symbol=config["symbol"],
            orderId=v["orderId"])
        catch
            nothing
        end
        if result !== nothing
            delete!(open_orders["BUY"], k)
            open(joinpath(bot_path, "open_orders.json"), "w") do f
                JSON.print(f, open_orders, 4)
            end
            Pinguu.create_notification("cycle closed", config, usr)
        end
    end
end
export first_order_takes_too_long

function add_fees_to_closed_order(order, config, bot_path, client)
    for t in order["trades"]
        sleep(0.5)
        commission = parse(Float64, t["commission"])
        commission_coin = parse(Float64, t["commission"])
        if commission != 0
            from = split(string(unix2datetime(t["time"]/1000)), ".")[1]
            to = split(string(unix2datetime(t["time"]/1000) + Minute(1)), ".")[1]
            #
            ## for commission
            if t["commissionAsset"] == config["farm_coin"]
                commission *= parse(Float64, t["price"])
            elseif t["commissionAsset"] != config["base_coin"]
                translator = string(t["commissionAsset"], config["base_coin"])
                klines = client.get_historical_klines(translator, client.KLINE_INTERVAL_1MINUTE, from, to)
                tt = Pinguu.klines2table(klines, time_warp=0)
                commission *= tt[1].close
                tmp_price_for_conversion = tt[1].close
            end
            #
            ## for commission_coin
            if t["commissionAsset"] != config["farm_coin"]
                translator = string(t["commissionAsset"], config["farm_coin"])
                # Translator exists
                klines = try client.get_historical_klines(translator, client.KLINE_INTERVAL_1MINUTE, from, to)
                    catch
                    false
                end
                if klines == false
                    # Translator exists in other order
                    translator = string(config["farm_coin"], t["commissionAsset"])
                    klines = try client.get_historical_klines(translator, client.KLINE_INTERVAL_1MINUTE, from, to)
                        catch
                        false
                    end
                else
                    tt = Pinguu.klines2table(klines, time_warp=0)
                    weightedAvgPrice = 1/tt[1].close
                end
                if klines == false
                    # Translator does not exist e.g. DOGEBNB
                    weightedAvgPrice = tmp_price_for_conversion / parse(Float64, t["price"])
                else
                    tt = Pinguu.klines2table(klines, time_warp=0)
                    weightedAvgPrice = 1/tt[1].close
                end

                if typeof(weightedAvgPrice) == String
                    weightedAvgPrice = parse(Float64, weightedAvgPrice)
                end
                commission_coin *= weightedAvgPrice
            end
            t["com"] = commission
            t["com_coin"] = commission_coin
        else
            t["com"] = 0
            t["com_coin"] = 0
        end
    end
    return order
end
export add_fees_to_closed_order

function check_ongoing_closed(id_str, config, closed_orders, bot_path, client)
    store = false
    for (id, order) in closed_orders[id_str]
        if !haskey(order, "trades")
            trades = client.get_my_trades(orderId=order["orderId"], symbol=order["symbol"])
            order["trades"] = trades
            order = try add_fees_to_closed_order(order, config, bot_path, client)
                catch
                @info("Getting fees failed.")
                return
            end
            store = true
        end
    end
    if store
        open(joinpath(bot_path, "closed_orders.json"), "w") do f
            JSON.print(f, closed_orders, 4)
        end
    end
end
export check_ongoing_closed


module Tarana

abstract type AbstractPitchClass end

struct PitchClass <: AbstractPitchClass
    s::Symbol
end

const Pitch{T <: AbstractPitchClass} = Tuple{T, Integer}

struct Note{T <: Number}
    t::T
    p::Pitch
end

struct Rest{T <: Number}
    t::T
end

pc = PitchClass

const
    Cff = pc(:Cff)
    Cf  = pc(:Cf)
    C   = pc(:C)
    Cs  = pc(:Cs)
    Css = pc(:Css)
    Dff = pc(:Dff)
    Df  = pc(:Df)
    D   = pc(:D)
    Ds  = pc(:Ds)
    Dss = pc(:Dss)
    Eff = pc(:Eff)
    Ef  = pc(:Ef)
    E   = pc(:E)
    Es  = pc(:Es)
    Ess = pc(:Ess)
    Fff = pc(:Fff)
    Ff  = pc(:Ff)
    F   = pc(:F)
    Fs  = pc(:Fs)
    Fss = pc(:Fss)
    Gff = pc(:Gff)
    Gf  = pc(:Gf)
    G   = pc(:G)
    Gs  = pc(:Gs)
    Gss = pc(:Gss)
    Aff = pc(:Aff)
    Af  = pc(:Af)
    A   = pc(:A)
    As  = pc(:As)
    Ass = pc(:Ass)
    Bff = pc(:Bff)
    Bf  = pc(:Bf)
    B   = pc(:B)
    Bs  = pc(:Bs)
    Bss = pc(:Bss)

cff(o, d) = Note(d, (Cff, o))
cf(o, d)  = Note(d, (Cf, o))
c(o, d)   = Note(d, (C, o))
cs(o, d)  = Note(d, (Cs, o))
css(o, d) = Note(d, (Css, o))
dff(o, d) = Note(d, (Dff, o))
df(o, d)  = Note(d, (Df, o))
d(o, d)   = Note(d, (D, o))
ds(o, d)  = Note(d, (Ds, o))
dss(o, d) = Note(d, (Dss, o))
eff(o, d) = Note(d, (Eff, o))
ef(o, d)  = Note(d, (Ef, o))
e(o, d)   = Note(d, (E, o))
es(o, d)  = Note(d, (Es, o))
ess(o, d) = Note(d, (Ess, o))
fff(o, d) = Note(d, (Fff, o))
ff(o, d)  = Note(d, (Ff, o))
f(o, d)   = Note(d, (F, o))
fs(o, d)  = Note(d, (Fs, o))
fss(o, d) = Note(d, (Fss, o))
gff(o, d) = Note(d, (Gff, o))
gf(o, d)  = Note(d, (Gf, o))
g(o, d)   = Note(d, (G, o))
gs(o, d)  = Note(d, (Gs, o))
gss(o, d) = Note(d, (Gss, o))
aff(o, d) = Note(d, (Aff, o))
af(o, d)  = Note(d, (Af, o))
a(o, d)   = Note(d, (A, o))
as(o, d)  = Note(d, (As, o))
ass(o, d) = Note(d, (Ass, o))
bff(o, d) = Note(d, (Bff, o))
bf(o, d)  = Note(d, (Bf, o))
b(o, d)   = Note(d, (B, o))
bs(o, d)  = Note(d, (Bs, o))
bss(o, d) = Note(d, (Bss, o))

export
    Note,
    Rest

export
    Cff, Cf, C, Cs, Css,
    Dff, Df, D, Ds, Dss,
    Eff, Ef, E, Es, Ess,
    Fff, Ff, F, Fs, Fss,
    Gff, Gf, G, Gs, Gss,
    Aff, Af, A, As, Ass,
    Bff, Bf, B, Bs, Bss

export
    cff, cf, c, cs, css,
    dff, df, d, ds, dss,
    eff, ef, e, es, ess,
    fff, ff, f, fs, fss,
    gff, gf, g, gs, gss,
    aff, af, a, as, ass,
    bff, bf, b, bs, bss

const pc_to_i = Dict(
    Cff => -2, Cf => -1, C =>  0, Cs =>  1, Css =>  2,
    Dff =>  0, Df =>  1, D =>  2, Ds =>  3, Dss =>  4,
    Eff =>  2, Ef =>  3, E =>  4, Es =>  5, Ess =>  6,
    Fff =>  3, Ff =>  4, F =>  5, Fs =>  6, Fss =>  7,
    Gff =>  5, Gf =>  6, G =>  7, Gs =>  8, Gss =>  9,
    Aff =>  7, Af =>  8, A =>  9, As => 10, Ass => 11,
    Bff =>  9, Bf => 10, B => 11, Bs => 12, Bss => 13)

const pc_to_str = Dict(
    Cff => "C♭♭", Cf => "C♭", C => "C", Cs => "C♯", Css => "C♯♯",
    Dff => "D♭♭", Df => "D♭", D => "D", Ds => "D♯", Dss => "D♯♯",
    Eff => "E♭♭", Ef => "E♭", E => "E", Es => "E♯", Ess => "E♯♯",
    Fff => "F♭♭", Ff => "F♭", F => "F", Fs => "F♯", Fss => "F♯♯",
    Gff => "G♭♭", Gf => "G♭", G => "G", Gs => "G♯", Gss => "G♯♯",
    Aff => "A♭♭", Af => "A♭", A => "A", As => "A♯", Ass => "D♯♯",
    Bff => "B♭♭", Bf => "B♭", B => "B", Bs => "B♯", Bss => "B♯♯")

include("./unicode.jl")

const eqtemp = [C, C♯, D, D♯, E, F, F♯, G, G♯, A, A♯, B]

ord(p) = ord(p...)
ord(n::Note) = ord(n.p)
ord(pc, oct) = 12 * (oct + 1) + pc_to_i[pc]

pitch(ap::Integer) =
    let (oct, n) = divrem(ap, 12)
        (eqtemp[n + 1], oct - 1)
    end

trans(i, p::Pitch) = ord(p) + i |> pitch
trans(i, n::Note) = Note(n.t, trans(i, n.p))

export
    ord,
    pitch,
    trans

# Base.show(io::IO, x::PitchClass) = print(io, pc_to_str[x])

Base.:+(a::Pitch, b) = trans(b, a)
Base.:-(a::Pitch, b) = trans(-b, a)

Base.:+(a::Note, b) = Note(a.t, a.p + b)
Base.:-(a::Note, b) = Note(a.t, a.p - b)

Base.isless(a::Pitch, b::Pitch) = ord(a) < ord(b)
Base.isless(a::PitchClass, b::PitchClass) = pc_to_i[a] < pc_to_i[b]

end # module


abstract type AbstractAlgorithm end

# trait dispatch
RatesCache(::AbstractAlgorithm) = HasRates()

# by default, initialization is the same as updating the rates vector
# algorithms should overload this behavior as needed
@inline function initialize!(algorithm::AbstractAlgorithm, state, model)
  update_jump_rates!(algorithm, state, model)
end

# we assume that `s` is a time increment rather than an absolute time
# as such, the default behavior is to return `t + s` as the next system time
# algorithms such as NRM can overload this behavior
@inline @fastmath function get_new_time(algorithm::AbstractAlgorithm, t, s)
  return t + s
end

abstract type DecoupledSearchAlgorithm <: AbstractAlgorithm end

# delegate the update procedure based on the RatesCacheType
@inline function update_jump_rates!(algorithm::DecoupledSearchAlgorithm, state, model)
  algorithm.total_rate = update_jump_rates!(RatesCache(algorithm), algorithm.rates, algorithm.total_rate, state, model)
end

@inline function update_jump_rates!(algorithm::DecoupledSearchAlgorithm, state, model, j)
  algorithm.total_rate = update_jump_rates!(RatesCache(algorithm), algorithm.rates, algorithm.total_rate, state, model, j)
end

# A DecoupledSearchSampler always generates the next reaction time based on an exponential distribution.
# The next reaction index depends exclusively on the sampler.
@inline function generate_jump(algorithm::DecoupledSearchAlgorithm)
  j = search_jumps(algorithm)
  s = randexp() / algorithm.total_rate

  return (j, s)
end

abstract type CoupledSearchAlgorithm <: AbstractAlgorithm end

# delegate the update procedure based on the RatesCacheType
@inline function update_jump_rates!(algorithm::CoupledSearchAlgorithm, state, model)
  algorithm.total_rate = update_jump_rates!(RatesCache(algorithm), algorithm.rates, algorithm.total_rate, state, model)
end

@inline function update_jump_rates!(algorithm::CoupledSearchAlgorithm, state, model, j)
  algorithm.total_rate = update_rates!(RatesCache(algorithm), algorithm.rates, algorithm.total_rate, state, model, j)
end

@inline function generate_jump(algorithm::CoupledSearchAlgorithm)
  (j, s) = search_jumps(algorithm)

  return (j, s)
end

abstract type UnsafeLeapAlgorithm <: AbstractAlgorithm end

@inline function update_jump_rates!(algorithm::UnsafeLeapAlgorithm, state, model)
  algorithm.total_rate = update_jump_rates!(RatesCache(algorithm), algorithm.rates, algorithm.total_rate, state, model)
end

function initialize!(algorithm::UnsafeLeapAlgorithm, state, model)
  update!(algorithm, state, model)
end

function generate_leap!(v, algorithm::UnsafeLeapAlgorithm, Δt_max)
  # unpack data
  rates = algorithm.rates
  total_rate = algorithm.total_rate
  validate_leap! = algorithm.validate_leap!

  # evaluate the leap formula to compute τ
  s = algorithm.leap_formula(rates, total_rate)
  s = min(s, Δt_max)

  # if there is a valid leap...
  if isfinite(s)
    generate_events!(algorithm, v, rates, s)
    validate_leap!(v, s)
  else
    # otherwise fill the jumps with zeros
    fill!(v, zero(eltype(v)))
  end

  return v, s
end

# abstract type SafeLeapAlgorithm <: AbstractAlgorithm end
#
# @inline function update_jump_rates!(algorithm::SafeLeapAlgorithm, state, model)
#   algorithm.total_rate = update_jump_rates!(RatesCache(algorithm), algorithm.rates, algorithm.total_rate, state, model)
# end
#
# function initialize!(algorithm::SafeLeapAlgorithm, state, model)
#   update!(algorithm, state, model)
# end
#
# function generate_leap!(v, algorithm::SafeLeapAlgorithm, Δt_max)
#   # unpack data
#   rates = algorithm.rates
#   total_rate = algorithm.total_rate
#
#   # evaluate the leap formula to compute τ
#   s = algorithm.leap_formula(rates, total_rate)
#   s = min(s, Δt_max)
#
#   # if there is a valid leap...
#   if isfinite(s)
#     generate_events!(algorithm, v, rates, s)
#   else
#     # otherwise fill the jumps with zeros
#     fill!(v, zero(eltype(v)))
#   end
#
#   return v, s
# end


function BodiesElements(arg0::AbsoluteDate, arg1::jdouble, arg10::jdouble, arg11::jdouble, arg12::jdouble, arg13::jdouble, arg14::jdouble, arg15::jdouble, arg16::jdouble, arg17::jdouble, arg18::jdouble, arg19::jdouble, arg2::jdouble, arg20::jdouble, arg21::jdouble, arg22::jdouble, arg23::jdouble, arg24::jdouble, arg25::jdouble, arg26::jdouble, arg27::jdouble, arg28::jdouble, arg29::jdouble, arg3::jdouble, arg30::jdouble, arg31::jdouble, arg4::jdouble, arg5::jdouble, arg6::jdouble, arg7::jdouble, arg8::jdouble, arg9::jdouble)
    return BodiesElements((AbsoluteDate, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble, jdouble), arg0, arg1, arg10, arg11, arg12, arg13, arg14, arg15, arg16, arg17, arg18, arg19, arg2, arg20, arg21, arg22, arg23, arg24, arg25, arg26, arg27, arg28, arg29, arg3, arg30, arg31, arg4, arg5, arg6, arg7, arg8, arg9)
end

function get_d(obj::DelaunayArguments)
    return jcall(obj, "getD", jdouble, ())
end

function get_d_dot(obj::DelaunayArguments)
    return jcall(obj, "getDDot", jdouble, ())
end

function get_date(obj::DelaunayArguments)
    return jcall(obj, "getDate", AbsoluteDate, ())
end

function get_f(obj::DelaunayArguments)
    return jcall(obj, "getF", jdouble, ())
end

function get_f_dot(obj::DelaunayArguments)
    return jcall(obj, "getFDot", jdouble, ())
end

function get_gamma(obj::DelaunayArguments)
    return jcall(obj, "getGamma", jdouble, ())
end

function get_gamma_dot(obj::DelaunayArguments)
    return jcall(obj, "getGammaDot", jdouble, ())
end

function get_l(obj::DelaunayArguments)
    return jcall(obj, "getL", jdouble, ())
end

function get_l_dot(obj::DelaunayArguments)
    return jcall(obj, "getLDot", jdouble, ())
end

function get_l_ju(obj::BodiesElements)
    return jcall(obj, "getLJu", jdouble, ())
end

function get_l_ju_dot(obj::BodiesElements)
    return jcall(obj, "getLJuDot", jdouble, ())
end

function get_l_ma(obj::BodiesElements)
    return jcall(obj, "getLMa", jdouble, ())
end

function get_l_ma_dot(obj::BodiesElements)
    return jcall(obj, "getLMaDot", jdouble, ())
end

function get_l_me(obj::BodiesElements)
    return jcall(obj, "getLMe", jdouble, ())
end

function get_l_me_dot(obj::BodiesElements)
    return jcall(obj, "getLMeDot", jdouble, ())
end

function get_l_ne(obj::BodiesElements)
    return jcall(obj, "getLNe", jdouble, ())
end

function get_l_ne_dot(obj::BodiesElements)
    return jcall(obj, "getLNeDot", jdouble, ())
end

function get_l_prime(obj::DelaunayArguments)
    return jcall(obj, "getLPrime", jdouble, ())
end

function get_l_prime_dot(obj::DelaunayArguments)
    return jcall(obj, "getLPrimeDot", jdouble, ())
end

function get_l_sa(obj::BodiesElements)
    return jcall(obj, "getLSa", jdouble, ())
end

function get_l_sa_dot(obj::BodiesElements)
    return jcall(obj, "getLSaDot", jdouble, ())
end

function get_l_ur(obj::BodiesElements)
    return jcall(obj, "getLUr", jdouble, ())
end

function get_l_ur_dot(obj::BodiesElements)
    return jcall(obj, "getLUrDot", jdouble, ())
end

function get_l_ve(obj::BodiesElements)
    return jcall(obj, "getLVe", jdouble, ())
end

function get_l_ve_dot(obj::BodiesElements)
    return jcall(obj, "getLVeDot", jdouble, ())
end

function get_le(obj::BodiesElements)
    return jcall(obj, "getLE", jdouble, ())
end

function get_le_dot(obj::BodiesElements)
    return jcall(obj, "getLEDot", jdouble, ())
end

function get_omega(obj::DelaunayArguments)
    return jcall(obj, "getOmega", jdouble, ())
end

function get_omega_dot(obj::DelaunayArguments)
    return jcall(obj, "getOmegaDot", jdouble, ())
end

function get_pa(obj::BodiesElements)
    return jcall(obj, "getPa", jdouble, ())
end

function get_pa_dot(obj::BodiesElements)
    return jcall(obj, "getPaDot", jdouble, ())
end

function get_tc(obj::DelaunayArguments)
    return jcall(obj, "getTC", jdouble, ())
end


module TestTreeSet

using Phylo
using DataFrames
using JuliaDB

using Test

species = ["Dog", "Cat", "Human"]
ntips = 10
df = DataFrame(species = species, count=[10, 20, 3])
observations = ["Dog", "Cat", "Dog", "Dog"]
jdb = table((species = observations, count = [1, 2, 3, 4]))

@testset "TreeSet" begin
    @test length(rand(Ultrametric(ntips), 10)) ==
        length(rand(Nonultrametric(ntips), 10))
    ts = rand(Ultrametric(ntips), 1:10)
    @test length(gettrees(ts)) == length(gettreenames(ts)) == ntrees(ts) == 10
    @test length(getleafnames(ts)) == ntips
    for name in gettreenames(ts)
        @test name ∈ 1:10
        @test length(getbranchnames(ts[name])) == ntips * 2 - 2
        @test length(getbranchnames(ts)[name]) == ntips * 2 - 2
        @test length(getleafnames(ts[name])) == ntips
    end
    for tree in gettrees(ts)
        @test length(getnodenames(tree)) == ntips * 2 - 1
        @test length(getbranchnames(tree)) == ntips * 2 - 2
        @test nleaves(tree) == ntips
    end
    @test nleaves(ts) == ntips
    @test_throws ArgumentError getleafnames(TreeSet(Dict{String, NamedTree}(),
                                                    Dict{String, Dict{String,
                                                                      Any}}()))
    @test nleaves(rand(Ultrametric(df), 20)) == length(species)
    @test nleaves(rand(Ultrametric(jdb), 10)) ==
        length(unique(observations))
    @test getleafinfo(rand(Ultrametric(df))) == df
    @test getleafinfo(rand(Ultrametric(jdb), 2)) == jdb
    @test nleaves(rand(Nonultrametric(df), 20)) == length(species)
    @test nleaves(rand(Nonultrametric(jdb), 10)) ==
        length(unique(observations))
    @test getleafinfo(rand(Nonultrametric(df))) == df
    @test getleafinfo(rand(Nonultrametric(jdb), 2)) == jdb

    @test nodetype(typeof(ts)) == nodetype(typeof(first(gettrees(ts))))
    @test branchtype(typeof(ts)) == branchtype(typeof(first(gettrees(ts))))
end
end


# The following contains just abstract types regarding models and "virtual" method

#The most basic type, needed in order to support DifferentialEquations.jl
abstract type BaseProcess{T <: Number} end

# There will inherit from this type just processes, i.e. models that know what is a zeroCurve and a divided
abstract type AbstractMonteCarloProcess{T <: Number} <: BaseProcess{T} end

# Just engines will inherit from this, Brownian Motion et al.
abstract type AbstractMonteCarloEngine{T <: Number} <: BaseProcess{T} end

# Abstract type representing scalar montecarlo processes
abstract type ScalarMonteCarloProcess{T <: Number} <: AbstractMonteCarloProcess{T} end

# Abstract type representing multi dimensional montecarlo processes
abstract type VectorialMonteCarloProcess{T <: Number} <: AbstractMonteCarloProcess{T} end

# Abstract type representing N dimensional montecarlo processes (??)
abstract type NDimensionalMonteCarloProcess{T <: Number} <: VectorialMonteCarloProcess{T} end

# Abstract type for Levy (useful in FinancialFFT.jl)
abstract type LevyProcess{T <: Number} <: ScalarMonteCarloProcess{T} end

# Abstract type for Ito, an Ito process is always a Levy
abstract type ItoProcess{T <: Number} <: LevyProcess{T} end

# Abstract type for FA processes
abstract type FiniteActivityProcess{T <: Number} <: LevyProcess{T} end

# Abstract type for IA processes
abstract type InfiniteActivityProcess{T <: Number} <: LevyProcess{T} end

# Utility for dividends (implemented just for mono dimensional processes)
dividend(x::mc) where {mc <: ScalarMonteCarloProcess} = x.underlying.d;

"""
General Interface for Stochastic Process simulation

		S=simulate(mcProcess,zeroCurve,mcBaseData,T)

Where:\n
		mcProcess          = Process to be simulated.
		zeroCurve  = Datas of the Zero Rate Curve.
		mcBaseData = Basic properties of MonteCarlo simulation
		T = Final time of the process

		S      = Matrix with path of underlying.

"""
function simulate(mcProcess::AbstractMonteCarloProcess, zeroCurve::AbstractZeroRateCurve, mcBaseData::AbstractMonteCarloConfiguration, T::Number)
    price_type = predict_output_type_zero_(mcProcess, zeroCurve, mcBaseData, T)
    S = get_matrix_type(mcBaseData, mcProcess, price_type)
    simulate!(S, mcProcess, zeroCurve, mcBaseData, T)
    return S
end

function simulate(mcProcess::VectorialMonteCarloProcess, zeroCurve::AbstractZeroRateCurve, mcBaseData::AbstractMonteCarloConfiguration, T::Number)
    price_type = predict_output_type_zero_(mcProcess, zeroCurve, mcBaseData, T)
    matrix_type = get_matrix_type(mcBaseData, mcProcess, price_type)
    S = matrix_type(undef, length(mcProcess.models))
    for i = 1:length(mcProcess.models)
        S[i] = eltype(S)(undef, mcBaseData.Nsim, mcBaseData.Nstep + 1)
    end
    simulate!(S, mcProcess, zeroCurve, mcBaseData, T)
    return S
end

"""
General Interface for Stochastic Engine simulation

		S=simulate(mcProcess,mcBaseData,T)

Where:\n
		mcProcess          = Process to be simulated.
		mcBaseData = Basic properties of MonteCarlo simulation
		T = Final time of the process

		S      = Matrix with path of underlying.

"""
function simulate(mcProcess::AbstractMonteCarloEngine, mcBaseData::AbstractMonteCarloConfiguration, T::Number)
    price_type = predict_output_type_zero_(mcProcess, mcBaseData, T)
    matrix_type = get_matrix_type(mcBaseData, mcProcess, price_type)
    S = matrix_type(undef, mcBaseData.Nsim, mcBaseData.Nstep + 1)
    simulate!(S, mcProcess, mcBaseData, T)
    return S
end


export AtmosFilterPerturbations
export AtmosSpecificFilterPerturbations

struct AtmosFilterPerturbations{M} <: AbstractFilterTarget
    atmos::M
end

vars_state_filtered(target::AtmosFilterPerturbations, FT) =
    vars_state(target.atmos, Prognostic(), FT)

function compute_filter_argument!(
    target::AtmosFilterPerturbations,
    filter_state::Vars,
    state::Vars,
    aux::Vars,
)
    # copy the whole state
    parent(filter_state) .= parent(state)
    # remove reference state
    filter_state.ρ -= aux.ref_state.ρ
    filter_state.energy.ρe -= aux.ref_state.ρe
    if !(moisture_model(target.atmos) isa DryModel)
        filter_state.moisture.ρq_tot -= aux.ref_state.ρq_tot
    end
    if (moisture_model(target.atmos) isa NonEquilMoist)
        filter_state.moisture.ρq_liq -= aux.ref_state.ρq_liq
        filter_state.moisture.ρq_ice -= aux.ref_state.ρq_ice
    end
end
function compute_filter_result!(
    target::AtmosFilterPerturbations,
    state::Vars,
    filter_state::Vars,
    aux::Vars,
)
    # copy the whole filter state
    parent(state) .= parent(filter_state)
    # add reference state
    state.ρ += aux.ref_state.ρ
    state.energy.ρe += aux.ref_state.ρe
    if !(moisture_model(target.atmos) isa DryModel)
        state.moisture.ρq_tot += aux.ref_state.ρq_tot
    end
    if (moisture_model(target.atmos) isa NonEquilMoist)
        filter_state.moisture.ρq_liq += aux.ref_state.ρq_liq
        filter_state.moisture.ρq_ice += aux.ref_state.ρq_ice
    end
end

struct AtmosSpecificFilterPerturbations{M} <: AbstractFilterTarget
    atmos::M
end

vars_state_filtered(target::AtmosSpecificFilterPerturbations, FT) =
    vars_state_filtered(target.atmos, FT)

function compute_filter_argument!(
    target::AtmosSpecificFilterPerturbations,
    filter_state::Vars,
    state::Vars,
    aux::Vars,
)

    ρ_inv = 1 / state.ρ
    ρ_ref_inv = 1 / aux.ref_state.ρ

    # copy the whole state
    parent(filter_state) .= parent(state) * ρ_inv

    # remove reference state
    filter_state.energy.e -= aux.ref_state.ρe * ρ_ref_inv
    if !(moisture_model(target.atmos) isa DryModel)
        filter_state.moisture.q_tot -= aux.ref_state.ρq_tot * ρ_ref_inv
    end
    if (moisture_model(target.atmos) isa NonEquilMoist)
        filter_state.moisture.q_liq -= aux.ref_state.ρq_liq * ρ_ref_inv
        filter_state.moisture.q_ice -= aux.ref_state.ρq_ice * ρ_ref_inv
    end

end

function compute_filter_result!(
    target::AtmosSpecificFilterPerturbations,
    state::Vars,
    filter_state::Vars,
    aux::Vars,
)

    ρ = state.ρ
    ρ_ρ_ref_ratio = ρ / aux.ref_state.ρ

    # copy the whole filter state
    parent(state) .= parent(filter_state) * ρ

    # add reference state
    state.energy.ρe += aux.ref_state.ρe * ρ_ρ_ref_ratio
    if !(moisture_model(target.atmos) isa DryModel)
        state.moisture.ρq_tot += aux.ref_state.ρq_tot * ρ_ρ_ref_ratio
    end
    if (moisture_model(target.atmos) isa NonEquilMoist)
        state.moisture.ρq_liq += aux.ref_state.ρq_liq * ρ_ρ_ref_ratio
        state.moisture.ρq_ice += aux.ref_state.ρq_ice * ρ_ρ_ref_ratio
    end
end


# Note: Any future concrete formulation requires the definition of

# construct_device!(
#     ::OptimizationContainer,
#     ::PSY.System,
#     ::DeviceModel{<:PSY.ACBranch, MyNewFormulation},
#     ::Union{Type{CopperPlatePowerModel}, Type{AreaBalancePowerModel}},
# ) = nothing

#

# Not implemented yet
# struct TapControl <: AbstractBranchFormulation end
# struct PhaseControl <: AbstractBranchFormulation end

#################################### Branch Variables ##################################################
# Because of the way we integrate with PowerModels, most of the time PowerSimulations will create variables
# for the branch flows either in AC or DC.

get_initial_conditions_device_model(
    ::OperationModel,
    ::DeviceModel{T, <:AbstractBranchFormulation},
) where {T <: PSY.ACBranch} = DeviceModel(T, StaticBranch)

get_initial_conditions_device_model(
    ::OperationModel,
    ::DeviceModel{T, <:AbstractBranchFormulation},
) where {T <: PSY.MonitoredLine} = DeviceModel(T, StaticBranchUnbounded)

add_variables!(
    container::OptimizationContainer,
    ::Type{<:AbstractPTDFModel},
    devices::IS.FlattenIteratorWrapper{<:PSY.ACBranch},
    formulation::AbstractBranchFormulation,
) = add_variable!(container, FlowActivePowerVariable(), devices, formulation)

get_variable_binary(
    ::FlowActivePowerVariable,
    ::Type{<:PSY.ACBranch},
    ::AbstractBranchFormulation,
) = false

get_variable_multiplier(_, ::Type{<:PSY.ACBranch}, _) = NaN

function get_default_time_series_names(
    ::Type{U},
    ::Type{V},
) where {U <: PSY.ACBranch, V <: AbstractBranchFormulation}
    return Dict{Type{<:TimeSeriesParameter}, String}()
end

function get_default_attributes(
    ::Type{U},
    ::Type{V},
) where {U <: PSY.ACBranch, V <: AbstractBranchFormulation}
    return Dict{String, Any}()
end
#################################### Flow Variable Bounds ##################################################

function branch_rate_bounds!(
    container::OptimizationContainer,
    devices::IS.FlattenIteratorWrapper{B},
    ::DeviceModel{B, <:AbstractBranchFormulation},
    ::Type{<:PM.AbstractDCPModel},
) where {B <: PSY.ACBranch}
    var = get_variable(container, FlowActivePowerVariable(), B)
    for d in devices
        name = PSY.get_name(d)
        for t in get_time_steps(container)
            JuMP.set_upper_bound(var[name, t], PSY.get_rate(d))
            JuMP.set_lower_bound(var[name, t], -1.0 * PSY.get_rate(d))
        end
    end
    return
end

function branch_rate_bounds!(
    container::OptimizationContainer,
    devices::IS.FlattenIteratorWrapper{B},
    ::DeviceModel{B, <:AbstractBranchFormulation},
    ::Type{<:PM.AbstractPowerModel},
) where {B <: PSY.ACBranch}
    vars = [
        get_variable(container, FlowActivePowerFromToVariable(), B),
        get_variable(container, FlowActivePowerToFromVariable(), B),
    ]
    for d in devices
        name = PSY.get_name(d)
        for t in get_time_steps(container), var in vars
            JuMP.set_upper_bound(var[name, t], PSY.get_rate(d))
            JuMP.set_lower_bound(var[name, t], -1.0 * PSY.get_rate(d))
        end
    end
    return
end

################################## Rate Limits constraint_infos ############################

"""
Min and max limits for Abstract Branch Formulation
"""
function get_min_max_limits(
    device,
    ::Type{<:ConstraintType},
    ::Type{<:AbstractBranchFormulation},
) #  -> Union{Nothing, NamedTuple{(:min, :max), Tuple{Float64, Float64}}}
    return (min=-1 * PSY.get_rate(device), max=PSY.get_rate(device))
end

"""
Add branch rate limit constraints for ACBranch with AbstractActivePowerModel
"""
function add_constraints!(
    container::OptimizationContainer,
    cons_type::Type{RateLimitConstraint},
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, U},
    X::Type{<:PM.AbstractActivePowerModel},
) where {T <: PSY.ACBranch, U <: AbstractBranchFormulation}
    add_range_constraints!(container, cons_type, FlowActivePowerVariable, devices, model, X)
    return
end

"""
Add rate limit from to constraints for ACBranch with AbstractPowerModel
"""
function add_constraints!(
    container::OptimizationContainer,
    cons_type::Type{RateLimitConstraintFromTo},
    devices::IS.FlattenIteratorWrapper{B},
    model::DeviceModel{B, <:AbstractBranchFormulation},
    ::Type{T},
) where {B <: PSY.ACBranch, T <: PM.AbstractPowerModel}
    rating_data = [(PSY.get_name(h), PSY.get_rate(h)) for h in devices]

    time_steps = get_time_steps(container)
    var1 = get_variable(container, FlowActivePowerFromToVariable(), B)
    var2 = get_variable(container, FlowReactivePowerFromToVariable(), B)
    add_constraints_container!(
        container,
        cons_type(),
        B,
        [r[1] for r in rating_data],
        time_steps,
    )
    constraint = get_constraint(container, cons_type(), B)

    for r in rating_data
        for t in time_steps
            constraint[r[1], t] = JuMP.@constraint(
                container.JuMPmodel,
                var1[r[1], t]^2 + var2[r[1], t]^2 <= r[2]^2
            )
        end
    end
end

"""
Add rate limit to from constraints for ACBranch with AbstractPowerModel
"""
function add_constraints!(
    container::OptimizationContainer,
    cons_type::Type{RateLimitConstraintToFrom},
    devices::IS.FlattenIteratorWrapper{B},
    model::DeviceModel{B, <:AbstractBranchFormulation},
    ::Type{T},
) where {B <: PSY.ACBranch, T <: PM.AbstractPowerModel}
    rating_data = [(PSY.get_name(h), PSY.get_rate(h)) for h in devices]

    time_steps = get_time_steps(container)
    var1 = get_variable(container, FlowActivePowerToFromVariable(), B)
    var2 = get_variable(container, FlowReactivePowerToFromVariable(), B)
    add_constraints_container!(
        container,
        cons_type(),
        B,
        [r[1] for r in rating_data],
        time_steps,
    )
    constraint = get_constraint(container, cons_type(), B)

    for r in rating_data
        for t in time_steps
            constraint[r[1], t] = JuMP.@constraint(
                container.JuMPmodel,
                var1[r[1], t]^2 + var2[r[1], t]^2 <= r[2]^2
            )
        end
    end
end

"""
Add network flow constraints for ACBranch and NetworkModel with StandardPTDFModel
"""
function add_constraints!(
    container::OptimizationContainer,
    cons_type::Type{NetworkFlowConstraint},
    devices::IS.FlattenIteratorWrapper{B},
    model::DeviceModel{B, <:AbstractBranchFormulation},
    network_model::NetworkModel{S},
) where {B <: PSY.ACBranch, S <: StandardPTDFModel}
    ptdf = get_PTDF(network_model)
    branches = PSY.get_name.(devices)
    time_steps = get_time_steps(container)
    branch_flow = add_constraints_container!(
        container,
        NetworkFlowConstraint(),
        B,
        branches,
        time_steps,
    )
    nodal_balance_expressions = get_expression(container, ActivePowerBalance(), S)
    flow_variables = get_variable(container, FlowActivePowerVariable(), B)
    jump_model = get_jump_model(container)
    for br in devices
        name = PSY.get_name(br)
        ptdf_col = ptdf[name, :]
        flow_variables_ = flow_variables[name, :]
        for t in time_steps
            branch_flow[name, t] = JuMP.@constraint(
                jump_model,
                sum(
                    ptdf_col[i] * nodal_balance_expressions.data[i, t] for
                    i in 1:length(ptdf_col)
                ) - flow_variables_[t] == 0.0
            )
        end
    end
end

"""
Min and max limits for monitored line
"""
function get_min_max_limits(
    device::PSY.MonitoredLine,
    ::Type{<:ConstraintType},
    ::Type{<:AbstractBranchFormulation},
)
    if PSY.get_flow_limits(device).to_from != PSY.get_flow_limits(device).from_to
        @warn(
            "Flow limits in Line $(PSY.get_name(device)) aren't equal. The minimum will be used in formulation $(T)"
        )
    end
    limit = min(
        PSY.get_rate(device),
        PSY.get_flow_limits(device).to_from,
        PSY.get_flow_limits(device).from_to,
    )
    minmax = (min=-1 * limit, max=limit)
    return minmax
end

############################## Flow Limits Constraints #####################################
# TODO: Write tests for these functions
"""
Add branch flow constraints for monitored lines with DC Power Model
"""
function branch_flow_constraints!(
    container::OptimizationContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, U},
    X::Type{<:PM.AbstractDCPModel},
) where {T <: PSY.MonitoredLine, U <: AbstractBranchFormulation}
    add_range_constraints!(
        container,
        FlowLimitConstraint,
        FlowActivePowerVariable,
        devices,
        model,
        X,
    )
    return
end

"""
Don't add branch flow constraints for monitored lines if formulation is StaticBranchUnbounded
"""
function branch_flow_constraints!(
    container::OptimizationContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, U},
    ::Type{<:PM.AbstractDCPModel},
) where {T <: PSY.MonitoredLine, U <: StaticBranchUnbounded}
    return
end

"""
Min and max limits for flow limit from-to constraint
"""
function get_min_max_limits(
    device::PSY.MonitoredLine,
    ::Type{FlowLimitFromToConstraint},
    ::Type{<:AbstractBranchFormulation},
)
    if PSY.get_flow_limits(device).to_from != PSY.get_flow_limits(device).from_to
        @warn(
            "Flow limits in Line $(PSY.get_name(device)) aren't equal. The minimum will be used in formulation $(T)"
        )
    end
    return (
        min=-1 * PSY.get_flow_limits(device).from_to,
        max=PSY.get_flow_limits(device).from_to,
    )
end

"""
Min and max limits for flow limit to-from constraint
"""
function get_min_max_limits(
    device::PSY.MonitoredLine,
    ::Type{FlowLimitToFromConstraint},
    ::Type{<:AbstractBranchFormulation},
)
    if PSY.get_flow_limits(device).to_from != PSY.get_flow_limits(device).from_to
        @warn(
            "Flow limits in Line $(PSY.get_name(device)) aren't equal. The minimum will be used in formulation $(T)"
        )
    end
    return (
        min=-1 * PSY.get_flow_limits(device).to_from,
        max=PSY.get_flow_limits(device).to_from,
    )
end

"""
Add branch flow constraints for monitored lines
"""
function branch_flow_constraints!(
    container::OptimizationContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, U},
    ::Type{<:PM.AbstractPowerModel},
) where {T <: PSY.MonitoredLine, U <: AbstractBranchFormulation}
    add_range_constraints!(
        container,
        FlowLimitFromToConstraint,
        FlowActivePowerFromToVariable,
        devices,
        model,
        X,
    )
    add_range_constraints!(
        container,
        FlowLimitToFromConstraint,
        FlowActivePowerToFromVariable,
        devices,
        model,
        X,
    )
    return
end

"""
Don't add branch flow constraints for monitored lines if formulation is StaticBranchUnbounded
"""
function branch_flow_constraints!(
    container::OptimizationContainer,
    devices::IS.FlattenIteratorWrapper{T},
    model::DeviceModel{T, U},
    ::Type{<:PM.AbstractPowerModel},
) where {T <: PSY.MonitoredLine, U <: StaticBranchUnbounded}
    return
end


#=##############################################################################
# DESCRIPTION
    Simulation interface connecting vehicles and maneuvers.

# AUTHORSHIP
  * Author    : Eduardo J. Alvarez
  * Email     : Edo.AlvarezR@gmail.com
  * Created   : Oct 2019
  * License   : MIT
=###############################################################################

################################################################################
# SIMULATION TYPE
################################################################################
"""
    `Simulation(vehicle, maneuver, Vref::Real, RPMref::R, ttot:R)`

Simulation interface. This type carries the simulation's options and connects
vehicle and maneuver together.
"""
mutable struct Simulation{V<:AbstractVehicle, M<:AbstractManeuver, R<:Real}
    # USER INPUTS: Simulation setup
    vehicle::V              # Vehicle
    maneuver::M             # Maneuver to be performed
    Vref::R                 # Reference velocity in this maneuver
    RPMref::R               # Reference RPM in this maneuver
    ttot::R                 # Total time in which to perform the maneuver

    # OPTION USER INPUTS
    Vinit::Any              # Initial vehicle velocity
    Winit::Any              # Initial vehicle angular velocity

    # INTERNAL PROPERTIES: Runtime parameters
    t::R                    # Dimensional time
    nt::Int                 # Current time step number

    Simulation{V, M, R}(
                            vehicle, maneuver, Vref, RPMref, ttot;
                            Vinit=nothing, Winit=nothing,
                            t=zero(R), nt=-1
                        ) where {V, M, R} = _check(vehicle, maneuver) ? new(
                            vehicle, maneuver, Vref, RPMref, ttot,
                            Vinit, Winit,
                            t, nt
                        ) : nothing
end

# Implicit V and M constructor
Simulation(v::AbstractVehicle, m::AbstractManeuver, n::Real, args...; optargs...
             ) = Simulation{typeof(v), typeof(m), typeof(n)}(v, m, n, args...;
                                                                     optargs...)

# Solvers of concrete AbstractVehicle implementations
for header_name in ["unsteady", "quasisteady"]
  include("FLOWUnsteady_simulation_types_"*header_name*".jl")
end

##### FUNCTIONS  ###############################################################

"""
    `nextstep_kinematic(self::Simulation, dt::Real)`

Takes a kinematic time step `dt` where the new velocity and angular velocity
is calculated and the vehicle is translated and rotated according to it. It also
updates the tilt angle and RPM of every system.

NOTE: No solver is run in this process, rather it uses the current aerodynamic
solution to calculate acceleration and moment.
"""
function nextstep_kinematic(self::Simulation, dt::Real)

    if self.nt==-1 # Setup step (first step of simulation)

        # Initial vehicle linear and angular velocity
        if self.Vinit!=nothing; set_V(self.vehicle, self.Vinit); end;
        if self.Winit!=nothing; set_W(self.vehicle, self.Winit); end;

    else
        # Linear velocity increment
        #  dV = calc_dV(self.maneuver, self.vehicle, self.t, dt, self.ttot, self.Vref)
          dV = calc_dV(self.maneuver, dt, self.Vref, self.t, self.nt)

        # Angular velocity increment
        #dW = calc_dW(self.maneuver, self.vehicle, self.t, dt, self.ttot)
        dW = calc_dW(self.maneuver, dt, self.t, self.nt)

        # Update vehicle linear and angular velocity
        add_dV(self.vehicle, dV)
        add_dW(self.vehicle, dW)

        # Translates and rotates the vehicle according to current velocity
        nextstep_kinematic(self.vehicle, dt)

        # Rotate tilting systems
        angles = get_angles(self.maneuver, self.t/self.ttot)
        tilt_systems(self.vehicle, angles)

        self.t += dt
    end

    self.nt += 1
end

function save_vtk(self::Simulation, filename; optargs...)
    return save_vtk(self.vehicle, filename; num=self.nt, optargs...)
end

"""
Precalculations before calling the solver.
"""
function precalculations(self::Simulation, Vinf::Function,
                                pfield::vpm.ParticleField, t::Real, dt::Real)
    # Rotate rotors
    rotate_rotors(self, dt)

    # Calculate kinematic velocities
    precalculations(self.vehicle, Vinf, pfield, t, dt, self.nt)

    # # Bring the rotors back to their initial positions. This is needed to avoid
    # # double accounting for the RPM when the solver is run
    # NOTE: I've moved this to after shedding the wake and before running the
    # solver, otherwise the particles shedding wouldn't see the kinematic
    # velocity due to rotor RPM
    # rotate_rotors(self, -dt)

    return nothing
end

function rotate_rotors(self::Simulation{V, M, R}, dt::Real
                        ) where {V<:AbstractVLMVehicle, M<:AbstractManeuver, R}

    for (si, rotors) in enumerate(self.vehicle.rotor_systems)

        # RPM of this rotor system
        RPM = self.RPMref*get_RPM(self.maneuver, si, self.t/self.ttot)

        for rotor in rotors

            # Save solution fields
            sol1 = deepcopy(rotor._wingsystem.sol)
            sol2 = deepcopy(rotor.sol)
            Vinf = rotor._wingsystem.Vinf

            # Rotate rotor
            vlm.rotate(rotor, 360*RPM/60*dt)

            # Force regeneration of horseshoes
            if "Gamma" in keys(sol1)
                vlm.setVinf(rotor, Vinf)
                vlm.setRPM(rotor, RPM)
                vlm.getHorseshoe(rotor, 1)
            end

            # Paste solution fields back
            for (key,val) in sol1; vlm._addsolution(rotor._wingsystem, key, val); end;
            for (key,val) in sol2; rotor.sol[key] = val; end;

        end
    end
    return nothing
end


using JLD2
using StatsPlots
using DataFrames

@load "paper/box_all.jld2" box_all_df
@load "paper/box_at.jld2" box_at_df
@load "paper/lg_all.jld2" lg_all_df

@df box_at_df scatter(:n, :mean)
p = @df box_all_df plot(:n, :mean, group=:type, marker=4,
  xlabel="T", ylabel="mean coupling time", xticks=200:200:1600, size=(450,250),
  legend=:topleft)

savefig(p, "paper/simpleComparisonATASBS.pdf")

p = @df lg_all_df plot(:n, :mean, group=:type, marker=4,
  xlabel="T", ylabel="mean coupling time", xticks=200:200:1600, size=(450,250),
  legend=:topleft)

savefig(p, "paper/simpleComparison_lg_ATASBS.pdf")


parametrify(x::Vector{<:NumDist}) = parametrify.(x)
parametrify(x::Number) = Dirac{Float64}(x)
parametrify(x::Distribution) = x
Base.Iterators.cycle(x::NumDist) = cycle([x])

function test_policy(is::InventorySystem, policy, n = 1000; logger = ISLogger(is))
    totreward = 0.
    for _ in 1:n
        reset!(is)
        for action in partition(policy, action_size(is))
            is(action)
            totreward += reward(is)
            if is_terminated(is) break end
        end
        logger(is)
    end
    totreward/=n
end


include("src/Blake3Hash.jl")

using .Blake3Hash

data = rand(UInt8, 2 * 8192)

sa     = Blake3Hash.SAImpl.Blake3Ctx()
sa_out = Vector{UInt8}(undef, 64)

Blake3Hash.SAImpl.update!(sa, data)
Blake3Hash.SAImpl.digest(sa, sa_out)

t     = Blake3Hash.Simd128.Blake3Ctx()
t_out = Vector{UInt8}(undef, 64)

Blake3Hash.Simd128.update!(t, data)
Blake3Hash.Simd128.digest(t, t_out)

println("E: $sa_out")
println("R: $t_out")


module MCCGA

import Optim
import Statistics

export Functions

export bits
export floats
export initialprobs
export sample
export mccga

include("stats.jl")
include("bitworks.jl")
include("thealgorithm.jl")
include("funcset.jl")

end # module


module Got
export demo
using Parameters

@with_kw mutable struct Cols
  x  = (all  = [], nums = [], syms = [])
  y = (all  = [], nums = [], syms = [],
           goals= [22], klass= nothing)
  all  = []
  nums = []
  syms = (10, 20,   30, 40)
  name=""
  pos=0
end

z=Cols(name="jane", pos=2)
#show(z)
#w = Cols(),
#dump(w)
#dump(nothing)

function num(s)
  v = tryparse(Float64,s)
  v == nothing ? s : v
end
function dd()
open("sherlock-holmes.txt") do f
   line = 1
   while !eof(f)
     x = readline(f)
     println("$line $x")
     line += 1
   end
 end
end

#[take(PSerie(2),10)...]
#
#struct PSerie
#           p
#       end
#
#
#function Base.iterate(ps::PSerie, (s,n) = (0,1))
#    s += 1 / (n^ps.p)
#           s, (s, n+1)
#       end
#

demo = begin
  open("got.jl") do file
      n=1
      old=""
      function prep(s)
        if sizeof(s) > 0
          println( map(num, split(s,",")))end end
      for line in eachline(file)
          line = replace(line, r"([ \t\n]|#.*)" => "")
          if sizeof(line) == 0  continue end
          print(line[end])
          if line[end] == ','
            old = old * line
          else
            prep(old*line)
            old=""
          end
      end
      prep(old)
  end
end
end


using DrWatson
import Images
import CSV

function load_dataset(seg_df_path::String, paper_seg_labels_path::String, watershed_path::String, dapi_path::String, prior_path::String="", pciseq_path::String="";
        name::String, min_area::Float64, min_mols_per_cell=50, paper_polygons::Bool=false, dapi::Bool=false, watershed::Bool=false, pciseq=true, prior=true,
        rescale::Bool=false, store_labels::Bool=false)
    df_spatial, gene_names = B.load_df(datadir(seg_df_path))
    res = Dict(:df => df_spatial, :gene_names => gene_names, :min_area => min_area,
        :min_mols_per_cell => min_mols_per_cell, :name => name)

    if rescale
        df_spatial.x .-= minimum(df_spatial.x);
        df_spatial.y .-= minimum(df_spatial.y);

        df_spatial.x .*= 10.0; # Specifically for Allen smFISH
        df_spatial.y .*= 10.0;
    end

    df_spatial[!, :gene_name] = gene_names[df_spatial.gene]

    paper_seg_labels = nothing
    if length(paper_seg_labels_path) > 0
        paper_seg_labels = B.load_segmentation_mask(datadir(paper_seg_labels_path));
        df_spatial[!, :cell_paper] = denserank(B.staining_value_per_transcript(df_spatial, paper_seg_labels)) .- 1;
        if store_labels
            res[:paper_labels] = paper_seg_labels
        end
    end

    # df_seg_prior = B.load_df(datadir("exp_pro/allen_smfish/baysor_prior/segmentation.csv"))[1];
    # df_spatial[!, :cell_prior] = df_seg_prior.cell;

    if watershed && length(watershed_path) > 0
        watershed_labels = B.load_segmentation_mask(datadir(watershed_path))
        df_spatial[!, :cell_watershed] = denserank(B.staining_value_per_transcript(df_spatial, watershed_labels)) .- 1
        if store_labels
            res[:watershed_labels] = watershed_labels
        end
    end

    if dapi
        res[:dapi_arr] = Float16.(Images.load(datadir(dapi_path)))
    end

    if paper_polygons && (length(paper_seg_labels_path) > 0)
        res[:paper_polys] = B.extract_polygons_from_label_grid(Matrix(paper_seg_labels[1:3:end, 1:3:end]), grid_step=3.0)
    end

    if pciseq && (length(pciseq_path) > 0)
        df_pciseq = DataFrame(CSV.File(datadir(pciseq_path)), copycols=false)
        df_spatial[!, :cell_pciseq] = df_pciseq.neighbour
    end

    if prior && (length(prior_path) > 0)
        df_prior = DataFrame(CSV.File(datadir(prior_path)), copycols=false)
        df_spatial[!, :cell_prior] = df_prior.cell
    end

    return res
end

function load_merfish(seg_df_path::String="exp_pro/merfish_moffit/baysor/segmentation.csv"; min_area::Float64=25.0, paper_polygons::Bool=false, pciseq::Bool=true, kwargs...)
    res = load_dataset(
        "exp_raw/merfish_moffit/merfish_coords_adj.csv",
        "",
        "exp_raw/merfish_moffit/dapi_merged_watershed.tif",
        "exp_raw/merfish_moffit/dapi_merged.tiff",
        "exp_pro/merfish_moffit/baysor_prior/segmentation.csv";
        paper_polygons=false, min_area=min_area, name="MERFISH", kwargs...
    )

    res[:df][!, :cell_paper] = ifelse.(ismissing.(res[:df].cell), 0, denserank(res[:df].cell));

    df_seg = B.load_df(datadir(seg_df_path))[1]
    res[:df][!, :cell] = df_seg.cell;
    res[:df][!, :confidence] = df_seg.confidence;
    res[:df][!, :cluster] = df_seg.cluster;
    res[:df][!, :ncv_color] = df_seg.ncv_color;

    if paper_polygons
        res[:paper_polys] = B.boundary_polygons(res[:df], res[:df].cell_paper, grid_step=5.0, bandwidth=5.0);
    end

    if pciseq
        df_pciseq = DataFrame!(CSV.File(datadir("exp_pro/merfish_moffit/pciseq/spots.csv")))
        res[:df] = res[:df][.!in.(res[:df].gene_name, Ref(Set(["Blank-1", "Blank-2", "Blank-3", "Blank-4", "Blank-5", "Krt90"]))),:]
        res[:df][!, :cell_pciseq] = df_pciseq.neighbour
    end

    return res
end

load_osmfish(; min_area::Float64=70.0, kwargs...) =
    load_dataset(
        "exp_pro/osmfish/baysor/segmentation.csv",
        "exp_raw/osmfish/paper_segmentation.tiff",
        "exp_raw/osmfish/nuclei_watershed.tif",
        "exp_raw/osmfish/nuclei.tif",
        "exp_pro/osmfish/baysor_prior/segmentation.csv",
        "exp_pro/osmfish/pciseq/spots.csv";
        min_area=min_area, name="osmFISH", kwargs...
    )

load_starmap1020(; min_area::Float64=25.0, kwargs...) =
    load_dataset(
        "exp_pro/starmap_vis1020/baysor/segmentation.csv",
        "exp_raw/starmap_vis1020/segmentation.tiff",
        "", "",
        "exp_pro/starmap_vis1020/baysor_prior/segmentation.csv";
        dapi=false, watershed=false,
        min_area=min_area, name="STARmap 1020", kwargs...
    )

load_allen_smfish(csv_path::String="exp_pro/allen_smfish/baysor/segmentation.csv"; min_area::Float64=25.0, kwargs...) =
    load_dataset(
        csv_path,
        "exp_raw/allen_smfish/segmentation_labels_from_json_transposed.tiff",
        "exp_raw/allen_smfish/dapi_merged_watershed.tif",
        "exp_raw/allen_smfish/dapi_merged.tiff",
        "exp_pro/allen_smfish/baysor_prior/segmentation.csv",
        "exp_pro/allen_smfish/pciseq/spots.csv";
        rescale=true, min_area=min_area, name="Allen smFISH", kwargs...
    )

function load_iss(csv_path::String="exp_pro/iss_hippo/baysor/segmentation.csv"; min_area::Float64=1.0, paper_polygons::Bool=false, kwargs...)
    res = load_dataset(
        csv_path, "",
        "exp_raw/iss_hippo/CA1DapiBoundaries_4-3_right_watershed.tif",
        "exp_raw/iss_hippo/CA1DapiBoundaries_4-3_right.tif",
        "exp_pro/iss_hippo/baysor_prior/segmentation.csv",
        "exp_pro/iss_hippo/pciseq/spots.csv";
        paper_polygons=false, min_area=min_area, min_mols_per_cell=3, name="ISS", kwargs...
    )

    res[:df][!, :cell_paper] = res[:df].parent_id;
    if paper_polygons
        res[:paper_polys] = B.boundary_polygons(res[:df], res[:df].cell_paper, grid_step=1.0, bandwidth=3.0);
    end

    return res
end


# Tests that the weighting scheme works for binary_classification

module TestWeights

using Test
using DelimitedFiles
using LightGBM

@testset "weightsTest.jl" begin
    try

        LGBM_PATH = ENV["LIGHTGBM_EXAMPLES_PATH"]
        LGBM_PATH = if isabspath(LGBM_PATH) LGBM_PATH else abspath(joinpath(pwd(), "..", LGBM_PATH)) end

        binary_test = readdlm(joinpath(LGBM_PATH, "examples", "binary_classification", "binary.test"), '\t')
        binary_train = readdlm(joinpath(LGBM_PATH, "examples", "binary_classification", "binary.train"), '\t')
        X_train = binary_train[:, 2:end]
        y_train = binary_train[:, 1]
        X_test = binary_test[:, 2:end]
        y_test = binary_test[:, 1]

        binary_test_weight = readdlm(joinpath(LGBM_PATH, "examples", "binary_classification", "binary.test.weight"), '\t')[:,1]
        binary_train_weight = readdlm(joinpath(LGBM_PATH, "examples", "binary_classification", "binary.train.weight"), '\t')[:,1]

        # Test binary estimator.
        estimator = LightGBM.LGBMClassification(
            objective = "binary",
            num_class = 1,
            num_iterations = 20,
            learning_rate = .1,
            early_stopping_round = 1,
            feature_fraction = .8,
            bagging_fraction = .9,
            bagging_freq = 1,
            num_leaves = 1000,
            metric = ["auc", "binary_logloss"],
            is_training_metric = true,
            max_bin = 255,
            min_sum_hessian_in_leaf = 0.,
            min_data_in_leaf = 1,
        )

        # Test fitting.
        LightGBM.fit!(estimator, X_train, y_train, verbosity = -1, weights=binary_train_weight[:,1])
        LightGBM.fit!(estimator, X_train, y_train, (X_test, y_test), verbosity = -1, weights=binary_train_weight[:,1])

        @test true
    catch
        @test false
    end
end

end # module


#=
    -------------------------------------------------------------------------
    Implements functionalities for setting up the Markov chain Monte Carlo
    algorithm. The main object is `MCMCSetup` and its members comprise of
    all objects that need to be passed to the `mcmc` function from `mcmc.jl`.
    The remaining routines are used to populate instances of `MCMCSetup` and
    verify its fields in a structured manner.
    --------------------------------------------------------------------------
=#

#===============================================================================
                                MCMC setup
===============================================================================#
"""
    mutable struct MCMCSetup

Implements functionalities for setting up the Markov chain Monte Carlo
algorithm. The main object is `MCMCSetup` and its members comprise of
all objects that need to be passed to the `mcmc` function from `mcmc.jl`.
The remaining routines are used to populate instances of `MCMCSetup` and
verify its fields in a structured manner.
"""
mutable struct MCMCSetup
    updates
    #readjust_params
    # add parameters for fusion, correlation matrix, historical acceptance rates
    function MCMCSetup(updates...)
        new([updates...])
    end
end

function set_readjust_params!(setup::MCMCSetup, readjust_params)
    setup.readjust_params = readjust_params
end

#===============================================================================
                                Diffusion setup
===============================================================================#

"""
    DiffusionSetup{ObsScheme} <: ModelSetup

Setup choices relevant to the path augmentation step to be passed to `mcmc` function from
`mcmc.jl`.

# Example:
```
         DiffusionSetup(P, P̃)
         DiffusionSetup(P, P̃, fptOrPartObs)
```
where P is the target process and P̃ is the linear approximation and possibly a vector of Booleans whether
the observations are first passage times.
"""
mutable struct DiffusionSetup{ObsScheme} <: ModelSetup
    setup_completion    # (internal) Indicates progress of setting-up DiffusionSetup
    P˟                  # Target diffusion law
    P̃                   # Vector of auxiliary diffusion laws
    adaptive_prop       # Object for adapting guided proposals
    skip_for_save       # When saving path, thin the grid by a factor of ...
    Ls                  # Vector of observation operators
    Σs                  # Vector of covariance matrices
    obs                 # Observations
    obs_times           # Recorded times of observations
    fpt                 # Vector with information about first-passage times
    dt                  # Granularity of the imputation grid
    τ                   # Time transformation for the imputation grid
    x0_prior            # Prior over the starting position
    x0_guess            # Guess for a position of a starting point
    Wnr                 # Definition of the driving Wiener law
    XX                  # Container for the path
    WW                  # Container for the driving noise
    P                   # Container for the guided proposals

    """
        DiffusionSetup(P˟, P̃, ::ObsScheme)

    Initialise `DiffusionSetup` with a given target law `P˟`, auxiliary laws `P̃` and
    observation scheme `ObsScheme`.
    """
    function DiffusionSetup(P˟, P̃, ::ObsScheme) where ObsScheme <: AbstractObsScheme
        new{ObsScheme}(Dict(:obs => false,
                            :imput => false,
                            :prior => false),
                       P˟, P̃, NoAdaptation(), 1)#NoBlocking(), ([], 0.1, NoChangePt()),
    end
end

adaptation_object(setup::DiffusionSetup) = deepcopy(setup.adaptive_prop)

"""
    set_observations!(setup::DiffusionSetup, Ls, Σs, obs, obs_times,
                      fpt=fill(nothing, length(obs)-1))

Store observations to `setup`. The observations follow the scheme `V=LX+η`,
where V are the observations in `obs`, observed at times given in `obs_times`,
`L` are the observation operators given in `Ls`, `X` is the unobserved,
underlying diffusion and `η` is a Gaussian noise with mean `0` and covariance
`Σ` with the last stored in `Σs`. `fpt` provides additional information in case
the nature of observations has to do with first-passage times
"""
function set_observations!(setup::DiffusionSetup, Ls, Σs, obs, obs_times,
                           fpt=fill(nothing, length(obs)-1))
    setup.Ls = Ls
    setup.Σs = Σs
    setup.obs = obs
    setup.obs_times = obs_times
    setup.fpt = fpt
    setup.setup_completion[:obs] = true
end

function incomplete_message(::Val{:obs})
    print("\nThe observations have not been set up. Please call ",
          "set_observations!() passing the data on observations and the type ",
          "of observation scheme.\n")
end

"""
    set_imputation_grid!(setup::DiffusionSetup, dt,
                         time_transf=(t₀,T) -> ((x) ->  t₀ + (x-t₀) * (2-(x-t₀)/(T-t₀))))

Define the imputation grid in `setup`. `dt` defines the granulatrity of the
imputation grid and `time_transf` defines a time transformation to use for
transforming equidistant grid.
"""
function set_imputation_grid!(setup::DiffusionSetup, dt,
                              time_transf=(t₀,T) -> ((x) ->  t₀ + (x-t₀) * (2-(x-t₀)/(T-t₀))))
    setup.dt = dt
    setup.τ = time_transf
    setup.setup_completion[:imput] = true
end

function incomplete_message(::Val{:imput})
    print("\nThe imputation grid has not been set up. Please call ",
          "set_imputation_grid!() passing the `delta-t` and the time ",
          "transformation.\n")
end

"""
    set_x0_prior!(setup::DiffusionSetup, x0_prior, x0_guess=nothing)

Store the priors over the starting point into the object `setup`.
"""
function set_x0_prior!(setup::DiffusionSetup, x0_prior, x0_guess=nothing)
    setup.x0_prior = x0_prior
    if x0_guess == nothing
        @assert typeof(x0_prior) <: KnownStartingPt
        x0_guess = start_pt(nothing, x0_prior)
    end
    setup.x0_guess = x0_guess
    setup.setup_completion[:prior] = true
end

function incomplete_message(::Val{:prior})
    print("\nThe priors have not been set up. Please call set_priors!() ",
          "passing the priors on the parameters and on the starting point.\n")
end

function set_auxiliary!(setup::DiffusionSetup; skip_for_save=nothing,
                        adaptive_prop=nothing)
    if skip_for_save !== nothing
        setup.skip_for_save = skip_for_save
    end
    if adaptive_prop !== nothing
        setup.adaptive_prop = adaptive_prop
    end
end

"""
    check_if_complete(setup::DiffusionSetup, labels=keys(setup.setup_completion))

Check if all the set-up steps listed in `labels` have been finalised in `setup`
"""
function check_if_complete(setup::DiffusionSetup,
                           labels=keys(setup.setup_completion))
    complete = true
    for label in labels
        if !setup.setup_completion[label]
            incomplete_message(Val(label))
            complete = false
        end
    end
    complete
end

"""
    determine_data_type(setup::DiffusionSetup)

Determine the data type of the containers with path and driving noise
"""
function determine_data_type(setup::DiffusionSetup)
    check_if_complete(setup, [:prior]) || throw(UndefRefError())
    x = setup.x0_guess
    drift = b(0.0, x, setup.P˟)
    vola = σ(0.0, x, setup.P˟)
    # @assert typeof(x) == typeof(drift) # maybe this assertion is too strong
    typeof(drift), typeof(vola' * drift)
end

"""
    prepare_containers!(setup::DiffusionSetup)

Set-up the containers for paths, driving noise, observations, observation noise
and observation operators, automatically choosing an appropriate data type based
on the return values of the drift and volatility coefficients
"""
function prepare_containers!(setup::DiffusionSetup)
    T, S = determine_data_type(setup)
    Wnr = Wiener{S}()
    TW = typeof(sample([0], Wnr))
    TX = typeof(SamplePath([], zeros(T, 0)))

    # TODO modify so that it will work with GPUArrays
    m = length(setup.obs)-1
    WW, XX = Vector{TW}(undef,m), Vector{TX}(undef,m)
    setup.Wnr, setup.WW, setup.XX = Wnr, WW, XX

    prepare_obs_containers!(T, setup)
end

"""
    prepare_obs_containers!(::Type{T}, setup::DiffusionSetup)

Define containers for the observations, observation noise and observation
operators based on the passed data-type
"""
function prepare_obs_containers!(::Type{T}, setup::DiffusionSetup) where T <: Number
    correct_data_type!(setup, T, T)
end

"""
    prepare_obs_containers!(::Type{T}, setup::DiffusionSetup)

Define containers for the observations, observation noise and observation
operators based on the passed data-type
"""
function prepare_obs_containers!(::Type{T}, setup::DiffusionSetup) where T <: Array
    correct_data_type!(setup, Array, Array)
end

"""
    prepare_obs_containers!(::Type{T}, setup::DiffusionSetup)

Define containers for the observations, observation noise and observation
operators based on the passed data-type
"""
function prepare_obs_containers!(::Type{T}, setup::DiffusionSetup) where T <:SArray
    f(x) = SMatrix{_dim(x)...}(x)

    g(x) = g(x, Val{ndims(x)}())
    g(x, ::Val{0}) = SVector{1}(x)
    g(x, ::Val{1}) = SVector{size(x)...}(x)
    g(x, ::Val{2}) = SMatrix{size(x)...}(x)

    correct_data_type!(setup, f, g)
end

"""
    correct_data_type!(setup::DiffusionSetup, f, g)

Transform the elements of observations, observation noise and observation
operators collections stored in `setup` according to functions `f` and `g`
"""
function correct_data_type!(setup::DiffusionSetup, f, g)
    setup.Σs = map(f, setup.Σs)
    setup.Ls = map(f, setup.Ls)
    setup.obs = map(g, setup.obs)
end

"""
    _dim(mat::T) where T <: Number

Size of a matrix corresponding to a scalar is (1,1)
"""
_dim(mat::T) where T <: Number = 1, 1

"""
    _dim(mat::T) where T <: Union{Array, SArray}

Size of a matrix corresponding to an element `mat`
"""
_dim(mat::T) where T <: Union{Array, SArray} = _dim(mat, Val{ndims(mat)}())

"""
    _dim(mat, ::Val{1})
Size of a matrix corresponding to a vector is (vector length, 1)
"""
_dim(mat, ::Val{1}) = size(mat)[1], 1

"""
    _dim(mat, ::Val{1})
Size of a matrix corresponding to a matrix `mat` is just size(mat)
"""
_dim(mat, ::Val{2}) = size(mat)

"""
    _dim(mat, ::T) where T
Size of a matrix corresponding to tensor with dimension larger than matrix is
undefined
"""
_dim(mat, ::T) where T = throw(ArgumentError())

function _build_time_grid(τ, dt, t0, T)
    num_pts = Int64(ceil((T-t0)/dt))+1
    tt = τ(t0, T).( range(t0, stop=T, length=num_pts) )
    tt
end

 #TODO remove this K, it's most likely not needed
"""
    find_proposal_law!(::Type{K}, setup::DiffusionSetup, solver, change_pt
                            ) where K

Initialise the object with proposal law and all the necessary containers needed
for the simulation of the guided proposals
"""
function find_proposal_law!(::Type{K}, setup::DiffusionSetup, solver, change_pt
                            ) where K
    xx, tt, P˟, P̃ = setup.obs, setup.obs_times, setup.P˟, setup.P̃
    Ls, Σs, dt, τ = setup.Ls, setup.Σs, setup.dt, setup.τ
    #NOTE remember to pre-allocate space for the M,L,mu containers for change_pt with blocking
    #change_pt = typeof(setup.change_pt)(get_change_pt(setup.blocking_params[3]))

    m = length(xx)-1
    t = _build_time_grid(τ, dt, tt[m], tt[m+1])
    P = GuidPropBridge(K, t, P˟, P̃[m], Ls[m], xx[m+1], Σs[m];
                       change_pt=change_pt, solver=solver)
    #P = Array{ContinuousTimeProcess,1}(undef,m)
    Ps = []
    push!(Ps, deepcopy(P))

    for i in m-1:-1:1
        t = _build_time_grid(τ, dt, tt[i], tt[i+1])
        P = GuidPropBridge(K, t, P˟, P̃[i], Ls[i], xx[i+1], Σs[i], P.H[1],
                           P.Hν[1], P.c[1]; change_pt=change_pt, solver=solver)
        push!(Ps, deepcopy(P))
    end
    Ps = [P for P in Ps]
    setup.P = reverse(Ps)
end

"""
    initialise(::Type{K}, setup::DiffusionSetup; verbose=false,
                        change_pt=NoChangePt())

Initialise the internal containers of `setup`. Check if all the necessary data
has been passed to `setup`
"""
function initialise!(::Type{K}, setup::DiffusionSetup, solver, verbose=false,
                     change_pt=NoChangePt()) where K
    verbose && print("Initialising MCMC setup...\nPreparing containers...\n")
    prepare_containers!(setup)
    verbose && print("Initialising proposal laws...\n")
    find_proposal_law!(K, setup, solver, change_pt) #TODO remove this K, it's most likely not needed
    #TODO initialise for computation of gradients
end

"""
    check_if_recompute_ODEs(Ps, updt_coord)

Utility function for checking if H,Hν,c need to be re-computed for a respective
parameter update
"""
function check_if_recompute_ODEs(Ps, updt_coord)
    any([any([uc in depends_on_params(P) for uc in updt_coord]) for P in Ps])
end

#NOTE code to adapt:
#θ = params(P[1].Target)
#ϑs = [[θ[j] for j in idx(uc)] for uc in updtCoord]
#result = [DiffResults.GradientResult(ϑ) for ϑ in ϑs]
#resultᵒ = [DiffResults.GradientResult(ϑ) for ϑ in ϑs]
#Q = eltype(result)


using JLD2, Base.Test
@test JLD2.Lookup3.hash(UInt8[]) == 0xdeadbeef
@test JLD2.Lookup3.hash(UInt8[], 1, 0, 0xdeadbeef) == 0xbd5b7dde
@test JLD2.Lookup3.hash(UInt8[23]) == 0xa209c931
@test JLD2.Lookup3.hash(UInt8[0]) == 0x8ba9414b
@test JLD2.Lookup3.hash(UInt8[23, 187]) == 0x8ba7a6c9
@test JLD2.Lookup3.hash(UInt8[0, 0]) == 0x62cd61b3
@test JLD2.Lookup3.hash(UInt8[23, 187, 98]) == 0xcebdf4f0
@test JLD2.Lookup3.hash(UInt8[0, 0, 0]) == 0x6bd0060f
@test JLD2.Lookup3.hash(UInt8[23, 187, 98, 217]) == 0x2c88bb51
@test JLD2.Lookup3.hash(UInt8[0, 0, 0, 0]) == 0x049396b8
large_buf = Vector{UInt8}(3093)
for i = 1:length(large_buf)
    large_buf[i] = ((i-1) * 3) % UInt8
end
@test JLD2.Lookup3.hash(large_buf) == 0x1bd2ee7b
@test JLD2.Lookup3.hash(fill!(large_buf, 0)) == 0x930c7afc


__precompile__()
module FINUFFT

## Export
export nufft1d1, nufft1d2, nufft1d3
export nufft2d1, nufft2d2, nufft2d3
export nufft3d1, nufft3d2, nufft3d3

export nufft1d1!, nufft1d2!, nufft1d3!
export nufft2d1!, nufft2d2!, nufft2d3!
export nufft3d1!, nufft3d2!, nufft3d3!

export nufft_opts
export nufft_c_opts        # backward-compatibility - remove?
export finufft_plan
export finufft_default_opts
export finufft_makeplan
export finufft_setpts
export finufft_exec
export finufft_destroy
export finufft_exec!
export BIGINT
export finufftReal

## External dependencies
using finufft_jll

const libfinufft = finufft_jll.libfinufft

const BIGINT = Int64        # must match that in FINUFFT include/dataTypes.h

# our allowed real array types...
const finufftReal = Union{Float64,Float32}

## FINUFFT opts struct, must bytewise match that in include/nufft_opts.h
"""
    mutable struct nufft_opts
        modeord            :: Cint
        chkbnds            :: Cint
        debug              :: Cint
        spread_debug       :: Cint
        showwarn           :: Cint
        nthreads           :: Cint
        fftw               :: Cint
        spread_sort        :: Cint
        spread_kerevalmeth :: Cint
        spread_kerpad      :: Cint
        upsampfac          :: Cdouble
        spread_thread      :: Cint
        maxbatchsize       :: Cint
        spread_nthr_atomic :: Cint
        spread_max_sp_size :: Cint
    end

Options struct passed to the FINUFFT library.

# Fields

This is a summary only; see FINUFFT documentation for full descriptions.

    modeord :: Cint
0: CMCL-style increasing mode ordering (neg to pos), or\\
1: FFT-style mode ordering (affects type-1,2 only)

    chkbnds :: Cint
0: don't check if input NU pts in [-3pi,3pi], 1: do

    debug :: Cint
0: silent, 1: text basic timing output

    spread_debug :: Cint
passed to spread_opts, 0 (no text) 1 (some) or 2 (lots)

    showwarn :: Cint
Whether to print warnings to stderr. 0: silent, 1: print warnings

    nthreads :: Cint
How many threads FINUFFT should use, or 0 (use max available in OMP)

    fftw :: Cint
0:`FFTW_ESTIMATE`, or 1:`FFTW_MEASURE` (slow plan but faster FFTs)

    spread_sort :: Cint
passed to spread_opts, 0 (don't sort) 1 (do) or 2 (heuristic)

    spread_kerevalmeth :: Cint
passed to spread_opts, 0: exp(sqrt()), 1: Horner ppval (faster)

    spread_kerpad :: Cint
passed to spread_opts, 0: don't pad to mult of 4, 1: do

    upsampfac :: Cdouble
upsampling ratio sigma: 2.0 (standard), or 1.25 (small FFT), or\\
0.0 (auto).

    spread_thread :: Cint
(for ntrans>1 only)\\
0: auto choice,\\
1: sequential multithreaded,\\
2: parallel singlethreaded spread.

    maxbatchsize :: Cint
(for ntrans>1 only). max blocking size for vectorized, 0 for auto-set

    spread_nthr_atomic :: Cint
if >=0, threads above which spreader OMP critical goes atomic

    spread_max_sp_size :: Cint
if >0, overrides spreader (dir=1 only) max subproblem size
"""
mutable struct nufft_opts{T}
    modeord            :: Cint
    chkbnds            :: Cint
    debug              :: Cint
    spread_debug       :: Cint
    showwarn           :: Cint
    nthreads           :: Cint
    fftw               :: Cint
    spread_sort        :: Cint
    spread_kerevalmeth :: Cint
    spread_kerpad      :: Cint
    upsampfac          :: Cdouble        # regardless of T
    spread_thread      :: Cint
    maxbatchsize       :: Cint
    spread_nthr_atomic :: Cint
    spread_max_sp_size :: Cint
    nufft_opts{T}() where T <: finufftReal = new{T}()
end
# The above must match include/nufft_opts.h in FINUFFT.
# Note that nufft_opts happens to be the same for Float32 vs Float64 types.
# This is as in FINUFFT. It may not always hold in the future.

const nufft_c_opts = nufft_opts        # for backward compatibility - remove?

"""
    p = finufft_default_opts()
    p = finufft_default_opts(dtype=Float32)

Return a [`nufft_opts`](@ref) struct with the default FINUFFT settings. Set up the double precision variant by default.\\
See: <https://finufft.readthedocs.io/en/latest/usage.html#options>
"""
function finufft_default_opts(dtype::DataType=Float64)
    opts = nufft_opts{dtype}()

    if dtype==Float64
        ccall( (:finufft_default_opts, libfinufft),
               Nothing,
               (Ref{nufft_opts},),
               opts
               )

    else
        ccall( (:finufftf_default_opts, libfinufft),
               Nothing,
               (Ref{nufft_opts},),
               opts
               )
    end

    return opts
end

### Error handling

# Following should match error codes in https://github.com/flatironinstitute/finufft/blob/master/include/defs.h
const ERR_EPS_TOO_SMALL        = 1
const ERR_MAXNALLOC            = 2
const ERR_SPREAD_BOX_SMALL     = 3
const ERR_SPREAD_PTS_OUT_RANGE = 4
const ERR_SPREAD_ALLOC         = 5
const ERR_SPREAD_DIR           = 6
const ERR_UPSAMPFAC_TOO_SMALL  = 7
const HORNER_WRONG_BETA        = 8
const ERR_NDATA_NOTVALID       = 9
const ERR_TYPE_NOTVALID        = 10
const ERR_ALLOC                = 11
const ERR_DIM_NOTVALID         = 12
const ERR_SPREAD_THREAD_NOTVALID = 13

struct FINUFFTError <: Exception
    errno::Cint
    msg::String
end
Base.showerror(io::IO, e::FINUFFTError) = print(io, "FINUFFT Error ($(e.errno)): ", e.msg)

function check_ret(ret)
    # Check return value and output corresponding error message to Julia side.
    # This should be kepts up to date with error code interpretation in
    # https://finufft.readthedocs.io/en/latest/error.html
    # which is generated from
    # https://github.com/flatironinstitute/finufft/blob/master/docs/error.rst
    if ret==0     # no error or warning
        return
    elseif ret==ERR_EPS_TOO_SMALL
        msg = "requested tolerance epsilon too small to achieve (warning only)"
    elseif ret==ERR_MAXNALLOC
        msg = "attemped to allocate internal array larger than MAX_NF (defined in defs.h)"
    elseif ret==ERR_SPREAD_BOX_SMALL
        msg = "spreader: fine grid too small compared to spread (kernel) width"
    elseif ret==ERR_SPREAD_PTS_OUT_RANGE
        msg = "spreader: if chkbnds=1, a nonuniform point is out of input range [-3pi,3pi]^d"
    elseif ret==ERR_SPREAD_ALLOC
        msg = "spreader: array allocation error"
    elseif ret==ERR_SPREAD_DIR
        msg = "spreader: illegal direction (should be 1 or 2)"
    elseif ret==ERR_UPSAMPFAC_TOO_SMALL
        msg = "upsampfac too small (should be >1.0)"
    elseif ret==HORNER_WRONG_BETA
        msg = "upsampfac not a value with known Horner eval: currently 2.0 or 1.25 only"
    elseif ret==ERR_NDATA_NOTVALID
        msg = "ntrans not valid in vectorized interface (should be >= 1)"
    elseif ret==ERR_TYPE_NOTVALID
        msg = "invalid transform type, type should be 1, 2, or 3"
    elseif ret==ERR_ALLOC
        msg = "general allocation failure"
    elseif ret==ERR_DIM_NOTVALID
        msg = "invalid dimension, should be 1, 2, or 3"
    elseif ret==ERR_SPREAD_THREAD_NOTVALID
        msg = "spread_thread option not valid"
    else
        msg = "error of type unknown to Julia interface! Check FINUFFT documentation"
    end
    throw(FINUFFTError(ret, msg))
end

# HELPER routines for guru interface...

### validate sizes of inputs for setpts
function valid_setpts(type::Integer,
                      dim::Integer,
                      x::Array{T},
                      y::Array{T}=T[],
                      z::Array{T}=T[],
                      s::Array{T}=T[],
                      t::Array{T}=T[],
                      u::Array{T}=T[]) where T <: finufftReal
    nj = length(x)
    if type==3
        nk = length(s)
    else
        nk = 0
    end

    if dim>1
        @assert nj == length(y)
        if type==3
            @assert nk == length(t)
        end
    end

    if dim>2
        @assert nj == length(z)
        if type==3
            @assert nk == length(u)
        end
    end

    return (nj, nk)
end

### validate number of transforms
function valid_ntr(x::Array{T},
                   c::Array{Complex{T}}) where T <: finufftReal
    ntrans = Cint(length(c) / length(x))
    @assert ntrans*length(x) == length(c)
    return ntrans
end

### infer number of modes from fk array
function get_nmodes_from_fk(dim::Integer,
                            fk::Array{Complex{T}}) where T <: finufftReal
    ndim = ndims(fk)
    @assert dim==1 || dim==2 || dim==3
    @assert ndim==dim || ndim==dim+1
    if ndim==dim
        ntrans = 1
        return (size(fk)...,ntrans)
    else
        return size(fk)
    end
end

### kwargs opt set
function setkwopts!(opts::nufft_opts; kwargs...)
    dtype = Float64
    for (key, value) in kwargs
        if hasproperty(opts, key::Symbol)
            setproperty!(opts, key, value)
        elseif String(key)=="dtype"
            @assert value <: finufftReal
            dtype = value
        else
            @warn "nufft_opts does not have attribute " * String(key)
        end
    end
    return dtype
end

### check kwargs with dtype
function checkkwdtype(dtype::DataType; kwargs...)
    for (key, value) in kwargs
        if String(key)=="dtype"
            @assert  value == dtype
        end
    end
end

# Finally, bring in the main user-facing interfaces...

include("guru.jl")
include("simple.jl")

end # module