#
#
# REAL MONEY ######################################################
# REAL MONEY ######################################################
#
#

function place_first_poly_order!(config, open_orders, bot_path, client, usr)
    @info("NEW order found!")
    for (k,v) in open_orders["NEW"]
        ticker = client.get_ticker(symbol=k)["lastPrice"]
        current = parse(Float64, ticker)

        id_str = v["trigger"] * "--" * string(now())
        symbol = k
        quant = floor(config["order_volume"] / current, digits=config["symbols"][symbol]["precision"])
        order = try client.order_limit_buy(
            symbol=symbol,
            quantity=quant,
            price=ticker);
        catch
            nothing
        end
        if order !== nothing
            open_orders["BUY"]["EXTRA-0_" * id_str] = order
            delete!(open_orders["NEW"], k)
            open(joinpath(bot_path, "open_orders.json"), "w") do f
                JSON.print(f, open_orders, 4)
            end
            p = Dict()
            p["ticker"] = ticker
            try create_notification("new cycle", config, usr, params = p)
                catch;
            end
        end
    end
    return open_orders
end

function check_open_poly_sell!(config, open_orders, closed_orders, bot_path, client, usr)
    for (k,v) in open_orders["SELL"]
        if length(open_orders["SELL"]) > 0
            fees = client.get_trade_fee(symbol=v["symbol"])["tradeFee"][1]
        end
        if v["status"] != "place order"
            skip = false
            tmp = try client.get_order(
                symbol=v["symbol"],
                orderId=v["orderId"])
                catch
                @info("Skipped..")
                sleep(0.25)
                skip = true
            end
            if !skip
                id_str = split(k, "_")[end]
                #
                # IS SELL CORRECT??
                #
                if tmp["status"] != "FILLED"
                    qty_new = 0
                    value   = 0
                    for (k2,v2) in closed_orders[id_str]
                        if v2["side"] == "BUY"
                            qty_new += parse(Float64, v2["executedQty"])
                            value += parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 + fees["taker"])
                        else
                            qty_new -= parse(Float64, v2["executedQty"])
                            value -= parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 - fees["maker"])
                        end
                    end
                    new_quant = floor(qty_new, digits=v["precision"])
                    new_price = string(floor(value / new_quant * (1 + parse(Float64, config["take_profit"])/100), digits=config["symbols"][v["symbol"]]["sell_precision"]))
                    if new_quant > parse(Float64, v["origQty"]) * 1.02
                        @info("SELL PLACED WITH WRONG QTY!")
                        result = try client.cancel_order(
                            symbol=v["symbol"],
                            orderId=v["orderId"])
                        catch
                            nothing
                        end
                        new_id    = "SELL_" * id_str
                        order = try client.order_limit_sell(
                            symbol=v["symbol"],
                            quantity=new_quant,
                            price=new_price);
                            catch
                            nothing
                        end
                        if order !== nothing
                            order["time"] = string(now())
                            open_orders["SELL"][new_id] = order
                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                        else
                            open_orders["SELL"][new_id] = Dict()
                            open_orders["SELL"][new_id]["origQty"] = new_quant
                            open_orders["SELL"][new_id]["price"]   = new_price
                            open_orders["SELL"][new_id]["status"]  = "place order"
                            open_orders["SELL"][new_id]["time"]    = string(now())
                            open_orders["SELL"][new_id]["fail"]    = 0
                            open_orders["SELL"][new_id]["symbol"]  = v["symbol"]
                            open_orders["SELL"][new_id]["if_fail"] = new_quant - 1/10^config["symbols"][v["symbol"]]["precision"]

                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                        end
                    end
                end
                #
                #
                #

                if tmp["status"] == "FILLED"
                    !(id_str in keys(closed_orders)) ? closed_orders[id_str] = Dict() : ""
                    tmp["time"] = string(now())

                    delete!(open_orders["SELL"], k)
                    for (k2, v2) in open_orders["BUY"]
                        if split(k2, "_")[end] == id_str && v2["symbol"] == v["symbol"]
                            sleep(0.2)
                            tmp_buy = try client.get_order(
                                symbol=v["symbol"],
                                orderId=v2["orderId"])
                            catch
                                nothing
                            end

                            if tmp_buy !== nothing
                                if !(tmp_buy["status"] in ["FILLED", "CANCELED"])
                                    sleep(0.2)
                                    result = try client.cancel_order(
                                        symbol=v["symbol"],
                                        orderId=v2["orderId"])
                                    catch
                                        nothing
                                    end
                                    if result !== nothing
                                        delete!(open_orders["BUY"], k2)
                                    else
                                        open_orders["BUY"][k2]["status"] = "cancel"
                                    end
                                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                                        JSON.print(f, open_orders, 4)
                                    end
                                else
                                    delete!(open_orders["BUY"], k2)
                                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                                        JSON.print(f, open_orders, 4)
                                    end
                                end
                                if parse(Float64, tmp_buy["executedQty"]) > 0
                                    closed_orders[id_str][k2] = tmp_buy
                                    open(joinpath(bot_path, "closed_orders.json"), "w") do f
                                        JSON.print(f, closed_orders, 4)
                                    end
                                end
                            else
                                open_orders["BUY"][k2]["status"] = "cancel"
                                open(joinpath(bot_path, "open_orders.json"), "w") do f
                                    JSON.print(f, open_orders, 4)
                                end
                            end
                        end
                    end
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end
                    qty = 0
                    net_qty = 0
                    val = 0
                    for (k2,v2) in closed_orders[id_str]
                        v2["side"] == "BUY" ? qty += parse(Float64, v2["executedQty"]) : ""
                        v2["side"] == "BUY" ? net_qty += parse(Float64, v2["executedQty"]) : net_qty -= parse(Float64, v2["executedQty"])
                        v2["side"] == "BUY" ? val -= parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 + fees["taker"]) : val += parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 - fees["maker"])
                    end

                    if val < 0 && val + parse(Float64, tmp["cummulativeQuoteQty"]) < 0
                        #
                        ## If there was an extra buy executed
                        closed_orders[id_str]["SELL-" * lpad(rand(1:1:100000), 6, "0")] = deepcopy(tmp);

                        open(joinpath(bot_path, "closed_orders.json"), "w") do f
                            JSON.print(f, closed_orders, 4)
                        end
                        #
                        ## Create new sell
                        if abs(val) > config["symbols"][v["symbol"]]["min_qty"]
                            #
                            ## qty is high enough to be sold
                            new_qty = floor(net_qty, digits=config["symbols"][v["symbol"]]["precision"])
                            new_price = string(floor(abs(val)/net_qty * (1 + parse(Float64, config["take_profit"])/100), digits=config["symbols"][v["symbol"]]["sell_precision"]))
                            sleep(0.2)
                            order = try client.order_limit_sell(
                                symbol=v["symbol"],
                                quantity=new_qty,
                                price=new_price);
                                catch
                                nothing
                            end
                            new_id = "SELL_" * id_str
                            if order !== nothing
                                order["time"] = string(now())
                                open_orders["SELL"][new_id] = order
                                open(joinpath(bot_path, "open_orders.json"), "w") do f
                                    JSON.print(f, open_orders, 4)
                                end
                                #
                                ## Place new buy
                                extra_iteration = 0
                                for (k2,v2) in closed_orders[id_str]
                                    if length(split(k2, "SELL")) < 2
                                        if parse(Int64, split(split(k2, "_")[1], "-")[end]) > extra_iteration
                                            extra_iteration = parse(Int64, split(split(k2, "_")[1], "-")[end])
                                        end
                                    end
                                end
                                extra_iteration += 1
                                if extra_iteration <= parse(Float64, config["extra_order_count"])
                                    new_price = string(round(parse(Float64, new_price) * (1 - config["extra_order_step"]), digits=config["symbols"][v["symbol"]]["sell_precision"]))
                                    quant = floor(config["order_volume"] * (config["extra_order_martingale"])^(extra_iteration + 1) / parse(Float64, new_price), digits=config["symbols"][v["symbol"]]["precision"])

                                    new_buy = Dict()
                                    new_buy["status"] = "place order"
                                    new_buy["origQty"] = quant
                                    new_buy["price"] = new_price
                                    new_buy["symbol"] = v["symbol"]
                                    new_buy_id = "EXTRA-" * string(extra_iteration) * "_" * id_str
                                    open_orders["BUY"][new_buy_id] = new_buy
                                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                                        JSON.print(f, open_orders, 4)
                                    end
                                end
                            else
                                #
                                ## If new sell could not be placed
                                open_orders["SELL"][new_id] = Dict()
                                open_orders["SELL"][new_id]["origQty"] = new_quant
                                open_orders["SELL"][new_id]["price"]   = new_price
                                open_orders["SELL"][new_id]["status"]  = "place order"
                                open_orders["SELL"][new_id]["time"]    = string(now())
                                open_orders["SELL"][new_id]["symbol"]  = v["symbol"]
                                open_orders["SELL"][new_id]["fail"]    = 0
                                open_orders["SELL"][new_id]["if_fail"] = new_quant - 1/10^config["symbols"][v["symbol"]]["precision"]

                                open(joinpath(bot_path, "open_orders.json"), "w") do f
                                    JSON.print(f, open_orders, 4)
                                end
                            end
                        else
                            #
                            ## qty is NOT high enough to be sold
                            new_price = string(floor(abs(val)/net_qty * (1 + parse(Float64, config["take_profit"])/100), digits=config["symbols"][v["symbol"]]["sell_precision"]))

                            extra_iteration = 0
                            for (k2,v2) in closed_orders[id_str]
                                if length(split(k2, "SELL")) < 2
                                    if parse(Int64, split(split(k2, "_")[1], "-")[end]) > extra_iteration
                                        extra_iteration = parse(Int64, split(split(k2, "_")[1], "-")[end])
                                    end
                                end
                            end
                            extra_iteration += 1
                            if extra_iteration <= parse(Float64, config["extra_order_count"])
                                new_price = string(round(parse(Float64, new_price) * (1 - config["extra_order_step"]), digits=config["symbols"][v["symbol"]]["sell_precision"]))
                                quant = floor(config["order_volume"] * (config["extra_order_martingale"])^(extra_iteration + 1) / parse(Float64, new_price), digits=config["symbols"][v["symbol"]]["precision"])

                                new_buy = Dict()
                                new_buy["status"]  = "place order"
                                new_buy["origQty"] = quant
                                new_buy["price"]   = new_price
                                new_buy["symbol"]  = v["symbol"]
                                new_buy_id = "EXTRA-" * string(extra_iteration) * "_" * id_str
                                open_orders["BUY"][new_buy_id] = new_buy
                                open(joinpath(bot_path, "open_orders.json"), "w") do f
                                    JSON.print(f, open_orders, 4)
                                end
                            end
                        end
                    else
                        #
                        ####################################################################
                        #
                        #
                        ## If everything went well and the profit is positive
                        sell_id = !haskey(closed_orders[id_str], "SELL") ? "SELL" : "SELL-" * lpad(rand(1:1:100000), 6, "0")
                        closed_orders[id_str][sell_id] = deepcopy(tmp);

                        open(joinpath(bot_path, "closed_order-" * id_str * ".json"), "w") do f
                            JSON.print(f, closed_orders[id_str], 4)
                        end

                        try for (k5,v5) in closed_orders
                                if DateTime(split(k5, "--")[end]) < now() - Day(42)
                                    delete!(closed_orders, k5)
                                end
                            end
                        catch;
                        end

                        open(joinpath(bot_path, "closed_orders.json"), "w") do f
                            JSON.print(f, closed_orders, 4)
                        end
                    end
                    #
                    ####################################################################
                    #
                elseif tmp["status"] == "CANCELED"
                    delete!(open_orders["SELL"], k)
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end
                    if parse(Float64, tmp["executedQty"]) > 0
                        closed_orders[id_str]["SELL-" * lpad(rand(1:1:100000), 6, "0")] = tmp
                        open(joinpath(bot_path, "closed_orders.json"), "w") do f
                            JSON.print(f, closed_orders, 4)
                        end
                    end
                elseif tmp["status"] != v["status"]
                    open_orders["SELL"][k] = tmp
                    open_orders["SELL"][k]["time"] = string(now())
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end
                end
            end
        else
            if v["fail"] == 0
                qty = floor(v["origQty"], digits=config["symbols"][v["symbol"]]["precision"])
            elseif v["fail"] == 1

                id_str = split(k, "_")[end]
                #
                # IS SELL CORRECT??
                #
                qty_new = 0
                value   = 0
                for (k2,v2) in closed_orders[id_str]
                    if v2["side"] == "BUY"
                        qty_new += parse(Float64, v2["executedQty"])
                        value += parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 + fees["taker"])
                    else
                        qty_new -= parse(Float64, v2["executedQty"])
                        value -= parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 - fees["maker"])
                    end
                end
                new_quant = floor(qty_new, digits=config["symbols"][v["symbol"]]["precision"])
                new_price = string(floor(value / new_quant * (1 + parse(Float64, config["take_profit"])/100), digits=config["symbols"][v["symbol"]]["sell_precision"]))
                result = try client.cancel_order(
                    symbol=v["symbol"],
                    orderId=v["orderId"])
                catch
                    nothing
                end
                qty = new_quant
            else
                account = client.get_account()["balances"];
                new = new_quant
                for bal in account
                    if bal["asset"] == v["symbol"][1:3]
                        if parse(Float64, bal["free"]) <= new_quant
                            new = parse(Float64, bal["free"])
                        end
                    end
                end
                qty = floor(new, digits=config["symbols"][v["symbol"]]["precision"])
            end
            order = try client.order_limit_sell(
                symbol=v["symbol"],
                quantity=qty,
                price=v["price"]);
                catch
                nothing
            end
            if order !== nothing
                order["time"] = string(now())
                open_orders["SELL"][k] = order
                open(joinpath(bot_path, "open_orders.json"), "w") do f
                    JSON.print(f, open_orders, 4)
                end
            else
                v["fail"] = 1
                open(joinpath(bot_path, "open_orders.json"), "w") do f
                    JSON.print(f, open_orders, 4)
                end
            end
        end
    end
    sleep(0.25)
    return open_orders, closed_orders
end

function check_open_poly_buy!(config, open_orders, closed_orders, bot_path, client, usr)
    for (k,v) in open_orders["BUY"]
        if v["status"] == "place order"
            quant = round(v["origQty"], digits=config["symbols"][v["symbol"]]["precision"])
            new_price = v["price"]
            new_id = k
            order = try client.order_limit_buy(
                symbol= v["symbol"],
                quantity=quant,
                price=new_price);
                catch
                nothing
            end
            if order !== nothing
                order["time"] = string(now())
                open_orders["BUY"][new_id] = order
            else
                open_orders["BUY"][new_id] = Dict()
                open_orders["BUY"][new_id]["origQty"] = quant
                open_orders["BUY"][new_id]["price"]   = new_price
                open_orders["BUY"][new_id]["symbol"]  = v["symbol"]
                open_orders["BUY"][new_id]["status"]  = "place order"
                open_orders["BUY"][new_id]["time"]    = string(now())
            end
            open(joinpath(bot_path, "open_orders.json"), "w") do f
                JSON.print(f, open_orders, 4)
            end
        else
            skip = false
            tmp = try client.get_order(
                symbol=v["symbol"],
                orderId=v["orderId"])
                catch
                @info("Skipped..")
                sleep(0.25)
                skip = true
            end
            if !skip
                if tmp["status"] == "FILLED" || v["status"] == "continue" || parse(Float64, tmp["executedQty"]) == parse(Float64, tmp["origQty"])
                    if tmp["status"] != "FILLED" && tmp["status"] != "CANCELLED"
                        result = try client.cancel_order(
                            symbol=v["symbol"],
                            orderId=tmp["orderId"])
                        catch
                            nothing
                        end
                    end

                    fees = client.get_trade_fee(symbol=v["symbol"])["tradeFee"][1]
                    id_str = split(k, "_")[end]
                    extra_iteration = parse(Int64, split(split(k, "_")[1], "-")[2])
                    !(id_str in keys(closed_orders)) ? closed_orders[id_str] = Dict() : ""
                    tmp["time"] = string(now())
                    closed_orders[id_str][split(k, "_")[1]] = deepcopy(tmp);
                    open(joinpath(bot_path, "closed_orders.json"), "w") do f
                        JSON.print(f, closed_orders, 4)
                    end

                    delete!(open_orders["BUY"], k)
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end

                    #
                    # EXTRA BUY
                    #
                    new_id = "EXTRA-" * string(extra_iteration + 1) * "_" * id_str
                    new_price = string(round(parse(Float64, v["price"]) * (1 - config["extra_order_step"]), digits=config["symbols"][v["symbol"]]["sell_precision"]))
                    quant = floor(config["order_volume"] * (config["extra_order_martingale"])^(extra_iteration + 1) / parse(Float64, new_price), digits=config["symbols"][v["symbol"]]["precision"])
                    if extra_iteration < parse(Float64, config["extra_order_count"])
                        order = try client.order_limit_buy(
                            symbol= v["symbol"],
                            quantity=quant,
                            price=new_price);
                            catch
                            nothing
                        end
                        if order !== nothing
                            order["time"] = string(now())
                            open_orders["BUY"][new_id] = order
                        else
                            open_orders["BUY"][new_id] = Dict()
                            open_orders["BUY"][new_id]["origQty"] = quant
                            open_orders["BUY"][new_id]["price"]   = new_price
                            open_orders["BUY"][new_id]["status"]  = "place order"
                            open_orders["BUY"][new_id]["symbol"]  = v["symbol"]
                            open_orders["BUY"][new_id]["time"]    = string(now())
                        end
                        open(joinpath(bot_path, "open_orders.json"), "w") do f
                            JSON.print(f, open_orders, 4)
                        end
                    end
                    #
                    # TAKE PROFIT
                    #
                    # Cancel current take profit orders
                    if length(open_orders["SELL"]) > 0
                        for (k2, v2) in open_orders["SELL"]
                            if split(k2, "_")[end] == id_str && v["symbol"] == v2["symbol"]
                                tmp_sell = try client.get_order(
                                    symbol=v["symbol"],
                                    orderId=v2["orderId"])
                                    catch
                                    try sleep(0.2)
                                        client.get_order(
                                            symbol=v["symbol"],
                                            orderId=v2["orderId"])
                                        catch nothing
                                    end
                                end
                                if !(tmp_sell["status"] in ["CANCELED", "FILLED"]) || tmp_sell == nothing
                                    sleep(0.1)
                                    tmp_sell = try client.cancel_order(
                                        symbol=v["symbol"],
                                        orderId=v2["orderId"])
                                    catch
                                        nothing
                                    end
                                end
                                if tmp_sell !== nothing
                                    if parse(Float64, v2["executedQty"]) > 0
                                        closed_orders[id_str]["SELL-" * lpad(rand(1:1:10000), 5, "0")] = result
                                        open(joinpath(bot_path, "closed_orders.json"), "w") do f
                                            JSON.print(f, closed_orders, 4)
                                        end
                                    end
                                    delete!(open_orders["SELL"], k2)
                                end
                            end
                        end
                    end

                    qty_new = 0
                    value   = 0
                    for (k2,v2) in closed_orders[id_str]
                        if v2["side"] == "BUY"
                            qty_new += parse(Float64, v2["executedQty"])
                            value += parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 + fees["taker"])
                        else
                            qty_new -= parse(Float64, v2["executedQty"])
                            value -= parse(Float64, v2["cummulativeQuoteQty"]) * (1.0 - fees["maker"])
                        end
                    end
                    new_id    = "SELL_" * id_str
                    new_quant = floor(qty_new, digits=config["symbols"][v["symbol"]]["precision"])
                    new_price = string(floor(value / new_quant * (1 + fees["maker"] + parse(Float64, config["take_profit"])/100), digits=config["symbols"][v["symbol"]]["sell_precision"]))
                    if value > config["symbols"][v["symbol"]]["min_qty"]
                        order = try client.order_limit_sell(
                            symbol=v["symbol"],
                            quantity=new_quant,
                            price=new_price);
                            catch
                            nothing
                        end
                        if order !== nothing
                            order["time"] = string(now())
                            open_orders["SELL"][new_id] = order
                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                        else
                            open_orders["SELL"][new_id] = Dict()
                            open_orders["SELL"][new_id]["origQty"] = new_quant
                            open_orders["SELL"][new_id]["price"]   = new_price
                            open_orders["SELL"][new_id]["status"]  = "place order"
                            open_orders["SELL"][new_id]["time"]    = string(now())
                            open_orders["SELL"][new_id]["fail"]    = 0
                            open_orders["SELL"][new_id]["symbol"]  = v["symbol"]
                            open_orders["SELL"][new_id]["if_fail"] = new_quant - 1/10^config["symbols"][v["symbol"]]["precision"]

                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                        end
                    end
                    p = Dict()
                    p["extra_iteration"] = extra_iteration
                    p["executedQty"] = tmp["executedQty"]
                    try create_notification("buy exec", config, usr, params = p)
                        catch;
                    end
                    if !(extra_iteration < parse(Float64, config["extra_order_count"]))
                        try create_notification("no volume", config, usr)
                            catch;
                        end
                    end
                elseif v["status"]  == "cancel"
                    result = try client.cancel_order(
                        symbol=v["symbol"],
                        orderId=v["orderId"])
                    catch
                        nothing
                    end
                    if result !== nothing
                        delete!(open_orders["BUY"], k)
                        open(joinpath(bot_path, "open_orders.json"), "w") do f
                            JSON.print(f, open_orders, 4)
                        end
                    end
                elseif v["status"]  == "buy now"
                    @info(v["status"])
                    id_str = split(k, "_")[end]
                    extra_iteration = parse(Int64, split(split(k, "_")[1], "-")[2])
                    if extra_iteration < parse(Float64, config["extra_order_count"])
                        #
                        # Cancel current BUY order
                        if tmp["status"] != "CANCELED"
                            result = try client.cancel_order(
                                symbol=v["symbol"],
                                orderId=v["orderId"])
                            catch
                                nothing
                            end
                        else
                            result = tmp
                        end
#                         @info(result)
                        if result !== nothing
                            @info("order cancelled")
                            #
                            # Check if that order was partially filled -> if so: move cancelled order to closed_orders

                            if parse(Float64, result["executedQty"]) > 0
                                result["time"] = string(now())
                                !haskey(closed_orders, id_str) ? closed_orders[id_str] = Dict() : ""
                                closed_orders[id_str][split(k, "_")[1]] = result
                                extra_iteration += 1
                                open(joinpath(bot_path, "closed_orders.json"), "w") do f
                                    JSON.print(f, closed_orders, 4)
                                end
                            end
                            #
                            # Remove from open_orders
                            delete!(open_orders["BUY"], k)
                            #
                            # Get new order details
                            ticker = client.get_ticker(symbol=v["symbol"])["lastPrice"]
                            quant = floor(parse(Float64, v["origQty"]), digits=config["symbols"][v["symbol"]]["precision"])
                            new_id = "EXTRA-" * string(extra_iteration) * "_" * id_str
                            #
                            # Place new order
                            order = try client.order_limit_buy(
                                symbol=v["symbol"],
                                quantity=quant,
                                price=ticker);
                                catch
                                nothing
                            end
                            if order !== nothing
                                order["time"] = string(now())
                                open_orders["BUY"][new_id] = order
                            else
                                open_orders["BUY"][new_id] = Dict()
                                open_orders["BUY"][new_id]["origQty"] = quant
                                open_orders["BUY"][new_id]["price"]   = ticker
                                open_orders["BUY"][new_id]["status"]  = "place order"
                                open_orders["BUY"][new_id]["symbol"]  = v["symbol"]
                                open_orders["BUY"][new_id]["time"]    = string(now())
                            end
                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                        end
                    else
                        open_orders["BUY"][k] = tmp
                        open(joinpath(bot_path, "open_orders.json"), "w") do f
                            JSON.print(f, open_orders, 4)
                        end
                    end
                elseif tmp["status"] == "CANCELED" && parse(Float64, tmp["executedQty"]) == 0
                    delete!(open_orders["BUY"], k)
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end
                elseif tmp["status"] == "PARTIALLY_FILLED" && v["status"] != tmp["status"]
                    v = deepcopy(tmp)
                    v["time"] = string(now())
                    v["partial_time"] = string(now())
                    open_orders["BUY"][k] = v
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end
                elseif parse(Float64, tmp["executedQty"]) != 0 && parse(Float64, tmp["executedQty"]) != parse(Float64, tmp["origQty"]) && haskey(v, "partial_time")
                    if v["partial_time"] < string(now() - Minute(2))
                        result = try client.cancel_order(
                            symbol=v["symbol"],
                            orderId=tmp["orderId"])
                        catch
                            nothing
                        end
                        if result !== nothing
                            tmp["partial_time"] = v["partial_time"]
                            v = deepcopy(tmp)
                            v["status"] = "continue"
                            open_orders["BUY"][k] = v
                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                            p = Dict()
                            p["extra_iteration"] = extra_iteration
                            p["executedQty"] = tmp["executedQty"]
                            try create_notification("partial buy exec", config, usr, params = p)
                                catch;
                            end
                        end
                        if tmp["status"] == "CANCELED"
                            v["status"] = "continue"
                            open_orders["BUY"][k] = v
                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                        end
                    end
                elseif tmp["status"] != v["status"]
                    tmp["time"] = string(now())
                    open_orders["BUY"][k] = tmp
                    if haskey(v, "partial_time")
                        open_orders["BUY"][k]["partial_time"] = v["partial_time"]
                    end
                    open(joinpath(bot_path, "open_orders.json"), "w") do f
                        JSON.print(f, open_orders, 4)
                    end
                elseif tmp["status"] == "NEW" && split(k, "_")[1] == "EXTRA-0" && parse(Float64, tmp["executedQty"]) == 0.0
                    # Maybe the first BUY has to be updated
                    id_str = split(k, "_")[end]
                    t = DateTime(split(id_str, "--")[end])
                    if now() - t > Minute(1)
                        result = try client.cancel_order(
                            symbol=v["symbol"],
                            orderId=v["orderId"])
                        catch
                            nothing
                        end
                        if result !== nothing
                            delete!(open_orders["BUY"], k)
                            open(joinpath(bot_path, "open_orders.json"), "w") do f
                                JSON.print(f, open_orders, 4)
                            end
                            try create_notification("cycle closed", config, usr)
                                catch;
                            end
                        end
                    end
                end
            end
        end
    end
    return open_orders, closed_orders
end


using HTTP
using Gumbo
using Cascadia

cache_path = "$(@__DIR__)/../assets/data.html.asset" # ".asset" is used to supress vscode and github wrong hint

function get_from_wiki(;cache=true)
    url = "https://hoi4.paradoxwikis.com/Land_units"
    res = HTTP.request("GET", url)
    data = String(res.body)

    if cache
        f = open(cache_path, "r")
        write(f, data)
        close(f)
    end

    return data
end

function get_from_cache()
    f = open(cache_path, "r")
    data = String(read(f))
    close(f)

    return data
end

function _unit_parse(::Type{Float64}, x::AbstractString)
    if length(x)==0 # empty string "" => 0
        return 0.0
    end

    # weired wiki "107.2.0" => "107.2"
    split_x = split(x, ".")
    if length(split_x) == 1
        x = split_x[1]
    else
        x = join(split_x[1:2], ".")
    end

    if x[1]=='+'
        x=x[2:end]
    end
    if x[end]=='%'
        return Base.parse(Float64, x[1:end-1]) / 100
    end
    return Base.parse(Float64, x)
end

function _unit_parse(::Type{Int64}, x::AbstractString)
    if length(x)==0
        return 0
    end
    if x[1]=='+'
        x=x[2:end]
    end
    return Base.parse(Int64, x)
end

function unit_parse(::Type{String}, x::AbstractString)
    return x
end

function unit_parse(typ, x::AbstractString)
    #global y
    y = split(x, "/")[1]
    _unit_parse(typ, y)
end

function extract_td(td)
    #global td_ = td
    match_img = eachmatch(sel"img", td);
    if length(match_img) > 0
        raw = attrs(match_img[1])["src"]
    else
        raw_list = []
        for child in td.children
            if child isa HTMLElement{:br}
                push!(raw_list, "/")
            else
                push!(raw_list, strip(nodeText(child)))
            end
        end
        raw = join(raw_list, "")
    end
    return raw
end

function colspan_sum(tr::HTMLElement{:tr})
    # determine cols by resolve first colspan.
    s = 0
    for el in tr.children # el ∈ {tr, th}
        if "colspan" in keys(attrs(el))
            colspan = Base.parse(Int, attrs(el)["colspan"])
        else
            colspan = 1
        end
        s += colspan
    end
    s
end

function table_index(table::HTMLElement{:table})
    tr_list = eachmatch(sel"tr", table)
    cols = colspan_sum(tr_list[1])
    rows = length(tr_list)
    idx = 1
    index_mat = zeros(Int, rows, cols) # Use 0 to denote "unassigned"
    content_list = []
    for (i, tr) in enumerate(tr_list)
        j = 1
        for el in tr.children # el ∈ tr, td
            while index_mat[i, j] != 0
                j += 1
            end
            colspan = Base.parse(Int, get(attrs(el), "colspan", "1"))
            rowspan = Base.parse(Int, get(attrs(el), "rowspan", "1"))
            for ii in i:(i+rowspan-1)
                for jj in j:(j+colspan-1)
                    index_mat[ii, jj] = idx
                end
            end
            idx += 1
            push!(content_list, el)
        end
    end
    return content_list, index_mat
end

struct HTMLTableMatrix <: AbstractArray{HTMLElement, 2}
    content_list::Vector{HTMLElement}
    index_mat::Array{Int, 2}
end

Base.size(A::HTMLTableMatrix) = size(A.index_mat)
Base.getindex(A::HTMLTableMatrix, i::Int, j::Int) = A.content_list[A.index_mat[i,j]]

function get_unit_list(;cache=true)
    if cache
        data = get_from_cache()
    else
        data = get_from_wiki()
    end

    data_parsed = parsehtml(data)
    table_selected = eachmatch(sel"table.wikitable", data_parsed.root)[1]
    content_list, index_mat = table_index(table_selected)
    html_table_matrix = HTMLTableMatrix(content_list, index_mat);
    mat_extracted = extract_td.(html_table_matrix)[3:end, :] # remove 2 row head
    mat_parsed = Matrix{Any}(undef, size(mat_extracted)...)

    for i in 1:size(mat_extracted, 1)
        for (f_name, f_type, j) in zip(fieldnames(Unit), fieldtypes(Unit), 1:size(mat_extracted, 2))
            #global i_ = i
            #global j_ = j
            raw = mat_extracted[i, j]
            #global raw_ = raw
            parsed = unit_parse(f_type, raw)
            mat_parsed[i,j] = parsed
        end
    end

    unit_list = [Unit(mat_parsed[i,:]...) for i in 1:size(mat_parsed, 1)]
end


export
    LazyInference,
    LazyState

"""
    mutable struct LazyState

Maintains the state of a lazy algorithm

# Fields

- `previous_algorithm`: The last instant algorithm used, if any
- `evidence`: The evidence supplied in `prepare`
- `interventions`: The interventions supplied in `prepare`
- `placeholder_beliefs`: The placeholder beliefs supplied in `prepare`
- `next_size`: The range size to use in the next call to `refine`
- `next_depth`: The depth to use in the next call to `refine`
- `next_iteration`: The number of the next iteration
- `is_complete`: A flag indicating whether the netwowk has been fully expanded
- `order`: The order of nodes used in computations
"""
mutable struct LazyState
    previous_algorithm :: Union{InstantAlgorithm, Nothing}
    evidence :: Dict{Symbol, Score}
    interventions :: Dict{Symbol, Dist}
    placeholder_beliefs :: Dict{Symbol, Dist}
    next_size :: Int
    next_depth :: Int
    next_iteration :: Int
    is_complete :: Bool
    order :: Vector{Node}
end

"""
    LazyState(ns, nd, ni, ic)

Intantiate `LazyState` with `next_size`, `next_depth`, `next_iterator`, and `is_complete`.
"""
LazyState(ns, nd, ni, ic) = LazyState(nothing, Dict{Symbol,Score}(), Dict{Symbol,Dist}(), Dict{Symbol,Dist}(), ns, nd, ni, ic, Node[])

"""
    struct LazyInference <: IterativeAlgorithm

An iterative algorithm that expands recursively and increases the ranges of instances on every iteration.
"""
struct LazyInference <: IterativeAlgorithm
    algorithm_maker :: Function # A function that takes the current depth and range size and returns the instant algorithm to use
    increment :: Int
    start_size :: Int
    max_iterations :: Int
    start_depth :: Int
    state :: LazyState

    """
    function LazyInference(maker::Function; increment = 10, start_size = increment, max_iterations = 100, start_depth = 1)

    # Arguments
    - `maker``:  A function that takes a range size and expansion depth and returns an `InstantAlgorithm`
    - `increment`:  The increment to range size on every iteration
    - `start_size`:  The starting range size
    - `max_iterations`:  The maximum number of refinement steps
    - `start_depth`:  The depth of recursive expansion in the first iteration
    """
    function LazyInference(maker::Function;
        increment = 10, start_size = increment, max_iterations = 100, start_depth = 1)
        new(maker, increment, start_size, max_iterations, start_depth,
            LazyState(start_size, start_depth, 1, false))
    end
end

function answer(query::Query, lazyalg::LazyInference, runtime::InstantRuntime, target::VariableInstance)
    state = lazyalg.state
    return answer(query, state.previous_algorithm, runtime, target)
end

function prepare(alg::LazyInference, runtime::InstantRuntime,
    evidence::Dict{Symbol, <:Score} = Dict{Symbol, Score}(),
    interventions::Dict{Symbol, <:Dist} = Dict{Symbol, Dist}(),
    placeholder_beliefs = get_placeholder_beliefs(runtime))
    ensure_all!(runtime)
    net = get_network(runtime)
    # The evidence, interventions, and placeholder_beliefs are punted to refine to pass to the underlying algorithm
    state = alg.state
    state.evidence = evidence
    state.interventions = interventions
    state.placeholder_beliefs = placeholder_beliefs
    state.order = topsort(get_initial_graph(net))
    clear_analysis!()
end

function refine(lazyalg::LazyInference, runtime::InstantRuntime)
    state = lazyalg.state
    if !(state.is_complete || state.next_iteration > lazyalg.max_iterations)
        set_ranges!(runtime, state.evidence, state.next_size, state.next_depth, state.order)
        inferencealg = lazyalg.algorithm_maker(state.next_size, state.next_depth)
        state.previous_algorithm = inferencealg
        infer(inferencealg, runtime,
            state.evidence, state.interventions, state.placeholder_beliefs)
        state.is_complete = _complete(runtime)
        state.next_iteration += 1
        state.next_depth += 1
        state.next_size += lazyalg.increment
    end
end


"A AbstractModel specifies all parameters of a system."
abstract type AbstractModel{T,N,P} <: AbstractParameter{T} end
abstract type AbstractODEModel{T,N,P} <: AbstractModel{T,N,P} end
abstract type AbstractModelwithDelay{T,N,P} <: AbstractModel{T,N,P} end
abstract type AbstractNoisyModel{T,N,P} <: AbstractModel{T,N,P} end
abstract type AbstractNoisySpaceAction{T,N} <: AbstractSpaceAction{T,N} end
abstract type AbstractSimulation{T} <: AbstractParameter{T} end
abstract type AbstractExecution{T,SIM} end
# FIXME shouldn't the second type of full execution be the type of the solution?
abstract type AbstractFullExecution{T,SIM} <: AbstractExecution{T,SIM} end
export AbstractFullExecution, AbstractModel, AbstractSimulation

#function (am::Type{<:AbstractModel})(fallback_args...; fallback_kwargs...)
#    @warn """Model $am undefined!
#    ---------------------
#    $fallback_args
#
#    $fallback_kwargs
#    ---------------------
#    """
#    missing
#end

struct FailedSimulation{T} <: AbstractSimulation{T} end
struct FailedExecution{T,S<:FailedSimulation{T}} <: AbstractExecution{T,S}
    sim::S
end

# export AbstractNoisyModel, NoisyInputModel, NoisyInputAction, AbstractODEModel

# struct NoisyInputModel{T,N,P,M<:AbstractModel{T,N,P}} <: AbstractNoisyModel{T,N,P}
#     model::M
#     noise_process::NoiseProcess
# end
# function Base.getproperty(wnm::NoisyInputModel, sym::Symbol)
#     if sym ∈ [:noise_process, :model]
#         return getfield(wnm, sym)
#     else
#         return getproperty(getfield(wnm, :model), sym)
#     end
# end

# struct NoisyInputAction{T,N,SA<:AbstractSpaceAction{T,N}} <: AbstractNoisySpaceAction{T,N}
#     space_action::SA
#     noise_process::NoiseProcess
# end
# function (act::NoisyInputAction)(du, u, p, t, W)
#     du .= W
#     act.space_action(du, u, p, t)
# end

# function (nim::NoisyInputModel)(args...)
#     inner_fn = nim.model(args...)
#     NoisyInputAction(inner_fn, nim.noise_process)
# end

n_populations(::AbstractModel{T,N,P}) where {T,N,P} = P

function initial_value(::AbstractModel{T,N,P}, space::AbstractSpace{T,N}) where {T,N,P}
    init_val = AxisArray(population_repeat(zeros(space), P), (coordinate_axes(space)..., 1:P))
    return init_val
end

"A Simulation holds an AbstractModel to be solved, the space on which to solve it, the time for which to solve it, the initial value, and various solver options."
struct Simulation{
        T,
        M<:AbstractModel{T},
        S<:AbstractSpace{T},
        IV<:AbstractArray{T},
        ALG,
        DT<:Union{T,Nothing},
        SV_IDX<:Union{AbstractArray,Nothing},
        CB<:Union{Tuple{Function,NamedTuple},Tuple{DECallback,NamedTuple},Nothing,Vector{Tuple{Function,NamedTuple}}},
        GR<:Function} <: AbstractSimulation{T}
    model::M
    space::S
    tspan::Tuple{T,T}
    initial_value::IV
    algorithm::ALG
    dt::DT
    save_idxs::SV_IDX
    callback::CB
    global_reduction::GR
    solver_options
end
function Simulation(model::M; space::S, tspan, initial_value::IV=initial_value(model,space),
                    algorithm::ALG=nothing, dt::DT=nothing, save_idxs::SV_IDX=nothing, callback::CB=nothing, global_reduction::GR=identity,
        opts...) where {T,N,P,M<:AbstractModel{T,N,P}, S<:AbstractSpace{T,N},IV,ALG,DT,SV_IDX,CB,GR}
    save_idxs = parse_save_idxs(space, P, save_idxs)
    return Simulation{T,M,S,IV,ALG,DT,typeof(save_idxs),CB,GR}(model, space, tspan, initial_value, algorithm, dt, save_idxs, callback, global_reduction, opts)
end
function Simulation(model::Missing; kwargs...)
    return FailedSimulation{Missing}()
end

"An Execution holds a Simulation and the solution obtained by running the Simulation."
struct Execution{T,S<:AbstractSimulation{T},D<:DESolution} <: AbstractFullExecution{T,S}
    simulation::S
    solution::D
end
struct ReducedExecution{T,ST,S<:AbstractSimulation{T},SV<:SavedValues{T,ST}} <: AbstractExecution{T,S}
    simulation::S
    saved_values::SV
end
struct AugmentedExecution{T,ST,S<:AbstractSimulation{T},D<:DESolution,SV<:SavedValues{T,ST}} <: AbstractFullExecution{T,S}
    simulation::S
    solution::D
    saved_values::SV
end
make_execution(s::S, sol::DESolution) where {T,S <: Simulation{T}} = Execution(s,sol)
make_execution(s::S, sv::SavedValues) where {T,S <: Simulation{T}} = ReducedExecution(s,sv)
make_execution(s::S, (sol,sv)::Tuple{<:DESolution,<:SavedValues}) where {T,S <: Simulation{T}} = AugmentedExecution(s,sol,sv)
export ReducedExecution, AugmentedExecution,make_execution
function execute(s::Simulation)
    return make_execution(s, solve(s))
end
function execute(s::FailedSimulation)
    return FailedExecution(s)
end

coordinates(sim::Simulation) = coordinates(space(sim))
coordinates(ex::AbstractExecution) = coordinates(space(ex))
timepoints(ex::AbstractFullExecution) = ex.solution.t
reduced_space(sim::Simulation) = subsample(sim.space, sim.save_idxs)
reduced_space(ex::AbstractExecution) = reduced_space(ex.simulation)
frame_xs(exec::AbstractExecution{T,<:Simulation{T,<:M}}) where {T,M<:AbstractModel{T,1}} = [x[1] for x in reduced_space(exec).arr]
frame_xs(sim::Simulation) = [x[1] for x in reduced_space(sim).arr]
origin_idx(sim::Simulation) = origin_idx(sim.space)
origin_idx(ex::AbstractExecution) = origin_idx(ex.simulation)
Base.extrema(ex::AbstractFullExecution) = extrema.(ex.solution.u) |> extrema_arr -> reduce(((max1, min1), (max2, min2)) -> (max(max1, max2), min(min1, min2)), extrema_arr)
Base.maximum(ex::AbstractFullExecution) = maximum(maximum.(ex.solution.u))
Base.minimum(ex::AbstractFullExecution) = minimum(minimum.(ex.solution.u))
stimulus_center(mod::AbstractModel) = center(mod.stimulus)
stimulus_center(sim::Simulation) = stimulus_center(sim.model)
export stimulus_center

"""
    make_system_mutator(simulation)

Construct the differential function to be provided to the ODE solver.
"""
function make_system_mutator(sim::SIM) where {SIM <: Simulation}
    sim.model(sim.space)
end

"""
    generate_problem(simulation)

Return an ODEProblem of the `simulation.model` with time span specified by `simulation.solver`.
"""

function generate_problem(simulation::Simulation{T,<:AbstractODEModel}, p::NamedTuple, callback::DECallback) where {T}
    system_fn! = make_system_mutator(simulation)# simulation.model(simulation.space)
    ode_fn = convert(ODEFunction{true}, system_fn!)
    return ODEProblem(ode_fn, simulation.initial_value, simulation.tspan, p;callback=callback)
end
function generate_problem(simulation::Simulation{T,<:AbstractODEModel}, callback::Nothing, p::NamedTuple) where {T}
    system_fn! = make_system_mutator(simulation)# simulation.model(simulation.space)
    ode_fn = convert(ODEFunction{true}, system_fn!)
    return ODEProblem(ode_fn, simulation.initial_value, simulation.tspan, p)
end

# function generate_problem(simulation::Simulation{T,<:AbstractNoisyModel}; callback=nothing) where {T}
#     system_fn! = make_system_mutator(simulation)# simulation.model(simulation.space)
#     return RODEProblem(system_fn!, simulation.initial_value, simulation.tspan, noise=simulation.model.noise_process, noise_prototype=zeros(size(simulation.initial_value)...), callback=callback)
# end

# TODO: Add history functionality
# function generate_problem(simulation::Simulation{T,MwD}) where {T, MwD<:AbstractModelwithDelay}
#     system_mutator! = make_system_mutator(simulation)
#     return DDEProblem(system_mutator!, simulation.initial_value, history(simulation), simulation.tspan)
# end
parse_save_idxs(::Any, P, save_idx) = save_idx
function parse_save_idxs(space::AbstractSpace, P, subsampler::Union{AbstractSubsampler,AbstractArray{<:AbstractSubsampler}})
	one_pop_coordinates = coordinate_indices(space, subsampler)
	population_coordinates(one_pop_coordinates, P)
end
export handle_callback
function handle_callback(sim::Simulation)
    return handle_callback(sim, sim.callback)
end
function handle_callback(sim::Simulation, cb::CB) where {CB<:Nothing}
    return ((;), nothing)
end
function handle_callback(sim::Simulation, unhandled_cbs::CB) where {CB<:Vector{<:Tuple}}
    cb_nts = map(unhandled_cbs) do unhandled_cb
        nt::NamedTuple, cb::DECallback = handle_callback(sim, unhandled_cb)
    end
    grand_cb = CallbackSet([cb_nt[2] for cb_nt in cb_nts]...)
    grand_nt = merge([cb_nt[1] for cb_nt in cb_nts]...) # FIXME should reduce
    return (grand_nt, grand_cb)
end

# TODO this could probably all be better served by dispatch; but that gets difficult
function _solve(simulation::Simulation, alg; dt=nothing, solver_options...)
    if dt != nothing
        solver_options = (solver_options..., dt=dt)
    end
    p, callback = handle_callback(simulation)
    problem = generate_problem(simulation, p, callback)
    solve(problem, alg; solver_options...)
end
function _solve(simulation::Simulation, alg, ensemble_alg; prob_func::Function, output_func=(sol, i) -> (simulation.global_reduction(sol), false), reduction=(u,data,i) -> (append!(u,data), false), dt=nothing, u_init=[], solver_options...)
    if dt != nothing
        solver_options = (solver_options..., dt=dt)
    end
    p, callback = handle_callback(simulation)
    initial_problem = generate_problem(simulation, p, callback)
    ensemble_problem = EnsembleProblem(initial_problem;
                    output_func=output_func,
                    prob_func=prob_func,
                    reduction=reduction,
                    u_init=u_init)
                    @show solver_options
    solve(ensemble_problem, alg, ensemble_alg; solver_options...)
end

function solve(simulation::Simulation, args...; kwargs...)
    sol = _solve(simulation, simulation.algorithm, args...; dt=simulation.dt, save_idxs=simulation.save_idxs, simulation.solver_options..., kwargs...)
    return sol
end


module DAQ

using NIDAQ
using DataStructures
import ThreadPools.@tspawnat
import NIDAQ: pulses!, pulses, pulsetiming, pulsetiming!, start, stop
export Recording

const NIDAQ_TID = Ref{Int64}(1)

if Threads.nthreads() > 1
    NIDAQ_TID[] = 2
end

mutable struct Recording
    task::DAQTask{AI}
    channels::Dict{Int64, String}
    fs::Int64
    refresh::Float64
    history_seconds::Float64
    history_samples::Int64
    signal::CircularBuffer{Float64}
    chan::Channel
    function Recording(; channels = Dict(2 => "XII", 3 => "Vm"), fs = 20000, refresh = 50, history_seconds = 8)
        history_samples = history_seconds * fs * length(channels)
        signal = CircularBuffer{Float64}(history_samples)
        fill!(signal, 0.0)
        chan = Channel(500)
        dev = DefaultDev()
        task = DAQTask{AI}()
        for (chan, name) in channels
            push!(task, dev.channels[AI][chan], alias = name, tcfg = DAQmx.Diff, range = (-1.0,1.0))
        end
        rec = new(task, channels, fs, refresh, history_seconds, history_samples, signal, chan)
        signal_updates(rec)
        finalizer(rec) do rec
            close(rec.chan)
        end
        return rec
    end
end

mutable struct Stimulus
    task::DAQTask{CO}
    function Stimulus()
        dev = DefaultDev()
        task = DAQTask{CO}()
        source = dev.channels[CO][1]
        pulses(task, source)
        return new(task)
    end
end

function signal_updates(rec::Recording)
    @async begin
        for x in rec.chan
            append!(rec.signal, vec(x))
        end
    end
end

function (rec::Recording)()
    @tspawnat NIDAQ_TID[] record!(rec.task, $(rec.fs), $(rec.refresh); feed = rec.chan)
end

fetch_history(rec::Recording) = fetch(@tspawnat NIDAQ_TID[] collect(rec.signal))

pulsetiming!(s::Stimulus, args...) = NIDAQ.pulsetiming!(s.task, args...)
pulsetiming(s::Stimulus, args...) = NIDAQ.pulsetiming(s.task, args...)
stop(s::Stimulus) = NIDAQ.stop(s.task)

end # module


###############################################################################
#
# Grid data
#
###############################################################################
"""
    grid_RAW!(param::Array)
    grid_RAW!(dt::MOD15A2Hv006LAI{FT},
              params::Array{Any,1}) where {FT<:AbstractFloat}
    grid_RAW!(dt::MOD15A2Hv006LAI{FT},
              params::Array{Array,1},
              nthread::Int) where {FT<:AbstractFloat}

Grid the data and save it to a csv file, given
- `param` Parameter sets to pass to different threads
- `dt` [`AbstractUngriddedData`](@ref) type ungridded data type
- `params` Array of `param`
- `nthread` Number of threads to run the code in parallel
"""
function grid_RAW!(param::Array{Any,1})
    grid_RAW!(param...);

    return nothing
end

function grid_RAW!(
            dt::MOD09A1v006NIRv{FT},
            nlayer::Int,
            path::String,
            prefix::String,
            band::Array{String,1},
            cache::String,
            mm::Array{FT,1}
) where {FT<:AbstractFloat}
    # transform the data to CSV file
    pos_start = findfirst(prefix, path);
    hv_string = path[pos_start[1]:end];
    outp_file = joinpath(cache, hv_string * ".csv");

    # if file exists, skip
    if !isfile(outp_file)
        h,v = parse_HV(hv_string);
        lat_mat,lon_mat = load_MODIS(dt, h, v);
        dat_out = DataFrame();
        try
            red_mat = read_nc(FT, path, band[1]);
            nir_mat = read_nc(FT, path, band[2]);
            dim_lon = size(red_mat,1);
            dim_lat = size(red_mat,2);
            (dat_out).lay = ones(Int, dim_lon * dim_lat) * nlayer;
            (dat_out).lat = ones(dim_lon * dim_lat) .* -9999;
            (dat_out).lon = ones(dim_lon * dim_lat) .* -9999;
            (dat_out).red = ones(dim_lon * dim_lat) .* -9999;
            (dat_out).nir = ones(dim_lon * dim_lat) .* -9999;
            for i in 1:dim_lon, j in 1:dim_lat
                _data = view(red_mat,i,j,:);
                _eata = view(nir_mat,i,j,:);
                _mask = (_data .> mm[1]) .* (_data .< mm[2]) .*
                        (_eata .> mm[1]) .* (_eata .< mm[2]);
                if sum(_mask) > 0
                    (dat_out).lat[dim_lat*(i-1)+j] = lat_mat[i,j];
                    (dat_out).lon[dim_lat*(i-1)+j] = lon_mat[i,j];
                    (dat_out).red[dim_lat*(i-1)+j] = nanmean(_data[_mask]);
                    (dat_out).nir[dim_lat*(i-1)+j] = nanmean(_eata[_mask]);
                end
            end
        catch e
            @warn "An error occurred when reading the dataset, use NaN...";
            @show e;
            (dat_out).lay = ones(Int, 1) * nlayer;
            (dat_out).lat = ones(1) .* -9999;
            (dat_out).lon = ones(1) .* -9999;
            (dat_out).red = ones(1) .* -9999;
            (dat_out).nir = ones(1) .* -9999;
        end

        # save data
        mask = ((dat_out).red .> -1) .* ((dat_out).nir .> -1);
        save_csv!(outp_file, dat_out[mask,:]);
    end

    return nothing
end

function grid_RAW!(
            dt::MOD15A2Hv006LAI{FT},
            nlayer::Int,
            path::String,
            prefix::String,
            band::String,
            cache::String,
            mm::Array{FT,1}
) where {FT<:AbstractFloat}
    # transform the data to CSV file
    pos_start = findfirst(prefix, path);
    hv_string = path[pos_start[1]:end];
    outp_file = joinpath(cache, hv_string * ".csv");

    # if file exists, skip
    if !isfile(outp_file)
        h,v = parse_HV(hv_string);
        lat_mat,lon_mat = load_MODIS(dt, h, v);
        dat_out = DataFrame();
        try
            lai_mat = read_nc(FT, path, band);
            dim_lon = size(lai_mat,1);
            dim_lat = size(lai_mat,2);
            (dat_out).lay = ones(Int, dim_lon * dim_lat) * nlayer;
            (dat_out).lat = ones(dim_lon * dim_lat) .* -9999;
            (dat_out).lon = ones(dim_lon * dim_lat) .* -9999;
            (dat_out).lai = ones(dim_lon * dim_lat) .* -9999;
            for i in 1:dim_lon, j in 1:dim_lat
                _data = view(lai_mat,i,j,:);
                _mask = (_data .> mm[1]) .* (_data .< mm[2]);
                if sum(_mask) > 0
                    (dat_out).lat[dim_lat*(i-1)+j] = lat_mat[i,j];
                    (dat_out).lon[dim_lat*(i-1)+j] = lon_mat[i,j];
                    (dat_out).lai[dim_lat*(i-1)+j] = nanmean(_data[_mask]);
                end
            end
        catch e
            @warn "An error occurred when reading the dataset, use NaN...";
            @show e;
            (dat_out).lay = ones(Int, 1) * nlayer;
            (dat_out).lat = ones(1) .* -9999;
            (dat_out).lon = ones(1) .* -9999;
            (dat_out).lai = ones(1) .* -9999;
        end

        # save data
        mask = ((dat_out).lai .> -1);
        save_csv!(outp_file, dat_out[mask,:]);
    end

    return nothing
end

function grid_RAW!(
            dt::AbstractUngriddedData{FT},
            params::Array{Any,1}
) where {FT<:AbstractFloat}
    # run the gridding process
    @info "Gridding RAW data..."
    @showprogress for param in params
        grid_RAW!(param);
    end

    return nothing
end

function grid_RAW!(
            dt::AbstractUngriddedData{FT},
            params::Array{Any,1},
            nthread::Int
) where {FT<:AbstractFloat}
    # create threads
    dynamic_workers!(nthread);

    # run the gridding process
    @info "Gridding RAW data..."
    @showprogress pmap(grid_RAW!, params);

    return nothing
end


import Base: convert
import Base: ==

@compat abstract type InferenceType end
@compat abstract type PackedInferenceType end

@compat abstract type FunctorInferenceType <: Function end

abstract type InferenceVariable end
abstract type ConvolutionObject <: Function end

# been replaced by Functor types, but may be reused for non-numerical cases
@compat abstract type Pairwise <: InferenceType end
@compat abstract type Singleton <: InferenceType end

@compat abstract type FunctorSingleton <: FunctorInferenceType end
# @compat abstract type FunctorPartialSingleton <: FunctorInferenceType end
@compat abstract type FunctorSingletonNH <: FunctorSingleton end

@compat abstract type FunctorPairwise <: FunctorInferenceType end
@compat abstract type FunctorPairwiseMinimize <: FunctorInferenceType end
@compat abstract type FunctorPairwiseNH <: FunctorPairwise end
# @compat abstract type FunctorPairwiseNHMinimize <: FunctorPairwiseMinimize end # TODO

struct ContinuousScalar <: InferenceVariable
  dims::Int
  labels::Vector{String}
  ContinuousScalar() = new(1, String["";])
end
struct ContinuousMultivariate <:InferenceVariable
  dims::Int
  labels::Vector{String}
  ContinuousMultivariate() = new()
  ContinuousMultivariate(x) = new(x, String["";])
end

const FGG = Graphs.GenericIncidenceList{Graphs.ExVertex,Graphs.Edge{Graphs.ExVertex},Array{Graphs.ExVertex,1},Array{Array{Graphs.Edge{Graphs.ExVertex},1},1}}
const FGGdict = Graphs.GenericIncidenceList{Graphs.ExVertex,Graphs.Edge{Graphs.ExVertex},Dict{Int,Graphs.ExVertex},Dict{Int,Array{Graphs.Edge{Graphs.ExVertex},1}}}

# Condensed representation of KernelDensityEstimate, by saving points and bandwidth
mutable struct EasyMessage
  pts::Array{Float64,2}
  bws::Array{Float64,1}
  EasyMessage() = new()
  EasyMessage(a::Array{Float64,2}, b::Array{Float64,1}) = new(a,b)
  EasyMessage(p::BallTreeDensity) = new(getPoints(p), getBW(p)[:,1])
end

mutable struct FactorGraph
  g::FGGdict
  bn
  IDs::Dict{Symbol,Int}
  fIDs::Dict{Symbol,Int}
  id::Int
  nodeIDs::Array{Int,1} # TODO -- ordering seems improved to use adj permutation -- pending merge JuliaArchive/Graphs.jl/#225
  factorIDs::Array{Int,1}
  bnverts::Dict{Int,Graphs.ExVertex} # TODO -- not sure if this is still used, remove
  bnid::Int # TODO -- not sure if this is still used
  dimID::Int
  cg
  cgIDs::Dict{Int,Int} # cgIDs[exvid] = neoid
  sessionname::String
  robotname::String
  username::String
  registeredModuleFunctions::VoidUnion{Dict{Symbol, Function}}
  reference::VoidUnion{Dict{Symbol, Tuple{Symbol, Vector{Float64}}}}
  stateless::Bool
  FactorGraph() = new()
  FactorGraph(
    x1,
    x2,
    x3,
    x4,
    x5,
    x6,
    x7,
    x8,
    x9,
    x10,
    x11,
    x12,
    x13,
    x14,
    x15,
    x16,
    x17
   ) = new(
    x1,
    x2,
    x3,
    x4,
    x5,
    x6,
    x7,
    x8,
    x9,
    x10,
    x11,
    x12,
    x13,
    x14,
    x15,
    x16,
    x17,
    false )
end

"""
    $(SIGNATURES)

Construct an empty FactorGraph object with the minimum amount of information / memory populated.
"""
function emptyFactorGraph(;reference::VoidUnion{Dict{Symbol, Tuple{Symbol, Vector{Float64}}}}=nothing)
    fg = FactorGraph(Graphs.incdict(Graphs.ExVertex,is_directed=false),
                     Graphs.incdict(Graphs.ExVertex,is_directed=true),
                    #  Dict{Int,Graphs.ExVertex}(),
                    #  Dict{Int,Graphs.ExVertex}(),
                     Dict{Symbol,Int}(),
                     Dict{Symbol,Int}(),
                     0,
                     [],
                     [],
                     Dict{Int,Graphs.ExVertex}(),
                     0,
                     0,
                     nothing,
                     Dict{Int,Int}(),
                     "",
                     "",
                     "",
                     Dict{Symbol, Function}(:IncrementalInference=>IncrementalInference.getSample), # TODO likely to be removed
                     reference  ) #evalPotential
    return fg
end

mutable struct VariableNodeData
  initval::Array{Float64,2}
  initstdev::Array{Float64,2}
  val::Array{Float64,2}
  bw::Array{Float64,2}
  BayesNetOutVertIDs::Array{Int,1}
  dimIDs::Array{Int,1}
  dims::Int
  eliminated::Bool
  BayesNetVertID::Int
  separator::Array{Int,1}
  groundtruth::VoidUnion{ Dict{ Tuple{Symbol, Vector{Float64}} } } # not packed yet
  softtype
  initialized::Bool
  VariableNodeData() = new()
  function VariableNodeData(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11)
    warn("Deprecated use of VariableNodeData(11 param), use 13 parameters instead")
    new(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11, nothing, true) # TODO ensure this is initialized true is working for most cases
  end
  VariableNodeData(x1::Array{Float64,2},
                   x2::Array{Float64,2},
                   x3::Array{Float64,2},
                   x4::Array{Float64,2},
                   x5::Vector{Int},
                   x6::Vector{Int},
                   x7::Int,
                   x8::Bool,
                   x9::Int,
                   x10::Vector{Int},
                   x11::VoidUnion{ Dict{ Tuple{Symbol, Vector{Float64}} } },
                   x12,
                   x13::Bool) =
    new(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13)
end

mutable struct FactorMetadata
  factoruserdata
  variableuserdata::Union{Vector, Tuple}
  variablesmalldata::Union{Vector, Tuple}
  solvefor::Union{Symbol, Void}
  variablelist::Union{Void, Vector{Symbol}}
  dbg::Bool
  FactorMetadata() = new() # [], []
  FactorMetadata(x1, x2::Union{Vector,Tuple},x3) = new(x1, x2, x3, nothing, nothing, false)
  FactorMetadata(x1, x2::Union{Vector,Tuple},x3,x4::Symbol) = new(x1, x2, x3, x4, nothing, false)
  FactorMetadata(x1, x2::Union{Vector,Tuple},x3,x4::Symbol,x5::Vector{Symbol};dbg::Bool=false) = new(x1, x2, x3, x4, x5, dbg)
end

struct SingleThreaded
end
struct MultiThreaded
end

mutable struct ConvPerThread
  thrid_::Int
  particleidx::Int # the actual particle being solved at this moment
  factormetadata::FactorMetadata # additional data passed to user function -- optionally used by user function
  activehypo::Union{UnitRange{Int},Vector{Int}} # subsection indices to select which params should be used for this hypothesis evaluation
  p::Vector{Int} # a permutation vector for low-dimension solves (FunctorPairwise only)
  perturb::Vector{Float64} # slight numerical perturbation for degenerate solver cases such as division by zero
  X::Array{Float64,2}
  Y::Vector{Float64}
  res::Vector{Float64}
  ConvPerThread() = new()
end

function ConvPerThread(X::Array{Float64,2},
                       zDim::Int;
                       factormetadata::FactorMetadata=FactorMetadata(),
                       particleidx::Int=1,
                       activehypo= 1:length(params),
                       p=collect(1:size(X,1)),
                       perturb=zeros(zDim),
                       Y=zeros(size(X,1)),
                       res=zeros(0)  )
  #
  cpt = ConvPerThread()
  cpt.thrid_ = 0
  cpt.X = X
  cpt.factormetadata = factormetadata
  cpt.particleidx = particleidx
  cpt.activehypo = activehypo
  cpt.p = p
  cpt.perturb = perturb
  cpt.Y = Y
  cpt.res = res
  return cpt
end

mutable struct CommonConvWrapper{T} <: ConvolutionObject where {T<:FunctorInferenceType}
  ### Values consistent across all threads during approx convolution
  usrfnc!::T # user factor / function
  # general setup
  xDim::Int
  zDim::Int
  # special case settings
  specialzDim::Bool # is there a special zDim requirement -- defined by user
  partial::Bool # is this a partial constraint -- defined by user
  # multi hypothesis settings
  hypotheses::Union{Void, Distributions.Categorical} # categorical to select which hypothesis is being considered during convolugtion operation
  # values specific to one complete convolution operation
  params::Vector{Array{Float64,2}} # parameters passed to each hypothesis evaluation event on user function
  varidx::Int # which index is being solved for in params?
  measurement::Tuple # user defined measurement values for each approxConv operation
  threadmodel::Union{Type{SingleThreaded}, Type{MultiThreaded}}

  ### particular convolution computation values per particle idx (varies by thread)
  cpt::Vector{ConvPerThread}
  # varidx::Int # which index is being solved for in params?
  # factormetadata::FactorMetadata # additional data passed to user function -- optionally used by user function
  # activehypo::Union{UnitRange{Int},Vector{Int}} # subsection indices to select which params should be used for this hypothesis evaluation
  # particleidx::Int # the actual particle being solved at this moment
  # p::Vector{Int} # a permutation vector for low-dimension solves (FunctorPairwise only)
  # perturb::Vector{Float64} # slight numerical perturbation for degenerate solver cases such as division by zero
  # X::Array{Float64,2}
  # Y::Vector{Float64}
  # res::Vector{Float64}

  CommonConvWrapper{T}() where {T<:FunctorInferenceType} = new{T}()
end

function CommonConvWrapper(fnc::T,
                           X::Array{Float64,2},
                           zDim::Int,
                           params::Vector{Array{Float64,2}};
                           factormetadata::FactorMetadata=FactorMetadata(),
                           specialzDim::Bool=false,
                           partial::Bool=false,
                           hypotheses=nothing,
                           activehypo= 1:length(params),
                           varidx::Int=1,
                           measurement::Tuple=(zeros(0,1),),
                           particleidx::Int=1,
                           p=collect(1:size(X,1)),
                           perturb=zeros(zDim),
                           Y=zeros(size(X,1)),
                           xDim=size(X,1),
                           res=zeros(0),
                           threadmodel=MultiThreaded  ) where {T<:FunctorInferenceType}
  #
  ccw = CommonConvWrapper{T}()

  ccw.usrfnc! = fnc
  ccw.xDim = xDim
  ccw.zDim = zDim
  ccw.specialzDim = specialzDim
  ccw.partial = partial
  ccw.hypotheses = hypotheses
  ccw.params = params
  ccw.varidx = varidx
  ccw.threadmodel = threadmodel

  # thread specific elements
  ccw.cpt = Vector{ConvPerThread}(Threads.nthreads())
  for i in 1:Threads.nthreads()
    ccw.cpt[i] = ConvPerThread(X, zDim,
                    factormetadata=factormetadata,
                    particleidx=particleidx,
                    activehypo=activehypo,
                    p=p,
                    perturb=perturb,
                    Y=Y,
                    res=res )
  end
  # ccw.varidx = varidx
  # ccw.measurement = measurement
  # ccw.activehypo = activehypo
  # ccw.factormetadata = factormetadata
  # ccw.particleidx = particleidx
  # ccw.p = p
  # ccw.perturb = perturb
  # ccw.X = X
  # ccw.Y = Y
  # ccw.res = res

  return ccw
end

mutable struct GenericFunctionNodeData{T, S}
  fncargvID::Array{Int,1}
  eliminated::Bool
  potentialused::Bool
  edgeIDs::Array{Int,1}
  frommodule::S #Union{Symbol, AbstractString}
  fnc::T
  multihypo::String # likely to moved when GenericWrapParam is refactored
  GenericFunctionNodeData{T, S}() where {T, S} = new{T,S}()
  GenericFunctionNodeData{T, S}(x1, x2, x3, x4, x5::S, x6::T, x7::String="") where {T, S} = new{T,S}(x1, x2, x3, x4, x5, x6, x7)
  GenericFunctionNodeData(x1, x2, x3, x4, x5::S, x6::T, x7::String="") where {T, S} = new{T,S}(x1, x2, x3, x4, x5, x6, x7)
  # GenericFunctionNodeData(x1, x2, x3, x4, x5::S, x6::T, x7::String) where {T, S} = new{T,S}(x1, x2, x3, x4, x5, x6, x7)
end

# where {T <: Union{InferenceType, FunctorInferenceType}}
const FunctionNodeData{T} = GenericFunctionNodeData{T, Symbol}
FunctionNodeData(x1, x2, x3, x4, x5::Symbol, x6::T, x7::String="") where {T <: Union{FunctorInferenceType, ConvolutionObject}}= GenericFunctionNodeData{T, Symbol}(x1, x2, x3, x4, x5, x6, x7)

# where {T <: PackedInferenceType}
const PackedFunctionNodeData{T} = GenericFunctionNodeData{T, <: AbstractString}
PackedFunctionNodeData(x1, x2, x3, x4, x5::S, x6::T, x7::String="") where {T <: PackedInferenceType, S <: AbstractString} = GenericFunctionNodeData(x1, x2, x3, x4, x5, x6, x7)

###

function addGraphsVert!(fgl::FactorGraph,
            exvert::Graphs.ExVertex;
            labels::Vector{<:AbstractString}=String[])
  #
  Graphs.add_vertex!(fgl.g, exvert)
end

function getVertNode(fgl::FactorGraph, id::Int; nt::Symbol=:var, bigData::Bool=false)
  return fgl.g.vertices[id] # check equivalence between fgl.v/f[i] and fgl.g.vertices[i]
  # return nt == :var ? fgl.v[id] : fgl.f[id]
end
function getVertNode(fgl::FactorGraph, lbl::Symbol; nt::Symbol=:var, bigData::Bool=false)
  return getVertNode(fgl, (nt == :var ? fgl.IDs[lbl] : fgl.fIDs[lbl]), nt=nt, bigData=bigData)
end
getVertNode{T <: AbstractString}(fgl::FactorGraph, lbl::T; nt::Symbol=:var, bigData::Bool=false) = getVertNode(fgl, Symbol(lbl), nt=nt, bigData=bigData)

# excessive function, needs refactoring
function updateFullVertData!(fgl::FactorGraph,
    nv::Graphs.ExVertex;
    updateMAPest::Bool=false )
  #

  # not required, since we using reference -- placeholder function CloudGraphs interface
  # getVertNode(fgl, nv.index).attributes["data"] = nv.attributes["data"]
  nothing
end

function makeAddEdge!(fgl::FactorGraph, v1::Graphs.ExVertex, v2::Graphs.ExVertex; saveedgeID::Bool=true)
  edge = Graphs.make_edge(fgl.g, v1, v2)
  Graphs.add_edge!(fgl.g, edge)
  if saveedgeID push!(getData(v2).edgeIDs,edge.index) end #.attributes["data"]
  edge
end

function graphsOutNeighbors(fgl::FactorGraph, vert::Graphs.ExVertex; ready::Int=1,backendset::Int=1, needdata::Bool=false)
  Graphs.out_neighbors(vert, fgl.g)
end
function graphsOutNeighbors(fgl::FactorGraph, exVertId::Int; ready::Int=1,backendset::Int=1, needdata::Bool=false)
  graphsOutNeighbors(fgl.g, getVert(fgl,exVertId), ready=ready, backendset=backendset, needdata=needdata)
end

function graphsGetEdge(fgl::FactorGraph, id::Int)
  nothing
end

function graphsDeleteVertex!(fgl::FactorGraph, vert::Graphs.ExVertex)
  warn("graphsDeleteVertex! -- not deleting Graphs.jl vertex id=$(vert.index)")
  nothing
end

#


export Obstacle, StateSpaceObstacle, ConfigSpaceObstacle, WorkspaceObstacle
export RobotCollisionBody, RobotCollisionComponent, RobotWorkspaceComponent, WorkspaceRobotBody, PointRobot
export CollisionChecker, ContinuousCollisionChecker, DiscreteCollisionChecker
export is_free_state, is_free_motion, is_free_config, is_free_sweep, rand_free_state, intersecting, sweep_intersecting

# Obstacle Sets
struct Obstacle{T,S}
    set::S
end
const StateSpaceObstacle{S}  = Obstacle{:statespace,S}
const ConfigSpaceObstacle{S} = Obstacle{:configspace,S}
const WorkspaceObstacle{S}   = Obstacle{:workspace,S}
StateSpaceObstacle(set::S) where {S}  = Obstacle{:statespace,S}(set)
ConfigSpaceObstacle(set::S) where {S} = Obstacle{:configspace,S}(set)
WorkspaceObstacle(set::S) where {S}   = Obstacle{:workspace,S}(set)
intersecting(o::Obstacle, x) = intersecting(o.set, x)
sweep_intersecting(o::Obstacle, x0, xf) = sweep_intersecting(o.set, x0, xf)
@recipe function f(o::Obstacle; dims=(1, 2))#, obstacle_color=:red, obstacle_alpha=1)
    dims  --> dims
    # color :=  obstacle_color
    # alpha :=  obstacle_alpha
    # label --> ""
    o.set
end

# Abstract Robot Types (speculative for now)
abstract type RobotCollisionBody end
abstract type RobotCollisionComponent end
abstract type RobotWorkspaceComponent <: RobotCollisionComponent end

# Collision Checker
struct CollisionChecker{C,O,R<:RobotCollisionBody,S2C}
    obstacles::O
    robot::R
    state2config::S2C
    motion_count::Base.RefValue{Int}
    edge_count::Base.RefValue{Int}
end
const ContinuousCollisionChecker{O,R,S2C} = CollisionChecker{true,O,R,S2C}
const DiscreteCollisionChecker{O,R,S2C}   = CollisionChecker{false,O,R,S2C}
@recipe function f(CC::CollisionChecker; dims=(1, 2))#, obstacle_color=:red, obstacle_alpha=1)
    dims           --> dims
    # obstacle_color --> obstacle_color
    # obstacle_alpha --> obstacle_alpha
    foreach(o -> @series(begin o end), CC.obstacles)
end

# General Methods for Dispatch
## is_free_state
is_free_state(CC::CollisionChecker, x) = all(o -> is_free_state(o, CC.robot, CC.state2config, x), CC.obstacles)
is_free_state(o::Obstacle, robot, s2c, x) = is_free_config(o, robot, s2c(x))
is_free_state(o::StateSpaceObstacle,  robot, s2c, x) = !intersecting(o, x)

## is_free_motion
function is_free_motion(CC::CollisionChecker, x0, xf)
    CC.motion_count[] += 1
    all(o -> is_free_motion(o, CC.robot, CC.state2config, x0, xf), CC.obstacles)
end
is_free_motion(o::Obstacle, robot, s2c, x0, xf) = is_free_sweep(o, robot, s2c(x0), s2c(xf))
is_free_motion(o::StateSpaceObstacle,  robot, s2c, x0, xf) = !sweep_intersecting(o, x0, xf)

## is_free_config
is_free_config(o::ConfigSpaceObstacle, robot, q) = !intersecting(o, q)

## is_free_sweep
is_free_sweep(o::ConfigSpaceObstacle, robot, q0, qf) = !sweep_intersecting(o, q0, qf)

## rand_free_state
rand_free_state(CC::CollisionChecker, X) = (x = rand(X); is_free_state(CC, x) ? x : rand_free_state(CC, X))

# WorkspaceRobotBody
struct WorkspaceRobotBody{C2W} <: RobotCollisionBody
    config2workspace::C2W
end
is_free_config(o::WorkspaceObstacle, robot::WorkspaceRobotBody, q) = !intersecting(o, robot.config2workspace(q))
function is_free_sweep(o::WorkspaceObstacle, robot::WorkspaceRobotBody, q0, qf)
    !sweep_intersecting(o, robot.config2workspace(q0), robot.config2workspace(qf))
end

PointRobot() = WorkspaceRobotBody(identity)

# SingleLinkRobot
struct SingleLinkRobot{L,C2T} <: RobotCollisionBody
    link::L
    config2transform::C2T
end
function is_free_config(o::WorkspaceObstacle, robot::SingleLinkRobot, q)
    !intersecting(o, robot.link, robot.config2transform(q))
end
function is_free_sweep(o::WorkspaceObstacle, robot::SingleLinkRobot, q0, qf)
    !sweep_intersecting(o, robot.link, robot.config2transform(q0), robot.config2transform(qf))
end

## Convenience Constructors
_type_sort(obstacles) = obstacles
_type_sort(obstacles::Vector{T}) where {T} = isconcretetype(T) ? obstacles : TypeSortedCollection(obstacles)
function ContinuousCollisionChecker(_obstacles, robot::R=PointRobot(), state2config::S2C=identity) where {R,S2C}
    obstacles = _type_sort(_obstacles)
    CollisionChecker{true,typeof(obstacles),R,S2C}(obstacles, robot, state2config, Ref(0), Ref(0))
end
function DiscreteCollisionChecker(_obstacles, robot::R=PointRobot(), state2config::S2C=identity) where {R,S2C}
    obstacles = _type_sort(_obstacles)
    CollisionChecker{false,typeof(obstacles),R,S2C}(obstacles, robot, state2config, Ref(0), Ref(0))
end

# Super basic collision checking; `using SeparatingAxisTheorem2D` or ... to load more advanced capabilities
intersecting(X::Union{SimpleConvexSet,SetComplement}, x) = x in X

function sweep_intersecting(B::AxisAlignedBox{N}, x0, xf) where {N}
    all(B.lo[i] <= max(x0[i], xf[i]) && min(x0[i], xf[i]) <= B.hi[i] for i in 1:N) && _sweep_intersecting(B, x0, xf)
end
function _sweep_intersecting(B::AxisAlignedBox{N}, x0, xf) where {N}
    any((λ = (ifelse(x0[i] < B.lo[i], B.lo[i], B.hi[i]) - x0[i])/(xf[i] - x0[i]);
        0 <= λ <= 1 && all(i == j || B.lo[j] <= x0[j] + (xf[j] - x0[j])*λ <= B.hi[j] for j in 1:N)) for i in 1:N)
end

function sweep_intersecting(B::Ball, x0, xf)
    all(B.c[i] - B.r <= max(x0[i], xf[i]) &&
        min(x0[i], xf[i]) <= B.c[i] + B.r for i in 1:N) && _sweep_intersecting(B, x0, xf)
end
function _sweep_intersecting(B::Ball, x0, xf)
    x0 in B && return true
    xf in B && return true
    X = xf - x0
    Y = B.c - x0
    a = dot(X, X)
    b = dot(Y, Y)
    c = dot(X, Y)
    a*C.r*C.r < a*b - c*c && return false
    0 <= c <= a
end

sweep_intersecting(S::SetComplement{<:SimpleConvexSet}, x0, xf) = x0 in S || xf in S    # S.set is convex


# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/MortarContact2D.jl/blob/master/LICENSE

"""
    2D mortar contact mechanics for JuliaFEM.

# Problem types

- `Mortar2D` to couple non-conforming meshes (mesh tying).
- `MortarContact2D` to add contact constraints.

"""
module MortarContact2D

using FEMBase, SparseArrays, LinearAlgebra, Statistics

include("mortar2d.jl")
include("contact2d.jl")
export Mortar2D, Contact2D

end


# Specify model parameters and/or initial values to optimize
function get_search_index()::Tuple{Array{Int64,1},Array{Int64,1}}
    # parameters
    search_idx_params::Vector{Int} = [
        C.VmaxPY
        C.KmPY
        C.kdeg
        C.kf47
        C.Vmaxr47
        C.Kmf47
        C.Kmr47
        C.kf48
        C.Kmf48
        C.Kmr48
        #C.PTEN
        C.kf49
        C.kr49
        C.Kmf49
        C.Kmr49
        C.Kmr49b
        C.kr49b
        C.kf51
        C.Vmaxr51
        C.Kmf51
        C.Kmr51
        C.Kmrb51
        C.kf52
        C.Vmaxr52
        C.Kmf52
        C.Kmr52
        C.kf54
        C.Vmaxr54
        C.Kmf54
        C.Kmr54
        C.kf55
        C.Vmaxr55
        C.Kmf55
        C.Kmr55
        C.kf38
        C.kf39
        C.kf50
        C.a98
        C.b98
        C.koff46
        C.EGF_off
        C.HRGoff_3
        C.HRGoff_4
        C.koff4
        C.koff5
        C.koff6
        C.koff7
        C.koff8
        C.koff9
        C.koff61
        C.koff62
        C.koff16
        C.koff17
        C.koff18
        C.koff19
        C.koff20
        C.koff21
        C.koff22
        C.koff23
        C.koff24
        C.koff25
        C.koff26
        C.koff27
        C.koff28
        C.koff29
        C.koff30
        C.koff31
        C.koff32
        C.koff33
        C.koff34
        C.koff35
        C.koff36
        C.koff37
        C.koff65
        C.koff66
        C.koff67
        C.koff68
        C.koff69
        C.koff70
        C.koff71
        C.koff72
        C.koff40
        C.koff41
        C.koff42
        C.koff43
        C.koff44
        C.koff45
        C.koff57
        C.koff58
        C.koff59
        C.koff60
        C.kPTP10
        C.kPTP11
        C.kPTP12
        C.kPTP13
        C.kPTP14
        C.kPTP15
        C.kPTP63
        C.kPTP64
        C.koff73
        C.koff74
        C.koff75
        C.koff76
        C.koff77
        C.koff78
        C.koff79
        C.koff80
        C.kPTP38
        C.kPTP39
        C.koff88
        C.kPTP50
        C.kf81
        C.Vmaxr81
        C.Kmf81
        C.Kmr81
        C.kf82
        C.Vmaxr82
        C.Kmf82
        C.Kmr82
        C.kf83
        C.Vmaxr83
        C.Kmf83
        C.Kmr83
        C.kf84
        C.Vmaxr84
        C.Kmf84
        C.Kmr84
        C.kf85
        C.Vmaxr85
        C.Kmf85
        C.Kmr85
        C.kcon49
        C.kon1
        C.kon86
        C.kon2
        C.kon3
        C.kon87
        C.kon4
        C.kon5
        C.kon6
        C.kon7
        C.kon8
        C.kon9
        C.kon61
        C.kon62
        C.kf10
        C.kf11
        C.kf12
        C.kf13
        C.kf14
        C.kf15
        C.kf63
        C.kf64
        C.kon16
        C.kon17
        C.kon18
        C.kon73
        C.kon19
        C.kon20
        C.kon21
        C.kon74
        C.kon22
        C.kon23
        C.kon24
        C.kon25
        C.kon75
        C.kon26
        C.kon27
        C.kon28
        C.kon29
        C.kon76
        C.kon30
        C.kon31
        C.kon32
        C.kon33
        C.kon77
        C.kon34
        C.kon35
        C.kon36
        C.kon37
        C.kon78
        C.kon65
        C.kon66
        C.kon67
        C.kon68
        C.kon79
        C.kon69
        C.kon70
        C.kon71
        C.kon72
        C.kon80
        C.kon40
        C.kon41
        C.kon42
        C.kon43
        C.kon44
        C.kon45
        C.kon88
        C.kon46
        C.kon57
        C.kon58
        C.kon59
        C.kon60
        #
        C.V1
        C.Km1
        C.V5
        C.Km5
        C.V10
        C.Km10
        #C.n10
        C.p11
        C.p12
        C.p13
        C.V14
        C.Km14
        C.V15
        C.Km15
        C.KimDUSP
        C.KexDUSP
        C.V20
        C.Km20
        C.V21
        C.Km21
        C.V24
        C.Km24
        C.V25
        C.Km25
        C.KimRSK
        C.KexRSK
        C.V27
        C.Km27
        C.V28
        C.Km28
        C.V29
        C.Km29
        C.V30
        C.Km30
        C.V31
        C.Km31
        #C.n31
        C.p32
        C.p33
        C.p34
        C.V35
        C.Km35
        C.V36
        C.Km36
        C.V37
        C.Km37
        C.KimFOS
        C.KexFOS
        C.V42
        C.Km42
        C.V43
        C.Km43
        C.V44
        C.Km44
        C.p47
        C.m47
        C.p48
        C.p49
        C.m49
        C.p50
        C.p51
        C.m51
        C.V57
        C.Km57
        #C.n57
        C.p58
        C.p59
        C.p60
        C.p61
        C.KimF
        C.KexF
        C.p63
        C.KF31
        #C.nF31
        #
        C.scale_E1
        C.scale_E2
        C.scale_E3
        C.scale_E4
        C.scale_G
        #C.scale_S
        C.a_SHC1
        C.a_SHC2
        C.a_SHC3
        C.a_SHC4
        #C.scale_I
        C.a_PIK3CA
        C.a_PIK3CB
        C.a_PIK3CD
        C.a_PIK3CG
        #ver.5
        C.scale_PTEN
        #C.scale_R
        C.a_RASA1
        C.a_RASA2
        C.a_RASA3
        #C.scale_O
        C.a_SOS1
        C.a_SOS2
        C.scale_A
        #C.scale_Akt
        C.a_AKT1
        C.a_AKT2
        #C.scale_RsD
        C.a_HRAS
        C.a_KRAS
        C.a_NRAS
        #C.scale_Raf
        C.a_ARAF
        C.a_BRAF
        C.a_RAF1
        #C.scale_MEK
        C.a_MAP2K1
        C.a_MAP2K2
        C.scale_T
        C.scale_CREBn
        #C.scale_ERKc
        C.a_MAPK1
        C.a_MAPK3
        C.scale_Elk1n
        #C.scale_RSKc
        C.a_RPS6KA1
        C.a_RPS6KA2
        C.a_RPS6KA3
        C.a_RPS6KA6
    ]

    # initial values
    search_idx_initials::Vector{Int} = [
        V.P2,
    ]

    return search_idx_params, search_idx_initials
end

function get_search_region()::Matrix{Float64}
    p::Vector{Float64} = param_values()
    u0::Vector{Float64} = initial_values()

    search_idx::Tuple{Array{Int64,1},Array{Int64,1}} = get_search_index()
    search_param::Vector{Float64} = init_search_param(search_idx, p, u0)

    search_rgn::Matrix{Float64} = zeros(2, length(p)+length(u0))

    # Default: 0.1 ~ 10x
    for (i,j) in enumerate(search_idx[1])
        search_rgn[1,j] = search_param[i]*0.1  # lower bound
        search_rgn[2,j] = search_param[i]*10.0  # upper bound
    end

    # Default: 0.5 ~ 2x
    for (i,j) in enumerate(search_idx[2])
        search_rgn[1,j+length(p)] = search_param[i+length(search_idx[1])]*0.5  # lower bound
        search_rgn[2,j+length(p)] = search_param[i+length(search_idx[1])]*2.0  # upper bound
    end

    # search_rgn[:, C.param_name] = [lower_bound, upper_bound]
    # search_rgn[:, V.var_name+length(p)] = [lower_bound, upper_bound]

    #search_rgn[:,C.PTEN] = [1.0,1000.0]
    search_rgn[:,V.P2+length(p)] = [1.0,1000.0]

    search_rgn[:,C.scale_E1] = [0.1, 100.0]
    search_rgn[:,C.scale_E2] = [0.1, 100.0]
    search_rgn[:,C.scale_E3] = [0.1, 100.0]
    search_rgn[:,C.scale_E4] = [0.1, 100.0]
    search_rgn[:,C.scale_G] = [0.1, 100.0]
    #search_rgn[:,C.scale_S] = [0.1, 100.0]
    search_rgn[:,C.a_SHC1] = [0.1, 100.0]
    search_rgn[:,C.a_SHC2] = [0.1, 100.0]
    search_rgn[:,C.a_SHC3] = [0.1, 100.0]
    search_rgn[:,C.a_SHC4] = [0.1, 100.0]
    #search_rgn[:,C.scale_I] = [0.1, 100.0]
    search_rgn[:,C.a_PIK3CA] = [0.1, 100.0]
    search_rgn[:,C.a_PIK3CB] = [0.1, 100.0]
    search_rgn[:,C.a_PIK3CD] = [0.1, 100.0]
    search_rgn[:,C.a_PIK3CG] = [0.1, 100.0]
    # ver.5
    search_rgn[:,C.scale_PTEN] = [0.1, 100.0]
    #search_rgn[:,C.scale_R] = [0.1, 100.0]
    search_rgn[:,C.a_RASA1] = [0.1, 100.0]
    search_rgn[:,C.a_RASA2] = [0.1, 100.0]
    search_rgn[:,C.a_RASA3] = [0.1, 100.0]
    #search_rgn[:,C.scale_O] = [0.1, 100.0]
    search_rgn[:,C.a_SOS1] = [0.1, 100.0]
    search_rgn[:,C.a_SOS2] = [0.1, 100.0]
    search_rgn[:,C.scale_A] = [0.1, 100.0]
    #search_rgn[:,C.scale_Akt] = [0.1, 100.0]
    search_rgn[:,C.a_AKT1] = [0.1, 100.0]
    search_rgn[:,C.a_AKT2] = [0.1, 100.0]
    #search_rgn[:,C.scale_RsD] = [0.1, 100.0]
    search_rgn[:,C.a_HRAS] = [0.1, 100.0]
    search_rgn[:,C.a_KRAS] = [0.1, 100.0]
    search_rgn[:,C.a_NRAS] = [0.1, 100.0]
    #search_rgn[:,C.scale_Raf] = [0.1, 100.0]
    search_rgn[:,C.a_ARAF] = [0.1, 100.0]
    search_rgn[:,C.a_BRAF] = [0.1, 100.0]
    search_rgn[:,C.a_RAF1] = [0.1, 100.0]
    #search_rgn[:,C.scale_MEK] = [0.1, 100.0]
    search_rgn[:,C.a_MAP2K1] = [0.1, 100.0]
    search_rgn[:,C.a_MAP2K2] = [0.1, 100.0]
    search_rgn[:,C.scale_T] = [0.1, 100.0]
    search_rgn[:,C.scale_CREBn] = [0.1, 100.0]
    #search_rgn[:,C.scale_ERKc] = [0.1, 100.0]
    search_rgn[:,C.a_MAPK1] = [0.1, 100.0]
    search_rgn[:,C.a_MAPK3] = [0.1, 100.0]
    search_rgn[:,C.scale_Elk1n] = [0.1, 100.0]
    #search_rgn[:,C.scale_RSKc] = [0.1, 100.0]
    search_rgn[:,C.a_RPS6KA1] = [0.1, 100.0]
    search_rgn[:,C.a_RPS6KA2] = [0.1, 100.0]
    search_rgn[:,C.a_RPS6KA3] = [0.1, 100.0]
    search_rgn[:,C.a_RPS6KA6] = [0.1, 100.0]

    search_rgn = conv_lin2log!(search_rgn, search_idx)

    return search_rgn
end

function update_param(indiv::Vector{Float64})::Tuple{Array{Float64,1},Array{Float64,1}}
    p::Vector{Float64} = param_values()
    u0::Vector{Float64} = initial_values()

    search_idx::Tuple{Array{Int64,1},Array{Int64,1}} = get_search_index()

    for (i,j) in enumerate(search_idx[1])
        @inbounds p[j] = indiv[i]
    end
    for (i,j) in enumerate(search_idx[2])
        @inbounds u0[j] = indiv[i+length(search_idx[1])]
    end

    # scaling
    (p, u0) = tpm2ival!(p, u0, "MCF7")
    #
    # constraints --------------------------------------------------------------
    p[C.V6] = p[C.V5]
    p[C.Km6] = p[C.Km5]
    p[C.KimpDUSP] = p[C.KimDUSP]
    p[C.KexpDUSP] = p[C.KexDUSP]
    p[C.KimpcFOS] = p[C.KimFOS]
    p[C.KexpcFOS] = p[C.KexFOS]
    p[C.p52] = p[C.p47]
    p[C.m52] = p[C.m47]
    p[C.p53] = p[C.p48]
    p[C.p54] = p[C.p49]
    p[C.m54] = p[C.m49]
    p[C.p55] = p[C.p50]
    p[C.p56] = p[C.p51]
    p[C.m56] = p[C.m51]
    # --------------------------------------------------------------------------

    return p, u0
end

function decode_gene2val(indiv_gene::Vector{Float64})::Vector{Float64}
    search_rgn::Matrix{Float64} = get_search_region()
    indiv::Vector{Float64} = zeros(length(indiv_gene))

    for i in eachindex(indiv_gene)
        indiv[i] = 10^(
            indiv_gene[i] * (
                search_rgn[2,i] - search_rgn[1,i]
            ) + search_rgn[1,i]
        )
    end

    return round.(indiv,sigdigits=7)
end

function encode_val2gene(indiv::Vector{Float64})
    search_rgn::Matrix{Float64} = get_search_region()
    indiv_gene::Vector{Float64} = zeros(length(indiv))

    for i in eachindex(indiv)
        indiv_gene[i] = (
            log10(indiv[i]) - search_rgn[1,i]
        ) / (
            search_rgn[2,i] - search_rgn[1,i]
        )
    end

    return indiv_gene
end

function encode_bestIndivVal2randGene(j::Int64, best_indiv::Vector{Float64},
                                        p0_bounds::Vector{Float64})::Float64
    search_rgn::Matrix{Float64} = get_search_region()
    rand_gene::Float64 = (
        log10(
            best_indiv[j]*10^(
                rand() * log10(p0_bounds[2]/p0_bounds[1]) + log10(p0_bounds[1])
            )
        ) - search_rgn[1,j]
    ) / (
        search_rgn[2,j] - search_rgn[1,j]
    )

    return rand_gene
end

function init_search_param(search_idx::Tuple{Array{Int64,1},Array{Int64,1}},
                            p::Vector{Float64}, u0::Vector{Float64})::Vector{Float64}
    search_param = zeros(
        length(search_idx[1]) + length(search_idx[2])
    )
    for (i,j) in enumerate(search_idx[1])
        @inbounds search_param[i] = p[j]
    end
    for (i,j) in enumerate(search_idx[2])
        @inbounds search_param[i+length(search_idx[1])] = u0[j]
    end

    if any(x -> x == 0.0, search_param)
        message::String = "search_param must not contain zero."
        for (_, idx) in enumerate(search_idx[1])
            if p[idx] == 0.0
                error(
                    @sprintf(
                        "`C.%s` in search_idx_params: ", C.parameters[idx]
                    ) * message
                )
            end
        end
        for (_, idx) in enumerate(search_idx[2])
            if u0[idx] == 0.0
                error(
                    @sprintf(
                        "`V.%s` in search_idx_initials: ", V.species[idx]
                    ) * message
                )
            end
        end
    end

    return search_param
end

function conv_lin2log!(search_rgn::Matrix{Float64},
                        search_idx::Tuple{Array{Int64,1},Array{Int64,1}}
                        )::Matrix{Float64}
    for i=1:size(search_rgn,2)
        if minimum(search_rgn[:,i]) < 0.0
            message = "search_rgn[lb,ub] must be positive.\n"
            if i <= C.n_parameters
                error(
                    @sprintf(
                        "`C.%s` ", C.parameters[i]
                    ) * message
                )
            else
                error(
                    @sprintf(
                        "`V.%s` ", V.species[i-C.n_parameters]
                    ) * message
                )
            end
        elseif minimum(search_rgn[:,i]) == 0.0 && maximum(search_rgn[:,i]) != 0.0
            message = "lower_bound must be larger than 0.\n"
            if i <= C.n_parameters
                error(
                    @sprintf(
                        "`C.%s` ", C.parameters[i]
                    ) * message
                )
            else
                error(
                    @sprintf(
                        "`V.%s` ", V.species[i-C.n_parameters]
                    ) * message
                )
            end
        elseif search_rgn[2,i] - search_rgn[1,i] < 0.0
            message = "lower_bound < upper_bound\n"
            if i <= C.n_parameters
                error(
                    @sprintf(
                        "`C.%s` ", C.parameters[i]
                    ) * message
                )
            else
                error(
                    @sprintf(
                        "`V.%s` ", V.species[i-C.n_parameters]
                    ) * message
                )
            end
        end
    end

    nonzero_idx::Vector{Int} = []
    for i=1:size(search_rgn,2)
        if search_rgn[:,i] != [0.0,0.0]
            push!(nonzero_idx,i)
        end
    end
    difference::Vector{Int} = collect(
        symdiff(
            Set(nonzero_idx),
            Set(
                append!(
                    search_idx[1], C.n_parameters .+ search_idx[2]
                )
            )
        )
    )
    if length(difference) > 0
        for (i,j) in enumerate(difference)
            if j <= C.n_parameters
                print(
                    @sprintf(
                        "`C.%s`\n", C.parameters[Int(j)]
                    )
                )
            else
                print(
                    @sprintf(
                        "`V.%s`\n", V.species[Int(j)-C.n_parameters]
                    )
                )
            end
        end
        error(
            "Set these search_params in both search_idx and search_rgn."
        )
    end

    search_rgn = search_rgn[:,nonzero_idx]

    return log10.(search_rgn)
end