using LinQuadOptInterface
const LQOI = LinQuadOptInterface
const MOI  = LQOI.MOI

const SUPPORTED_OBJECTIVES = [
    LQOI.Linear
    LQOI.SinVar
]

const SUPPORTED_CONSTRAINTS = [
    (LQOI.Linear, LQOI.EQ),
    (LQOI.Linear, LQOI.LE),
    (LQOI.Linear, LQOI.GE),
    (LQOI.SinVar, LQOI.EQ),
    (LQOI.SinVar, LQOI.LE),
    (LQOI.SinVar, LQOI.GE),
    (LQOI.SinVar, LQOI.IV),
    (LQOI.SinVar, MOI.ZeroOne),
    (LQOI.SinVar, MOI.Integer),
    (LQOI.VecVar, MOI.Nonnegatives),
    (LQOI.VecVar, MOI.Nonpositives),
    (LQOI.VecVar, MOI.Zeros),
    (LQOI.VecLin, MOI.Nonnegatives),
    (LQOI.VecLin, MOI.Nonpositives),
    (LQOI.VecLin, MOI.Zeros)
]

mutable struct Optimizer <: LQOI.LinQuadOptimizer
    LQOI.@LinQuadOptimizerBase
    env::Env # Cplex environment
    Optimizer(::Void) = new()
end

function LQOI.LinearQuadraticModel(::Type{Optimizer},env)
    env = Env()
    return Model(env::Env)
end

function Optimizer(;kwargs...)
    env = Env()
    for (name,value) in kwargs
        set_param!(env, string(name), value)
    end
    model = Optimizer(nothing)
    model.env = env
    MOI.empty!(model)
    return model
end

LQOI.supported_constraints(::Optimizer) = SUPPORTED_CONSTRAINTS
LQOI.supported_objectives(::Optimizer)  = SUPPORTED_OBJECTIVES

LQOI.backend_type(model::Optimizer, ::MOI.EqualTo{Float64})     = Cchar('E')
LQOI.backend_type(model::Optimizer, ::MOI.LessThan{Float64})    = Cchar('L')
LQOI.backend_type(model::Optimizer, ::MOI.GreaterThan{Float64}) = Cchar('G')

LQOI.backend_type(model::Optimizer, ::MOI.Zeros)                = Cchar('B')
LQOI.backend_type(model::Optimizer, ::MOI.Nonpositives)         = Cchar('L')
LQOI.backend_type(model::Optimizer, ::MOI.Nonnegatives)         = Cchar('E')

# TODO - improve single type
function LQOI.change_variable_bounds!(model::Optimizer,
        columns::Vector{Int}, values::Vector{Float64}, senses::Vector{Cchar})

    c_api_chgbds(model.inner, ivec(columns), senses, values)
end

function LQOI.get_variable_lowerbound(model::Optimizer, column::Int)
    c_api_getlb(model.inner, Cint(column), Cint(column))[1]
end

function LQOI.get_variable_upperbound(model::Optimizer, column::Int)
    c_api_getub(model.inner, Cint(column), Cint(column))[1]
end

function LQOI.get_number_linear_constraints(model::Optimizer)
    c_api_getnumrows(model.inner)
end

function LQOI.add_linear_constraints!(model::Optimizer,
        A::LQOI.CSRMatrix{Float64}, sense::Vector{Cchar}, rhs::Vector{Float64})

    c_api_addrows(model.inner, ivec(A.row_pointers), ivec(A.columns),
                  A.coefficients, sense, rhs)
end

function LQOI.get_rhs(model::Optimizer, row::Int)
    rhs = Vector{Cdouble}(1)
    c_api_getrhs(model.inner, rhs, Cint(row), Cint(row))
    return rhs[1]
end

function LQOI.get_linear_constraint(model::Optimizer, row::Int)
    (nzcnt, rmatbeg, rmatind, rmatval) =
            c_api_getrows(model.inner, Cint(row), Cint(row))
    return rmatind[1:nzcnt], rmatval[1:nzcnt]
end

function LQOI.change_matrix_coefficient!(model::Optimizer,
                                         row::Int, col::Int, coef::Float64)

    c_api_chgcoef(model.inner, Cint(row), Cint(col), coef)
end

function LQOI.change_objective_coefficient!(model::Optimizer, col::Int,
                                            coef::Float64)

    c_api_chgobj(model.inner, [Cint(col)], [coef])
end

function LQOI.change_rhs_coefficient!(model::Optimizer, row::Int,
                                      coef::Float64)

    c_api_chgrhs(model.inner, [Cint(row)], [coef])
end

function LQOI.delete_linear_constraints!(model::Optimizer,
                                         first_row::Int, last_row::Int)

    c_api_delrows(model.inner, Cint(first_row), Cint(last_row))
end

function LQOI.change_variable_types!(model::Optimizer,
        columns::Vector{Int}, vtypes::Vector{Cchar})

    c_api_chgctype(model.inner, ivec(columns), vtypes)
end

function LQOI.change_linear_constraint_sense!(model::Optimizer,
        rows::Vector{Int}, senses::Vector{Cchar})

    c_api_chgsense(model.inner, ivec(rows), senses)
end

function LQOI.set_linear_objective!(model::Optimizer,
        columns::Vector{Int}, coefficients::Vector{Float64})

    n = num_var(model.inner)
    all_coefs = zeros(Float64, n)
    for (col, coef) in zip(columns, coefficients)
        all_coefs[col] += coef
    end
    c_api_chgobj(model.inner, Cint[1:n;], all_coefs)
end

function LQOI.change_objective_sense!(model::Optimizer, symbol)
    if symbol == :min
        c_api_chgobjsen(model.inner, Cint(1))
    else
        c_api_chgobjsen(model.inner, Cint(-1))
    end
end

function LQOI.get_linear_objective!(model::Optimizer, x)
    c_api_getobj(model.inner, x, Cint(1), c_api_getnumcols(model.inner))
end

function LQOI.get_objectivesense(model::Optimizer)
    s = c_api_getobjsen(model.inner)
    if s == 1
        return MOI.MinSense
    else
        return MOI.MaxSense
    end
end

function LQOI.get_number_variables(model::Optimizer)
    return c_api_getnumcols(model.inner)
end

function LQOI.add_variables!(model::Optimizer, N::Int)
    add_vars!(model.inner, zeros(Float64, N), fill(-Inf, N), fill(Inf, N))
end

function LQOI.delete_variables!(model::Optimizer,
                                first_col::Int, last_col::Int)

    c_api_delcols(model.inner, Cint(first_col), Cint(last_col))
end

function LQOI.solve_mip_problem!(model::Optimizer)
    model.inner.has_int = true
    LQOI.solve_linear_problem!(model)
end

function LQOI.solve_linear_problem!(model::Optimizer)
    optimize!(model.inner)
end

function LQOI.get_termination_status(model::Optimizer)
    stat = c_api_getstat(model.inner)

    if stat == 1 # CPX_STAT_OPTIMAL
        return MOI.Success
    elseif stat == 3 # CPX_STAT_INFEASIBLE
        return MOI.InfeasibleNoResult
    elseif stat == 4 # CPX_STAT_INForUNBD
        return MOI.InfeasibleOrUnbounded
    elseif stat == 2 # CPX_STAT_UNBOUNDED
        return MOI.UnboundedNoResult
    elseif stat in [12, 21, 22, 36] # CPX_STAT_*ABORT*_OBJ_LIM
        return MOI.ObjectiveLimit
    elseif stat in [10,34] # CPX_STAT_*ABORT_IT_LIM
        return MOI.IterationLimit
    elseif stat == 53 # CPX_STAT_CONFLICT_ABORT_NODE_LIM
        return MOI.NodeLimit
    elseif stat in [11, 25, 33, 39] # CPX_STAT_*ABORT*TIME_LIM
        return MOI.TimeLimit
    elseif stat == 5 # CPX_STAT_OPTIMAL_INFEAS
        return MOI.NumericalError

    # MIP STATUS
    elseif stat in [101, 102] # CPXMIP_OPTIMAL, CPXMIP_OPTIMAL_TOL
        return MOI.Success
    elseif stat == 103 # CPXMIP_INFEASIBLE
        return MOI.InfeasibleNoResult
    elseif stat == 119 # CPXMIP_INForUNBD
        return MOI.InfeasibleOrUnbounded
    elseif stat == 118 # CPXMIP_UNBOUNDED
        return MOI.UnboundedNoResult
    elseif stat == [105, 106] # CPXMIP_NODE_LIM*
        return MOI.NodeLimit
    elseif stat in [107, 108, 131, 132] # CPXMIP_*TIME_LIM*
        return MOI.TimeLimit
    else
        return MOI.OtherError
    end
end

function LQOI.get_primal_status(model::Optimizer)
    stat = c_api_solninfo(model.inner)[3]
    if stat == 1
        return MOI.FeasiblePoint
    else
        return MOI.UnknownResultStatus
    end
end

function LQOI.get_dual_status(model::Optimizer)
    if model.inner.has_int
        return MOI.UnknownResultStatus
    end
    stat = c_api_solninfo(model.inner)[4]
    if stat == 1
        return MOI.FeasiblePoint
    else
        return MOI.UnknownResultStatus
    end
end

function LQOI.get_variable_primal_solution!(model::Optimizer, result)
    return c_api_getx(model.inner, result)
end

function LQOI.get_linear_primal_solution!(model::Optimizer, result)
    return c_api_getax(model.inner, result)
end

function LQOI.get_variable_dual_solution!(model::Optimizer, place)
    return c_api_getdj(model.inner, place)
end

function LQOI.get_linear_dual_solution!(model::Optimizer, place)
    return c_api_getpi(model.inner, place)
end

function LQOI.get_objective_value(model::Optimizer)
    return c_api_getobjval(model.inner)
end


function get_price_data_single(currency::String)

    date = Dates.today()

    filename = "$(currency)_EUR_data_$(date).csv"
    filepath = joinpath("data", filename)

    raw_df = DataFrame()

    # Look for present day's CSV file, if not found, download and save data to a new file
    if isfile(filepath)
        @info "Reading $(currency) price/vol data from CSV file on disk"
        raw_df = CSV.File(filepath) |> DataFrame
    else
        try
            @info "Fetching $(currency) price/vol data from Alpha Vantage"
            raw = AlphaVantage.digital_currency_daily(currency, "EUR", datatype="csv")
            raw_df = raw_to_df(raw)
            CSV.write(filepath, raw_df)
        catch
            @info "Could not fetch data, try again later!"
        end
    end

    # Return processed DataFrame only when raw data has been fetched successfully
    if ~isempty(raw_df)
        df_out_price, df_out_candle = average_price_df(currency, raw_df)
        df_out_vol = vol_df(currency, raw_df)

        return df_out_price, df_out_candle, df_out_vol
    else
        return DataFrame[], DataFrame[], DataFrame[]
    end
end

function get_ratings_data(currency::String)

    date = Dates.today()

    filename = "$(currency)_metrics_data_$(date).csv"
    filepath = joinpath("data", filename)

    metrics_df = DataFrame()

    # Look for present day's CSV file, if not found, download and save data to a new file
    if isfile(filepath)
        @info "Reading $(currency) FCAS data from CSV file on disk"
        metrics_df = CSV.File(filepath) |> DataFrame
    else
        try
            @info "Fetching $(currency) FCAS data from Alpha Vantage"
            rating = AlphaVantage.crypto_rating(currency)
            scores = rating["Crypto Rating (FCAS)"]
            CSV.write(filepath, scores)
            metrics_df = CSV.File(filepath) |> DataFrame
        catch err
            if isa(err, KeyError)
                @info "Could not retrieve data. Something wrong with the API, try again later!"
            else
                @info "This is a new error: $(err)"
            end
        end
    end

    ratings = ["utility", "fcas score", "developer", "market", "fcas rating"]
    index = Array{Int64}(undef,0)

    # Return scores only when metrics data has been fetched successfully
    if ~isempty(metrics_df)

        # Find the row(index) of a string match
        for rating in ratings
            i = findall(occursin.(rating, metrics_df[!,1]))
            push!(index, i[1])
        end

        # Variables should be assigned in the same order as the list of ratings above
        utility_score, fcas_score, dev_score = metrics_df[!,2][index[1]], metrics_df[!,2][index[2]], metrics_df[!,2][index[3]]
        mark_score, fcas_rating = metrics_df[!,2][index[4]], metrics_df[!,2][index[5]]

        return utility_score, fcas_score, dev_score, mark_score, fcas_rating
    else
        return Number[], Number[], Number[], Number[], Number[]
    end

end

################# Functions for CoinGecko API #################

function get_API_response(params::String, url::String = URL)

	CG_request = HTTP.request("GET", url * params; verbose = 0, retries = 2)
	response_text = String(CG_request.body)
	response_dict = JSON.parse(response_text)

	return response_dict
end

function get_coin_id(currency::String)

    date = Dates.today()

    filename = "List_of_all_coins_$(date).csv"
    filepath = joinpath("data", filename)

    df_coins, df_filter, df_filter_1 = [DataFrame() for i = 1:3]

    # Look for present day's CSV file, if not found, download and save data to a new file
    if isfile(filepath)
        @info "Reading list of coins from CSV file on disk"
        df_coins = CSV.File(filepath) |> DataFrame
    else
        try
            @info "Fetching list of coins from CoinGecko"
            coins_dict = get_API_response("/coins/list")
            df_coins = vcat(DataFrame.(coins_dict)...)
            CSV.write(filepath, df_coins)
        catch
            @info "Could not fetch data, try again later!"
        end
    end

    # Return valid coin id only when list of coins is available
    if ~isempty(df_coins)

        # Filter on matching currency
        df_filter = df_coins |> @filter(_.symbol == currency) |> DataFrame

        try
            # For multiple matches, first filter on coin ids and then on names,
            # which do not have "-" in them
            if size(df_filter)[1] > 1

                df_filter_1 = df_filter |>
                            @filter(~occursin("-", _.id)) |> DataFrame

                if isempty(df_filter_1)
                    df_filter_1 = df_filter |>
                            @filter(~occursin("-", _.name)) |> DataFrame
                end

                return df_filter_1[!, :id][1]
            end

            return df_filter[!, :id][1]

        catch err
            if isa(err, BoundsError)
                @info "Could not find an id for the given currency"
            else
                @info "Something went wrong, check this error: $(err)"
            end
        end

    else
        return ""
    end
end

function dict_to_df(data_dict::Dict, df::DataFrame)

    # Collect only the key-value data which is suitable for plotting
    for key in collect(keys(data_dict))
        if ~isnothing(data_dict[key]) && length(data_dict[key]) == 1
            push!(df, [key Float64(data_dict[key])])
        end
    end

    return df
end

function get_dev_comm_data(currency::String)

    coin_id = get_coin_id(currency)

    coin_dict, dev_dict, comm_dict = [Dict() for i = 1:3]

    try
        @info "Fetching coin data from CoinGecko"
        coin_dict = get_API_response("/coins/$(coin_id)")
    catch
        @info "Could not fetch data, try again later!"
    end

    # Get developer data
    try
        dev_dict = coin_dict["developer_data"]
    catch err
        if isa(err, KeyError)
            @info "Could not find developer data!"
        else
            @info "This is a new error: $(err)"
        end
    end

    # Get community data
    try
        comm_dict = coin_dict["community_data"]
    catch err
        if isa(err, KeyError)
            @info "Could not find community data!"
        else
            @info "This is a new error: $(err)"
        end
    end

    # Convert dict to DataFrame
    df_dev, df_comm = [DataFrame(Metric = String[], Value = Float64[]) for i = 1:2]

    if ~isempty(dev_dict)
        df_dev = dict_to_df(dev_dict, df_dev)
    end

    if ~isempty(comm_dict)
        df_comm = dict_to_df(comm_dict, df_comm)
    end

    return df_dev, df_comm
end

function get_list_of_exchanges(num_exchanges::Int64)

    date = Dates.today()

    filename = "List_of_top_$(num_exchanges)_exchanges_$(date).csv"
    filepath = joinpath("data", filename)

    df_all_ex, df_ex = [DataFrame() for i = 1:2]

    # Look for present day's CSV file, if not found, download and save data to a new file
    if isfile(filepath)
        @info "Reading list of exchanges from CSV file on disk"
        df_ex = CSV.File(filepath) |> DataFrame
    else
        try
            @info "Fetching list of exchanges from CoinGecko"
            ex_dict = get_API_response("/exchanges")
            df_all_ex = vcat(DataFrame.(ex_dict)...)

            # Keep only name and id columns
            df_ex = DataFrame(name = df_all_ex[!, :name], id = df_all_ex[!, :id])
            CSV.write(filepath, df_ex)
        catch err
            @info "Could not fetch data, try again later!"
            @info "$(err)"
        end
    end

    # Return filtered list of exchanges only when data is available
    if ~isempty(df_ex)
        df_ex_vol = DataFrame(Name = df_ex[!, :name][1:num_exchanges],
                               ID  = df_ex[!, :id][1:num_exchanges])

        return df_ex_vol
    else
        return DataFrame()
    end
end

function get_exchange_vol_data(currency::String, num_exchanges::Int64)

    coin_id = get_coin_id(currency)

    df_ex_vol = get_list_of_exchanges(num_exchanges)
    allowmissing!(df_ex_vol)

    exchange_coin_vol, exchange_usd_vol = [Union{Missing, Float64}[] for i = 1:2]

    # Extract volume data only when a list of exchanges is available
    if ~isempty(df_ex_vol)

        for exchange in df_ex_vol[!, :ID]

            coin_vol, usd_vol = [Float64[] for i = 1:2]
            coin_vol_tickers_dict = Dict()

            try
                coin_vol_tickers_dict = get_API_response("/exchanges/$(exchange)/tickers?coin_ids=$(coin_id)")
            catch err
                @info "Could not find $(coin_id) volume data on $(exchange)!"
                @info "$(err)"
            end

            if ~isempty(coin_vol_tickers_dict)

                for i = 1:length(coin_vol_tickers_dict["tickers"])
                    push!(coin_vol, coin_vol_tickers_dict["tickers"][i]["volume"])
                    push!(usd_vol, coin_vol_tickers_dict["tickers"][i]["converted_volume"]["usd"])
                end

                push!(exchange_coin_vol, sum(coin_vol))
                push!(exchange_usd_vol, sum(usd_vol))

            else
                push!(exchange_coin_vol, missing)
                push!(exchange_usd_vol, missing)
            end

        end
    end

    insertcols!(df_ex_vol, 3, :Coin_volume => exchange_coin_vol, :USD_volume => exchange_usd_vol)

    return df_ex_vol
end

function get_vol_chart(exchange::String)

    date = Dates.today()

    filename = "$(exchange)_vol_data_$(date).txt"
    filepath = joinpath("data", filename)

    ex_vol_chart = Vector{Any}[]

    # Historical data is fetched and saved for these many days. Takes more time for longer duration,
    # and queries for many exchanges don't even return more points.
    days = 365

    if isfile(filepath)
        @info "Reading $(exchange) vol data from file on disk"
        ex_vol_chart = readdlm(filepath, ';')

        # Drop the dimension and convert to Float64
        ex_vol_chart = Float64.(ex_vol_chart[:])
    else
        try
            # Fetch and save data for 365 days
            ex_vol_chart = get_API_response("/exchanges/$(exchange)/volume_chart?days=$(days)")

            open(filepath, "w") do f
                for i = 1:length(ex_vol_chart)
                    writedlm(f, [ex_vol_chart[i]], ";")
                end
            end

        catch
            @info "Could not find volume data for $(exchange)"
            @info "$(err)"
        end
    end

    if length(ex_vol_chart) > days
        start_index = Int64(length(ex_vol_chart)/2) + 1
        return ex_vol_chart[start_index:end]

    else
        ex_vol = Union{Missing, Float64}[]

        for i = 1:length(ex_vol_chart)
            try
                push!(ex_vol, round(parse(Float64, ex_vol_chart[i][2]); digits = 2))
            catch
                push!(ex_vol, missing)
            end
        end

        return ex_vol
    end
end

function get_overall_vol_data(duration::Int64, num_exchanges::Int64)

    df_ex_list = get_list_of_exchanges(num_exchanges)

    # Check on duration, current maximum is set to 365
    if duration > 365
        duration = 365
    end

    # Create a column with dates
    f_day = Dates.today()
    i_day = f_day - Dates.Day(duration-1)

    time = collect(i_day:Dates.Day(1):f_day)

    df_ex_vol = DataFrame(Time = time)
    allowmissing!(df_ex_vol)

    # Extract volume data only when a list of exchanges is available
    if ~isempty(df_ex_list)

        for exchange in df_ex_list[!, :ID]

            ex_vol = Vector{Any}[]

            try
                ex_vol = get_vol_chart(exchange)
            catch err
                @info "Could not find volume data for $(exchange), will continue to next!"
                @info "$(err)"

                # Skip next part of the code, and continue to next exchange
                continue
            end

            # Check and filter on duration
            if duration > length(ex_vol)
                duration = length(ex_vol)
            end
            ex_vol = ex_vol[end-duration+1:end]

            # Find name of the exchange corresponding to its ID
            df_row = df_ex_list |> @filter(_.ID == exchange) |> DataFrame
            name = df_row[!, :Name][1]

            try
                insertcols!(df_ex_vol, 2, Symbol(name) => ex_vol)
            catch err
                if isa(err, DimensionMismatch)
                    @info "Data is missing for $(exchange) for the requested duration"
                else
                    @info "Something went wrong, check this error: $(err)"
                end
            end

        end

        return df_ex_vol
    else
        return DataFrame()
    end

end


"""
    GainEquation

Abstract type for a container representing a gain (vector field).
"""
abstract type GainEquation end

"""
    GainEstimationMethod

Abstract type for a method used to solve an equation of `GainEquation` type.
"""
abstract type GainEstimationMethod end

"""
    solve!(eq::GainEquation, method::GainEstimationMethod) --> eq

Solves the gain equation `eq` using method `method`.
"""
function solve!(eq::GainEquation, method::GainEstimationMethod) end

"""
    update!(eq::GainEquation)

Updates the gain equation `eq` such that all information contained in it is self-consistent.

    update!(eq::GainEquation, ens::ParticleRepresentation)

Updates the gain equation `eq` by incorporating new information from the ensemble `ens`.
"""
function update!(eq::GainEquation) end
function update!(eq::GainEquation, ens::ParticleRepresentation) end

function state_dim(eq::GainEquation) end
function obs_dim(eq::GainEquation) end


AnyIntType = Union{Int8, Int32, Int64, Int128, Int}
AnyDoubleType = Union{Int8, Int32, Int64, Int128, Int, Float16, Float32, Float64}

@enum AggOperations begin

    MEAN = 0
    MEDIAN = 1
    EACH = 2

end

@enum FilterTypes begin

    BUTTERWORTH = 0
    CHEBYSHEV_TYPE_1 = 1
    BESSEL = 2

end

@enum WindowFunctions begin

    NO_WINDOW = 0
    HANNING = 1
    HAMMING = 2
    BLACKMAN_HARRIS = 3

end

@enum DetrendOperations begin

    # NONE = 0 # will lead to invalid enum redefinitions since enums entries are global constants in fact
    # its a temp hack, lets wait for fixed enums in julia
    CONSTANT = 1
    LINEAR = 2

end

function perform_lowpass(data, sampling_rate::AnyIntType, cutoff::AnyDoubleType, order::AnyIntType,
    filter_type::AnyIntType, ripple::AnyDoubleType)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_lowpass, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(cutoff), Int32(order), Int32(filter_type), Float64(ripple))
    elseif Sys.isapple()
        ec = ccall((:perform_lowpass, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(cutoff), Int32(order), Int32(filter_type), Float64(ripple))
    else
        ec = ccall((:perform_lowpass, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(cutoff), Int32(order), Int32(filter_type), Float64(ripple))
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_lowpass ", ec), ec))
    end
    return
end

function perform_highpass(data, sampling_rate::AnyIntType, cutoff::AnyDoubleType, order::AnyIntType,
    filter_type::AnyIntType, ripple::AnyDoubleType)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_highpass, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(cutoff), Int32(order), Int32(filter_type), Float64(ripple))
    elseif Sys.isapple()
        ec = ccall((:perform_highpass, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(cutoff), Int32(order), Int32(filter_type), Float64(ripple))
    else
        ec = ccall((:perform_highpass, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(cutoff), Int32(order), Int32(filter_type), Float64(ripple))
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_highpass ", ec), ec))
    end
    return
end

function perform_bandpass(data, sampling_rate::AnyIntType, center_freq::AnyDoubleType,
    band_width::AnyDoubleType, order::AnyIntType, filter_type::AnyIntType, ripple::AnyDoubleType)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_bandpass, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(center_freq), Float64(band_width), Int32(order), Int32(filter_type), Float64(ripple))
    elseif Sys.isapple()
        ec = ccall((:perform_bandpass, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(center_freq), Float64(band_width), Int32(order), Int32(filter_type), Float64(ripple))
    else
        ec = ccall((:perform_bandpass, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(center_freq), Float64(band_width), Int32(order), Int32(filter_type), Float64(ripple))
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_bandpass ", ec), ec))
    end
    return
end

function perform_bandstop(data, sampling_rate::AnyIntType, center_freq::AnyDoubleType,
    band_width::AnyDoubleType, order::AnyIntType, filter_type::AnyIntType, ripple::AnyDoubleType)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_bandstop, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(center_freq), Float64(band_width), Int32(order), Int32(filter_type), Float64(ripple))
    elseif Sys.isapple()
        ec = ccall((:perform_bandstop, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(center_freq), Float64(band_width), Int32(order), Int32(filter_type), Float64(ripple))
    else
        ec = ccall((:perform_bandstop, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Float64, Float64, Cint, Cint, Float64),
            data, length(data), Int32(sampling_rate), Float64(center_freq), Float64(band_width), Int32(order), Int32(filter_type), Float64(ripple))
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_bandstop ", ec), ec))
    end
    return
end

function perform_rolling_filter(data, period::AnyIntType, operation::AnyIntType)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_rolling_filter, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Cint),
            data, length(data), Int32(period), Int32(operation))
    elseif Sys.isapple()
        ec = ccall((:perform_rolling_filter, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Cint),
            data, length(data), Int32(period), Int32(operation))
    else
        ec = ccall((:perform_rolling_filter, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Cint),
            data, length(data), Int32(period), Int32(operation))
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_rolling_filter ", ec), ec))
    end
    return
end

function detrend(data, operation::AnyIntType)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:detrend, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint),
            data, length(data), Int32(operation))
    elseif Sys.isapple()
        ec = ccall((:detrend, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint),
            data, length(data), Int32(operation))
    else
        ec = ccall((:detrend, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint),
            data, length(data), Int32(operation))
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in detrend ", ec), ec))
    end
    return
end

function perform_wavelet_denoising(data, wavelet::String, decomposition_level::AnyIntType)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_wavelet_denoising, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint),
            data, length(data), wavelet, Int32(decomposition_level))
    elseif Sys.isapple()
        ec = ccall((:perform_wavelet_denoising, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint),
            data, length(data), wavelet, Int32(decomposition_level))
    else
        ec = ccall((:perform_wavelet_denoising, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint),
            data, length(data), wavelet, Int32(decomposition_level))
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_wavelet_denoising ", ec), ec))
    end
    return
end

function perform_downsampling(data, period::AnyIntType, operation::AnyIntType)
    len = Integer(floor(length(data) / period))
    downsampled_data = Vector{Float64}(undef, len)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_downsampling, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}),
            data, length(data), Int32(period), Int32(operation), downsampled_data)
    elseif Sys.isapple()
        ec = ccall((:perform_downsampling, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}),
            data, length(data), Int32(period), Int32(operation), downsampled_data)
    else
        ec = ccall((:perform_downsampling, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}),
            data, length(data), Int32(period), Int32(operation), downsampled_data)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_downsampling ", ec), ec))
    end
    downsampled_data
end

function write_file(data, file_name::String, file_mode::String)
    shape = size(data)
    flatten = reshape(transpose(data), (1, shape[1] * shape[2]))
    flatten = copy(flatten)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:write_file, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Ptr{UInt8}, Ptr{UInt8}),
            flatten, shape[1], shape[2], file_name, file_mode)
    elseif Sys.isapple()
        ec = ccall((:write_file, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Ptr{UInt8}, Ptr{UInt8}),
            flatten, shape[1], shape[2], file_name, file_mode)
    else
        ec = ccall((:write_file, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Ptr{UInt8}, Ptr{UInt8}),
            flatten, shape[1], shape[2], file_name, file_mode)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in write_file ", ec), ec))
    end
    return
end

function get_nearest_power_of_two(value::AnyIntType)
    power_of_two = Vector{Cint}(undef, 1)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:get_nearest_power_of_two, "DataHandler.dll"), Cint, (Cint, Ptr{Cint}),
                Int32(value), power_of_two)
    elseif Sys.isapple()
        ec = ccall((:get_nearest_power_of_two, "libDataHandler.dylib"), Cint, (Cint, Ptr{Cint}),
                Int32(value), power_of_two)
    else
        ec = ccall((:get_nearest_power_of_two, "libDataHandler.so"), Cint, (Cint, Ptr{Cint}),
                Int32(value), power_of_two)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in get_nearest_power_of_two ", ec), ec))
    end

    power_of_two[1]
end

function read_file(file_name::String)
    num_elements = Vector{Cint}(undef, 1)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:get_num_elements_in_file, "DataHandler.dll"), Cint, (Ptr{UInt8}, Ptr{Cint}),
                file_name, num_elements)
    elseif Sys.isapple()
        ec = ccall((:get_num_elements_in_file, "libDataHandler.dylib"), Cint, (Ptr{UInt8}, Ptr{Cint}),
                file_name, num_elements)
    else
        ec = ccall((:get_num_elements_in_file, "libDataHandler.so"), Cint, (Ptr{UInt8}, Ptr{Cint}),
                file_name, num_elements)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in get_num_elements_in_file ", ec), ec))
    end

    data = Vector{Float64}(undef, num_elements[1])
    num_cols = Vector{Cint}(undef, 1)
    num_rows = Vector{Cint}(undef, 1)

    if Sys.iswindows()
        ec = ccall((:read_file, "DataHandler.dll"), Cint, (Ptr{Float64}, Ptr{Cint}, Ptr{Cint}, Ptr{UInt8}, Cint),
                data, num_rows, num_cols, file_name, num_elements[1])
    elseif Sys.isapple()
        ec = ccall((:read_file, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Ptr{Cint}, Ptr{Cint}, Ptr{UInt8}, Cint),
                data, num_rows, num_cols, file_name, num_elements[1])
    else
        ec = ccall((:read_file, "libDataHandler.so"), Cint, (Ptr{Float64}, Ptr{Cint}, Ptr{Cint}, Ptr{UInt8}, Cint),
                data, num_rows, num_cols, file_name, num_elements[1])
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in read_file ", ec), ec))
    end
    value = transpose(reshape(data, (num_cols[1], num_rows[1])))
    value
end

function perform_wavelet_transform(data, wavelet::String, decomposition_level::AnyIntType)
    wavelet_coeffs = Vector{Float64}(undef, length(data) + 2 * (40 + 1))
    lengths = Vector{Cint}(undef, decomposition_level + 1)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_wavelet_transform, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint, Ptr{Float64}, Ptr{Cint}),
            data, length(data), wavelet, Int32(decomposition_level), wavelet_coeffs, lengths)
    elseif Sys.isapple()
        ec = ccall((:perform_wavelet_transform, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint, Ptr{Float64}, Ptr{Cint}),
            data, length(data), wavelet, Int32(decomposition_level), wavelet_coeffs, lengths)
    else
        ec = ccall((:perform_wavelet_transform, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint, Ptr{Float64}, Ptr{Cint}),
            data, length(data), wavelet, Int32(decomposition_level), wavelet_coeffs, lengths)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_wavelet_transform ", ec), ec))
    end
    wavelet_coeffs[1:sum(lengths)], lengths
end

function perform_inverse_wavelet_transform(wavelet_output, original_data_len::AnyIntType, wavelet::String, decomposition_level::AnyIntType)
    original_data = Vector{Float64}(undef, original_data_len)
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_inverse_wavelet_transform, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint, Ptr{Float64}, Ptr{Float64}),
            wavelet_output[1], Int32(original_data_len), wavelet, Int32(decomposition_level), wavelet_output[2], original_data)
    elseif Sys.isapple()
        ec = ccall((:perform_inverse_wavelet_transform, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint, Ptr{Float64}, Ptr{Float64}),
            wavelet_output[1], Int32(original_data_len), wavelet, Int32(decomposition_level), wavelet_output[2], original_data)
    else
        ec = ccall((:perform_inverse_wavelet_transform, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Ptr{UInt8}, Cint, Ptr{Float64}, Ptr{Float64}),
            wavelet_output[1], Int32(original_data_len), wavelet, Int32(decomposition_level), wavelet_output[2], original_data)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_inverse_wavelet_transform ", ec), ec))
    end
    original_data
end

function perform_fft(data, window::AnyIntType)

    function is_power_of_two(value)
        (value != 0) && (value & (value - 1) == 0)
    end

    if !is_power_of_two(length(data))
        throw(BrainFlowError(string("Data Len must be power of two ", INVALID_ARGUMENTS_ERROR), INVALID_ARGUMENTS_ERROR))
    end

    temp_re = Vector{Float64}(undef, Integer(length(data) / 2) + 1)
    temp_im = Vector{Float64}(undef, Integer(length(data) / 2) + 1)
    res = Vector{Complex}(undef, Integer(length(data) / 2) + 1)

    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_fft, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(window), temp_re, temp_im)
    elseif Sys.isapple()
        ec = ccall((:perform_fft, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(window), temp_re, temp_im)
    else
        ec = ccall((:perform_fft, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(window), temp_re, temp_im)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_fft ", ec), ec))
    end
    for i in 1:Integer(length(data) / 2) + 1
        res[i] = Complex(temp_re[i], temp_im[i])
    end
    res
end

function perform_ifft(data)

    temp_re = Vector{Float64}(undef, length(data))
    temp_im = Vector{Float64}(undef, length(data))
    res = Vector{Float64}(undef, 2 * (length(data) - 1))

    for i in 1:length(data)
        temp_re[i] = data[i].re
        temp_im[i] = data[i].im
    end
    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:perform_ifft, "DataHandler.dll"), Cint, (Ptr{Float64}, Ptr{Float64}, Cint, Ptr{Float64}),
            temp_re, temp_im, length(res), res)
    elseif Sys.isapple()
        ec = ccall((:perform_ifft, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Ptr{Float64}, Cint, Ptr{Float64}),
            temp_re, temp_im, length(res), res)
    else
        ec = ccall((:perform_ifft, "libDataHandler.so"), Cint, (Ptr{Float64}, Ptr{Float64}, Cint, Ptr{Float64}),
            temp_re, temp_im, length(res), res)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in perform_ifft ", ec), ec))
    end
    res
end

function get_psd(data, sampling_rate::AnyIntType, window::AnyIntType)

    function is_power_of_two(value)
        (value != 0) && (value & (value - 1) == 0)
    end

    if !is_power_of_two(length(data))
        throw(BrainFlowError(string("Data Len must be power of two ", INVALID_ARGUMENTS_ERROR), INVALID_ARGUMENTS_ERROR))
    end

    temp_ampls = Vector{Float64}(undef, Integer(length(data) / 2) + 1)
    temp_freqs = Vector{Float64}(undef, Integer(length(data) / 2) + 1)

    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:get_psd, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    elseif Sys.isapple()
        ec = ccall((:get_psd, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    else
        ec = ccall((:get_psd, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in get_psd ", ec), ec))
    end
    temp_ampls, temp_freqs
end

function get_psd_welch(data, nfft::AnyIntType, overlap::AnyIntType, sampling_rate::AnyIntType, window::AnyIntType)

    function is_power_of_two(value)
        (value != 0) && (value & (value - 1) == 0)
    end

    if !is_power_of_two(nfft)
        throw(BrainFlowError(string("nfft must be power of two ", INVALID_ARGUMENTS_ERROR), INVALID_ARGUMENTS_ERROR))
    end

    temp_ampls = Vector{Float64}(undef, Integer(nfft / 2) + 1)
    temp_freqs = Vector{Float64}(undef, Integer(nfft / 2) + 1)

    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:get_psd_welch, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(nfft), Int32(overlap), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    elseif Sys.isapple()
        ec = ccall((:get_psd_welch, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(nfft), Int32(overlap), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    else
        ec = ccall((:get_psd_welch, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(nfft), Int32(overlap), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in get_psd_welch ", ec), ec))
    end
    temp_ampls, temp_freqs
end

function get_log_psd_welch(data, nfft::AnyIntType, overlap::AnyIntType, sampling_rate::AnyIntType, window::AnyIntType)

    function is_power_of_two(value)
        (value != 0) && (value & (value - 1) == 0)
    end

    if !is_power_of_two(nfft)
        throw(BrainFlowError(string("nfft must be power of two ", INVALID_ARGUMENTS_ERROR), INVALID_ARGUMENTS_ERROR))
    end

    temp_ampls = Vector{Float64}(undef, Integer(nfft / 2) + 1)
    temp_freqs = Vector{Float64}(undef, Integer(nfft / 2) + 1)

    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:get_log_psd_welch, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(nfft), Int32(overlap), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    elseif Sys.isapple()
        ec = ccall((:get_log_psd_welch, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(nfft), Int32(overlap), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    else
        ec = ccall((:get_log_psd_welch, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(nfft), Int32(overlap), Int32(sampling_rate), Int32(window), temp_ampls, temp_freqs)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in get_log_psd_welch ", ec), ec))
    end
    temp_ampls, temp_freqs
end

function get_log_psd(data, sampling_rate::AnyIntType, window::AnyIntType)

    function is_power_of_two(value)
        (value != 0) && (value & (value - 1) == 0)
    end

    if !is_power_of_two(length(data))
        throw(BrainFlowError(string("Data Len must be power of two ", INVALID_ARGUMENTS_ERROR), INVALID_ARGUMENTS_ERROR))
    end

    temp_ampls = Vector{Float64}(undef, Integer(length(data) / 2) + 1)
    temp_freqs = Vector{Float64}(undef, Integer(length(data) / 2) + 1)

    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:get_log_psd, "DataHandler.dll"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(sampling_rate), Int32(window), temp_re, temp_im)
    elseif Sys.isapple()
        ec = ccall((:get_log_psd, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(sampling_rate), Int32(window), temp_re, temp_im)
    else
        ec = ccall((:get_log_psd, "libDataHandler.so"), Cint, (Ptr{Float64}, Cint, Cint, Cint, Ptr{Float64}, Ptr{Float64}),
            data, length(data), Int32(sampling_rate), Int32(window), temp_re, temp_im)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in get_log_psd ", ec), ec))
    end
    temp_ampls, temp_freqs
end

function get_band_power(psd, freq_start::AnyDoubleType, freq_end::AnyDoubleType)
    band_power = Vector{Float64}(undef, 1)

    ec = STATUS_OK
    # due to this bug https://github.com/JuliaLang/julia/issues/29602 libname should be hardcoded
    if Sys.iswindows()
        ec = ccall((:get_band_power, "DataHandler.dll"), Cint, (Ptr{Float64}, Ptr{Float64}, Cint, Float64, Float64, Ptr{Float64}),
            psd[1], psd[2], length(psd[1]), Float64(freq_start), Float64(freq_end), band_power)
    elseif Sys.isapple()
        ec = ccall((:get_band_power, "libDataHandler.dylib"), Cint, (Ptr{Float64}, Ptr{Float64}, Cint, Float64, Float64, Ptr{Float64}),
            psd[1], psd[2], length(psd[1]), Float64(freq_start), Float64(freq_end), band_power)
    else
        ec = ccall((:get_band_power, "libDataHandler.so"), Cint, (Ptr{Float64}, Ptr{Float64}, Cint, Float64, Float64, Ptr{Float64}),
            psd[1], psd[2], length(psd[1]), Float64(freq_start), Float64(freq_end), band_power)
    end
    if ec != Integer(STATUS_OK)
        throw(BrainFlowError(string("Error in get_band_power ", ec), ec))
    end
    band_power[1]
end


#const background_img = zeros(UInt8,640*480)

#const background=Ref{WT_Image}(WT_Image(1,640,480,C_NULL,pointer(background_img)))

#1st dim = w
#2nd dim = h
function JT_trace(iFrame::Int,image_data::AbstractArray{UInt8,2},background_img=zeros(UInt8,length(image_data)),background=Ref{WT_Image}(WT_Image(1,size(image_data,1),size(image_data,2),C_NULL,pointer(background_img))))

    img=Ref{WT_Image}(WT_Image(1,size(image_data,1),size(image_data,2),C_NULL,pointer(image_data)))

    pnseg=Ref{Int32}(0)

    data=ccall((:find_segments,libwhisk_path),Ptr{Whisker2},(Int32,Ref{WT_Image},Ref{WT_Image},Ref{Int32}),iFrame,img,background,pnseg)

    wts=Array{Whisker1,1}()

    for i=1:pnseg[]
        ww=Whisker1(unsafe_load(data,i))
        push!(wts,deepcopy(ww))
        #push!(wts,Whisker1(unsafe_load(data,i)))
    end

    ccall((:Free_Whisker_Seg_Vec,libwhisk_path),Void,(Ptr{Whisker2}, Int32),data,pnseg[])

    wts
end

function WT_trace(iFrame::Int,image_data::AbstractArray{UInt8,2},min_length::Int,pad_pos::Tuple{Float32,Float32},mask::BitArray{2})

    whiskers=JT_trace(iFrame,image_data)

    WT_length_constraint(whiskers,min_length)

    WT_reorder_whisker(whiskers,pad_pos)

    #Apply mask
    apply_mask(whiskers,mask,min_length)

    apply_roi(whiskers,pad_pos)

    eliminate_redundant(whiskers)

    whiskers
end

function get_JT_params()
    ccall((:Params,libwhisk_path),Ptr{JT_Params},())
end

function change_JT_param(f_name,value)
    xx=get_JT_params()
    jt=unsafe_load(xx,1)

    setfield!(jt,f_name,value)

    unsafe_store!(xx,jt,1)

    nothing
end

#returns measurements for single
function JT_measure(wt::Tracker,frame_id,whisker_num,face_axis='x')

    facex=round(Int32,wt.pad_pos[1])
    facey=round(Int32,wt.pad_pos[2])
    whisk_num = convert(Int32,1)
    whisk_array = Ref{Whisker2}(Whisker2(wt.all_whiskers[frame_id][whisker_num]))

    mm=ccall((:Whisker_Segments_Measure,libwhisk_path),Ptr{JT_Measurements},
    (Ref{Whisker2},Int32,Int32,Int32,Cuchar),whisk_array,whisk_num,facex,facey,face_axis)

    unsafe_load(mm,1)
end

function JT_measure(w::Whisker1,facex::Float32,facey::Float32,face_axis='x')
    JT_measure(w,round(Int32,facex),round(Int32,facey),face_axis)
end

function JT_measure(w::Whisker1,facex::Int32,facey::Int32,face_axis='x')
    #This should be an option depending on how the whisker is oriented

    whisk_num = convert(Int32,1)

    whisk_array = Ref{Whisker2}(Whisker2(w))

    mm=ccall((:Whisker_Segments_Measure,libwhisk_path),Ptr{JT_Measurements},
    (Ref{Whisker2},Int32,Int32,Int32,Cuchar),whisk_array,whisk_num,facex,facey,face_axis)

    unsafe_load(mm,1)
end

function JT_measure(wt::Tracker,w::Whisker1,face_axis='x')

    facex=round(Int32,wt.pad_pos[1])
    facey=round(Int32,wt.pad_pos[2])

    JT_measure(w,facex,facey,face_axis)
end

function get_JT_measurements(wt::Tracker,face_axis='x')

    mymeas=Array{JT_Measurements,1}(0)

    for i=1:length(wt.all_whiskers)
        for j=1:length(wt.all_whiskers[i])
            xx=JT_measure(wt,i,j,face_axis)
            push!(mymeas,xx)
        end
    end

    mymeas
end

#Finds the best length threshold to discriminate whiskers and non whiskers for a known number
#of whiskers to track
function JT_Best_Threshold(wt,table,num_whisk)

    n_rows=convert(Int32,size(wt.vid,3))
    n_cols=convert(Int32,0)

    #Ranges for length threshold (I'm basically making these giant and ignoring them)
    low=1.0
    high=600.0

    is_gt=convert(Int32,1)

    #Number of whiskers that should be in frame
    target_count=convert(Int32,num_whisk)

    ccall((:Measurements_Table_Estimate_Best_Threshold_For_Known_Count,WhiskerTracking.libwhisk_path),Float64,
        (Ref{WhiskerTracking.JT_Measurements},Int32,Int32,Float64,Float64,Int32,Int32),
        table,n_rows,n_cols,low,high,is_gt,target_count)
end

function JT_find_segments(img,h,th,s,facemask)

    ccall(Libdl.dlsym(libwhisk,:compute_seed_from_point_field_on_grid),Void,(Ref{WT_Image},Int32,Int32,Int32,Float32,Float32,Ref{WT_Image},Ref{WT_Image},Ref{WT_Image}),
    img,50,4,5,0.0f0,0.0f0,h,th,s)

    mystride = img[].height

    s_j=unsafe_wrap(Array,convert(Ptr{Float32},s[].array),(length(img),))
    h_j=unsafe_wrap(Array,h[].array,(length(img),))
    th_j=unsafe_wrap(Array,convert(Ptr{Float32},th[].array),(length(img),))
    for i=1:length(h_j)
        if h_j[i]>0
            th_j[i] = th_j[i] / convert(Float32,h_j[i])
        end
    end

    mask=falses(length(img))

    nseeds=0
    for i=1:length(h_j)
        if (s_j[i] > 0.80f0)&(!facemask[i]) #SEED_THRESH
            nseeds+=1
            mask[i]=true
        end
    end

    myline=Ref{Line_Params}(Line_Params(0.5f0,convert(Float32,0.0f0),2.0f0,0.0f0))

    pi_cycle = pi / 4.0f0 / 18.0f0
    myangle=0.0f0

    scores=zeros(Float32,nseeds)
    inds=find(mask)
    j=1

    for i=inds

        offset=0.5f0
        xdir=100.0f0 * convert(Float32,cos(th_j[i]))
        ydir=100.0f0 * convert(Float32,sin(th_j[i]))

        if xdir < 0.0f0
            myangle = round(atan(-1.0f0 * ydir, -1.0f0 * xdir) / pi_cycle) * pi_cycle
        else
            myangle = round(atan(ydir,xdir) / 18.0f0) * pi_cycle
        end

        width = 2.0f0

        myline[].angle=myangle

        scores[j]=ccall(Libdl.dlsym(libwhisk,:eval_line),Float32,(Ref{Line_Params},Ref{WT_Image},Int32),myline,img,convert(Int32,i-1))
        j+=1
    end

    score_ord=sortperm(scores)

    nsegs=0
    ss=Ref{Seed}(Seed(0,0,0,0))
    for i=score_ord
        ss[].xpnt=(inds[i]-1) % mystride
        ss[].ypnt=div(inds[i]-1,mystride)
        ss[].xdir=round(Int32,100 * cos(th_j[inds[i]]))
        ss[].ydir=round(Int32,100 * sin(th_j[inds[i]]))
        w=ccall(Libdl.dlsym(libwhisk,:trace_whisker),Ptr{Whisker2},(Ref{Seed},Ref{WT_Image}),ss,img)

        if w == C_NULL
            ss[].ydir=round(Int32,100 * cos(th_j[inds[i]]))
            ss[].xdir=round(Int32,100 * sin(th_j[inds[i]]))
            w=ccall(Libdl.dlsym(libwhisk,:trace_whisker),Ptr{Whisker2},(Ref{Seed},Ref{WT_Image}),ss,img)
        end

        if w != C_NULL
            nsegs+=1
        end
    end

    nsegs
end


# provide a way to invoke any solver

"""
  This function provides a way to invoke any physics solver based on the
  specification of the physics in the input file.

  The physics module must have already been registered using [`register_physics`](@ref)

  This function is equivalent to [`solvePDE`](@ref), it is maintained here
  for legacy purposes only.

  **Inputs**

   * input_file: an AbstractString specifying the path to the input file

  **Outputs**

   * mesh: the AbstractMesh object used during the solve
   * sbp: the SBP operator used by the solver
   * eqn: the AbstractSolutionData object during the solve.  At exit,
          eqn.q_vec should have the final solution in it
   * opts: the options dictionary
"""
function run_solver(input_file::Union{AbstractString, Dict})

  return solvePDE(input_file)

end


export
    term_pl02,
    termpt,
    timdef,
    timout,
    tipbod,
    tisbod,
    tkvrsn,
    tparse,
    tpictr,
    trace,
    tsetyr,
    twopi,
    twovec,
    tyear

"""

Find terminator points on a target body. The caller specifies
half-planes, bounded by the illumination source center-target center
vector, in which to search for terminator points.

The terminator can be either umbral or penumbral. The umbral
terminator is the boundary of the region on the target surface
where no light from the source is visible. The penumbral
terminator is the boundary of the region on the target surface
where none of the light from the source is blocked by the target
itself.

The surface of the target body may be represented either by a
triaxial ellipsoid or by topographic data.

### Arguments ###

- `method`: Computation method
- `ilusrc`: Illumination source
- `target`: Name of target body
- `et    `: Epoch in ephemeris seconds past J2000 TDB
- `fixref`: Body-fixed, body-centered target body frame
- `abcorr`: Aberration correction
- `corloc`: Aberration correction locus
- `obsrvr`: Name of observing body
- `refvec`: Reference vector for cutting half-planes
- `rolstp`: Roll angular step for cutting half-planes
- `ncuts `: Number of cutting planes
- `schstp`: Angular step size for searching
- `soltol`: Solution convergence tolerance
- `maxn  `: Maximum number of entries in output arrays

### Output ###

- `npts  `: Counts of terminator points corresponding to cuts
- `points`: Terminator points
- `epochs`: Times associated with terminator points
- `trmvcs`: Terminator vectors emanating from the observer

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/termpt_c.html)
"""
function termpt(method, ilusrc, target, et, fixref, abcorr, corloc, obsrvr, refvec, rolstp,
                ncuts, schstp, soltol, maxn)
    @checkdims 3 refvec
    npts = zeros(SpiceInt, maxn)
    points = Array{SpiceDouble}(undef, 3, maxn)
    epochs = Array{SpiceDouble}(undef, maxn)
    trmvcs = Array{SpiceDouble}(undef, 3, maxn)

    ccall((:termpt_c, libcspice), Cvoid,
          (Cstring, Cstring, Cstring, SpiceDouble, Cstring, Cstring, Cstring, Cstring,
           Ref{SpiceDouble}, SpiceDouble, SpiceInt, SpiceDouble, SpiceDouble, SpiceInt,
           Ref{SpiceInt}, Ref{SpiceDouble}, Ref{SpiceDouble}, Ref{SpiceDouble}),
          method, ilusrc, target, et, fixref, abcorr, corloc, obsrvr, refvec, rolstp,
          ncuts, schstp, soltol, maxn, npts, points, epochs, trmvcs)
    handleerror()
    valid_points = npts .>= 1
    npts[valid_points], cmatrix_to_array(points)[valid_points],
        epochs, cmatrix_to_array(trmvcs)[valid_points]
end

@deprecate term_pl02 termpt

"""
    timdef(action, item, value="")

Set and retrieve the defaults associated with calendar input strings.

### Arguments ###

- `action`: The kind of action to take, either `:SET` or `:GET`
- `item`: The default item of interest. The items that may be requested are:
    - `:CALENDAR` with allowed values:
        - `"GREGORIAN"`
        - `"JULIAN"`
        - `"MIXED"`
    - `:SYSTEM` with allowed values:
        - `"TDB"`
        - `"TDT"`
        - `"UTC"`
    - `:ZONE` with allowed values (`0 <= HR < 13` and `0 <= MN < 60`):
        - `"EST"`
        - `"EDT"`
        - `"CST"`
        - `"CDT"`
        - `"MST"`
        - `"MDT"`
        - `"PST"`
        - `"PDT"`
        - `"UTC+\$HR"`
        - `"UTC-\$HR"`
        - `"UTC+\$HR:\$MN"`
        - `"UTC-\$HR:\$MN"`

### Output ###

Returns the value associated with the default item.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/timdef_c.html)
"""
function timdef(action, item, value="")
    lenout = 10
    val = fill(SpiceChar(0), lenout)
    val[1:length(value)] .= collect(value)
    ccall((:timdef_c, libcspice), Cvoid,
          (Cstring, Cstring, SpiceInt, Ref{SpiceChar}),
          string(action), string(item), lenout, val)
    handleerror()
    chararray_to_string(val)
end

"""
    timout(et, pictur, lenout=128)

This routine converts an input epoch represented in TDB seconds past the TDB
epoch of J2000 to a character string formatted to the specifications of a
user's format picture.

### Arguments ###

- `et`: An epoch in seconds past the ephemeris epoch J2000
- `pictur`: A format specification for the output string
- `lenout`: The length of the output string plus 1 (default: 128)

### Output ###

Returns a string representation of the input epoch.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/timout_c.html)
"""
function timout(et, pictur, lenout=128)
    string = Array{SpiceChar}(undef, lenout)
    ccall((:timout_c, libcspice), Cvoid,
          (Cdouble, Cstring, SpiceInt, Ref{SpiceChar}),
          et, pictur, lenout, string)
    handleerror()
    chararray_to_string(string)
end

"""
    tipbod(ref, body, et)

Return a 3x3 matrix that transforms positions in inertial coordinates to
positions in body-equator-and-prime-meridian coordinates.

### Arguments ###

- `ref`: Name of inertial reference frame to transform from
- `body`: ID code of body
- `et`: Epoch of transformation

### Output ###

Returns transformation matrix from intertial position to prime meridian.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/tipbod_c.html)
"""
function tipbod(ref, body, et)
    tipm = Array{SpiceDouble}(undef, 3, 3)
    ccall((:tipbod_c, libcspice), Cvoid,
          (Cstring, SpiceInt, SpiceDouble, Ref{SpiceDouble}),
          ref, body, et, tipm)
    handleerror()
    permutedims(tipm)
end

"""
    tisbod(ref, body, et)

Return a 6x6 matrix that transforms states in inertial coordinates to
states in body-equator-and-prime-meridian coordinates.

### Arguments ###

- `ref`: Name of inertial reference frame to transform from
- `body`: ID code of body
- `et`: Epoch of transformation

### Output ###

Returns transformation matrix from intertial state to prime meridian.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/tisbod_c.html)
"""
function tisbod(ref, body, et)
    tsipm = Array{SpiceDouble}(undef, 6, 6)
    ccall((:tisbod_c, libcspice), Cvoid, (Cstring, SpiceInt, SpiceDouble, Ref{SpiceDouble}),
        ref, body, et, tsipm)
    handleerror()
    permutedims(tsipm)
end

"""
    tkvrsn(item=:TOOLKIT)

Given an item such as the Toolkit or an entry point name, return the latest
version string.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/tkvrsn_c.html)
"""
function tkvrsn(item=:TOOLKIT)
    out = ccall((:tkvrsn_c, libcspice), Cstring, (Cstring,), string(item))
    return unsafe_string(out)
end

"""
    tparse(string)

Parse a time string and return seconds past the J2000 epoch on a formal calendar.

### Arguments ###

- `string`: Input time string in UTC

### Output ###

Returns UTC expressed in seconds since J2000.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/tparse_c.html)
"""
function tparse(string)
    lenout = 128
    sp2000 = Ref{SpiceDouble}()
    errmsg = Array{SpiceChar}(undef, lenout)
    ccall((:tparse_c, libcspice), Cvoid,
          (Cstring, SpiceInt, Ref{SpiceDouble}, Ref{SpiceChar}),
          string, lenout, sp2000, errmsg)
    msg = chararray_to_string(errmsg)
    if !isempty(msg)
        throw(SpiceError(msg))
    end
    sp2000[]
end

"""
    tpictr(sample, lenout=80)

Given a sample time string, create a time format picture suitable for use by
the routine [`timout`](@ref).

### Arguments ###

- `sample`: A sample time string
- `lenout`: The length for the output picture string (default: 80)

### Output ###

Returns a format picture that describes sample.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/tpictr_c.html)
"""
function tpictr(sample, lenout=80)
    lenerr = 128
    pictur = Array{SpiceChar}(undef, lenout)
    ok = Ref{SpiceBoolean}()
    errmsg = Array{SpiceChar}(undef, lenerr)
    ccall((:tpictr_c, libcspice), Cvoid,
          (Cstring, SpiceInt, SpiceInt, Ref{SpiceChar}, Ref{SpiceBoolean}, Ref{SpiceChar}),
          sample, lenout, lenerr, pictur, ok, errmsg)
    if !Bool(ok[])
        throw(SpiceError(chararray_to_string(errmsg)))
    end
    chararray_to_string(pictur)
end

function _trace(matrix)
    ccall((:trace_c, libcspice), SpiceDouble, (Ref{SpiceDouble},), matrix)
end

"""
    trace(matrix)

!!! warning "Deprecated"
    Use `LinearAlgebra.tr(matrix)` instead.
"""
trace

@deprecate trace LinearAlgebra.tr

"""
    tsetyr(year)

Set the lower bound on the 100 year range.

### Arguments ###

-`year`: Lower bound on the 100 year interval of expansion

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/tsetyr_c.html)
"""
function tsetyr(year)
    ccall((:tsetyr_c, libcspice), Cvoid, (SpiceInt,), year)
end

function _twopi()
    ccall((:twopi_c, libcspice), SpiceDouble, ())
end

"""
    twopi()

!!! warning "Deprecated"
    Use `2π` instead.
"""
twopi

@deprecate twopi() 2π

"""
    twovec(axdef, indexa, plndef, indexp)

Find the transformation to the right-handed frame having a given vector as a
specified axis and having a second given vector lying in a specified coordinate
plane.

### Arguments ###

- `axdef`: Vector defining a principal axis
- `indexa`: Principal axis number of axdef (X=1, Y=2, Z=3)
- `plndef`: Vector defining (with axdef) a principal plane
- `indexp`: Second axis number (with indexa) of principal plane

### Output ###

Returns output rotation matrix.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/twovec_c.html)
"""
function twovec(axdef, indexa, plndef, indexp)
    @checkdims 3 axdef plndef
    mout = Array{SpiceDouble}(undef, 3, 3)
    ccall((:twovec_c, libcspice), Cvoid,
          (Ref{SpiceDouble}, SpiceInt, Ref{SpiceDouble}, SpiceInt, Ref{SpiceDouble}),
          axdef, indexa, plndef, indexp, mout)
    handleerror()
    permutedims(mout)
end

"""
    tyear()

Returns the number of seconds per tropical year.

### References ###

- [NAIF Documentation](https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/tyear_c.html)
"""
function tyear()
    ccall((:tyear_c, libcspice), SpiceDouble, ())
end


# this user supplied file creates a dictionary of arguments
# if a key is repeated, the last use of the key is used
# it is a little bit dangerous letting the user run arbitrary code
# as part of the solver
# now that this file is read inside a function, it is better encapsulated

arg_dict = Dict{String, Any}(
"physics" => "Advection",
"run_type" => 1,
"jac_type" => 2,
"order" => 4,
"real_time" => false,
"t_max" => 1000.0,
"IC_name" => "ICp4",
"use_DG" => true,
"Flux_name" => "LFFlux",
"LFalpha" => 0,
"numBC" => 1,
"BC1" => [0, 1, 2, 3],
"BC1_name" => "p4BC",
"SRCname" => "SRCp4",
#"BC2" => [4, 10],
#"BC2_name" => "noPenetrationBC",
"delta_t" => 2*pi/5000,
#"smb_name" => "SRCMESHES/channel.smb",
#"smb_name" => "/users/creanj/test2/p1/m1/parallel2.smb",
#"dmg_name" => "parallel2.dmg",
"smb_name" => "SRCMESHES/serial2.smb",
"dmg_name" => "SRCMESHES/serial2.dmg",
#"smb_name" => "SRCMESHES/channel_verylarge.smb",
#"dmg_name" => "../../mesh_files/vortex.dmg",
"dmg_name" => ".null",
"res_abstol" => 1e-10,
# "step_tol" => 1e-6,
"itermax" => 100,
"res_abstol" => 1e-9,
#"res_reltol" => 1e-9,
#"step_tol" => 1e-10,
"writeq" => false,
"write_edge_vertnums" => false,
"write_face_vertnums" => false,
"write_qic" => false,
"writeboundary" => false,
"write_res" => false,
"print_cond" => false,
"write_counts" => false,
"write_vis" => true,
"output_freq" => 100,
"write_vis" => false,
"solve" => false,
"do_postproc" => true,
"exact_soln_func" => "ICp4",
)


using OrdinaryDiffEq: OrdinaryDiffEq

"""
    Classical <: DynamicsMethods.Method

Type for performing classical molecular dynamics.

```jldoctest
sim = Simulation{Classical}(Atoms(:H), Harmonic())

# output

Simulation{Classical}:
  Atoms{Float64}([:H], [1], [1837.4715941070515])
  Harmonic{Float64, Float64, Float64}
  m: Float64 1.0
  ω: Float64 1.0
  r₀: Float64 0.0
  dofs: Int64 1
```
"""
struct Classical <: DynamicsMethods.Method end

function NQCDynamics.Simulation{Classical}(atoms::Atoms, model::Model; kwargs...)
    NQCDynamics.Simulation(atoms, model, Classical(); kwargs...)
end
function NQCDynamics.RingPolymerSimulation{Classical}(atoms::Atoms, model::Model, n_beads::Integer; kwargs...)
    NQCDynamics.RingPolymerSimulation(atoms, model, Classical(), n_beads::Integer; kwargs...)
end

function NQCDynamics.Simulation(atoms::Atoms, model::Model; kwargs...)
    NQCDynamics.Simulation{Classical}(atoms, model; kwargs...)
end
function NQCDynamics.RingPolymerSimulation(atoms::Atoms, model::Model, n_beads::Integer; kwargs...)
    NQCDynamics.RingPolymerSimulation{Classical}(atoms, model, n_beads; kwargs...)
end

function DynamicsMethods.motion!(du, u, sim::AbstractSimulation{<:Classical}, t)
    dr = DynamicsUtils.get_positions(du)
    dv = DynamicsUtils.get_velocities(du)
    r = DynamicsUtils.get_positions(u)
    v = DynamicsUtils.get_velocities(u)
    DynamicsUtils.velocity!(dr, v, r, sim, t)
    acceleration!(dv, v, r, sim, t)
end

function DynamicsMethods.motion!(du, u, sim::RingPolymerSimulation{<:Classical}, t)
    dr = DynamicsUtils.get_positions(du)
    dv = DynamicsUtils.get_velocities(du)
    r = DynamicsUtils.get_positions(u)
    v = DynamicsUtils.get_velocities(u)
    DynamicsUtils.velocity!(dr, v, r, sim, t)
    ring_polymer_acceleration!(dv, v, r, sim, t)
end

"""
`f1` in `DifferentialEquations.jl` docs.
"""
function acceleration!(dv, v, r, sim::AbstractSimulation, t)
    Calculators.evaluate_derivative!(sim.calculator, r)
    dv .= -sim.calculator.derivative
    DynamicsUtils.divide_by_mass!(dv, sim.atoms.masses)
end

function ring_polymer_acceleration!(dv, v, r, sim::RingPolymerSimulation, t)
    Calculators.evaluate_derivative!(sim.calculator, r)
    dv .= -sim.calculator.derivative
    DynamicsUtils.divide_by_mass!(dv, sim.atoms.masses)
    DynamicsUtils.apply_interbead_coupling!(dv, r, sim)
end

function DynamicsMethods.create_problem(u0, tspan::Tuple, sim::Simulation{<:Classical})
    OrdinaryDiffEq.DynamicalODEProblem(acceleration!, DynamicsUtils.velocity!,
        DynamicsUtils.get_velocities(u0), DynamicsUtils.get_positions(u0), tspan, sim)
end

function DynamicsMethods.create_problem(u0, tspan::Tuple, sim::RingPolymerSimulation{<:Classical})
    OrdinaryDiffEq.DynamicalODEProblem(ring_polymer_acceleration!, DynamicsUtils.velocity!,
        DynamicsUtils.get_velocities(u0), DynamicsUtils.get_positions(u0), tspan, sim)
end

DynamicsMethods.select_algorithm(::AbstractSimulation{<:Classical}) = OrdinaryDiffEq.VelocityVerlet()


const DATA_FILE = joinpath(@__DIR__, "dummy.csv")
const NUM_DIMENSIONS, NUM_OBSERVATIONS = size(load_data(DATA_FILE)[1])
const SOLVER = optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0)
const INIT_STRAT = SimpleCombinedStrategy(RuleOfThumbScott(), BoundedTaxErrorEstimate(0.05, 0.02, 0.98))

experiment = Dict{Symbol, Any}(
    :hash => 1,
    :data_file => DATA_FILE,
    :data_set_name => "Dummy",
    :output_file => joinpath(@__DIR__, "example_query_synthesis.json"),
    :log_dir => joinpath(@__DIR__, "example_query_synthesis.log"),
    :split_strategy_name => "Sf",
    :initial_pool_strategy_name => "Pu",
    :model => Dict(:type => :SVDDneg,
                   :param => Dict{Symbol, Any}(),
                   :init_strategy => INIT_STRAT),
    :query_strategy => Dict(:type => :DecisionBoundaryQss,
                            :param => Dict{Symbol, Any}(
                                :optimizer => ParticleSwarmOptimization()
                            )),
    :split_strategy => OneClassActiveLearning.DataSplits(trues(NUM_OBSERVATIONS)),
    :oracle => QuerySynthesisOCCOracle(SVDDneg, INIT_STRAT, DATA_FILE, SOLVER),
    :param => Dict(:num_al_iterations => 10,
                   :solver => SOLVER,
                   :initial_pools => fill(:U, NUM_OBSERVATIONS),
                   :adjust_K => true,
                   :initial_pool_resample_version => 1))


# Copyright (c) 2018: Matthew Wilhelm & Matthew Stuber.
# This code is licensed under MIT license (see LICENSE.md for full details)
#############################################################################
# EAGO
# A development environment for robust and global optimization
# See https://github.com/PSORLab/EAGO.jl
#############################################################################
# src/eago_optimizer/constraints.jl
# Defines constraints supported by optimizer and how to store them.
#############################################################################

const INEQ_SETS = Union{LT, GT, ET}
const VAR_SETS = Union{LT, GT, ET, ZO, MOI.Integer}

##### Utilities for checking that JuMP model contains variables used in expression
function check_inbounds!(m::Optimizer, vi::VI)
    if !(1 <= vi.value <= m._input_problem._variable_count)
        error("Invalid variable index $vi. ($(m._input_problem._variable_count) variables in the model.)")
    end
    return nothing
end
check_inbounds!(m::Optimizer, f::SAF) = foreach(x -> check_inbounds!(m, x.variable), f.terms)
check_inbounds!(m::Optimizer, f::VECOFVAR) = foreach(x -> check_inbounds!(m, x), f.variables)
function check_inbounds!(m::Optimizer, f::SQF)
    foreach(x -> check_inbounds!(m, x.variable), f.affine_terms)
    for term in f.quadratic_terms
        check_inbounds!(m, term.variable_1)
        check_inbounds!(m, term.variable_2)
    end
    return nothing
end

MOI.supports_constraint(::Optimizer, ::Type{VI}, ::Type{S}) where S <: VAR_SETS = true
MOI.supports_constraint(::Optimizer,::Type{T},::Type{S}) where {T<:Union{SAF,SQF},S<:INEQ_SETS} = true

MOI.is_valid(m::Optimizer, v::VI) = (1 <= v.value <= m._input_problem._variable_count)
MOI.is_valid(m::Optimizer, c::CI{VI,S}) where S <: VAR_SETS = (1 <= c.value <= m._input_problem._variable_count)
MOI.is_valid(m::Optimizer, c::CI{F,S}) where {F<:Union{SAF,SQF}, S<:INEQ_SETS} = (1 <= c.value <= length(_constraints(m,F,S)))

MOI.get(m::Optimizer, ::MOI.NumberOfConstraints{VI,S}) where S<:VAR_SETS = length(_constraints(m,VI,S))
MOI.get(m::Optimizer, ::MOI.NumberOfConstraints{F,S}) where {F<:Union{SAF,SQF},S<:INEQ_SETS} = length(_constraints(m,F,S))

MOI.get(m::Optimizer, ::MOI.ListOfConstraintIndices{VI,S}) where S<:VAR_SETS = collect(keys(_constraints(m,VI,S)))
MOI.get(m::Optimizer, ::MOI.ListOfConstraintIndices{F,S}) where {F<:Union{SAF,SQF},S<:INEQ_SETS} = collect(keys(_constraints(m,F,S)))

MOI.add_variable(m::Optimizer) = VI(m._input_problem._variable_count += 1)

function MOI.add_constraint(m::Optimizer, f::F, s::S) where {F<:Union{SAF,SQF},S<:INEQ_SETS}
    check_inbounds!(m, f)
    ci = CI{F, S}(m._input_problem._constraint_count += 1)
    _constraints(m, F, S)[ci] = (f, s)
    return ci
end
function MOI.add_constraint(m::Optimizer, f::VI, s::S) where S<:VAR_SETS
    check_inbounds!(m, f)
    ci = CI{VI,S}(f.value)
    _constraints(m,VI,S)[ci] = (f, s)
    return ci
end

result_index_1_error(v::T) where T = throw(MOI.ResultIndexBoundsError{T}(v, 1))
function MOI.get(model::Optimizer, v::MOI.VariablePrimal, vi::MOI.VariableIndex)
    check_inbounds!(model, vi)
    (v.result_index != 1) && result_index_1_error(v)
    return model._global_optimizer._continuous_solution[vi.value]
end
function MOI.get(m::Optimizer{R,S,T}, v::MOI.ConstraintPrimal, c::CI{VI,<:Any}) where {R,S,T}
    (v.result_index != 1) && result_index_1_error(v)
    return MOI.get(m, MOI.VariablePrimal(), MOI.VariableIndex(c.value))
end
function MOI.get(m::Optimizer{R,Q,T}, v::MOI.ConstraintPrimal, c::CI{F,S}) where {R,Q,T,F,S}
    (v.result_index != 1) && result_index_1_error(v)
    return _constraint_primal(m._global_optimizer, F, S)[c]
end

MOI.get(m::Optimizer, ::MOI.ConstraintFunction, c::CI{F,S}) where {F,S} = _constraints(m,F,S)[c][1]
MOI.get(m::Optimizer, ::MOI.ConstraintSet, c::CI{F,S}) where {F,S} = _constraints(m,F,S)[c][2]

function MOI.empty!(m::Optimizer{R,S,T}) where {R,S,T}

    MOI.empty!(m.subsolver_block)
    MOI.empty!(m._global_optimizer)
    m._input_problem = InputProblem()
    m._working_problem = ParsedProblem()

    m._termination_status_code = MOI.OPTIMIZE_NOT_CALLED
    m._result_status_code = MOI.OTHER_RESULT_STATUS
    m._run_time = 0.0
    m._objective_value  = -Inf
    m._objective_bound  =  Inf
    m. _relative_gap     = Inf
    m._iteration_count  = 0
    m._node_count       = 0

    return nothing
end

function MOI.is_empty(m::Optimizer{R,S,T}) where {R,S,T}

    flag = true
    flag &= MOI.is_empty(m._global_optimizer)
    flag &= isempty(m._input_problem)
    flag &= isempty(m._working_problem)
    flag &= isempty(m.subsolver_block)
    flag &= m._termination_status_code == MOI.OPTIMIZE_NOT_CALLED
    flag &= m._result_status_code == MOI.OTHER_RESULT_STATUS

    # set constructor reset on empty! and  to zero in initial parse! in parse.jl
    flag &= iszero(m._run_time)
    flag &= iszero(m._iteration_count)
    flag &= iszero(m._node_count)
    flag &= m._objective_value == -Inf
    flag &= m._objective_bound ==  Inf
    flag &= m. _relative_gap   == Inf

    return flag
end

MOI.supports_incremental_interface(m::Optimizer) = true
MOI.copy_to(model::Optimizer, src::MOI.ModelLike) = MOIU.default_copy_to(model, src)

#####
##### Set & get attributes of model
#####
MOI.supports(::Optimizer, ::MOI.Silent) = true

function MOI.set(m::Optimizer, ::MOI.Silent, value)
    if value
        m._parameters.verbosity = 0
        m._parameters.log_on = false
    else
        m._parameters.verbosity = 1
    end
    return
end

MOI.set(m::Optimizer, ::MOI.TimeLimitSec, ::Nothing) = m._parameters.time_limit = Inf
MOI.set(m::Optimizer, ::MOI.TimeLimitSec, v::Float64) = m._parameters.time_limit = v

function MOI.get(m::Optimizer, ::MOI.ListOfConstraintTypesPresent)
    constraint_types = []
    for S in (ZO, MOI.Integer)
        if MOI.get(m, MOI.NumberOfConstraints{VI,S}()) > 0
            push!(constraint_types, (VI,S))
        end
    end
    for S in (LT, GT, ET), T in (VI, SAF, SQF)
        if MOI.get(m, MOI.NumberOfConstraints{T,S}()) > 0
            push!(constraint_types, (VI,S))
        end
    end
    return constraint_types
end

MOI.get(m::Optimizer, v::MOI.ObjectiveValue) = !isone(v.result_index) ? result_index_1_error(v) : m._objective_value
MOI.get(m::Optimizer, v::MOI.PrimalStatus) = !isone(v.result_index) ? MOI.NO_SOLUTION : m._result_status_code
MOI.get(m::Optimizer, ::MOI.DualStatus) = MOI.NO_SOLUTION
MOI.get(m::Optimizer, ::MOI.ObjectiveBound) = m._objective_bound
MOI.get(m::Optimizer, ::MOI.NumberOfVariables) = m._input_problem._variable_count
MOI.get(m::Optimizer, ::MOI.SolverName) = "EAGO: Easy Advanced Global Optimization"
MOI.get(m::Optimizer, ::MOI.SolverVersion) = "0.7.0"
MOI.get(m::Optimizer, ::MOI.TerminationStatus) = m._termination_status_code
MOI.get(m::Optimizer, ::MOI.SolveTimeSec) = m._run_time
MOI.get(m::Optimizer, ::MOI.NodeCount) = m._node_count
MOI.get(m::Optimizer, ::MOI.ResultCount) = (m._result_status_code === MOI.FEASIBLE_POINT) ? 1 : 0
MOI.get(m::Optimizer, ::MOI.TimeLimitSec) = m._parameters.time_limit
MOI.get(m::Optimizer, ::MOI.Silent) = m._parameters.verbosity == 0
MOI.get(m::Optimizer, ::MOI.ListOfVariableIndices) = [VI(i) for i = 1:m._input_problem._variable_count]

function MOI.get(m::Optimizer, ::MOI.RelativeGap)
    b = MOI.get(m, MOI.ObjectiveBound())
    v = MOI.get(m, MOI.ObjectiveValue())
    return relative_gap(b,v)
end

_to_sym(d) = error("EAGO only supports raw parameters with Symbol or String names.")
_to_sym(d::String) = Symbol(d)
_to_sym(d::Symbol) = d
function MOI.get(m::Optimizer, p::MOI.RawOptimizerAttribute)
    s = _to_sym(p.name)
    s in EAGO_PARAMETERS ? getfield(m._parameters, s) : getfield(m, s)
end
function MOI.set(m::Optimizer, p::MOI.RawOptimizerAttribute , x)
    s = _to_sym(p.name)
    if (s == :relaxed_optimizer) || (s == :upper_optimizer)
        setfield!(m, s, Incremental(x))
    else
        s in EAGO_PARAMETERS ? setfield!(m._parameters, s, x) : setfield!(m, s, x)
    end
end
MOI.get(m::Optimizer, p::MOI.RawStatusString) = string(m._global_optimizer._end_state)

#####
##### Support, set, and evaluate objective functions
#####
MOI.supports(::Optimizer, ::MOI.TimeLimitSec) = true
MOI.supports(::Optimizer, ::MOI.ObjectiveSense) = true
MOI.supports(::Optimizer, ::MOI.ObjectiveFunction{F}) where {F <: Union{VI, SAF, SQF}} = true

function MOI.set(m::Optimizer, ::MOI.NLPBlock, nlp_data::MOI.NLPBlockData)
    if nlp_data.has_objective
        m._input_problem._objective = nothing
    end
    m._input_problem._nlp_data = nlp_data
end

function MOI.set(m::Optimizer, ::MOI.ObjectiveFunction{T}, f::T) where T <: Union{VI,SAF,SQF}
    check_inbounds!(m, f)
    m._input_problem._objective = f
end
MOI.get(m::Optimizer, ::MOI.ObjectiveFunction{T}) where T <: Union{VI,SAF,SQF} = m._input_problem._objective
MOI.get(m::Optimizer, ::MOI.ObjectiveFunctionType) = typeof(m._input_problem._objective)

MOI.set(m::Optimizer, ::MOI.ObjectiveSense, s::MOI.OptimizationSense) = m._input_problem._optimization_sense = s
MOI.get(m::Optimizer, ::MOI.ObjectiveSense) = m._input_problem._optimization_sense


struct SerialExecution{T <: AbstractFloat,
                       A <: AbstractVector,
                       S <: LQSolver} <: AbstractExecution
    subproblems::Vector{SubProblem{T,A,S}}

    function SerialExecution(::Type{T}, ::Type{A}, ::Type{S}) where {T <: AbstractFloat, A <: AbstractVector, S <: LQSolver}
        return new{T,A,S}(Vector{SubProblem{T,A,S}}())
    end
end

function init_subproblems!(ph::AbstractProgressiveHedgingSolver, subsolver::QPSolver, execution::SerialExecution)
    for i = 1:ph.nscenarios
        push!(execution.subproblems,SubProblem(WS(ph.stochasticprogram, scenario(ph.stochasticprogram,i); solver = subsolver),
                                               i,
                                               probability(ph.stochasticprogram,i),
                                               decision_length(ph.stochasticprogram),
                                               subsolver))
    end
    update_iterate!(ph)
    return ph
end

function iterate!(ph::AbstractProgressiveHedgingSolver, ::SerialExecution)
    return iterate_nominal!(ph)
end

function init_workers!(::AbstractProgressiveHedgingSolver, ::SerialExecution)
    return nothing
end

function close_workers!(::AbstractProgressiveHedgingSolver, ::SerialExecution)
    return nothing
end

function resolve_subproblems!(ph::AbstractProgressiveHedgingSolver, execution::SerialExecution{T,A}) where {T <: AbstractFloat, A <: AbstractVector}
    Qs = A(undef, length(execution.subproblems))
    # Update subproblems
    reformulate_subproblems!(execution.subproblems, ph.ξ, penalty(ph))
    # Solve sub problems
    for (i, subproblem) ∈ enumerate(execution.subproblems)
        Qs[i] = subproblem()
    end
    # Return current objective value
    return sum(Qs)
end

function update_iterate!(ph::AbstractProgressiveHedgingSolver, execution::SerialExecution)
    # Update the estimate
    ξ_prev = copy(ph.ξ)
    ph.ξ[:] = sum([subproblem.π*subproblem.x for subproblem in execution.subproblems])
    # Update δ₁
    ph.data.δ₁ = norm(ph.ξ-ξ_prev, 2)^2
    return nothing
end

function update_subproblems!(ph::AbstractProgressiveHedgingSolver, execution::SerialExecution)
    # Update dual prices
    update_subproblems!(execution.subproblems, ph.ξ, penalty(ph))
    return nothing
end

function update_dual_gap!(ph::AbstractProgressiveHedgingSolver, execution::SerialExecution)
    # Update δ₂
    ph.data.δ₂ = sum([s.π*norm(s.x-ph.ξ,2)^2 for s in execution.subproblems])
    return nothing
end

function calculate_objective_value(ph::AbstractProgressiveHedgingSolver, execution::SerialExecution)
    return sum([get_objective_value(s) for s in execution.subproblems])
end

function fill_submodels!(ph::AbstractProgressiveHedgingSolver, scenarioproblems::StochasticPrograms.ScenarioProblems, execution::SerialExecution)
    for (i, submodel) in enumerate(scenarioproblems.problems)
        fill_submodel!(submodel, execution.subproblems[i])
    end
end

function fill_submodels!(ph::AbstractProgressiveHedgingSolver, scenarioproblems::StochasticPrograms.DScenarioProblems, execution::SerialExecution)
    j = 0
    @sync begin
        for w in workers()
            n = remotecall_fetch((sp)->length(fetch(sp).problems), w, scenarioproblems[w-1])
            for i in 1:n
                k = i+j
                @async remotecall_fetch((sp,i,x,μ,λ) -> fill_submodel!(fetch(sp).problems[i],x,μ,λ),
                                        w,
                                        scenarioproblems[w-1],
                                        i,
                                        get_solution(execution.subproblems[k])...)
            end
            j += n
        end
    end
end
# API
# ------------------------------------------------------------
struct Serial <: Execution end

function (execution::Serial)(::Type{T}, ::Type{A}, ::Type{S}) where {T <: AbstractFloat, A <: AbstractVector, S <: LQSolver}
    return SerialExecution(T,A,S)
end

function str(::Serial)
    return "serial execution"
end


using PyCall

@pyimport modeller
@pyimport modeller.scripts as scripts
@pyimport _modeller

mod_lib = _modeller.mod_libdir_get()

modeller.log[:verbose]()    # request verbose output
env = modeller.environ()
env[:libs][:topology][:read](file=string(mod_lib, "/top_heav.lib")) # read topology
env[:libs][:parameters][:read](file=string(mod_lib, "/par.lib")) # read parameters

# read model file
mdl = scripts.complete_pdb(env, "TvLDH.B99990002.pdb")

# Assess with DOPE:
s = modeller.selection(mdl)   # all atom selection
s[:assess_dope](output="ENERGY_PROFILE NO_REPORT", file="TvLDH.profile",
              normalize_profile=true, smoothing_window=15)


using .GeneticVariation

export bcf, vcf

"""
    openvcf(::String)
Open VCF file (`.vcf(.gz)`, or `.bcf(.gz)`) and return an `IO` stream in reading mode `"r"`.
"""
function openvcf(infile::String)
    if endswith(infile, ".gz")
        throw(ArgumentError("Please load in GZip.jl with \`using GZip\`"))
    else
        return open(infile, "r")
    end
end

function bcf(infile::String; rename_loci::Bool = false, silent::Bool = false, allow_monomorphic::Bool = false)
    bases = (A = Int8(1), T = Int8(2), C = Int8(3), G = Int8(4), miss = Int8(0))
    stream = BCF.Reader(openvcf(infile))
    nmarkers = countlines(openvcf(infile)) - length(BCF.header(stream)) - 1
    sample_ID = header(stream).sampleID
    nsamples = length(sample_ID)
    loci_names = fill("marker", nmarkers)
    geno_df = DataFrame(:name => sample_ID, :population =>  "missing")
    if silent == false
        @info "\n$(abspath(infile))\n$nsamples samples detected\n$nmarkers markers detected\npopulation info must be added <---"
    end
        for record in stream
        ref_alt = Dict(-1 => "miss", 0 => BCF.ref(record), [i => j for (i,j) in enumerate(BCF.alt(record))]...)
        raw_geno = BCF.genotype(record, 1:nsamples, "GT")
        conv_geno = map(raw_geno) do rg
            tmp = replace.(rg, "." => "-1")
            ig = collect(parse.(Int8, split(tmp, r"\/|\|")))
            [bases[Symbol(ref_alt[i])] for i in ig] |> sort |> Tuple
        end
        insertcols!(geno_df, Symbol(BCF.chrom(record) * "_" * string(BCF.pos(record))) => conv_geno)
    end
    close(stream)
    if rename_snp
        rnm = append!([:name, :population], [Symbol.("snp_" * i) for i in string.(1:nmarkers)])
        rename!(geno_df, rnm)
    end
    stacked_geno_df = DataFrames.stack(geno_df, DataFrames.Not(1:2))
    rename!(stacked_geno_df, [:name, :population, :locus, :genotype])
    # set columns as PooledArrays
    select!(
        stacked_geno_df,
        :name => PooledArray => :name,
        :population => PooledArray => :population,
        :locus => (i -> PooledArray(i |> Vector{String})) => :locus,
        :genotype
    )
    # replace missing genotypes as missing
    stacked_geno_df.genotype = map(stacked_geno_df.genotype) do geno
        if all(0 .== geno)
            return missing
        else
            return geno
        end
    end
    sort!(stacked_geno_df, [:name, :locus])
    #meta_df = generate_meta(stacked_geno_df)
    # ploidy finding
    meta_df = DataFrames.combine(DataFrames.groupby(stacked_geno_df, :name),
        :genotype => (i -> Int8(length(first(skipmissing(i))))) => :ploidy
    )
    insertcols!(meta_df, 2, :population => "missing")
    insertcols!(meta_df, 4, :longitude => Vector{Union{Missing, Float32}}(undef, nsamples), :latitude => Vector{Union{Missing, Float32}}(undef, nsamples))

    pd_out = PopData(meta_df, stacked_geno_df)
    !allow_monomorphic && drop_monomorphic!(pd_out)
    return pd_out
end

### VCF parsing ###

function vcf(infile::String; rename_snp::Bool = false, silent::Bool = false, allow_monomorphic::Bool = false)
    bases = (A = Int8(1), T = Int8(2), C = Int8(3), G = Int8(4), miss = Int8(0))
    stream = VCF.Reader(openvcf(infile))
    nmarkers = countlines(openvcf(infile)) - length(VCF.header(stream)) - 1
    sample_ID = header(stream).sampleID
    nsamples = length(sample_ID)
    loci_names = fill("marker", nmarkers)
    geno_df = DataFrame(:name => sample_ID, :population =>  "missing")
    if silent == false
        @info "\n$(abspath(infile))\n$nsamples samples detected\n$nmarkers markers detected\npopulation info must be added <---"
    end
    for record in stream
        ref_alt = Dict(-1 => "miss", 0 => VCF.ref(record), [i => j for (i,j) in enumerate(VCF.alt(record))]...)
        raw_geno = VCF.genotype(record, 1:nsamples, "GT")
        conv_geno = map(raw_geno) do rg
            tmp = replace.(rg, "." => "-1")
            ig = collect(parse.(Int8, split(tmp, r"\/|\|")))
            [bases[Symbol(ref_alt[i])] for i in ig] |> sort |> Tuple
        end
        insertcols!(geno_df, Symbol(VCF.chrom(record) * "_" * string(VCF.pos(record))) => conv_geno)
    end
    close(stream)
    if rename_snp
        rnm = append!([:name, :population], [Symbol.("snp_" * i) for i in string.(1:nmarkers)])
        rename!(geno_df, rnm)
    end
    stacked_geno_df = DataFrames.stack(geno_df, DataFrames.Not(1:2))
    rename!(stacked_geno_df, [:name, :population, :locus, :genotype])
    # set columns as PooledArrays
    select!(
        stacked_geno_df,
        :name => PooledArray => :name,
        :population => PooledArray => :population,
        :locus => (i -> PooledArray(i |> Vector{String})) => :locus,
        :genotype
    )
    # replace missing genotypes as missing
    stacked_geno_df.genotype = map(stacked_geno_df.genotype) do geno
        if all(0 .== geno)
            return missing
        else
            return geno
        end
    end
    sort!(stacked_geno_df, [:name, :locus])
    # ploidy finding
    #meta_df = generate_meta(stacked_geno_df)
    meta_df = DataFrames.combine(DataFrames.groupby(stacked_geno_df, :name),
        :genotype => (i -> Int8(length(first(skipmissing(i))))) => :ploidy
    )
    insertcols!(meta_df, 2, :population => "missing")
    insertcols!(meta_df, 4, :longitude => Vector{Union{Missing, Float32}}(undef, nsamples), :latitude => Vector{Union{Missing, Float32}}(undef, nsamples))

    pd_out = PopData(meta_df, stacked_geno_df)
    !allow_monomorphic && drop_monomorphic!(pd_out)
    return pd_out
end


import IncrementalInference: selectFactorType
# import ApproxManifoldProducts: getManifold

## ============================================================================
# Starting integration with Manifolds.jl, via ApproxManifoldProducts.jl first
## ============================================================================

getManifold(::InstanceType{Point2Point2}) = Point2 |> getManifold
getManifold(::InstanceType{Pose2Point2}) = Point2 |> getManifold
getManifold(::InstanceType{Pose2Pose2}) = Pose2 |> getManifold
# getManifold(::InstanceType{Pose3Point3}) = Point3
getManifold(::InstanceType{Pose3Pose3}) = Pose3 |> getManifold
getManifold(::InstanceType{Pose2Point2BearingRange}) = BearingRange2 |> getManifold

function selectFactorType(T1::Type{<:InferenceVariable}, T2::Type{<:InferenceVariable})
  # hacky version
  if T1 == Pose2 && T2 == Pose2
    return Pose2Pose2
  elseif T1 == Pose2 && T2 == Point2
    return Pose2Point2
  elseif T1 == DynPose2 && T2 == DynPose2
      return DynPose2DynPose2
  elseif T1 == Point2 && T2 == Point2
    return Point2Point2
  elseif T1 == Point3 && T2 == Point3
    return Point3Point3
  elseif T1 == Pose3 && T2 == Pose3
    return Pose3Pose3
  else
    error("dont know which Factor type to select between $T1 and $T2")
  end
end


# ******************************************************************************************
# Notices:
#
# Copyright © 2022 United States Government as represented by the Administrator of the
# National Aeronautics and Space Administration.  All Rights Reserved.
#
# Disclaimers
#
# No Warranty: THE SUBJECT SOFTWARE IS PROVIDED "AS IS" WITHOUT ANY WARRANTY OF ANY KIND,
# EITHER EXPRESSED, IMPLIED, OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTY
# THAT THE SUBJECT SOFTWARE WILL CONFORM TO SPECIFICATIONS, ANY IMPLIED WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR FREEDOM FROM INFRINGEMENT, ANY
# WARRANTY THAT THE SUBJECT SOFTWARE WILL BE ERROR FREE, OR ANY WARRANTY THAT DOCUMENTATION,
# IF PROVIDED, WILL CONFORM TO THE SUBJECT SOFTWARE. THIS AGREEMENT DOES NOT, IN ANY MANNER,
# CONSTITUTE AN ENDORSEMENT BY GOVERNMENT AGENCY OR ANY PRIOR RECIPIENT OF ANY RESULTS,
# RESULTING DESIGNS, HARDWARE, SOFTWARE PRODUCTS OR ANY OTHER APPLICATIONS RESULTING FROM
# USE OF THE SUBJECT SOFTWARE.  FURTHER, GOVERNMENT AGENCY DISCLAIMS ALL WARRANTIES AND
# LIABILITIES REGARDING THIRD-PARTY SOFTWARE, IF PRESENT IN THE ORIGINAL SOFTWARE, AND
# DISTRIBUTES IT "AS IS."
#
# Waiver and Indemnity:  RECIPIENT AGREES TO WAIVE ANY AND ALL CLAIMS AGAINST THE UNITED
# STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL AS ANY PRIOR RECIPIENT.
# IF RECIPIENT'S USE OF THE SUBJECT SOFTWARE RESULTS IN ANY LIABILITIES, DEMANDS, DAMAGES,
# EXPENSES OR LOSSES ARISING FROM SUCH USE, INCLUDING ANY DAMAGES FROM PRODUCTS BASED ON,
# OR RESULTING FROM, RECIPIENT'S USE OF THE SUBJECT SOFTWARE, RECIPIENT SHALL INDEMNIFY AND
# HOLD HARMLESS THE UNITED STATES GOVERNMENT, ITS CONTRACTORS AND SUBCONTRACTORS, AS WELL
# AS ANY PRIOR RECIPIENT, TO THE EXTENT PERMITTED BY LAW.  RECIPIENT'S SOLE REMEDY FOR ANY
# SUCH MATTER SHALL BE THE IMMEDIATE, UNILATERAL TERMINATION OF THIS AGREEMENT.
# ******************************************************************************************

const TODO_COUNTER = Threads.Atomic{Int64}(0)
const DONE_COUNTER = Threads.Atomic{Int64}(0)

add_todo(n::Int64=1) = Threads.atomic_add!(TODO_COUNTER, n)

function add_done(n::Int64=1)
    Threads.atomic_add!(DONE_COUNTER, n)
    Threads.atomic_sub!(TODO_COUNTER, n)
end

"""
Cell information pertaining to proof and processing status.
"""
Base.@kwdef mutable struct CellStatus
    depth::Int64 = 0
    detached::Bool = false
    hash::UInt64 = hash(0)
    member::Bool = false
    proven::Bool = false
    pid::Int64 = 0
end

"""
    BinaryRefinery <: AbstractRefinery

Binary refinement process.
"""
Base.@kwdef struct BinaryRefinery <: AbstractRefinery
    network::Network
    solver = ReluVal(max_iter=100)
    val::Float64 = 0.0                # boundary value
    output = HalfSpace([1.0], val)    # a⋅x ≤ b (region of interest)
    coutput = HalfSpace([-1.0], -val) # a⋅x ≥ b (complement region)
    tol::Float64 = 0.1                # maximum unproven length scale
    ks::Vector{Int64} = []            # detachment depths
end

"""
    IntervalRefinery <: AbstractRefinery

Interval refinement process.
"""
Base.@kwdef struct IntervalRefinery <: AbstractRefinery
    network::Network
    solver = ReluVal(max_iter=100)
    interval::Vector{Float64} = [-1.0, 1.0]
    output = Hyperrectangle(low=interval[1:1], high=interval[2:2])
    tol::Float64 = 0.1
    ks::Vector{Int64} = []
end

"""
Determine whether condition can be proven true or false over entire cell
or if further refinement is needed.
"""
function needs_refinement(cell::Cell, r::AbstractRefinery)
    # Checks status.
    cell.data.proven && return false

    # Checks tolerance.
    w = cell.boundary.widths
    maximum(w) < r.tol && return false

    # Determines corresponding input set.
    l = cell.boundary.origin
    h = l + w
    input = Hyperrectangle(low=convert(Vector{Float64}, l), high=convert(Vector{Float64}, h))

    # Attempts verification of condition.
    problem = Problem(r.network, input, r.output)
    result = solve(r.solver, problem)
    if result.status == :holds
        cell.data.member = true
        cell.data.proven = true
        return false
    end

    if r isa BinaryRefinery
        # Attempts verification of complement.
        problem = Problem(r.network, input, r.coutput)
        result = solve(r.solver, problem)
        if result.status == :holds
            cell.data.member = false
            cell.data.proven = true
            return false
        end
    end

    return true
end

"""
Generate data for newly-created cell.
"""
function child_data(c::Cell, idx::Tuple)
    return CellStatus(depth = c.data.depth + 1, hash = hash((c.data.hash, idx)))
end

"""
    refine!(cell::Cell, r::AbstractRefinery)

Recursively build miniminal k-d tree that defines volumes proven to satisfy given condition
and its complement. This version is single-process.
"""
function refine!(cell::Cell, r::AbstractRefinery)
    add_todo()
    yield()
    if needs_refinement(cell, r)
        split!(cell, child_data)
        for c in children(cell)
           refine!(c, r)
        end
        merge!(cell)
    end
    add_done()
    yield()
end

"""
Merge children of cell if all are proven and belong to the same set, or if all are unproven
and attached (not awaiting computation by another process). If `recursive`, all descendents
are merged.
"""
function merge!(cell::Cell; recursive::Bool=false)
    if !isleaf(cell)
        cs = children(cell)
        provens = [recursive ? merge!(c; recursive=true) : c.data.proven for c in cs]
        members = (c -> c.data.member).(cs)
        detachs = (c -> c.data.detached).(cs)
        if all(provens) && reduce(==, members) || !any(provens) && !any(detachs) && all(isleaf.(cs))
            cell.data.member = members[1]
            cell.data.proven = provens[1]
            cell.children = nothing
        end
    end
    return cell.data.proven
end

"""
Verify that tree is well-formed after recombination; i.e., no cells are marked as detached
and all children recognize the correct parent.
"""
function verify(cell::Cell)
    for c in allcells(cell)
        if c.data.detached
            return false
        end
        if !isleaf(c)
            if !all((x->x.parent == c).(children(c)))
                return false
            end
        end
    end
    return true
end

"""
Generate root from region limits.
"""
function get_root(limits::NTuple{2, Vector{<:Real}})
    lows, highs = limits
    widths = highs - lows
    root = Cell(SVector(lows...), SVector(widths...), CellStatus())
    return root
end

"""
Perform PVV analysis, using multiple processes if available.
"""
function analyze(r::AbstractRefinery, limits::Tuple{Vector, Vector}; progress::Bool=true, multiproc::Bool=true)
    TODO_COUNTER[] = 1
    tree = get_root(limits)

    @sync begin
        # Refinement process
        @async begin
            try
                if nprocs() == 1 || !multiproc
                    refine!(tree, r)
                else
                    refine_multiprocess!(tree, r)
                end
            finally
                TODO_COUNTER[] = 0
            end
        end

        # Progress meter
        if progress
            @async begin
                p = ProgressUnknown("Cells awaiting processing:")
                while TODO_COUNTER[] > 0
                    ProgressMeter.update!(p, TODO_COUNTER[]; ignore_predictor=true)
                    sleep(0.1)
                end
                ProgressMeter.update!(p, 0)
                ProgressMeter.finish!(p)
            end
        end
    end

    try
        print_metrics(tree)
    catch
        println("Unable to calculate metrics.")
    end

    return tree
end