#=
This file is auto-generated. Do not edit.
=#

#! format: off

"""
    mutable struct HydroEnergyReservoir <: HydroGen
        name::String
        available::Bool
        bus::Bus
        active_power::Float64
        reactive_power::Float64
        rating::Float64
        prime_mover::PrimeMovers
        active_power_limits::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
        reactive_power_limits::Union{Nothing, Min_Max}
        ramp_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
        time_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
        base_power::Float64
        storage_capacity::Float64
        inflow::Float64
        initial_storage::Float64
        operation_cost::OperationalCost
        storage_target::Float64
        conversion_factor::Float64
        time_at_status::Float64
        services::Vector{Service}
        dynamic_injector::Union{Nothing, DynamicInjection}
        ext::Dict{String, Any}
        time_series_container::InfrastructureSystems.TimeSeriesContainer
        internal::InfrastructureSystemsInternal
    end

# Arguments
- `name::String`
- `available::Bool`
- `bus::Bus`
- `active_power::Float64`
- `reactive_power::Float64`, validation range: `reactive_power_limits`, action if invalid: `warn`
- `rating::Float64`: Thermal limited MVA Power Output of the unit. <= Capacity, validation range: `(0, nothing)`, action if invalid: `error`
- `prime_mover::PrimeMovers`: Prime mover technology according to EIA 923
- `active_power_limits::NamedTuple{(:min, :max), Tuple{Float64, Float64}}`
- `reactive_power_limits::Union{Nothing, Min_Max}`, action if invalid: `warn`
- `ramp_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}`: ramp up and ramp down limits in MW (in component base per unit) per minute, validation range: `(0, nothing)`, action if invalid: `error`
- `time_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}`: Minimum up and Minimum down time limits in hours, validation range: `(0, nothing)`, action if invalid: `error`
- `base_power::Float64`: Base power of the unit in MVA, validation range: `(0, nothing)`, action if invalid: `warn`
- `storage_capacity::Float64`: Maximum storage capacity in the reservoir (units can be p.u-hr or m^3)., validation range: `(0, nothing)`, action if invalid: `error`
- `inflow::Float64`: Baseline inflow into the reservoir (units can be p.u. or m^3/hr), validation range: `(0, nothing)`, action if invalid: `error`
- `initial_storage::Float64`: Initial storage capacity in the reservoir (units can be p.u-hr or m^3)., validation range: `(0, nothing)`, action if invalid: `error`
- `operation_cost::OperationalCost`: Operation Cost of Generation [`OperationalCost`](@ref)
- `storage_target::Float64`: Storage target at the end of simulation as ratio of storage capacity.
- `conversion_factor::Float64`: Conversion factor from flow/volume to energy: m^3 -> p.u-hr.
- `time_at_status::Float64`
- `services::Vector{Service}`: Services that this device contributes to
- `dynamic_injector::Union{Nothing, DynamicInjection}`: corresponding dynamic injection device
- `ext::Dict{String, Any}`
- `time_series_container::InfrastructureSystems.TimeSeriesContainer`: internal time_series storage
- `internal::InfrastructureSystemsInternal`: power system internal reference, do not modify
"""
mutable struct HydroEnergyReservoir <: HydroGen
    name::String
    available::Bool
    bus::Bus
    active_power::Float64
    reactive_power::Float64
    "Thermal limited MVA Power Output of the unit. <= Capacity"
    rating::Float64
    "Prime mover technology according to EIA 923"
    prime_mover::PrimeMovers
    active_power_limits::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
    reactive_power_limits::Union{Nothing, Min_Max}
    "ramp up and ramp down limits in MW (in component base per unit) per minute"
    ramp_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
    "Minimum up and Minimum down time limits in hours"
    time_limits::Union{Nothing, NamedTuple{(:up, :down), Tuple{Float64, Float64}}}
    "Base power of the unit in MVA"
    base_power::Float64
    "Maximum storage capacity in the reservoir (units can be p.u-hr or m^3)."
    storage_capacity::Float64
    "Baseline inflow into the reservoir (units can be p.u. or m^3/hr)"
    inflow::Float64
    "Initial storage capacity in the reservoir (units can be p.u-hr or m^3)."
    initial_storage::Float64
    "Operation Cost of Generation [`OperationalCost`](@ref)"
    operation_cost::OperationalCost
    "Storage target at the end of simulation as ratio of storage capacity."
    storage_target::Float64
    "Conversion factor from flow/volume to energy: m^3 -> p.u-hr."
    conversion_factor::Float64
    time_at_status::Float64
    "Services that this device contributes to"
    services::Vector{Service}
    "corresponding dynamic injection device"
    dynamic_injector::Union{Nothing, DynamicInjection}
    ext::Dict{String, Any}
    "internal time_series storage"
    time_series_container::InfrastructureSystems.TimeSeriesContainer
    "power system internal reference, do not modify"
    internal::InfrastructureSystemsInternal
end

function HydroEnergyReservoir(name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, storage_capacity, inflow, initial_storage, operation_cost=TwoPartCost(0.0, 0.0), storage_target=1.0, conversion_factor=1.0, time_at_status=INFINITE_TIME, services=Device[], dynamic_injector=nothing, ext=Dict{String, Any}(), time_series_container=InfrastructureSystems.TimeSeriesContainer(), )
    HydroEnergyReservoir(name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, storage_capacity, inflow, initial_storage, operation_cost, storage_target, conversion_factor, time_at_status, services, dynamic_injector, ext, time_series_container, InfrastructureSystemsInternal(), )
end

function HydroEnergyReservoir(; name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, storage_capacity, inflow, initial_storage, operation_cost=TwoPartCost(0.0, 0.0), storage_target=1.0, conversion_factor=1.0, time_at_status=INFINITE_TIME, services=Device[], dynamic_injector=nothing, ext=Dict{String, Any}(), time_series_container=InfrastructureSystems.TimeSeriesContainer(), internal=InfrastructureSystemsInternal(), )
    HydroEnergyReservoir(name, available, bus, active_power, reactive_power, rating, prime_mover, active_power_limits, reactive_power_limits, ramp_limits, time_limits, base_power, storage_capacity, inflow, initial_storage, operation_cost, storage_target, conversion_factor, time_at_status, services, dynamic_injector, ext, time_series_container, internal, )
end

# Constructor for demo purposes; non-functional.
function HydroEnergyReservoir(::Nothing)
    HydroEnergyReservoir(;
        name="init",
        available=false,
        bus=Bus(nothing),
        active_power=0.0,
        reactive_power=0.0,
        rating=0.0,
        prime_mover=PrimeMovers.HY,
        active_power_limits=(min=0.0, max=0.0),
        reactive_power_limits=nothing,
        ramp_limits=nothing,
        time_limits=nothing,
        base_power=0.0,
        storage_capacity=0.0,
        inflow=0.0,
        initial_storage=0.0,
        operation_cost=TwoPartCost(nothing),
        storage_target=0.0,
        conversion_factor=0.0,
        time_at_status=INFINITE_TIME,
        services=Device[],
        dynamic_injector=nothing,
        ext=Dict{String, Any}(),
        time_series_container=InfrastructureSystems.TimeSeriesContainer(),
    )
end

"""Get [`HydroEnergyReservoir`](@ref) `name`."""
get_name(value::HydroEnergyReservoir) = value.name
"""Get [`HydroEnergyReservoir`](@ref) `available`."""
get_available(value::HydroEnergyReservoir) = value.available
"""Get [`HydroEnergyReservoir`](@ref) `bus`."""
get_bus(value::HydroEnergyReservoir) = value.bus
"""Get [`HydroEnergyReservoir`](@ref) `active_power`."""
get_active_power(value::HydroEnergyReservoir) = get_value(value, value.active_power)
"""Get [`HydroEnergyReservoir`](@ref) `reactive_power`."""
get_reactive_power(value::HydroEnergyReservoir) = get_value(value, value.reactive_power)
"""Get [`HydroEnergyReservoir`](@ref) `rating`."""
get_rating(value::HydroEnergyReservoir) = get_value(value, value.rating)
"""Get [`HydroEnergyReservoir`](@ref) `prime_mover`."""
get_prime_mover(value::HydroEnergyReservoir) = value.prime_mover
"""Get [`HydroEnergyReservoir`](@ref) `active_power_limits`."""
get_active_power_limits(value::HydroEnergyReservoir) = get_value(value, value.active_power_limits)
"""Get [`HydroEnergyReservoir`](@ref) `reactive_power_limits`."""
get_reactive_power_limits(value::HydroEnergyReservoir) = get_value(value, value.reactive_power_limits)
"""Get [`HydroEnergyReservoir`](@ref) `ramp_limits`."""
get_ramp_limits(value::HydroEnergyReservoir) = get_value(value, value.ramp_limits)
"""Get [`HydroEnergyReservoir`](@ref) `time_limits`."""
get_time_limits(value::HydroEnergyReservoir) = value.time_limits
"""Get [`HydroEnergyReservoir`](@ref) `base_power`."""
get_base_power(value::HydroEnergyReservoir) = value.base_power
"""Get [`HydroEnergyReservoir`](@ref) `storage_capacity`."""
get_storage_capacity(value::HydroEnergyReservoir) = get_value(value, value.storage_capacity)
"""Get [`HydroEnergyReservoir`](@ref) `inflow`."""
get_inflow(value::HydroEnergyReservoir) = get_value(value, value.inflow)
"""Get [`HydroEnergyReservoir`](@ref) `initial_storage`."""
get_initial_storage(value::HydroEnergyReservoir) = get_value(value, value.initial_storage)
"""Get [`HydroEnergyReservoir`](@ref) `operation_cost`."""
get_operation_cost(value::HydroEnergyReservoir) = value.operation_cost
"""Get [`HydroEnergyReservoir`](@ref) `storage_target`."""
get_storage_target(value::HydroEnergyReservoir) = value.storage_target
"""Get [`HydroEnergyReservoir`](@ref) `conversion_factor`."""
get_conversion_factor(value::HydroEnergyReservoir) = value.conversion_factor
"""Get [`HydroEnergyReservoir`](@ref) `time_at_status`."""
get_time_at_status(value::HydroEnergyReservoir) = value.time_at_status
"""Get [`HydroEnergyReservoir`](@ref) `services`."""
get_services(value::HydroEnergyReservoir) = value.services
"""Get [`HydroEnergyReservoir`](@ref) `dynamic_injector`."""
get_dynamic_injector(value::HydroEnergyReservoir) = value.dynamic_injector
"""Get [`HydroEnergyReservoir`](@ref) `ext`."""
get_ext(value::HydroEnergyReservoir) = value.ext
"""Get [`HydroEnergyReservoir`](@ref) `time_series_container`."""
get_time_series_container(value::HydroEnergyReservoir) = value.time_series_container
"""Get [`HydroEnergyReservoir`](@ref) `internal`."""
get_internal(value::HydroEnergyReservoir) = value.internal

"""Set [`HydroEnergyReservoir`](@ref) `available`."""
set_available!(value::HydroEnergyReservoir, val) = value.available = val
"""Set [`HydroEnergyReservoir`](@ref) `bus`."""
set_bus!(value::HydroEnergyReservoir, val) = value.bus = val
"""Set [`HydroEnergyReservoir`](@ref) `active_power`."""
set_active_power!(value::HydroEnergyReservoir, val) = value.active_power = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `reactive_power`."""
set_reactive_power!(value::HydroEnergyReservoir, val) = value.reactive_power = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `rating`."""
set_rating!(value::HydroEnergyReservoir, val) = value.rating = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `prime_mover`."""
set_prime_mover!(value::HydroEnergyReservoir, val) = value.prime_mover = val
"""Set [`HydroEnergyReservoir`](@ref) `active_power_limits`."""
set_active_power_limits!(value::HydroEnergyReservoir, val) = value.active_power_limits = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `reactive_power_limits`."""
set_reactive_power_limits!(value::HydroEnergyReservoir, val) = value.reactive_power_limits = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `ramp_limits`."""
set_ramp_limits!(value::HydroEnergyReservoir, val) = value.ramp_limits = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `time_limits`."""
set_time_limits!(value::HydroEnergyReservoir, val) = value.time_limits = val
"""Set [`HydroEnergyReservoir`](@ref) `base_power`."""
set_base_power!(value::HydroEnergyReservoir, val) = value.base_power = val
"""Set [`HydroEnergyReservoir`](@ref) `storage_capacity`."""
set_storage_capacity!(value::HydroEnergyReservoir, val) = value.storage_capacity = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `inflow`."""
set_inflow!(value::HydroEnergyReservoir, val) = value.inflow = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `initial_storage`."""
set_initial_storage!(value::HydroEnergyReservoir, val) = value.initial_storage = set_value(value, val)
"""Set [`HydroEnergyReservoir`](@ref) `operation_cost`."""
set_operation_cost!(value::HydroEnergyReservoir, val) = value.operation_cost = val
"""Set [`HydroEnergyReservoir`](@ref) `storage_target`."""
set_storage_target!(value::HydroEnergyReservoir, val) = value.storage_target = val
"""Set [`HydroEnergyReservoir`](@ref) `conversion_factor`."""
set_conversion_factor!(value::HydroEnergyReservoir, val) = value.conversion_factor = val
"""Set [`HydroEnergyReservoir`](@ref) `time_at_status`."""
set_time_at_status!(value::HydroEnergyReservoir, val) = value.time_at_status = val
"""Set [`HydroEnergyReservoir`](@ref) `services`."""
set_services!(value::HydroEnergyReservoir, val) = value.services = val
"""Set [`HydroEnergyReservoir`](@ref) `ext`."""
set_ext!(value::HydroEnergyReservoir, val) = value.ext = val
"""Set [`HydroEnergyReservoir`](@ref) `time_series_container`."""
set_time_series_container!(value::HydroEnergyReservoir, val) = value.time_series_container = val


using DFControl, Test

testdir = joinpath(dirname(dirname(pathof(DFControl))), "test")
testjobpath = joinpath(testdir, "testassets", "test_job")

name = "Pt"
local_dir = testjobpath
server_dir = testjobpath
bin_dir = joinpath(homedir(), "bin")
excs = [Exec("mpirun", bin_dir, :np => 24), Exec("pw.x", bin_dir, Dict{Symbol, Any}(:nk => 2))]

pseudoset = :test

header = ["#SBATCH -N 1", "#SBATCH --ntasks-per-node=24",
          "#SBATCH --time=24:00:00", "#SBATCH -p defpart",
          "module load open-mpi/gcc/1.10.4-hfi", "module load mkl/2016.1.056"
         ]

scf_data = Dict(:kpoints => [6, 6, 6, 1, 1, 1], :flags => [:verbosity => "low"])
bands_data = Dict(:kpoints => [[0.5, 0.5, 0.5, 100.],
                                [0.0, 0.0, 0.0, 100.],
                                [0.0, 0.5, 0.0, 1.]],
                  :flags => [:verbosity => "high", :nbnd => 8])

nscf_data = merge(bands_data, Dict(:kpoints => [10, 10, 10]))
calculations = [:vc_relax => (excs, scf_data), :scf => (excs, scf_data), :bands => (excs, bands_data), :nscf =>(excs, nscf_data)]

job = DFJob(name, local_dir, joinpath(testjobpath, "Pt.cif"), calculations,
      :prefix       => "$name",
      :restart_mode => "from_scratch",
      :mixing_mode  => "plain",
      :mixing_beta  => 0.7,
      :conv_thr     => 1.0e-8,
      #kwargs
      server_dir  = server_dir,
      bin_dir     = bin_dir,
      header      = header,
      pseudoset  = :test,
     )

save(job)
show(job)
@test data(job["scf"], :k_points).data == [6, 6, 6, 1, 1, 1]
@test data(job, "nscf", :k_points).data == DFControl.kgrid(10, 10, 10, :nscf)
@test all(values(job[:ecutwfc]) .== 32.0)
@test job["scf"][:prefix] == job["nscf"][:prefix] == "$name"
@test job["bands"][:verbosity] == "high"

setflags!(job, :prefix => "blabla", print=false)
@test job["scf"][:prefix] == job["nscf"][:prefix] == "blabla"
setflags!(job, :Hubbard_U => [4], print=false)
setflags!(job, :Hubbard_J => [4 4 5], print=false)
@test job["scf"][:Hubbard_U] == job["nscf"][:Hubbard_U] == [4.0]
@test job["scf"][:Hubbard_J] == job["nscf"][:Hubbard_J] == [4.0  4.0  5.0]

struct2 = DFControl.create_supercell(job.structure, 1, 2, 1)
newpositions = [DFControl.position_cart(at) for at in atoms(struct2)]
oldposition = atoms(job.structure)[1].position_cart
cell_ = DFControl.cell(job.structure)

@test atoms(job.structure)[1].position_cart == job.structure.cell' * atoms(job.structure)[1].position_cryst
@test oldposition == newpositions[1]
@test oldposition + cell_[:, 1] ∈ newpositions
@test oldposition + 2*cell_[:, 2] ∈ newpositions
@test oldposition + 1*cell_[:, 3] ∈ newpositions
@test oldposition + (1*cell_[:, 3] + 1*cell_[:, 2]) ∈ newpositions


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct LCLFilter <: Filter
        lf::Float64
        rf::Float64
        cf::Float64
        lg::Float64
        rg::Float64
        ext::Dict{String, Any}
        states::Vector{Symbol}
        n_states::Int64
    end

Parameters of a LCL filter outside the converter, the states are in the grid's reference frame

# Arguments
- `lf::Float64`: Series inductance in p.u. of converter filter, validation range: (0, nothing)
- `rf::Float64`: Series resistance in p.u. of converter filter, validation range: (0, nothing)
- `cf::Float64`: Shunt capacitance in p.u. of converter filter, validation range: (0, nothing)
- `lg::Float64`: Series inductance in p.u. of converter filter to the grid, validation range: (0, nothing)
- `rg::Float64`: Series resistance in p.u. of converter filter to the grid, validation range: (0, nothing)
- `ext::Dict{String, Any}`
- `states::Vector{Symbol}`
- `n_states::Int64`
"""
mutable struct LCLFilter <: Filter
    "Series inductance in p.u. of converter filter"
    lf::Float64
    "Series resistance in p.u. of converter filter"
    rf::Float64
    "Shunt capacitance in p.u. of converter filter"
    cf::Float64
    "Series inductance in p.u. of converter filter to the grid"
    lg::Float64
    "Series resistance in p.u. of converter filter to the grid"
    rg::Float64
    ext::Dict{String, Any}
    states::Vector{Symbol}
    n_states::Int64
end

function LCLFilter(lf, rf, cf, lg, rg, ext=Dict{String, Any}(), )
    LCLFilter(lf, rf, cf, lg, rg, ext, [:ir_cnv, :ii_cnv, :vr_filter, :vi_filter, :ir_filter, :ii_filter], 6, )
end

function LCLFilter(; lf, rf, cf, lg, rg, ext=Dict{String, Any}(), )
    LCLFilter(lf, rf, cf, lg, rg, ext, )
end

# Constructor for demo purposes; non-functional.
function LCLFilter(::Nothing)
    LCLFilter(;
        lf=0,
        rf=0,
        cf=0,
        lg=0,
        rg=0,
        ext=Dict{String, Any}(),
    )
end

"""Get LCLFilter lf."""
get_lf(value::LCLFilter) = value.lf
"""Get LCLFilter rf."""
get_rf(value::LCLFilter) = value.rf
"""Get LCLFilter cf."""
get_cf(value::LCLFilter) = value.cf
"""Get LCLFilter lg."""
get_lg(value::LCLFilter) = value.lg
"""Get LCLFilter rg."""
get_rg(value::LCLFilter) = value.rg
"""Get LCLFilter ext."""
get_ext(value::LCLFilter) = value.ext
"""Get LCLFilter states."""
get_states(value::LCLFilter) = value.states
"""Get LCLFilter n_states."""
get_n_states(value::LCLFilter) = value.n_states

"""Set LCLFilter lf."""
set_lf!(value::LCLFilter, val::Float64) = value.lf = val
"""Set LCLFilter rf."""
set_rf!(value::LCLFilter, val::Float64) = value.rf = val
"""Set LCLFilter cf."""
set_cf!(value::LCLFilter, val::Float64) = value.cf = val
"""Set LCLFilter lg."""
set_lg!(value::LCLFilter, val::Float64) = value.lg = val
"""Set LCLFilter rg."""
set_rg!(value::LCLFilter, val::Float64) = value.rg = val
"""Set LCLFilter ext."""
set_ext!(value::LCLFilter, val::Dict{String, Any}) = value.ext = val
"""Set LCLFilter states."""
set_states!(value::LCLFilter, val::Vector{Symbol}) = value.states = val
"""Set LCLFilter n_states."""
set_n_states!(value::LCLFilter, val::Int64) = value.n_states = val


"""
Constant that represents document term vector (DTV) models used in text embedding.
"""
const DTVModel{T,S} = Union{
    StringAnalysis.RPModel{S,T,<:AbstractMatrix{T},<:Integer},
    StringAnalysis.LSAModel{S,T,<:AbstractMatrix{T},<:Integer}
}

"""
Structure for document embedding using DTV's.
"""
struct DTVEmbedder{T,S} <: AbstractEmbedder{T,S}
    model::DTVModel{T,S}
    config::NamedTuple
end

DTVEmbedder(mtype::Type{<:DTVModel}, dtm, config; kwargs...) = DTVEmbedder(mtype(dtm; kwargs...), config)

# Document to vector embedding function
function __document2vec(embedder::DTVEmbedder{T,S},
                        document::Vector{String};  # a vector of sentences
                        isregex::Bool=false,
                        kwargs...  # for the unused arguments
                       )::SparseVector{T, Int} where {T,S}
    dtv_function = ifelse(isregex, dtv_regex, dtv)
    words = Vector{String}()
    for sentence in document
        for word in tokenize(sentence, method=:stringanalysis)
            push!(words, word)
        end
    end
    vocab_hash = embedder.model.vocab_hash
    model = embedder.model
    v = dtv_function(words, vocab_hash, T;
                     ngram_complexity=embedder.config.ngram_complexity,
                     tokenizer=DEFAULT_TOKENIZER,
                     lex_is_row_indices=true)
    embedded_document = StringAnalysis.embed_document(model, v)
    return embedded_document
end

function document2vec(embedder::DTVEmbedder{T,S},
                      document::Vector{String};  # a vector of sentences
                      isregex::Bool=false,
                      kwargs...  # for the unused arguments
                     )::Tuple{SparseVector{T, Int}, Bool} where {T,S}
    embedded_document = __document2vec(embedder,
                                       document;
                                       isregex=isregex,
                                       kwargs...)
    is_embedded = !iszero(embedded_document)
    # Check for OOV (out-of-vocabulary) policy
    if embedder.config.oov_policy == :large_vector && !is_embedded
        embedded_document .= T(DEFAULT_OOV_VAL)
    end
    return embedded_document, is_embedded
end

# Dimensionality function
function dimensionality(embedder::DTVEmbedder)
    # Return output dimensionality (second dim)
    return size(embedder.model)[2]
end


module CubicEoS

export AbstractEoSComponent, AbstractEoSMixture
export BrusilovskyEoSComponent, BrusilovskyEoSMixture, thermo_buffer
export molar_mass, carbon_number, name, describe, load, components # Do we need to export `load`?
export ncomponents, pressure, wilson_saturation_pressure, compressibility
export log_c_activity!, log_c_activity, log_c_activity_wj!, log_c_activity_wj
export vt_stability, vt_stability_buffer
export vt_flash

include("constants.jl")
include("types.jl")
include("interface.jl")
include("dbload.jl")
include("basic_thermo.jl")
include("chempotential.jl")
include("vt_stability.jl")
include("vt_flash.jl")
include("newton.jl")

end # module


module FixedEffects

##############################################################################
##
## Dependencies
##
##############################################################################
using Base
using LinearAlgebra
using Distributed
using CategoricalArrays
using FillArrays
using Reexport
using CUDAapi
if has_cuda()
	try
		using CuArrays
		@eval has_cuarrays() = true
	catch
		@info "CUDA is installed, but CuArrays.jl fails to load"
		@eval has_cuarrays() = false
	end
else
	has_cuarrays() = false
end
@reexport using StatsBase # used for Weights
##############################################################################
##
## Exported methods and types
##
##############################################################################

export
group,
FixedEffect,
AbstractFixedEffectSolver,
solve_residuals!,
solve_coefficients!

##############################################################################
##
## Load files
##
##############################################################################
include("utils/lsmr.jl")
include("FixedEffect.jl")
include("FixedEffectSolvers/FixedEffectLinearMap.jl")
include("FixedEffectSolvers/AbstractFixedEffectSolver.jl")
include("FixedEffectSolvers/FixedEffectSolverLSMR.jl")
include("FixedEffectSolvers/FixedEffectSolverLSMRParallel.jl")
if has_cuarrays()
	include("FixedEffectSolvers/FixedEffectSolverLSMRGPU.jl")
end

end  # module FixedEffects


@enum Building Smelter Constructor Assembler Manufacturer Refinery Miner Foundry WaterExtractor OilExtractor Packager

struct Recipe
    name::String
    out::Vector{Tuple{DataType, Float64}}
    in::Vector{Tuple{DataType, Float64}}
    building::Building
    duration::Float64
end

dataPath = joinpath(@__DIR__, "data.json")

function readData()
    data = JSON3.read(read(dataPath))

    _buildings = [data.buildings[b] for b in first.(filter(!isempty, unique(r.producedIn for r in values(data.recipes))))]
    _recipes = filter(x -> !isempty(x.producedIn), collect(values(data.recipes)))
    _products = unique(union((p.item for r in _recipes for p in r.products), (p.item for r in _recipes for p in r.ingredients)))

    classNameToProduct = Dict{String, Type{<:Product}}(data.items[p].className => nameToType(data.items[p].name) for p in _products)
    classNameToBuilding = Dict{String, Building}(b.className => nameToType(b.name) for b in _buildings)

    empty!(allRecipes)
    empty!(dictProductDependantRecipes)
    empty!(dictProductRecipes)
    empty!(baseRecipes)

    for r in _recipes
        out = [(classNameToProduct[p.item], p.amount * 60 / r.time) for p in r.products]
        in = [(classNameToProduct[p.item], p.amount * 60 / r.time) for p in r.ingredients]
        building = classNameToBuilding[only(r.producedIn)]
        push!(allRecipes, Recipe(r.name, out, in, building, r.time))
    end

    minerMK1 = data.miners.Build_MinerMk1_C
    for p in minerMK1.allowedResources
        product = classNameToProduct[p]
        qty = minerMK1.itemsPerCycle / minerMK1.extractCycleTime * 60
        push!(allRecipes, Recipe("$product", [(product, qty)], [], Miner, 60))
    end
    oilPump = data.miners.Build_OilPump_C
    for p in oilPump.allowedResources
        product = classNameToProduct[p]
        qty = oilPump.itemsPerCycle / oilPump.extractCycleTime * 60 / 1000
        push!(allRecipes, Recipe("$product", [(product, qty)], [], OilExtractor, 60))
    end
    waterPump = data.miners.Build_WaterPump_C
    for p in waterPump.allowedResources
        product = classNameToProduct[p]
        qty = waterPump.itemsPerCycle / waterPump.extractCycleTime * 60 / 1000
        push!(allRecipes, Recipe("$product", [(product, qty)], [], WaterExtractor, 60))
    end

    for r in allRecipes
        for (p, qty) in r.out
            push!(get!(dictProductRecipes, p, Set()), (r, qty))
        end
    end
    for r in allRecipes
        for (p, qty) in r.in
            push!(get!(dictProductDependantRecipes, p, Set()), (r, qty))
        end
    end

    for p in subtypes(Product)
        if p in harvestedProducts
            dictProductRecipes[p] = Set()
        end
        if !haskey(dictProductDependantRecipes, p)
            dictProductDependantRecipes[p] = Set()
        end
        if !haskey(dictProductRecipes, p)
            @warn "No recipe found to make $p"
        end
    end

    append!(baseRecipes, filter(x -> !occursin("Alternate", x.name), allRecipes))
    nothing
end

nameToType(s::AbstractString) = getfield(Satisfactory, Symbol(replace(s, r" |-|\." => ""))) # remove spaces, dashes and dots

recipes(::Type{T}) where T <: Product = dictProductRecipes[T]
recipes(::T) where T <: Product = recipes(T)

dependantRecipes(::Type{T}) where T <: Product = dictProductDependantRecipes[T]
dependantRecipes(::T) where T <: Product = dictProductDependantRecipes(T)

import Base.show, Base.==, Base.hash
show(io::IO, r::Recipe) = print(io, "$(r.name)")
hash(r::Recipe, h::UInt) = hash(r.name, h)
==(r1::Recipe, r2::Recipe) = r1.name == r2.name


# TODO: this should end up in BioSequences.jl?

"Extract the element stored in a packed bitarray referred to by bidx."
@inline function BioSequences.extract_encoded_element(bidx::BioSequences.BitIndex{N,W}, data::NTuple{n,W}) where {N,n,W}
    @inbounds chunk = data[BioSequences.index(bidx)]
    offchunk = chunk >> (BioSequences.bitwidth(W) - N - BioSequences.offset(bidx))
    return offchunk & BioSequences.bitmask(bidx)
end

"""
    _cliphead(by::Integer, head::UInt64, tail...)

A method used to mask the first `by` MSB's in `head`, before catting it with
tail to return a NTuple.

This is used internally to mask the first `by` bits in the first word of a
NTuple of UInt64's.

Notably it's used when constructing a Kmer from an existing NTuple of UInt64
"""
@inline function _cliphead(by::Integer, head::UInt64, tail...)
    return (head & (typemax(UInt64) >> by), tail...)
end

#=
rightshift_carry & leftshift_carry

These methods are micro-optimised (or should be!!!) for shifting the bits in
an NTuple of unsigned integers, carrying the bits "shifted off" one word
over to the next word. The carry can also be "seeded" so as other methods like
pushfirst and pushlast can be efficiently implemented without duplication of code
or less efficient implementations that first shift and then insert an element.
=#

@inline function rightshift_carry(x::NTuple{N,UInt64}, nbits::Integer, prevcarry = zero(UInt64)) where {N}
    return _rightshift_carry(nbits, prevcarry, x...)
end

@inline function _rightshift_carry(nbits::Integer, carry::UInt64, head::UInt64, tail...)
    return ((head >> nbits) | carry, _rightshift_carry(nbits, (head & ((one(UInt64) << nbits) - 1)) << (64 - nbits), tail...)...)
end

@inline _rightshift_carry(nbits::Integer, carry::UInt64) = ()

@inline function leftshift_carry(x::NTuple{N,UInt64}, nbits::Integer, prevcarry::UInt64 = zero(UInt64)) where {N}
    _, newbits = _leftshift_carry(nbits, prevcarry, x...)
    return newbits
end

@inline function _leftshift_carry(nbits::Integer, prevcarry::UInt64, head::UInt64, tail...)
    carry, newtail = _leftshift_carry(nbits, prevcarry, tail...)
    return head >> (64 - nbits), ((head << nbits) | carry, newtail...)
end

@inline _leftshift_carry(nbits::Integer, prevcarry::UInt64) = prevcarry, ()

@inline function _reverse(bpe::BioSequences.BitsPerSymbol{N}, head::UInt64, tail...) where {N}
    return (_reverse(bpe, tail...)..., BioSequences.reversebits(head, bpe))
end

@inline _reverse(::BioSequences.BitsPerSymbol{N}) where {N} = ()

#=
@inline function _reverse(f::F, bpe::BioSequences.BitsPerSymbol{N}, head::UInt64, tail...) where {N,F<:Function}
    return (_reverse(f, bpe, tail...)..., f(reversebits(head, bpe)))
end

@inline _reverse(f::F, ::BioSequences.BitsPerSymbol{N}) where {N,F<:Function} = ()
=#


module DeepCFR

using CounterfactualRegret
const CFR = CounterfactualRegret
using Random
using ProgressMeter
using Flux
using FileIO
using RecipesBase

include("memory.jl")
include("solver.jl")
include("ESCFR.jl")
include("OSCFR.jl")
include("train.jl")
include("check.jl")
include("callback.jl")
include("fitting.jl")

export DeepCFRSolver

end


module Vann

using BlackBoxOptim
using Distributions
using DataAssim
using PyPlot
using CSV
using DataFrames

abstract Hydro
abstract Snow

export Hbv, Gr4j
export TinBasic, TinStandard

export run_timestep, run_timestep
export load_data, crop_data, load_operational
export init_states, get_states
export run_model_calib, calib_wrapper
export run_model
export get_param_range, assign_param
export get_input
export plot_sim
export epot_zero, epot_monthly
export particle_filter, enkf_filter
export split_prec, compute_ddf, pot_melt
export kge, nse

include("models/gr4j.jl")
include("models/hbv.jl")
include("models/tinbasic.jl")
include("models/tinstandard.jl")
include("utils_calib.jl")
include("utils_data_assim.jl")
include("utils_data.jl")
include("utils_epot.jl")
include("utils_model.jl")
include("utils_snow.jl")
include("utils_plot.jl")

end


struct GradFreeLocalOptimizer <: LocalOptimizer
    algo::Symbol
    options::Dict{Symbol,Any}
end

need_gradient(::GradFreeLocalOptimizer) = false

function BOBYQAOptimizer(; options...)
    GradFreeLocalOptimizer(:LN_BOBYQA, Dict{Symbol,Any}(options))
end

function minimize(o::GradFreeLocalOptimizer, f, lb, ub, x0)
    opt = create_nlopt(o.algo, lb, ub; min_objective = f, o.options...)
    fx, x, ret = NLopt.optimize(opt, x0)
    ret == :FORCED_STOP && throw(InterruptException())
    return fx, x, ret
end

function maximize(o::GradFreeLocalOptimizer, f, lb, ub, x0)
    opt = create_nlopt(o.algo, lb, ub; max_objective = f, o.options...)
    fx, x, ret = NLopt.optimize(opt, x0)
    ret == :FORCED_STOP && throw(InterruptException())
    return fx, x, ret
end


@testset "MurmurHash3" begin
      tests = [
      Dict{String,Any}("x86_32"=>"00000000","text"=>""),
      Dict{String,Any}("x86_32"=>"087fcd5c","seed"=>42,"text"=>""),
      Dict{String,Any}("x86_32"=>"1cb30d2d","text"=>"!eDoZj \$XVe>vtaxn<x^ qU]m([aWSI^Hy>@E#hX}v mbkz#gB"),
      Dict{String,Any}("x86_32"=>"e6aa80e1","seed"=>10,"text"=>"o.SH}f]tVS\$deSKiOgE?#V %EP&]V!oyjlq}{BO)\$cz&fRGPrGUtz@b]{C"),
      Dict{String,Any}("x86_32"=>"100949f7","text"=>"[)<{vbjRgcp"),
      Dict{String,Any}("x86_32"=>"79b365c1","text"=>"R"),
      Dict{String,Any}("x86_32"=>"e70207f0","seed"=>5489776,"text"=>"mB"),
      Dict{String,Any}("x86_32"=>"c4b773e5","seed"=>111117,"text"=>"mExonGybCN{n# iVmrVI!?ahk]q&}BbHyx"),
      Dict{String,Any}("x86_32"=>"92bbd066","text"=>"TqDa}YvgP!K&P\$vsbeGt<Q{zVuBHQ>h}tBxvVlYVO\$LAgxNIik{)WeJtX]i?yom!.KoNB*sOUJv/.pF,lncuM@EXBTgw%VuY>\$!l!TFGe?HEO>NFiK,RSS%o#)lhy&Mw{QFGN![qKcM!eSXxfQ&eYeTtq[m<h#rvwuaNFTv e@kXzD##[%.JDYlKhQzrGV{/bB)^dCT&GDum#V{cVgAn)xjahnT[E\$<)yyElAhV/imQ!P%JL\$&tB%oaj*L\$<qzsD\$*oKT^/ykA([\$(sI}G{blBT>SSo UnOlclQkAJ(@FLXP&w/EKY?Kn}ndbY\$!#IC%IlcWF]^UX*iVejGrzX%ox@b{ &KUm>q>dP,k\$(&mmE,f.Kp/R[.FnjDPJ?FhniNuAi<o%*m./BjypWIaao*&@<kGICn!qP(uTjavSxzKfaT(H]!NmRWgejf?#q%l&ulr oxLv])NsNM>Q#bEAXy[lZXod?Yeu][SYaO DZ^doDLmgrCuBBWy"),
      Dict{String,Any}("x86_32"=>"8ba3f73c","text"=>"#Xu"),
      Dict{String,Any}("x86_32"=>"fb73894d","seed"=>1,"text"=>"MMBh*"),
      Dict{String,Any}("x86_32"=>"c83626b3","text"=>"QDJ{gyEAJMbhd\$,iRU(!uv>zCpzIDjcel F#/?/Tux?nX{^n*?a"),
      Dict{String,Any}("x86_32"=>"91032392","text"=>"GeBcX^k?C,]d*nx##RL*#*lZ{ &b{.I?Z]#qMyMiMT"),
      Dict{String,Any}("x86_32"=>"1b9cb3e5","seed"=>160753,"text"=>"KA.f#)K?#%by#G(w?ag^at{jbW)Q(QJtvX\$Gs{@UMWhhzYdX# A{uPrvm#"),
      Dict{String,Any}("x86_32"=>"54c379d4","seed"=>2,"text"=>"%w.I#vFp"),
      Dict{String,Any}("x86_32"=>"950a02a4","seed"=>0,"text"=>"th!S?L*A[VUtZZ)[%Zw/&!ZO!MsJDcFjV/uGb^yuZLZ\$)mlf}#N\$<b,!k t@R"),
      Dict{String,Any}("x86_32"=>"c7d8827f","seed"=>999983,"text"=>"IqP ]t (OrK"),
      Dict{String,Any}("x86_32"=>"704a49be","text"=>"PKqG47bFcNTKKTE"),
      Dict{String,Any}("x86_32"=>"52042560","text"=>"Qs7i2oN0kr10dOf")]
      for test in tests
            if "seed" in keys(test)
                  x86_32 = lpad(string(Bitcoin.Murmur3.hash32(test["text"], test["seed"]), base=16), 8, "0")
            else
                  x86_32 = lpad(string(Bitcoin.Murmur3.hash32(test["text"]), base=16), 8, "0")
            end
            @test test["x86_32"] == x86_32
      end
end


struct EconomicDispatchProblem <: PowerSimulationsOperationsProblem end
struct UnitCommitmentProblem <: PowerSimulationsOperationsProblem end
struct AGCReserveDeployment <: PowerSimulationsOperationsProblem end

function _generic_template(; kwargs...)
    network = get(kwargs, :network, CopperPlatePowerModel)

    devices = get(
        kwargs,
        :devices,
        Dict(
            :Generators => DeviceModel(PSY.ThermalStandard, ThermalBasicUnitCommitment),
            :RE => DeviceModel(PSY.RenewableDispatch, RenewableFullDispatch),
            :DistRE => DeviceModel(PSY.RenewableFix, FixedOutput),
            :Hydro => DeviceModel(PSY.HydroEnergyReservoir, HydroDispatchRunOfRiver),
            :HydroROR => DeviceModel(PSY.HydroDispatch, HydroDispatchRunOfRiver),
            :Loads => DeviceModel(PSY.PowerLoad, StaticPowerLoad),
            :ILoads => DeviceModel(PSY.InterruptibleLoad, InterruptiblePowerLoad),
        ),
    )

    branches = get(
        kwargs,
        :branches,
        Dict(
            :L => DeviceModel(PSY.Line, StaticLine),
            :T => DeviceModel(PSY.Transformer2W, StaticTransformer),
            :TT => DeviceModel(PSY.TapTransformer, StaticTransformer),
            :DC => DeviceModel(PSY.HVDCLine, HVDCDispatch),
        ),
    )

    services = get(
        kwargs,
        :services,
        Dict(
            :ReserveUp =>
                ServiceModel(PSY.VariableReserve{PSY.ReserveUp}, RangeReserve),
            :ReserveDown =>
                ServiceModel(PSY.VariableReserve{PSY.ReserveDown}, RangeReserve),
        ),
    )

    template = OperationsProblemTemplate(network, devices, branches, services)

    return template
end
"""
    template_unit_commitment(; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for a Unit Commitment
problem.

# Example
```julia
template = template_unit_commitment()
```

# Accepted Key Words
- `network::Type{<:PM.AbstractPowerModel}` : override default network model settings
- `devices::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `branches::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `services::Dict{Symbol, ServiceModel}` : override default `ServiceModel` settings
"""
function template_unit_commitment(; kwargs...)
    template = _generic_template(; kwargs...)
    return template
end

"""
    template_economic_dispatch(; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for an Economic Dispatch
problem.

# Example
```julia
template = template_economic_dispatch()
```

# Accepted Key Words
- `network::Type{<:PM.AbstractPowerModel}` : override default network model settings
- `devices::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `branches::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `services::Dict{Symbol, ServiceModel}` : override default `ServiceModel` settings
"""
function template_economic_dispatch(; kwargs...)
    devices = get(
        kwargs,
        :devices,
        Dict(
            :Generators => DeviceModel(PSY.ThermalStandard, ThermalRampLimited),
            :RE => DeviceModel(PSY.RenewableDispatch, RenewableFullDispatch),
            :DistRE => DeviceModel(PSY.RenewableFix, FixedOutput),
            :Hydro =>
                DeviceModel(PSY.HydroEnergyReservoir, HydroDispatchReservoirBudget),
            :HydroROR => DeviceModel(PSY.HydroDispatch, FixedOutput),
            :Loads => DeviceModel(PSY.PowerLoad, StaticPowerLoad),
            :ILoads => DeviceModel(PSY.InterruptibleLoad, InterruptiblePowerLoad),
        ),
    )

    services = get(kwargs, :services, Dict())

    template = _generic_template(devices = devices, services = services; kwargs...)

    return template
end

"""
    template_agc_reserve_deployment(; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for an AGC Reserve Deplyment Problem. This model doesn't support customization

# Example
```julia
template = agc_reserve_deployment()
```
"""
function template_agc_reserve_deployment(; kwargs...)
    if !isempty(kwargs)
        throw(ArgumentError("AGC Template doesn't currently support customization"))
    end
    devices = Dict(
        :Generators => DeviceModel(PSY.ThermalStandard, FixedOutput),
        :Ren => DeviceModel(PSY.RenewableDispatch, FixedOutput),
        :Loads => DeviceModel(PSY.PowerLoad, StaticPowerLoad),
        :Hydro => DeviceModel(PSY.HydroEnergyReservoir, FixedOutput),
        :HydroROR => DeviceModel(PSY.HydroDispatch, FixedOutput),
        :RenFx => DeviceModel(PSY.RenewableFix, FixedOutput),
        :Regulation_thermal => DeviceModel(
            PSY.RegulationDevice{PSY.ThermalStandard},
            DeviceLimitedRegulation,
        ),
        :Regulation_hydro_dispatch => DeviceModel(
            PSY.RegulationDevice{PSY.HydroDispatch},
            ReserveLimitedRegulation,
        ),
        :Regulation_hydro_reservoir => DeviceModel(
            PSY.RegulationDevice{PSY.HydroEnergyReservoir},
            ReserveLimitedRegulation,
        ),
    )
    services = Dict(:AGC => ServiceModel(PSY.AGC, PIDSmoothACE))
    template = _generic_template(
        network = PSI.AreaBalancePowerModel,
        devices = devices,
        branches = Dict(),
        services = services;
        kwargs...,
    )
end

"""
    EconomicDispatchProblem(system::PSY.System; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for an EconomicDispatch
problem. Uses the template to create an `OperationsProblem`.

# Example
```julia
ed_problem = EconomicDispatchProblem(system)
```

# Accepted Key Words
- `network::Type{<:PM.AbstractPowerModel}` : override default network model settings
- `devices::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `branches::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `services::Dict{Symbol, ServiceModel}` : override default `ServiceModel` settings
- Key word arguments supported by `OperationsProblem`
"""

function EconomicDispatchProblem(system::PSY.System; kwargs...)
    kwargs = Dict(kwargs)
    template_kwargs = Dict()
    for kw in setdiff(keys(kwargs), OPERATIONS_ACCEPTED_KWARGS)
        template_kwargs[kw] = pop!(kwargs, kw)
    end

    template = template_economic_dispatch(; template_kwargs...)
    op_problem = OperationsProblem(EconomicDispatchProblem, template, system; kwargs...)
    return op_problem
end

"""
    UnitCommitmentProblem(system::PSY.System; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for a Unit Commitment
problem. Uses the template to create an `OperationsProblem`.

# Example
```julia
uc_problem = UnitCommitmentProblem(system)
```

# Accepted Key Words
- `network::Type{<:PM.AbstractPowerModel}` : override default network model settings
- `devices::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `branches::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `services::Dict{Symbol, ServiceModel}` : override default `ServiceModel` settings
- Key word arguments supported by `OperationsProblem`
"""
function UnitCommitmentProblem(system::PSY.System; kwargs...)
    kwargs = Dict(kwargs)
    template_kwargs = Dict()
    for kw in setdiff(keys(kwargs), OPERATIONS_ACCEPTED_KWARGS)
        template_kwargs[kw] = pop!(kwargs, kw)
    end

    template = template_unit_commitment(; template_kwargs...)
    op_problem = OperationsProblem(UnitCommitmentProblem, template, system; kwargs...)
    return op_problem
end

"""
    AGCReserveDeployment(system::PSY.System; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for an AGC Reserve Deplyoment Problem.
Uses the template to create an `OperationsProblem`.

# Example
```julia
agc_problem = AGCReserveDeployment(system)
```

# Accepted Key Words
- Key word arguments supported by `OperationsProblem`
"""
function AGCReserveDeployment(system::PSY.System; kwargs...)
    kwargs = Dict(kwargs)
    template_kwargs = Dict()
    for kw in setdiff(keys(kwargs), OPERATIONS_ACCEPTED_KWARGS)
        template_kwargs[kw] = pop!(kwargs, kw)
    end

    template = template_agc_reserve_deployment(; template_kwargs...)
    op_problem = OperationsProblem(AGCReserveDeployment, template, system; kwargs...)
    return op_problem
end

"""
    run_unit_commitment(system::PSY.System; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for a Unit Commitment
problem. Uses the template to create an `OperationsProblem`. Solves the created operations problem.

# Example
```julia
results = run_unit_commitment(system; optimizer = optimizer)
```

# Accepted Key Words
- `network::Type{<:PM.AbstractPowerModel}` : override default network model settings
- `devices::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `branches::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `services::Dict{Symbol, ServiceModel}` : override default `ServiceModel` settings
- `optimizer::JuMP Optimizer` : An optimizer is a required key word
- `savepath::AbstractString`  : Path to save results
- Key word arguments supported by `OperationsProblem`
"""

function run_unit_commitment(sys::PSY.System; kwargs...)
    solve_kwargs = Dict()
    for kw in OPERATIONS_SOLVE_KWARGS
        haskey(kwargs, kw) && (solve_kwargs[kw] = kwargs[kw])
    end
    op_problem = UnitCommitmentProblem(sys; kwargs...)
    results = solve!(op_problem; solve_kwargs...)
    return results
end

"""
    run_economic_dispatch(system::PSY.System; kwargs...)

Creates an `OperationsProblemTemplate` with default DeviceModels for an EconomicDispatch
problem. Uses the template to create an `OperationsProblem`.

# Example
```julia
results = run_economic_dispatch(system; optimizer = optimizer)
```

# Accepted Key Words
- `network::Type{<:PM.AbstractPowerModel}` : override default network model settings
- `devices::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `branches::Dict{Symbol, DeviceModel}` : override default `DeviceModel` settings
- `services::Dict{Symbol, ServiceModel}` : override default `ServiceModel` settings
- `optimizer::JuMP optimizer` : a JuMP optimizer is a required key word
- `savepath::AbstractString`  : Path to save results
- Key word arguments supported by `OperationsProblem`
"""

function run_economic_dispatch(sys::PSY.System; kwargs...)
    solve_kwargs = Dict()
    for kw in OPERATIONS_SOLVE_KWARGS
        haskey(kwargs, kw) && (solve_kwargs[kw] = kwargs[kw])
    end
    op_problem = EconomicDispatchProblem(sys; kwargs...)
    results = solve!(op_problem; solve_kwargs...)
    return results
end


const CV_CN_MAX = 512
const CV_CN_SHIFT = 3
const CV_DEPTH_MAX = (1 << CV_CN_SHIFT)

const CV_8U = 0
const CV_8S = 1
const CV_16U = 2
const CV_16S = 3
const CV_32S = 4
const CV_32F = 5
const CV_64F = 6

const CV_MAT_DEPTH_MASK = (CV_DEPTH_MAX - 1)
CV_MAT_DEPTH(flags) = ((flags) & CV_MAT_DEPTH_MASK)

CV_MAKETYPE(depth,cn) = (CV_MAT_DEPTH(depth) + (((cn)-1) << CV_CN_SHIFT))
CV_MAKE_TYPE = CV_MAKETYPE

function cpp_to_julia(mat::CxxMat)
    rets = jlopencv_core_Mat_mutable_data(mat)
    if rets[2] == CV_MAKE_TYPE(CV_8U, rets[3])
        dtype = UInt8
    elseif rets[2]==CV_MAKE_TYPE(CV_8S, rets[3])
        dtype = Int8
    elseif rets[2]==CV_MAKE_TYPE(CV_16U, rets[3])
        dtype = UInt16
    elseif rets[2]==CV_MAKE_TYPE(CV_16S, rets[3])
        dtype = Int16
    elseif rets[2]==CV_MAKE_TYPE(CV_32S, rets[3])
        dtype = Int32
    elseif rets[2]==CV_MAKE_TYPE(CV_32F, rets[3])
        dtype = Float32
    elseif rets[2]==CV_MAKE_TYPE(CV_64F, rets[3])
        dtype = Float64
    else
        error("Bad type returned from OpenCV")
    end
    steps = [rets[6]/sizeof(dtype), rets[7]/sizeof(dtype)]
    # println(steps[1]/rets[3], steps[2]/rets[3]/rets[4])
    #TODO: Implement views when steps do not result in continous memory
    arr = Base.unsafe_wrap(Array{dtype, 3}, Ptr{dtype}(rets[1].cpp_object), (rets[3], rets[4], rets[5]))

    #Preserve Mat so that array allocated by C++ isn't deallocated
    return Mat{dtype}(mat, arr)
end

function julia_to_cpp(img::InputArray)
    if typeof(img) <: CxxMat
        return img
    end
    steps = 0
    try
        steps = strides(img)
    catch
        # Copy array since array is not strided
        img = img[:, :, :]
        steps = strides(img)
    end

    if steps[1] <= steps[2] <= steps[3] && steps[1]==1
        steps_a = Array{size_t, 1}()
        ndims_a = Array{Int32, 1}()
        sz = sizeof(eltype(img))
        push!(steps_a, UInt64(steps[3]*sz))
        push!(steps_a, UInt64(steps[2]*sz))
        push!(steps_a, UInt64(steps[1]*sz))

        push!(ndims_a, Int32(size(img)[3]))
        push!(ndims_a, Int32(size(img)[2]))
        if eltype(img) == UInt8
            return CxxMat(2, pointer(ndims_a), CV_MAKE_TYPE(CV_8U, size(img)[1]), Ptr{Nothing}(pointer(img)), pointer(steps_a))
        elseif eltype(img) == UInt16
            return CxxMat(2, pointer(ndims_a), CV_MAKE_TYPE(CV_16U, size(img)[1]), Ptr{Nothing}(pointer(img)), pointer(steps_a))
        elseif eltype(img) == Int8
            return CxxMat(2, pointer(ndims_a), CV_MAKE_TYPE(CV_8S, size(img)[1]), Ptr{Nothing}(pointer(img)), pointer(steps_a))
        elseif eltype(img) == Int16
            return CxxMat(2, pointer(ndims_a), CV_MAKE_TYPE(CV_16S, size(img)[1]), Ptr{Nothing}(pointer(img)), pointer(steps_a))
        elseif eltype(img) == Int32
            return CxxMat(2, pointer(ndims_a), CV_MAKE_TYPE(CV_32S, size(img)[1]), Ptr{Nothing}(pointer(img)), pointer(steps_a))
        elseif eltype(img) == Float32
            return CxxMat(2, pointer(ndims_a), CV_MAKE_TYPE(CV_32F, size(img)[1]), Ptr{Nothing}(pointer(img)), pointer(steps_a))
        elseif eltype(img) == Float64
            return CxxMat(2, pointer(ndims_a), CV_MAKE_TYPE(CV_64F, size(img)[1]), Ptr{Nothing}(pointer(img)), pointer(steps_a))
        end
    else
        # Copy array, invalid config
        return julia_to_cpp(img[:, :, :])
    end
end

function julia_to_cpp(var::Array{T, 1}) where {T <: InputArray}
    ret = CxxWrap.StdVector{CxxMat}()
    for x in var
        push!(ret, julia_to_cpp(x))
    end
    return ret
end

function cpp_to_julia(var::CxxWrap.StdVector{T}) where {T <: CxxMat}
    ret = Array{Mat, 1}()
    for x in var
        push!(ret, cpp_to_julia(x))
    end
    return ret
end


# specific to Julia, that is, related to the output file format (language).

#=
F: read the file containing constants
I:
O:
BA
=#
function jl_include_constants_from_literature(src_file_name, pad_string)
  # create src_buffer -
  src_buffer = ""
  # path to distrubtion -
  path_to_src_file = src_file_name
  open(path_to_src_file,"r") do src_file
    for line in eachline(src_file)
      src_buffer *= pad_string*"$line"
    end
  end
  return src_buffer
end

#=
F: generate kinetic buffer
I:
O: return monod affinity constant symbol, mRNA species array,
  protein species array, W_string_array, disassociation_const_string_array.
BA
=#
function jl_build_kinetics_buffer(all_species_dict::Dict{String, Int},
  all_rnx_list::Array, all_txtl_dict::Dict, sys2user::Dict)
  kinetics = "function calculate_kinetics(X, data_dictionary)" *
             "\n\n\t# load all species dictionary" *
             "\n\tall_species_dict = data_dictionary[\"all_species_dict\"]  # String-->Int"
  # for signaling
  kinetics *= "\n\n\t# generate kinetics for signaling, assume saturation kinetics"
  totalRnxNo = length(all_rnx_list)
  kinetics *= "\n\t# load Monod affinity constants" *
              "\n\tMonodK = data_dictionary[\"MonodAffinityConstantDict\"]  # String-->Float"
  MonodAffinityConstant_String_Array = Array{String,1}()  # collect monod affinity constant symbol
  kinetics *= "\n\t# load reaction kinetic constants" *
              "\n\tkcat = data_dictionary[\"kcat_signaling\"]  # in order of rnx" *
              "\n\n\t# write reaction rate equations" *
              "\n\trnx_rate_vector = zeros($totalRnxNo)"
  for (index, rnx) in enumerate(all_rnx_list)  # go thru every rnx
    tmp_rnx_rate = "kcat[$index]"
    if isdefined(rnx, :catalysts)  # enzyme
      for token in rnx.catalysts
        tmp_rnx_rate *= "*X[$(all_species_dict[token.oriBioName])]"
      end
    end
    if isdefined(rnx,:reactants)  # substrate
      for token in rnx.reactants
        tmp_rnx_rate *= "*X[$(all_species_dict[token.oriBioName])]/(MonodK[\"MonodK~rnx$(index)~$(token.oriBioName)\"]+X[$(all_species_dict[token.oriBioName])])"
        push!(MonodAffinityConstant_String_Array, "MonodK~rnx$(index)~$(token.oriBioName)")
      end
    end
    kinetics *= "\n\trnx_rate_vector[$index] = "*tmp_rnx_rate
  end

  # for TXTL
  W_string_array = Array{String,1}()  # W: weight of protein action on transcription
  disassociation_const_string_array = Array{String,1}()  # disassociation constant in tranfer function
  kinetics *= "\n\n\t# generate kinetics & control terms for TXTL" *
              "\n\t# load data from data dictionary" *
              "\n\tW_value_dict = data_dictionary[\"W_value_dict\"]  #String-->Float" *
              "\n\tcoop = data_dictionary[\"cooperativity\"]" *
              "\n\tdisassociation_constant_dict = data_dictionary[\"transferFunctionDisassociationConstantDict\"]" *
              "\n\tbackground_control_term = data_dictionary[\"backgroundGeneExpressionControlTermDict\"]" *
              "\n\tkcatTX = data_dictionary[\"kcatTranscription\"]" *
              "\n\tRNAPconc = data_dictionary[\"RNAPConcentration\"]" *
              "\n\tgeneConc = data_dictionary[\"avgGeneConcentration\"]" *
              "\n\tsaturationTX = data_dictionary[\"transcriptionSaturationConstant\"]" *
              "\n\tkcatTL = data_dictionary[\"kcatTranslation\"]" *
              "\n\tRIBOconc = data_dictionary[\"RIBOConcentration\"]" *
              "\n\tsaturationTL = data_dictionary[\"translationSaturationConstant\"]"
  totalTXTLNo = length(all_txtl_dict)
  kinetics *= "\n\n\t# write control terms, transcription rate equations, and translation rate equations" *
             "\n\tTX_control_term = Dict{String, Float64}()  # control term" *
             "\n\tTX_rate_vector = Dict{String, Float64}()  # transcription rate" *
             "\n\tTL_rate_vector = Dict{String, Float64}()  # translation rate"
  for (key, txtl) in all_txtl_dict # go thru every txtl
    tmp_protein_string = replace(key, sys2user["MRNA"] => sys2user["PROTEIN"], count=1)
    kinetics *= "\n\t# $key and $tmp_protein_string"
    up_factors_array = Array{String,1}()  # collection of upregulation factors name: targetedmRNA_factor(s)
    if !isempty(txtl.activationProtein)  # upregulation
      for token_array in txtl.activationProtein
        tmp_up = ""
        tmp_up_name = key*"_"
        for token in token_array
          tmp_W = "W~$(token.oriBioName)~$(key)"
          tmp_up *= "W_value_dict[\"$(tmp_W)\"] * X[$(all_species_dict[token.oriBioName])]^coop/((disassociation_constant_dict[\"KD~$(token.oriBioName)\"])^coop+X[$(all_species_dict[token.oriBioName])]^coop)*"
          push!(W_string_array, tmp_W)
          push!(disassociation_const_string_array, "KD~$(token.oriBioName)")
          tmp_up_name *= token.oriBioName*"_"
        end
        tmp_up = chop(tmp_up)
        tmp_up_name = chop(tmp_up_name)
        push!(up_factors_array, tmp_up_name)
        kinetics *= "\n\t$(tmp_up_name) = $(tmp_up)"
      end
    end
    down_factors_array = Array{String,1}()  # collection of downregulation factors name: targetedmRNA_factor(s)
    if !isempty(txtl.inhibitionProtein)  # downregulation
      for token_array in txtl.inhibitionProtein
        tmp_down = ""
        tmp_down_name = key*"_"
        for token in token_array
          tmp_W = "W~$(token.oriBioName)~$(key)"
          tmp_down *= "W_value_dict[\"$(tmp_W)\"] * X[$(all_species_dict[token.oriBioName])]^coop/((disassociation_constant_dict[\"KD~$(token.oriBioName)\"])^coop+X[$(all_species_dict[token.oriBioName])]^coop)*"
          push!(W_string_array, tmp_W)
          push!(disassociation_const_string_array, "KD~$(token.oriBioName)")
          tmp_down_name *= token.oriBioName*"_"
        end
        tmp_down = chop(tmp_down)
        tmp_down_name = chop(tmp_down_name)
        push!(down_factors_array, tmp_down_name)
        kinetics *= "\n\t$(tmp_down_name) = $(tmp_down)"
      end
    end
    # combine to form up and down, then merge up and down to form control term
    if !isempty(up_factors_array) && !isempty(down_factors_array)
      up_action = ""
      for up in up_factors_array
        up_action *= up*" +"
      end
      kinetics *= "\n\tup_action_on_$key = $(chop(up_action))"
      down_action = ""
      for down in down_factors_array
        down_action *= down*" +"
      end
      kinetics *= "\n\tdown_action_on_$key = $(chop(down_action))" *
                  "\n\tTX_control_term[\"$key\"] = (background_control_term[\"$key\"] + up_action_on_$key)/(1 + background_control_term[\"$key\"] + up_action_on_$key + down_action_on_$key)"
    elseif !isempty(up_factors_array)
      up_action = ""
      for up in up_factors_array
        up_action *= up*" +"
      end
      kinetics *= "\n\tup_action_on_$key = $(chop(up_action))" *
                  "\n\tTX_control_term[\"$key\"] = (background_control_term[\"$key\"] + up_action_on_$key)/(1 + background_control_term[\"$key\"] + up_action_on_$key)"
    elseif !isempty(down_factors_array)
      down_action = ""
      for down in down_factors_array
        down_action *= down*" +"
      end
      kinetics *= "\n\tdown_action_on_$key = $(chop(down_action))" *
                  "\n\tTX_control_term[\"$key\"] = background_control_term[\"$key\"]/(1 + background_control_term[\"$key\"] + down_action_on_$key)"
    else
      # seems impossible, a TXTL dict is constructed b/c something acts on the key
    end
    kinetics *= "\n\tTX_rate_vector[\"$key\"] = TX_control_term[\"$key\"]*kcatTX*RNAPconc*geneConc/(saturationTX+geneConc)" *
                "\n\tTL_rate_vector[\"$tmp_protein_string\"] = kcatTL*RIBOconc*X[$(all_species_dict[key])]/(saturationTL+X[$(all_species_dict[key])])"
  end

  kinetics *= "\n\n\treturn (rnx_rate_vector, TX_rate_vector, TL_rate_vector)" *
              "\nend"
  return (kinetics, MonodAffinityConstant_String_Array, W_string_array, disassociation_const_string_array)
end

function jl_build_data_dictionary_buffer(host_type::AbstractString, all_species_array::Array,
  all_species2index_dict::Dict,
  rnx_species_array::Array, all_rnx_list::Array, all_txtl_dict::Dict,
  Monod_affinity_constant_array::Array, W_string_array::Array,
  disassociation_const_array::Array, mRNA_species_array::Array, protein_species_array::Array)
  # data dictionary
  buffer = "function generate_model_parameters_dictionary()"
  # stoichiometry
  buffer *= "\n\t# load stoichiometry"
  buffer *= "\n\tstoichiometric_matrix = readdlm(\"./stoichiometry.dat\")"

  No_species = length(all_species_array)
  # initial concentration
  buffer *= "\n\n\t# initial species concentration" *
    "\n\tinitial_condition = [\n\t\t"
  buffer *= join(["0.0,  # $(i)  $(all_species_array[i])" for i = 1:(No_species - 1)], "\n\t\t")
  buffer *= "\n\t\t0.0   # $(No_species) $(all_species_array[No_species])"
  buffer *= "\n\t]"

  # all species dictionary
  buffer *= "\n\n\t# all species dictionary"
  buffer *= "\n\tall_species_dict = Dict{String, Int64}()  # String --> Int"
  #all_species_dict = Dict{String, Int64}()  # create all species dict: string --> int64
  for (st, index) in all_species2index_dict
    buffer *= "\n\tall_species_dict[\"$st\"] = $index"
    #all_species_dict[st] = index
  end
  # all species dictionary_reversed
  buffer *= "\n\n\t# all species dictionary_reversed"
  buffer *= "\n\tall_species_reversed_dict = Dict{Int64, String}()  # Int --> String"
  buffer *= "\n\tfor (key,val) in all_species_dict"
  buffer *= "\n\t\tall_species_reversed_dict[val] = key"
  buffer *= "\n\tend"
  # all rnx species array
  buffer *= "\n\n\t# all rnx species array"
  buffer *= "\n\trnx_species_array::Array{String,1} = ["
  for st in rnx_species_array
    buffer *= "\n\t\t\"$st\","
  end
  buffer = chop(buffer)*"\n\t]"
  # all mRNA species array
  buffer *= "\n\n\t# all mRNA species array"
  buffer *= "\n\tmRNA_species_array::Array{String,1} = ["
  for st in mRNA_species_array
    buffer *= "\n\t\t\"$st\","
  end
  buffer = chop(buffer)*"\n\t]"
  # all protein species array
  buffer *= "\n\n\t# all protein species array"
  buffer *= "\n\tprotein_species_array::Array{String,1} = ["
  for st in protein_species_array
    buffer *= "\n\t\t\"$st\","
  end
  buffer = chop(buffer)*"\n\t]"

  # signaling reaction kinetic constants, reaction name: reactant(s)<rnx type:catalyst(s)>product(s)
  buffer *= "\n\n\t# signaling reaction kinetic constants, reaction name: reactant(s)<rnx type:catalyst(s)>product(s)"
  buffer *= "\n\tkcat_signaling = ones($(length(all_rnx_list)))  # kcat[#reaction]: reaction name"
  for (id, rnx) in enumerate(all_rnx_list)
    buffer*= "\n\tkcat_signaling[$id] = 1e-4  # kcat: $(rnx.rnxName)"
  end
  # Monod affinity constant in signaling reaction, nomenclature: MonodK_#reaction_Substrate
  buffer *= "\n\n\t# Monod affinity constant in signaling reaction, nomenclature: MonodK_#reaction_Substrate"
  buffer *= "\n\tMonodAffinityConstantDict = Dict{String, Float64}()"
  for MK in Monod_affinity_constant_array
    # buffer *= "\n\t$MK = 1.0"
    # buffer *= "\n\tMonodAffinityConstant_dict[\"$MK\"] = $MK"
    buffer *= "\n\tMonodAffinityConstantDict[\"$MK\"] = 0.1"
  end

  # W_value in transcription control term, nomenclature: W_targetmRNA_actor
  buffer *= "\n\n\n\t# W_value in transcription control term, nomenclature: W_targetmRNA_actor"
  buffer *= "\n\tW_value_dict = Dict{String, Float64}()"
  for WS in W_string_array
    buffer *= "\n\tW_value_dict[\"$WS\"] = 1.0"
  end
  # disassociation constants in transfer function, nomenclature: KD_speciesName
  buffer *= "\n\n\t# disassociation constants in transfer function, nomenclature: KD_speciesName"
  buffer *= "\n\ttransferFunctionDisassociationConstantDict = Dict{String, Float64}()"
  for DC in disassociation_const_array
    buffer *= "\n\ttransferFunctionDisassociationConstantDict[\"$DC\"] = 1.0"
  end
  # transcription specific correction factor
  buffer *= "\n\n\t# transcription specific correction factor"
  buffer *= "\n\ttranscriptionSpecificCorrectionFactorDict = Dict{String, Float64}()"
  for mrna in mRNA_species_array
    buffer *= "\n\ttranscriptionSpecificCorrectionFactorDict[\"$mrna\"] = 1.0  # $mrna"
  end
  # background gene expression control term
  buffer *= "\n\n\t# background gene expression control term"
  buffer *= "\n\tbackgroundGeneExpressionControlTermDict = Dict{String, Float64}()"
  for mrna in mRNA_species_array
    buffer *= "\n\tbackgroundGeneExpressionControlTermDict[\"$mrna\"] = 0.001  # $mrna"
  end
  # translation specific correction factor
  buffer *= "\n\n\t# translation specific correction factor"
  buffer *= "\n\ttranslationSpecificCorrectionFactor = Dict{String, Float64}()"
  for pro in protein_species_array
    buffer *= "\n\ttranslationSpecificCorrectionFactor[\"$pro\"] = 1.0  # $pro"
  end
  # cooperativity number in transfer function
  buffer *= "\n\n\t# cooperativity number in transfer function"
  buffer *= "\n\tcooperativity = 1"
  #buffer *= "\n\n\tbackground_mRNA_synthesis_rate_vector = 0.01*ones($length_TXTL)"
  # load txtl constants buffer
  buffer *= "\n\n"
  if host_type == "bacteria"
    buffer *= jl_include_constants_from_literature(
              joinpath(Base.@__DIR__, "txtl_constants_ecoli.jl"),"\n\t")
  else
    buffer *= jl_include_constants_from_literature(
              joinpath(Base.@__DIR__, "txtl_constants_hl60.jl"), "\n\t")
  end

  #---------------------------------
  # put all stuff in a Dictionary
  buffer *= "\n\n\t####################################"
  buffer *= "\n\t# put all stuff in a Dictionary"
  buffer *= "\n\tdataDictionary = Dict{String, Any}()"
  buffer *= "\n\tdataDictionary[\"stoichiometric_matrix\"] = stoichiometric_matrix"
  buffer *= "\n\tdataDictionary[\"all_species_dict\"] = all_species_dict"
  buffer *= "\n\tdataDictionary[\"all_species_reversed_dict\"] = all_species_reversed_dict"
  buffer *= "\n\tdataDictionary[\"rnx_species_array\"] = rnx_species_array"
  buffer *= "\n\tdataDictionary[\"mRNA_species_array\"] = mRNA_species_array"
  buffer *= "\n\tdataDictionary[\"protein_species_array\"] = protein_species_array"
  buffer *= "\n\tdataDictionary[\"initial_condition\"] = initial_condition"

  buffer *= "\n\tdataDictionary[\"kcat_signaling\"] = kcat_signaling"
  buffer *= "\n\tdataDictionary[\"MonodAffinityConstantDict\"] = MonodAffinityConstantDict"
  buffer *= "\n\tdataDictionary[\"W_value_dict\"] = W_value_dict"
  buffer *= "\n\tdataDictionary[\"transferFunctionDisassociationConstantDict\"] = transferFunctionDisassociationConstantDict"
  buffer *= "\n\tdataDictionary[\"transcriptionSpecificCorrectionFactorDict\"] = transcriptionSpecificCorrectionFactorDict"
  buffer *= "\n\tdataDictionary[\"backgroundGeneExpressionControlTermDict\"] = backgroundGeneExpressionControlTermDict"
  buffer *= "\n\tdataDictionary[\"translationSpecificCorrectionFactor\"] = translationSpecificCorrectionFactor"
  buffer *= "\n\tdataDictionary[\"cooperativity\"] = cooperativity"
  buffer *= "\n\tdataDictionary[\"RNAPConcentration\"] = rnapII_concentration"
  buffer *= "\n\tdataDictionary[\"RIBOConcentration\"] = ribosome_concentration"
  buffer *= "\n\tdataDictionary[\"degradationConstantmRNA\"] = degradation_constant_mRNA"
  buffer *= "\n\tdataDictionary[\"degradationConstantProtein\"] = degradation_constant_protein"
  buffer *= "\n\tdataDictionary[\"kcatTranscription\"] = kcat_transcription"
  buffer *= "\n\tdataDictionary[\"kcatTranslation\"] = kcat_translation"
  buffer *= "\n\tdataDictionary[\"avgGeneConcentration\"] = avg_gene_concentration"
  buffer *= "\n\tdataDictionary[\"transcriptionSaturationConstant\"] = saturation_transcription"
  buffer *= "\n\tdataDictionary[\"translationSaturationConstant\"] = saturation_translation"
  buffer *= "\n\tdataDictionary[\"specificGrowthRate\"] = maximum_specific_growth_rate - death_rate_constant"

  buffer *= "\n\n\treturn dataDictionary"
  buffer *= "\nend"

  return buffer
end

function jl_build_simulation_buffer(NoExtracellularSpecies::Int64)
  # buffer = build_copyright_header_buffer()
  buffer = "\n# set up ODE, get the derivatives
function Balances(t,y,dataDictionary)
  # correct for negatives
  idx_small = findall(y.<0)
  y[idx_small] .= 0.0

  # load data from data dictionary
  rnx_species = dataDictionary[\"rnx_species_array\"]
  mRNA_species = dataDictionary[\"mRNA_species_array\"]
  protein_species = dataDictionary[\"protein_species_array\"]
  stoichiometry = dataDictionary[\"stoichiometric_matrix\"]
  all_species_dict = dataDictionary[\"all_species_dict\"]  # String --> number
  specific_growth_rate = dataDictionary[\"specificGrowthRate\"]
  transcription_correction = dataDictionary[\"transcriptionSpecificCorrectionFactorDict\"]
  translation_correction = dataDictionary[\"translationSpecificCorrectionFactor\"]
  degradation_constant_mRNA = dataDictionary[\"degradationConstantmRNA\"]
  degradation_constant_protein = dataDictionary[\"degradationConstantProtein\"]

  # kinetics
  (rnx_rate, transcription_rate, translation_rate) = calculate_kinetics(y, dataDictionary)

  # calculate the derivatives
  dydt = zeros(length(all_species_dict))

  # signaling reaction network
  X2 = zeros(length(rnx_species)-$(NoExtracellularSpecies))  # species inside the cell
  for (id, st) in enumerate(rnx_species[$(NoExtracellularSpecies)+1:end])  # get X2 value from y
    X2[id] = y[all_species_dict[st]]
  end
  dX1 = stoichiometry[1:$(NoExtracellularSpecies), :]*rnx_rate  # extracellular metabolites
  dX2 = stoichiometry[$(NoExtracellularSpecies)+1:end, :]*rnx_rate - specific_growth_rate*X2  # INTRA
  dX = [dX1; dX2]
  for (id, st) in enumerate(rnx_species)  # return derivatives back to dydt
    dydt[all_species_dict[st]] = dX[id]
  end
  dydt[1] = specific_growth_rate*y[1]  # for Biomass

  # TXTL network
  for i = 1:length(mRNA_species)
  # seems easy to add additional terms to account for the signaling effect
    mRNA_id = all_species_dict[mRNA_species[i]]
    p_id = all_species_dict[protein_species[i]]
    dydt[mRNA_id] = transcription_rate[mRNA_species[i]] - (specific_growth_rate + transcription_correction[mRNA_species[i]]*degradation_constant_mRNA) * y[mRNA_id]
    dydt[p_id] = translation_rate[protein_species[i]] - (specific_growth_rate + translation_correction[protein_species[i]]*degradation_constant_protein) * y[p_id]
  end

  return dydt
end"
  return buffer
end

function jl_build_solveODEBalances_buffer(all_species_array::Array,
  all_species_dict::Dict{String, Int}, mRNA_species::Array, protein_species::Array)

  buffer = "include(\"./include.jl\")" *
    "\n\nfunction SolveBalances(TStart, TStop, TStep, dataDictionary)" *
    "\n\tt = collect(TStart:TStep:TStop)" *
    "\n\tall_species_reversed_dict = dataDictionary[\"all_species_reversed_dict\"]"
  # initial concentration
  buffer *= "\n\t# initial species concentration"
  buffer *= "\n\ty0 = dataDictionary[\"initial_condition\"]"
  # run simulation
  buffer *= "\n\t# call ODE solver"
  buffer *= "\n\tf(t, y) = Balances(t, y, dataDictionary)" *
    "\n\tt, y = ode23s(f, y0, t; points=:specified)"
  # data transfer
  buffer *= "\n\n\t# data transfer" *
    "\n\trow = length(t)" *
    "\n\tcol = length(y0)" *
    "\n\tY = zeros(row, col)" *
    "\n\tforeach(x->(Y[x, :] = y[x]), collect(1:row))"
  # plotting
  buffer *= "\n\t# plot results" *
    "\n\tplt.figure(\"simulation \$TStart to \$TStop\")" *
    "\n\tsubplt_col = ceil(Int, col/4)" *
    "\n\tfor i = 1:col" *
    "\n\t\tplt.subplot(4, subplt_col, i)" *
    "\n\t\tplt.plot(t, Y[:,i])" *
    "\n\t\tplt.title(all_species_reversed_dict[i])" *
    "\n\tend" *
    "\n\tplt.show()"
  buffer *= "\n\n\treturn (t, Y)" *
    "\nend"

  # add a test
  buffer *= "\n\n\n"
  buffer *= "# # uncomment to test"
  buffer *= "\n# dataDictionary = generate_model_parameters_dictionary()"
  buffer *= "\n# SolveBalances(0, 10, 0.1, dataDictionary)"

  return buffer
end

# function build_copyright_header_buffer()
#   current_year = string(Dates.year(now()))
#   # Get comment data from
#   buffer = ""
#   buffer *=
# "# ----------------------------------------------------------------------------------- #
# # Copyright (c) $(current_year) Varnerlab
# # Robert Frederick Smith School of Chemical and Biomolecular Engineering
# # Cornell University, Ithaca NY 14850
# #
# # Permission is hereby granted, free of charge, to any person obtaining a copy
# # of this software and associated documentation files (the \"Software\"), to deal
# # in the Software without restriction, including without limitation the rights
# # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# # copies of the Software, and to permit persons to whom the Software is
# # furnished to do so, subject to the following conditions:
# #
# # The above copyright notice and this permission notice shall be included in
# # all copies or substantial portions of the Software.
# #
# # THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# # THE SOFTWARE.
# # ----------------------------------------------------------------------------------- #"
#
#   return buffer
# end

# FBA data dictionary generation
function jl_generate_FBA_data_dictionary(all_rnx_list::Array,
  rnx_species_array::Array, extra_species_num::Int)
  secrete_id = Set{Int64}()  # For initialize the coefficient array
  # data dictionary
  buffer = "function DataDictionary(time_start,time_stop,time_step)"
  # stoichiometry
  buffer *= "\n\n\t# load stoichiometry"
  buffer *= "\n\tstoichiometric_matrix = readdlm(\"./stoichiometry.dat\")"
  # Setup default flux bounds array
  buffer *= "\n\t# Setup default flux bounds array"
  buffer *= "\n\tdefault_bounds_array = ["
  for (id, rnx) in enumerate(all_rnx_list)
    buffer *= "\n\t\t0 100.0; # $(id) $(rnx.rnxName)"
  end
  buffer *= "\n\t]"
  # Setup default species bounds array
  buffer *= "\n\n\t# Setup default species bounds array"
  buffer *= "\n\tspecies_bounds_array = ["
  for (id, sp) in enumerate(rnx_species_array)
    if id > extra_species_num
      buffer *= "\n\t\t 0.0 0.0; # $(id) $(sp)"
    else
      buffer *= "\n\t\t-1.0 1.0; # $(id) $(sp)"
    end
  end
  buffer *= "\n\t]"
  # Setup the objective coefficient array
  buffer *= "\n\n\t# Setup the objective coefficient array"
  buffer *= "\n\tobjective_coefficient_array = ["
  for (id, rnx) in enumerate(all_rnx_list)
    buffer *= "\n\t\t0.0; # $(id) $(rnx.rnxName)"
    if rnx.rnxType == "secrete"
        push!(secrete_id, id)
    end
  end
  buffer *= "\n\t]"
  # Min/Max flag - default is minimum -
  buffer *= "\n\n\t# Min/Max flag - default is minimum -"
  buffer *= "\n\tis_minimum_flag = true"
  # List of reation strings - used to write flux report
  buffer *= "\n\n\t# List of reation strings - used to write flux report"
  buffer *= "\n\tlist_of_reaction_strings = ["
  for rnx in all_rnx_list
    buffer *= "\n\t\t\"$(rnx.rnxName)\""
  end
  buffer *= "\n\t]"
  # List of metabolite strings - used to write flux report
  buffer *= "\n\n\t# List of metabolite strings - used to write flux report"
  buffer *= "\n\tlist_of_metabolite_symbols = ["
  for sp in rnx_species_array
    buffer *= "\n\t\t\"$(sp)\""
  end
  buffer *= "\n\t]"
  # return block -
  buffer *= "\n\n"
  buffer *= "\t# =============================== DO NOT EDIT BELOW THIS LINE ============================== #\n"
  buffer *= "\tdata_dictionary = Dict{AbstractString,Any}()\n"
  buffer *= "\tdata_dictionary[\"stoichiometric_matrix\"] = stoichiometric_matrix\n"
  buffer *= "\tdata_dictionary[\"objective_coefficient_array\"] = objective_coefficient_array\n"
  buffer *= "\tdata_dictionary[\"default_flux_bounds_array\"] = default_bounds_array;\n"
  buffer *= "\tdata_dictionary[\"species_bounds_array\"] = species_bounds_array\n"
  buffer *= "\tdata_dictionary[\"list_of_reaction_strings\"] = list_of_reaction_strings\n"
  buffer *= "\tdata_dictionary[\"list_of_metabolite_symbols\"] = list_of_metabolite_symbols\n"
  buffer *= "\tdata_dictionary[\"is_minimum_flag\"] = is_minimum_flag\n"
  buffer *= "\t# =============================== DO NOT EDIT ABOVE THIS LINE ============================== #\n"
  buffer *= "\treturn data_dictionary\n"
  buffer *= "end\n"

  # maximize_product_dictionary
  buffer *= "\n\n\n# maximize_product_dictionary"
  buffer *= "\nfunction maximize_product_dictionary(time_start,time_stop,time_step)"
  # load the data dictionary -
  buffer *= "\n\t# load the data dictionary -"
  buffer *= "\n\tdata_dictionary = DataDictionary(time_start,time_stop,time_step)"
  # Modify the data dictionary -
  buffer *= "\n\t# Modify the data dictionary -"
  buffer *= "\n\tobjective_coefficient_array = data_dictionary[\"objective_coefficient_array\"]"
  secrete_id = sort(collect(secrete_id))
  for id in secrete_id
      buffer *= "\n\tobjective_coefficient_array[$(id)] = -1"
  end
  buffer *= "\n\t# return -"
  buffer *= "\n\treturn data_dictionary"
  buffer *= "\nend"

  return buffer
end


module Cronbach

using LinearAlgebra: diag

export cronbach

include("tauequivalent.jl")

end


export set_lagrange_term!, set_mayer_term!

function set_lagrange_term!( model::Model, lagrange::Union{Nothing,Function}; dx::CallbackArg=:autodiff, du::CallbackArg=:autodiff, dp::CallbackArg=:autodiff )
    # Allow the lagrange term to be disabled
    if isa( lagrange, Nothing )
        model.integralCost = false

        model.callbacks.lfct_cb = nothing
        model.callbacks.dldx_cb = nothing
        model.callbacks.dldu_cb = nothing
        model.callbacks.dldp_cb = nothing

        return
    else
        model.integralCost = true
        model.callbacks.lfct_cb = lagrange
    end

    if isa( dx, Function )
        model.callbacks.dldx_cb = dx
    elseif dx == :autodiff
        # TODO
    else
        throw( ArgumentError( "Unknown option $(String(dx)) for dx" ) )
    end

    if isa( du, Function )
        model.callbacks.dldu_cb = du
    elseif du == :autodiff
        # TODO
    else
        throw( ArgumentError( "Unknown option $(String(du)) for du" ) )
    end

    if isa( dp, Function )
        model.callbacks.dldp_cb = dp
    elseif dp == :autodiff
        # TODO
    else
        throw( ArgumentError( "Unknown option $(String(dp)) for dp" ) )
    end
end

function set_mayer_term!( model::Model, mayer::Union{Nothing,Function}; dx::CallbackArg=:autodiff, dp::CallbackArg=:autodiff, dT::CallbackArg=:autodiff )
    # Allow the mayer term to be disabled
    if isa( mayer, Nothing )
        model.terminalCost = false

        model.callbacks.Vfct_cb = nothing
        model.callbacks.dVdx_cb = nothing
        model.callbacks.dVdp_cb = nothing
        model.callbacks.dVdT_cb = nothing

        return
    else
        model.terminalCost = true
        model.callbacks.Vfct_cb = mayer
    end

    if isa( dx, Function )
        model.callbacks.dVdx_cb = dx
    elseif dx == :autodiff
        # TODO
    else
        throw( ArgumentError( "Unknown option $(String(dx)) for dx" ) )
    end

    if isa( dp, Function )
        model.callbacks.dVdp_cb = dp
    elseif dp == :autodiff
        # TODO
    else
        throw( ArgumentError( "Unknown option $(String(dp)) for dp" ) )
    end

    if isa( dT, Function )
        model.callbacks.dVdT_cb = dT
    elseif dT == :autodiff
        # TODO
    else
        throw( ArgumentError( "Unknown option $(String(dT)) for dT" ) )
    end
end


###
### Counting
###
### Counting operations on biological sequence types.
###
### This file is a part of BioJulia.
### License is MIT: https://github.com/BioJulia/BioSequences.jl/blob/master/LICENSE.md

###
### Naive counting
###

function count_naive(pred, seq::BioSequence)
    n = 0
    @inbounds for i in eachindex(seq)
        n += pred(seq[i])::Bool
    end
    return n
end

function count_naive(pred, seqa::BioSequence, seqb::BioSequence)
    n = 0
    @inbounds for i in 1:min(length(seqa), length(seqb))
        n += pred(seqa[i], seqb[i])::Bool
    end
    return n
end

"""
Count how many positions in a sequence satisfy a condition (i.e. f(seq[i]) -> true).

The first argument should be a function which accepts an element of the sequence
as its first parameter, additional arguments may be passed with `args...`.
"""
Base.count(pred, seq::BioSequence) = count_naive(pred, seq)

Base.count(pred, seqa::BioSequence, seqb::BioSequence) = count_naive(pred, seqa, seqb)

BioSymbols.isambiguous(x::T, y::T) where {T<:NucleicAcid} = isambiguous(x) | isambiguous(y)
BioSymbols.isgap(x::T, y::T) where {T<:NucleicAcid} = isgap(x) | isgap(y)
BioSymbols.iscertain(x::T, y::T) where {T<:NucleicAcid} = iscertain(x) & iscertain(y)

Base.count(::typeof(isambiguous), seqa::S, seqb::S) where {S<:BioSequence{<:NucleicAcidAlphabet{2}}} = 0
Base.count(::typeof(isgap), seqa::S, seqb::S) where {S<:BioSequence{<:NucleicAcidAlphabet{2}}} = 0
Base.count(::typeof(iscertain), seqa::S, seqb::S) where {S<:BioSequence{<:NucleicAcidAlphabet{2}}} = min(length(seqa), length(seqb))

###
### Aliases for various uses of `count`.
###

"""
    gc_content(seq::BioSequence)

Calculate GC content of `seq`.
"""
gc_content(seq::NucleotideSeq) = isempty(seq) ? 0.0 : count(isGC, seq) / length(seq)

n_ambiguous(seq) = count(isambiguous, seq)
n_ambiguous(seqa::BioSequence, seqb::BioSequence) = count(isambiguous, seqa, seqb)

n_certain(seq) = count(iscertain, seq)
n_certain(seqa::BioSequence, seqb::BioSequence) = count(iscertain, seqa, seqb)

n_gaps(seq::BioSequence) = count(isgap, seq)
n_gaps(seqa::BioSequence, seqb::BioSequence) = count(isgap, seqa, seqb)

mismatches(seqa::BioSequence, seqb::BioSequence) = count(!=, seqa, seqb)
matches(seqa::BioSequence, seqb::BioSequence) = count(==, seqa, seqb)


using Chain
using DataFrames
using Dates
using Test

using Stonks:
  UpdatableSymbol,
  AssetPrice,
  ExchangeRate,
  is_weekday,
  last_sunday,
  last_workday,
  build_fx_pair

function fake_stock_data(days=30, ref_date=today(), symbols=["AAPL", "IBM", "TSLA"])
  dates = @chain begin
    [ref_date - Day(i) for i in reverse(0:(days - 1))]
    filter(x -> is_weekday(x), _)
  end
  nrows = length(dates)
  return @chain begin
    symbols
    map(
      t -> DataFrame(;
        symbol=repeat([t], nrows),
        date=dates,
        close=[100 + (rand(-8:10) * 0.1) for i in 1:nrows],
      ),
      _,
    )
    vcat(_...)
  end
end

function fake_price_data(
  days=30, ref_date=today(), symbols=["AAPL", "IBM", "TSLA"]
)::Vector{AssetPrice}
  dates = @chain begin
    [ref_date - Day(i) for i in reverse(0:(days - 1))]
    filter(x -> is_weekday(x), _)
  end
  data = AssetPrice[]
  for symbol in symbols
    append!(
      data, map(d -> AssetPrice(; symbol=symbol, date=d, close=100 + rand() * 10), dates)
    )
  end
  return data
end

function fake_exchange_data(
  days=30, ref_date=today(), symbols=["EUR/USD", "USD/CAD", "USD/JPY"]
)::Vector{ExchangeRate}
  dates = @chain begin
    [ref_date - Day(i) for i in reverse(0:(days - 1))]
    filter(x -> is_weekday(x), _)
  end
  data = ExchangeRate[]
  for symbol in symbols
    base, target = build_fx_pair(symbol)
    append!(
      data,
      map(
        d -> ExchangeRate(; base=base, target=target, date=d, rate=1 + rand() * 10), dates
      ),
    )
  end
  return data
end

function test_info_data()
  return [
    AssetInfo(;
      symbol="AAPL",
      currency="USD",
      name="Apple Inc.",
      type="EQUITY",
      exchange="NMS",
      country="United States",
      industry="Consumer Electronics",
      sector="Technology",
      timezone="America/New_York",
      employees=100000,
    ),
    AssetInfo(;
      symbol="MSFT",
      currency="USD",
      name="Microsoft Corporation",
      type="EQUITY",
      country="United States",
      industry="Software—Infrastructure",
      sector="Technology",
      timezone="America/New_York",
      employees=181000,
    ),
  ]
end

function test_price_data()
  return [
    AssetPrice(; symbol="MSFT", date=Date("2022-02-16"), close=299.5),
    AssetPrice(; symbol="MSFT", date=Date("2022-02-17"), close=290.73),
    AssetPrice(; symbol="AAPL", date=Date("2022-02-17"), close=168.88),
    AssetPrice(; symbol="AAPL", date=Date("2022-02-18"), close=167.3),
  ]
end

get_test_data(path::String) = open(f -> read(f, String), joinpath(@__DIR__, path))

function complex_tickers()
  return [
    UpdatableSymbol("AAPL"; from="2022-01-01"),
    UpdatableSymbol("MSFT"; from="2022-02-10"),
    UpdatableSymbol("TSLA"; from="2022-02-10"),
    UpdatableSymbol("IBM"),
    UpdatableSymbol("GOOG"),
    UpdatableSymbol("NFLX"),
  ]
end


function parse_ref(ref_file::String, chroms::UnitRange)
    df = CSV.File(ref_file; types=Dict(:A1=>Char,:A2=>Char)) |> DataFrame
    df.A1 = tochar.(df.A1)
    df.A2 = tochar.(df.A2)
    @assert df.CHR isa Vector{Int}
    @assert df.BP isa Vector{Int}
    @assert df.MAF isa Vector{T} where T<:Real
    filter!(row->row.CHR in chroms, df)
    return df
end
parse_ref(ref_file::String, chrom::Integer) =
    parse_ref(ref_file, chrom:chrom)
function tochar(x)::Union{Char,Missing}
    if x isa String
        if length(x) == 1
            return first.(x)
        else
            return missing
        end
    else
        x
    end
end

function parse_bim(bim_file::String, chroms::UnitRange)
    header = [:CHR, :SNP, :POS, :BP, :A1, :A2]
    df = CSV.File(bim_file*".bim"; header=header, types=Dict(:A1=>Char,:A2=>Char)) |> DataFrame
    df.A1 = tochar.(df.A1)
    df.A2 = tochar.(df.A2)
    @assert df.CHR isa Vector{Int}
    filter!(row->row.CHR in chroms, df)
    return df
end
parse_bim(bim_file::String, chrom::Integer) =
    parse_bim(bim_file, chrom:chrom)

nuc_map(char::Char) = nuc_map(Val(char))
nuc_map(::Val{'A'}) = 'T'
nuc_map(::Val{'T'}) = 'A'
nuc_map(::Val{'C'}) = 'G'
nuc_map(::Val{'G'}) = 'C'
function permute_snps(df)
    unique(vcat(
        df[:,[:SNP,:A1,:A2]],
        DataFrame(SNP=df.SNP,
                  A1=df.A2,
                  A2=df.A1),
        DataFrame(SNP=df.SNP,
                  A1=nuc_map.(first.(df.A1)),
                  A2=nuc_map.(first.(df.A2))),
        DataFrame(SNP=df.SNP,
                  A1=nuc_map.(first.(df.A2)),
                  A2=nuc_map.(first.(df.A1)))
    ))
end
function join_snps(ref_df, vld_df, sst_df; verbose=false)
    # TODO: Be more efficient, don't allocate all this memory
    vld_snps = vld_df[:,[:SNP,:A1,:A2]]
    ref_snps = permute_snps(ref_df)
    sst_snps = permute_snps(sst_df)
    snps = innerjoin(vld_snps, ref_snps, sst_snps, on=[:SNP,:A1,:A2], makeunique=true)
    verbose && @info "$(nrow(snps)) common SNPs"
    return snps
end
norm_ppf(x) = quantile(Normal(), x)
function parse_sumstats(ref_df, vld_df, sst_file, n_subj; verbose=false, missingstring="")
    sst_df = CSV.File(sst_file; missingstring=missingstring, types=Dict(:A1=>Char,:A2=>Char)) |> DataFrame
    sst_df.A1 = tochar.(sst_df.A1)
    sst_df.A2 = tochar.(sst_df.A2)
    nucs = Set(['A','C','T','G'])
    filter!(row->(row.A1 in nucs) && (row.A2 in nucs), sst_df)
    filter!(row->!(row.P isa Missing) && !(row.BETA isa Missing), sst_df)
    sst_df.P = convert(Vector{Float64}, sst_df.P)
    sst_df.BETA = convert(Vector{Float64}, sst_df.BETA)
    snps = join_snps(ref_df, vld_df, sst_df; verbose=verbose)
    sort!(snps, [:SNP, :A1, :A2])

    n_sqrt = sqrt(n_subj)
    sst_eff = Dict{String,Float64}()
    for row in Tables.namedtupleiterator(sst_df)
        if hassnp(snps, (row.SNP,row.A1,row.A2)) ||
           hassnp(snps, (row.SNP,nuc_map.(row.A1),nuc_map.(row.A2)))
            effect_sign = 1
        elseif hassnp(snps, (row.SNP,row.A2,row.A1)) ||
               hassnp(snps, (row.SNP,nuc_map.(row.A2),nuc_map.(row.A1)))
            effect_sign = -1
        else
            continue
        end
        if hasproperty(row, :BETA)
            beta = row.BETA
        elseif hasproperty(row, :OR)
            beta = log(row.OR)
        end
        p = max(row.P, 1e-323)
        beta_std = effect_sign*sign(beta)*abs(norm_ppf(p/2))/n_sqrt
        sst_eff[row.SNP] = beta_std
    end
    _sst_df = DataFrame(SNP=String[],CHR=Int[],BP=Int[],BETA=Float64[],A1=Char[],A2=Char[],MAF=Float64[],FLP=Int[])
    for (idx,row) in enumerate(Tables.namedtupleiterator(ref_df))
        haskey(sst_eff, row.SNP) || continue

        SNP = row.SNP
        CHR = row.CHR
        BP = row.BP
        BETA = sst_eff[row.SNP]
        A1,A2 = row.A1,row.A2
        if hassnp(snps, (SNP,A1,A2))
            MAF = row.MAF
            FLP = 1
        elseif hassnp(snps, (SNP,A2,A1))
            A1, A2 = A2, A1
            MAF = 1-row.MAF
            FLP = -1
        elseif hassnp(snps, (SNP,nuc_map(A1),nuc_map(A2)))
            A1, A2 = nuc_map(A1), nuc_map(A2)
            MAF = row.MAF
            FLP = 1
        elseif hassnp(snps, (SNP,nuc_map(A2),nuc_map(A1)))
            A1, A2 = nuc_map(A2), nuc_map(A1)
            MAF = 1-row.MAF
            FLP = -1
        else
            verbose && @warn "(Chromosome $CHR) Didn't find ($SNP,$A1,$A2) in snps"
            # FIXME: Skip?
        end
        push!(_sst_df, (SNP=SNP,CHR=CHR,BP=BP,BETA=BETA,A1=A1,A2=A2,MAF=MAF,FLP=FLP))
    end
    return _sst_df
end

function findsnp(snps, (snp,a1,a2))
    SNP_range = binary_range_search(snps, snp, :SNP)
    SNP_range === nothing && return nothing
    SNP_L, SNP_R = SNP_range
    SNP_sub = snps[SNP_L:SNP_R,:]

    A1_range = binary_range_search(SNP_sub, a1, :A1)
    A1_range === nothing && return nothing
    A1_L, A1_R = A1_range
    A1_sub = SNP_sub[A1_L:A1_R,:]

    A2_range = binary_range_search(A1_sub, a2, :A2)
    A2_range === nothing && return nothing
    A2_L, A2_R = A2_range
    @assert A2_L == A2_R
    return SNP_L + (A1_L-1) + (A2_L-1)
end
hassnp(snps, row) = findsnp(snps, row) !== nothing
# TODO: Allow warm-restarts
function binary_range_search(snps, x, col)
    _snps = snps[!,col]
    L = 1
    R = nrow(snps)
    while true
        (L > R) && return nothing
        M = floor(Int, (L+R)/2)
        _x = _snps[M]
        if _x == x
            L,R = M,M
            snps_rows = nrow(snps)
            while (L > 1) && (_snps[L - 1] == x)
                L -= 1
            end
            while R < (snps_rows) && (_snps[R + 1] == x)
                R += 1
            end
            return L,R
        elseif _x < x
            L = M+1
        elseif _x > x
            R = M-1
        end
    end
end

function parse_ldblk(ldblk_dir, sst_df, chrom)
    chr_name = ldblk_dir * "/ldblk_1kg_chr" * string(chrom) * ".hdf5"
    hdf_chr = h5open(chr_name, "r")
    n_blk = length(hdf_chr)
    ld_blk = [read(hdf_chr["blk_"*string(blk)]["ldblk"]) for blk in 1:n_blk]

    snp_blk = Vector{String}[]
    for blk in 1:n_blk
        push!(snp_blk, read(hdf_chr["blk_"*string(blk)]["snplist"]))
    end

    blk_size = Int[]
    _ld_blk = Matrix{Float64}[]
    mm = 1
    for blk in 1:n_blk
        idx = [(ii,findfirst(s->s==snp, sst_df.SNP)) for (ii, snp) in enumerate(snp_blk[blk]) if snp in sst_df.SNP]
        push!(blk_size, length(idx))
        if !isempty(idx)
            idx_blk = mm:(mm+length(snp_blk[blk])-1)
            flip = [sst_df.FLP[jj] for jj in last.(idx)]
            flipM = flip' .* flip
            _blk = Matrix{Float64}(undef, length(idx), length(idx))
            P = collect(Iterators.product(first.(idx),first.(idx)))
            for icol in 1:size(P,2)
                for irow in 1:size(P,1)
                    row,col = P[irow,icol]
                    _blk[irow,icol] = ld_blk[blk][row,col] * flipM[irow,icol]
                end
            end
            push!(_ld_blk, _blk)
            mm += length(snp_blk[blk])
        else
            push!(_ld_blk,  Matrix{Float64}(undef, 0, 0))
        end
    end

    return _ld_blk, blk_size
end