verbose = true
 using Dates
 if verbose println(now()) end
 if verbose && !isdefined(Main,:RvSpectML)  println("# Loading RvSpecML")    end
 using PDMats  # SOme dependancy needs this explicit first
 using RvSpectMLBase
 using EchelleCCFs
 using EchelleInstruments#, EchelleInstruments.NEID
 using RvSpectML
 if verbose println("# Loading NeidSolarScripts")    end
 using SunAsAStar
 using NeidSolarScripts
 println("# Loading other packages 1/2")
 using ArgParse
 using Markdown

println("# Parsing arguments...")
   #lsf_width = 3.0e3
   function parse_commandline()
     s = ArgParseSettings( description = "Calculate RVs from JLD2 file with NEID's daily order CCFs.")
     #import_settings!(s, s_files_only, args_only=false)
     add_arg_group!(s, "Files", :argg_files)
     @add_arg_table! s begin
         "input"
             help = "Filename for input with daily order CCFs (jld2)"
             arg_type = String
             default = "daily_ccfs.jld2"
         "output"
             help = "Filename for output dailiy RVs (csv)"
             arg_type = String
             default = "daily_rvs.csv"
         "--overwrite"
            help = "Specify it's ok to overwrite the output file."
            #default = true
            action = :store_true
      end
      #=
      add_arg_group!(s, "CCF parameters", :argg_ccf_param)
      @add_arg_table! s begin
        "--orders_to_use"
           help = "First and last order _index_ to compute CCFs for"
           nargs = 2
           arg_type = Int64
           #default = [ min_order(NEID2D()), max_order(NEID2D()) ]
           #default = [ first(orders_to_use_default(NEID2D())), last(orders_to_use_default(NEID2D())) ]
        "--mask_scale_factor"
            help = "Specify CCF mask width scale as multiple of NEID default v width " * string(default_ccf_mask_v_width(NEID2D()))
            arg_type = Float64
            #default = round(lsf_width/default_ccf_mask_v_width(NEID2D()), sigdigits=3)
        "--line_width_50_default"
            help = "Specify default line full width half maximum"
            arg_type = Float64
            #default = 7.9e3 # m/2
      end
      =#
      add_arg_group!(s, "Algorithm for measuring RVs ", :argg_alg_param)
      @add_arg_table! s begin
         "--rv_alg"
            help = "Algorithm for measuring RVs."
            arg_type = String
            default = "template"
            #default = "gaussian"
         "--frac_of_width_to_fit"
            help = "Regions of CCF to fit, as fraction of measured width."
            arg_type = Float64
            default = 0.5
         "--measure_width_at_frac_depth"
            help = "Measure CCF width at this fractional depth."
            arg_type = Float64
            default = 0.5
         "--template_file"
            help = "Filename with CCF template stored as 'order_ccfs' with 'clean_obs_mask' for which observations to include (jld2)."
            arg_type = String
      end
      add_arg_group!(s, "Filter manifest for ", :argg_filter_param)
      @add_arg_table! s begin
         "--target"
            help = "Target field"
            arg_type = String
            default = "Sun"
         "--datestr"
            help = "Filenames that contain date string."
            arg_type = String
        "--max_solar_hour_angle"
           help = "Maximum absolute value of solar hour angle."
           arg_type = Float64
           default = 3.12
         "--max_airmass"
            help = "Maximum airmass."
            arg_type = Float64
            default = 2.0
         "--min_expmeter"
            help = "Minimum exposure meter flux relative to model flux."
            arg_type = Float64
            default = 6e4
#=
         "--min_expmeter_factor"
            help = "Minimum exposure meter flux relative to model flux."
            arg_type = Float64
            default = 0.9
         "--max_expmeter_factor"
            help = "Maximum exposure meter flux relative to model flux."
            arg_type = Float64
            default = 1.1
=#
         "--max_expmeter_frac_rms"
            help = "Maximum fractional RMS of exposure meter flux."
            arg_type = Float64
            default = 0.01
#=
         "--min_pyrhelio_factor"
            help = "Minimum pyrheliometer flux relative to model flux."
            arg_type = Float64
            default = 0.9
         "--max_pyrhelio_factor"
            help = "Maximum pyrheliometer flux relative to model flux."
            arg_type = Float64
            default = 1.1
=#
         "--max_pyrhelio_frac_rms"
            help = "Maximum fractional RMS of pyrheliometer meter flux."
            arg_type = Float64
            default = 0.0035
         #=
         "--min_drp_snr"
            help = "Minimum extracted SNR reported by NEID DRP."
            arg_type = Float64
            #default = 0.0
         =#
            #=
         "--start_time"
            help = "Specify daily start time for CCFs to be processed (HH MM)"
            nargs = 2
            arg_type = Int64
            default = [0, 0]
            #default = [18, 30]
        "--stop_time"
           help = "Specify daily stop time for CCFs to be processed (HH MM)"
           nargs = 2
           arg_type = Int64
           #default = [ min_order(NEID2D()), max_order(NEID2D()) ]
           #default = [22, 30]
           default = [23, 59]
           =#
         "--max_num_spectra"
            help = "Maximum number of spectra to process."
            arg_type = Int
            default = 300   # TODO: Increase  after finish testing
      end
      return parse_args(s)
 end
 args = parse_commandline()
 @assert any(map(alg->occursin(alg,args["rv_alg"]), ["template","gaussian","quadratic"] ))

if verbose   println("# Loading other packages 2/2")    end
 using CSV, DataFrames, Query, Dates
 using JLD2, FileIO, MD5 #, SHA
 using StatsBase, Statistics, NaNMath

 # Filename arguments
 @assert isfile(args["input"]) || islink(args["input"])
 @assert match(r"\.jld2$",args["input"]) != nothing
 daily_ccf_filename = args["input"]

 args["overwrite"] = true   # TODO: Comment after done testing
 if isfile(args["output"]) && !args["overwrite"]
    error("# Output file " * args["output"] * " already exists (size = " * string(filesize(args["output"])) * " ).")
 end
 @assert !isfile(args["output"]) || args["overwrite"]
 @assert match(r"\.csv$",args["output"]) != nothing
 daily_rvs_filename = args["output"]
 touch(daily_rvs_filename)

#=
 if isfile(args["report"]) && !args["overwrite"]
    error("# Report file " * args["report"] * " already exists (size = " * string(filesize(args["report"])) * " ).")
 end
 @assert !isfile(args["report"]) || args["overwrite"] == true
 @assert match(r"\.md$",args["report"]) != nothing
 daily_report_filename = args["report"]
 touch(daily_report_filename)
=#

 file_hashes = Dict{String,String}()
 start_processing_time = now()

if filesize(daily_ccf_filename) > 0
   println("# Reading daily CCFs from ", daily_ccf_filename)
   input_data = load(daily_ccf_filename)
else
   println("# Empy daily CCF file.  Creating empty ", daily_rvs_filename)
   touch(daily_rvs_filename)
   exit(0)
end

println("# Filtering for usable observations...")
manifest = input_data["manifest"]

manifest_use = manifest |>
 @filter( args["target"] == nothing || _.target == args["target"] ) |>
 @filter( args["datestr"] == nothing || occursin(args["datestr"],_.target) ) |>
 @filter( _.driftfun == "dailymodel0" ) |>
 @filter( args["max_airmass"] == nothing || _.airmass <= args["max_airmass"] ) |>
 @filter( args["max_solar_hour_angle"] == nothing || abs(_.hour_angle) <= args["max_solar_hour_angle"] ) |>
 #@filter( args["start_time"] == nothing || Time(julian2datetime(_.bjd)) >= start_time ) |>
 #@filter( args["stop_time"] == nothing || Time(julian2datetime(_.bjd)) <= stop_time ) |> # TODO for other instruments may need to deal wtih cross end of 24 UTC
 @filter( args["min_expmeter"] == nothing || _.expmeter_mean >= args["min_expmeter"] ) |>
 DataFrame

if hasproperty(manifest_use,:mean_pyroflux) && hasproperty(manifest_use,:rms_pyroflux)
   manifest_use = manifest_use |>
      @filter( args["max_pyrhelio_frac_rms"] == nothing || _.rms_pyroflux <= args["max_pyrhelio_frac_rms"] * _.mean_pyroflux ) |>
      DataFrame
else
   println("# Warning:  mean_pyroflux and rms_pyroflux not found.  Reverting to expmeter_mean and expmeter_rms.")
   manifest_use = manifest_use |>
      @filter( args["max_expmeter_frac_rms"] == nothing || _.expmeter_rms <= args["max_expmeter_frac_rms"] * _.expmeter_mean ) |>
      DataFrame
end

manifest_use = manifest_use |>
   @take(args["max_num_spectra"] ) |> @orderby(_.bjd) |>
   DataFrame

if !(size(manifest_use,1) >= 1)
   println("# No usable observations.  Creating empty ", daily_rvs_filename)
   touch(daily_rvs_filename)
   exit(0)
end

obs_to_use = map(i->searchsortedfirst(manifest.bjd,manifest_use.bjd[i]),1:size(manifest_use,1) )
(num_vels, num_orders_to_use, num_obs) = size(input_data["order_ccfs"])

order_ccfs = input_data["order_ccfs"]
order_ccf_vars = input_data["order_ccf_vars"]
non_nan_mask = map(i->(!any(isnan.(view(order_ccfs,:,:,i)))) && (!any(isnan.(view(order_ccf_vars,:,:,i)))), obs_to_use)
obs_to_use = obs_to_use[non_nan_mask]
   mask = falses(1:num_obs)
   mask[obs_to_use] .= true

if !(sum(mask) >= 1)
   println("# No usable observations without any NaNs in CCFs.  Creating empty ", daily_rvs_filename)
   touch(daily_rvs_filename)
   exit(0)
end

println("# Found ", length(obs_to_use), " files of ",  size(manifest,1), " to use for RVs.")
   #df_out = manifest_use |> @rename(:drp_ccfjdmod=>:jd_drp,:drp_ccfrvmod => :rv_drp,:drp_dvrmsmod => :σrv_drp,
   df_out = manifest |> @rename(:drp_ccfjdmod=>:jd_drp,:drp_ccfrvmod => :rv_drp,:drp_dvrmsmod => :σrv_drp,
                                    :mean_pyroflux=>:pyrflux_mean, :rms_pyroflux=>:pyrflux_rms ) |> DataFrame
   df_out[!,"mask"] = mask

ccf_sum = reshape(sum(order_ccfs,dims=2),num_vels,num_obs)
ccf_var_sum = reshape(sum(input_data["order_ccf_vars"],dims=2),num_vels,num_obs)

println("# Measuring RVs from RvSpectML's CCFs.")
v_grid = collect(input_data["v_grid"])
@assert num_vels == length(v_grid)
#rvs = measure_rv_from_ccf(v_grid,view(ccf_sum,:,1),view(ccf_var_sum,:,1))

if occursin("template",args["rv_alg"])
   if isnothing(args["template_file"])
      template_ccf = view(ccf_sum,:,obs_to_use)
      template_ccf_var = view(ccf_var_sum,:,obs_to_use)
   else
      template_filename = args["template_file"]
      @assert isfile(template_filename) || islink(template_filename)
      @assert match(r"\.jld2$",template_filename) != nothing
      @assert filesize(template_filename) > 0
      println("# Reading template CCFs from ", template_filename)
      template_data = load(template_filename)
      template_order_ccfs = template_data["order_ccfs"]
      template_order_ccf_vars = template_data["order_ccf_vars"]
      @assert size(template_order_ccfs,1) == size(order_ccfs,1)
      @assert size(template_order_ccfs,2) == size(order_ccfs,2) # Could relax and check which orders are in each
      template_obs_to_use = template_data["clean_obs_mask"]
      template_num_obs = sum(template_obs_to_use)
      @assert template_num_obs >= 1
      template_ccf = reshape(sum(view(template_order_ccfs,:,:,template_obs_to_use),dims=2),num_vels,template_num_obs)
      template_ccf_var = reshape(sum(view(template_order_ccf_vars,:,:,template_obs_to_use),dims=2),num_vels,template_num_obs)
   end
   #ccf_template = EchelleCCFs.calc_ccf_template(view(ccf_sum,:,obs_to_use), view(ccf_var_sum,:,obs_to_use) )
   ccf_template = EchelleCCFs.calc_ccf_template(template_ccf, template_ccf_var,frac_of_width_to_fit=args["frac_of_width_to_fit"], measure_width_at_frac_depth=args["measure_width_at_frac_depth"])
   alg_template = MeasureRvFromCCFTemplate(v_grid=collect(v_grid),template=ccf_template)
   rvs_template = DataFrame(map(i->any(isnan.(view(ccf_sum,:,i))) ? (rv=NaN, σ_rv=NaN) : measure_rv_from_ccf(v_grid,view(ccf_sum,:,i),view(ccf_var_sum,:,i),alg=alg_template),1:num_obs))
   df_out[!,Symbol("rv_template")] = rvs_template.rv
   df_out[!,Symbol("σrv_template")] = rvs_template.σ_rv
end
if occursin("gaussian",args["rv_alg"])
   alg_gauss = MeasureRvFromCCFGaussian(frac_of_width_to_fit=args["frac_of_width_to_fit"], measure_width_at_frac_depth=args["measure_width_at_frac_depth"])
   rvs_gauss = DataFrame(map(i->any(isnan.(view(ccf_sum,:,i))) ? (rv=NaN, σ_rv=NaN)  : measure_rv_from_ccf(v_grid,view(ccf_sum,:,i),view(ccf_var_sum,:,i),alg=alg_gauss),1:num_obs))
   df_out[!,Symbol("rv_template")]= rvs_gauss.rv
   df_out[!,Symbol("σrv_gaussian")] = rvs_gauss.σ_rv
end
if occursin("quadratic",args["rv_alg"])
   alg_quad = MeasureRvFromCCFQuadratic(frac_of_width_to_fit=args["frac_of_width_to_fit"], measure_width_at_frac_depth=args["measure_width_at_frac_depth"])
   rvs_quad = DataFrame(map(i->any(isnan.(view(ccf_sum,:,i))) ? (rv=NaN, σ_rv=NaN)  : measure_rv_from_ccf(v_grid,view(ccf_sum,:,i),view(ccf_var_sum,:,i),alg=alg_quad),1:num_obs))
   df_out[!,Symbol("rv_quadratic")] = rvs_quad.rv
   df_out[!,Symbol("σrv_quadratic")] = rvs_quad.σ_rv
end

orders_physical = collect(174 .-input_data["orders_to_use"])
for j in 1:num_orders_to_use
   if iszero(maximum(view(order_ccfs,:,j,:))) || any(isnan.(view(order_ccfs,:,j,:)))   continue    end
   #println(" j = ", j, "  order_fits_idx = ", input_data["orders_to_use"][j], "  order_physical = ", orders_physical[j])
   if occursin("template",args["rv_alg"])
   # Templated-based
   if isnothing(args["template_file"])
      global template_ccf = view(order_ccfs,:,j,obs_to_use)
      global template_ccf_var = view(order_ccf_vars,:,j,obs_to_use)
   else
      global template_ccf = view(template_order_ccfs,:,j,template_obs_to_use)
      global template_ccf_var = view(template_order_ccf_vars,:,j,template_obs_to_use)
   end
   #local ccf_template = EchelleCCFs.calc_ccf_template(view(order_ccfs,:,j,obs_to_use), view(order_ccf_vars,:,j,obs_to_use) )
   local ccf_template = EchelleCCFs.calc_ccf_template(template_ccf, template_ccf_var)
   local alg_template = MeasureRvFromCCFTemplate(v_grid=collect(v_grid),template=ccf_template, frac_of_width_to_fit=args["frac_of_width_to_fit"], measure_width_at_frac_depth=args["measure_width_at_frac_depth"])
   order_rvs_template = DataFrame(map(i->any(isnan.(view(order_ccfs,:,j,i))) ? (rv=NaN, σ_rv=NaN) : measure_rv_from_ccf(v_grid,view(order_ccfs,:,j,i),view(order_ccf_vars,:,j,i),alg=alg_template),1:num_obs))
   df_out[!,Symbol("rv_" * string(orders_physical[j]) * "_template")] = order_rvs_template.rv
   df_out[!,Symbol("σrv_" * string(orders_physical[j]) * "_template")] = order_rvs_template.σ_rv
   end
   if occursin("gaussian",args["rv_alg"])
   # Gaussian fitting
   order_rvs_gauss = DataFrame(map(i->any(isnan.(view(order_ccfs,:,j,i))) ? (rv=NaN, σ_rv=NaN) : measure_rv_from_ccf(v_grid,view(order_ccfs,:,j,i),view(order_ccf_vars,:,j,i),alg=alg_gauss) ,1:num_obs))
   df_out[!,Symbol("rv_" * string(orders_physical[j]) * "_gauss")] = order_rvs_gauss.rv
   df_out[!,Symbol("σrv_" * string(orders_physical[j]) * "_gauss")] = order_rvs_gauss.σ_rv
   end
   if occursin("quadratic",args["rv_alg"])
   # Quadratic fitting
   order_rvs_quad = DataFrame(map(i->any(isnan.(view(order_ccfs,:,j,i))) ? (rv=NaN, σ_rv=NaN) : measure_rv_from_ccf(v_grid,view(order_ccfs,:,j,i),view(order_ccf_vars,:,j,i),alg=alg_quad),1:num_obs))
   df_out[!,Symbol("rv_" * string(orders_physical[j]) * "_quad")] = order_rvs_quad.rv
   df_out[!,Symbol("σrv_" * string(orders_physical[j]) * "_quad")] = order_rvs_quad.σ_rv
   end
end

println("# Writing daily RVs.")
 CSV.write(daily_rvs_filename,df_out)


## FastForward (FF) delete-relaxation heuristic ##
export FFHeuristic

"Precomputed domain information for FF heuristic."
struct FFCache
    domain::Domain # Preprocessed domain
    axioms::Vector{Clause} # Preprocessed axioms
    preconds::Dict{Symbol,Vector{Vector{Term}}} # Preconditions in DNF
    additions::Dict{Symbol,Vector{Term}} # Action add lists
end

"FastForward (FF) delete-relaxation heuristic."
mutable struct FFHeuristic <: Heuristic
    cache::FFCache # Precomputed domain information
    pre_key::UInt64 # Key to check if information needs to be precomputed again
    FFHeuristic() = new()
end

Base.hash(heuristic::FFHeuristic, h::UInt) = hash(FFHeuristic, h)

function precompute!(h::FFHeuristic,
                     domain::Domain, state::State, spec::Specification)
    # Check if cache has already been computed
    if is_precomputed(h, domain, state, spec) return h end
    h.pre_key = objectid(domain) # Precomputed data is unique to each domain
    domain = copy(domain) # Make a local copy of the domain
    # Preprocess axioms
    axioms = regularize_clauses(domain.axioms) # Regularize domain axioms
    axioms = [Clause(ax.head, [t for t in ax.body if t.name != :not])
              for ax in axioms] # Remove negative literals
    domain.axioms = Clause[] # Remove axioms so they do not affect execution
    # Preprocess actions
    preconds = Dict{Symbol,Vector{Vector{Term}}}()
    additions = Dict{Symbol,Vector{Term}}()
    for (act_name, act_def) in domain.actions
        # Convert preconditions to DNF without negated literals
        conds = get_preconditions(act_def; converter=to_dnf)
        conds = [c.args for c in conds.args]
        for c in conds filter!(t -> t.name != :not, c) end
        preconds[act_name] = conds
        # Extract additions from each effect
        diff = effect_diff(act_def.effect)
        additions[act_name] = diff.add
    end
    h.cache = FFCache(domain, axioms, preconds, additions)
    return h
end

function is_precomputed(h::FFHeuristic,
                        domain::Domain, state::State, spec::Specification)
    return (isdefined(h, :cache) && objectid(domain) == h.pre_key)
end

function compute(h::FFHeuristic,
                 domain::Domain, state::State, spec::Specification)
    # Precompute if necessary
    if !is_precomputed(h, domain, state, spec)
        precompute!(h, domain, state, spec) end
    @unpack cache = h
    @unpack domain = cache
    @unpack types, facts = state
    goals = get_goal_terms(spec)
    # Initialize fact levels and achievers in a GraphPlan-style planning graph
    levels = Dict{Term,Int}(f => 1 for f in facts)
    achievers = Dict{Term,Vector{Term}}(f => Term[] for f in facts)
    cur_level = 1
    while true
        facts = Set(keys(levels))
        state = State(types, facts, Dict{Symbol,Any}())
        # Break out of loop once all goals are achieved
        if is_goal(spec, domain, state) break end
        cur_level += 1
        # Add all one-step derivations of domain axioms
        for ax in cache.axioms
            _, subst = resolve(ax.body, [Clause(f, []) for f in facts])
            for s in subst
                derived = substitute(ax.head, s)
                if haskey(levels, derived) continue end
                levels[derived] = cur_level
                achievers[derived] = Term[]
            end
        end
        # Add effects of available actions
        actions = available(state, domain)
        for act in actions
            act_vars = domain.actions[act.name].args
            subst = Subst(var => val for (var, val) in zip(act_vars, act.args))
            additions = [substitute(a, subst) for a in cache.additions[act.name]]
            for fact in additions
                if get(levels, fact, Inf) < cur_level continue end
                levels[fact] = cur_level # Update fact level
                push!(get!(achievers, fact, Term[]), act) # Add achieving actions
            end
        end
        # Terminate if we hit a fixed point before satisfying all goals
        if length(levels) == length(facts) && keys(levels) == facts
            return Inf end
    end
    # Backout relaxed partial-order plan from the planning graph
    plan = Set{Term}[]
    while cur_level > 1
        pushfirst!(plan, Set{Term}())
        next_goals = Set{Term}()
        for goal in goals
            if levels[goal] < cur_level
                push!(next_goals, goal) # Handle at next level
                continue
            end
            actions = achievers[goal] # Get all achieving actions
            if length(actions) == 0 continue end # No achievers, continue
            # TODO: match actions with their achieving preconditions
            act = actions[1] # Select the first achiever
            push!(plan[1], act) # Add to plan
            preconds = cache.preconds[act.name][1] # Select the first DNF clause
            act_vars = domain.actions[act.name].args
            s = Subst(var => val for (var, val) in zip(act_vars, act.args))
            preconds = [substitute(p, s) for p in preconds]
            union!(next_goals, preconds) # Handle preconds at next level
        end
        goals = next_goals
        cur_level -= 1
    end
    # Return length of relaxed plan
    # TODO : Return helpful actions
    return sum(length.(plan))
end


using  AlphaGo
using AlphaGo: PlayerMove, GoPosition, BLACK, WHITE, to_flat, from_kgs
using AlphaGo: MCTSNode, select_leaf, incorporate_results!, maybe_add_child!,
               inject_noise!, child_action_score, N, Q, child_Q, set_N!,
               add_virtual_loss!
using Base.Test

include("test_utils.jl")

@testset "MCTS" begin
  env = GoEnv(9)
  ALMOST_DONE_BOARD = load_board("""
                                .XO.XO.OO
                                X.XXOOOO.
                                XXXXXOOOO
                                XXXXXOOOO
                                .XXXXOOO.
                                XXXXXOOOO
                                .XXXXOOO.
                                XXXXXOOOO
                                XXXXOOOOO
                                """, env)

  TEST_POSITION = Position(env,
      board = ALMOST_DONE_BOARD,
      n = 105,
      komi = 2.5,
      caps = (1, 4),
      ko = nothing,
      recent = [PlayerMove(BLACK, (1, 2)),
              PlayerMove(WHITE, (1, 9))],
      to_play = BLACK);

  SEND_TWO_RETURN_ONE = Position(env,
      board = ALMOST_DONE_BOARD,
      n = 75,
      komi =  0.5,
      caps = (0, 0),
      ko = nothing,
      recent = [PlayerMove(BLACK, (1, 2)),
              PlayerMove(WHITE, (1, 9)),
              PlayerMove(BLACK, (2, 1))],
      to_play = WHITE);

  @testset "action_flipping" begin
    srand(1)
    probs = 0.02 * ones(env.N ^ 2 + 1)
    probs = probs + rand(env.N ^ 2 + 1) * 0.001
    black_root = MCTSNode(Position(env))
    white_root = MCTSNode(Position(env,to_play = WHITE))
    incorporate_results!(select_leaf(black_root), probs, 0, black_root)
    incorporate_results!(select_leaf(white_root), probs, 0, white_root)
    # No matter who is to play, when we know nothing else, the priors
    # should be respected, and the same move should be picked
    black_leaf = select_leaf(black_root)
    white_leaf = select_leaf(white_root)
    @test black_leaf.fmove == white_leaf.fmove
    @test assertEqualArray(child_action_score(black_root), child_action_score(white_root))
  end

  @testset "select_leaf" begin
    flattened = to_flat(from_kgs("D9", env), env)
    probs = 0.02 * ones(env.N ^ 2 + 1)
    probs[flattened] = 0.4
    root = MCTSNode(SEND_TWO_RETURN_ONE)
    incorporate_results!(select_leaf(root), probs, 0, root)

    @test root.position.to_play == WHITE
    @test select_leaf(root) == root.children[flattened]
  end

  @testset "backup_incorporate_results" begin
    probs = 0.02 * ones(env.N ^ 2 + 1)
    root = MCTSNode(SEND_TWO_RETURN_ONE)
    incorporate_results!(select_leaf(root), probs, 0, root)

    leaf = select_leaf(root)
    incorporate_results!(leaf, probs, -1, root)  # white wins!

    # Root was visited twice: first at the root, then at this child.
    @test N(root) == 2
    # Root has 0 as a prior and two visits with value 0, -1
    @test Q(root) ≈ -1/3  # average of 0, 0, -1
    # Leaf should have one visit
    @test root.child_N[leaf.fmove] == 1
    @test N(leaf) == 1
    # And that leaf's value had its parent's Q (0) as a prior, so the Q
    # should now be the average of 0, -1
    @test child_Q(root)[leaf.fmove] == -0.5
    @test Q(leaf) ≈ -0.5

    # We're assuming that select_leaf() returns a leaf like:
    #   root
    #     \
    #     leaf
    #       \
    #       leaf2
    # which happens in this test because root is W to play and leaf was a W win.
    @test root.position.to_play == WHITE
    leaf2 = select_leaf(root)
    incorporate_results!(leaf2, probs, -0.2, root)  # another white semi-win
    @test N(root) == 3
    # average of 0, 0, -1, -0.2
    @test Q(root) ≈ -0.3

    @test N(leaf) == 2
    @test N(leaf2) == 1
    # average of 0, -1, -0.2
    @test Q(leaf) ≈ child_Q(root)[leaf.fmove]
    @test Q(leaf) ≈ -0.4
    # average of -1, -0.2
    @test child_Q(leaf)[leaf2.fmove] ≈ -0.6
    @test Q(leaf2) ≈ -0.6
  end

  @testset "do_not_explore_past_finish" begin
    probs = 0.02 * ones(Float32, env.N ^ 2 + 1)
    root = MCTSNode(Position(env))
    incorporate_results!(select_leaf(root), probs, 0, root)
    first_pass = maybe_add_child!(root, to_flat(nothing, env))
    incorporate_results!(first_pass, probs, 0, root)
    second_pass = maybe_add_child!(first_pass, to_flat(nothing, env))
    @test_throws AssertionError incorporate_results!(second_pass, probs, 0, root)
    node_to_explore = select_leaf(second_pass)
    # should just stop exploring at the end position.
    @test node_to_explore == second_pass
  end

  @testset "add_child" begin
    root = MCTSNode(Position(env,))
    child = maybe_add_child!(root, 17)
    @test 17 ∈ keys(root.children)
    @test child.parent == root
    @test child.fmove == 17
  end

  @testset "add_child_idempotency" begin
    root = MCTSNode(Position(env,))
    child = maybe_add_child!(root, 17)
    current_children = copy(root.children)
    child2 = maybe_add_child!(root, 17)
    @test child == child2
    @test current_children == root.children
  end

  @testset "never_select_illegal_moves" begin
    probs = 0.02 * ones(env.N ^ 2 + 1)
    # let's say the NN were to accidentally put a high weight on an illegal move
    probs[2] = 0.99
    root = MCTSNode(SEND_TWO_RETURN_ONE)
    incorporate_results!(root, probs, 0, root)
    # and let's say the root were visited a lot of times, which pumps up the
    # action score for unvisited moves...
    set_N!(root, 10000)
    root.child_N[Bool.(all_legal_moves(root.position))] = 10000
    # this should not throw an error...
    leaf = select_leaf(root)
    # the returned leaf should not be the illegal move
    @test leaf.fmove != 2

    # and even after injecting noise, we should still not select an illegal move
    for i = 1:10
      inject_noise!(root)
      leaf = select_leaf(root)
      @test leaf.fmove != 2
    end
  end

  @testset "dont_pick_unexpanded_child" begin
    probs = 0.02 * ones(env.N ^ 2 + 1)
    # make one move really likely so that tree search goes down that path twice
    # even with a virtual loss
    probs[18] = 0.999
    root = MCTSNode(Position(env,))
    incorporate_results!(root, probs, 0, root)
    leaf1 = select_leaf(root)
    @test leaf1.fmove == 18
    add_virtual_loss!(leaf1, root)
    # the second select_leaf pick should return the same thing, since the child
    # hasn't yet been sent to neural net for eval + result incorporation
    leaf2 = select_leaf(root)
    @test leaf1 == leaf2
  end
end


module CausalityTools

    using DynamicalSystems
    using StaticArrays

    export Dataset, trajectory, DiscreteDynamicalSystem, ContinuousDynamicalSystem
    export SVector, @SVector, MVector, @MVector
    using Requires

    using Reexport
    @reexport using CausalityToolsBase
    @reexport using StateSpaceReconstruction
    @reexport using PerronFrobenius
    @reexport using TransferEntropy
    @reexport using CrossMappings

    # Modules defined in CausalityTools
    include("JointDistanceDistribution/JointDistanceDistribution.jl")

    display_update = true
    version = "v0.9.0"
    update_name = "update_$version"

    if display_update
    if !isfile(joinpath(@__DIR__, update_name))
    printstyled(stdout,
    """
    \nUpdate message: CausalityTools $version
    ----------------
    New functionality
    ----------------
    #### TransferEntropy.jl

    - Added [`NearestNeighbourMI`](@ref) transfer entropy estimator.
    - Added [`transferentropy(::Any, ::TEVars, ::NearestNeighbourMI)`](@ref) method.
    - Transfer entropy estimators now contain a field `b` which gives the base of the logarithm
        used during transfer entropy computations, and hence dictates the unit of the transfer
        entropy. By default, `b = 2`, which gives the transfer entropy in bits. The keyword `b`
        is thus obsolete in all transfer entropy methods that used it before.

    #### CausalityTools.jl

    - Added [`NearestNeighbourMITest`](@ref) transfer entropy test.
    - Added [`NormalisedPredictiveAsymmetryTest`](@ref), for which the predictive asymmetry
        is normalised to some fraction of the mean of the raw values of the predictive statistic.
    - All subtypes of `CausalityTest` are now mutable. This allows adjusting test
        parameters during sensitivity analyses without creating a new test every time.

    #### TimeseriesSurrogates.jl

    - Plotting functionality is behind a Requires-block. To use the plotting functionality, you
        now need to do `using Plots` beforehand.

    ### Documentation

    - Fixed small error in documentation for `DiscreteSystemSetup`.
    - Created high-level pages for transfer entropy. Re-did documentation.
    - Created high-level pages for cross mappings.
    """; color = :light_magenta)
    touch(joinpath(@__DIR__, update_name))
    end
    end

    # Example systems
    include("systems/Systems.jl")

    # SystemModels module
    include("SystemModels/SystemModels.jl")

    # Wrappers of the different methods.
    include("method_wrappers/highlevel_methods.jl")
    include("method_wrappers/transferentropy_from_timeseries.jl")

    # Various algorithsm that are implemented here and not in subpackages
    include("algorithms/smeasure.jl")

    # High-level estimators
    include("CausalityTests/CausalityTests.jl")

    ################################################################
    # Integration with DynamicalSystems.jl (for all causality tests)
    ################################################################
    include("integration_dynamicalsystems/IntegrationDynamicalSystems.jl")

    ################################################################
    # Integration with UncertainData.jl (for all causality tests)
    ################################################################
    include("integration_uncertaindata/IntegrationUncertainData.jl")

    function __init__()

        # Surrogate wrappers for embeddings and Datasets
        @require TimeseriesSurrogates="c804724b-8c18-5caa-8579-6025a0767c70" @eval include("surrogates/surrogates.jl")
    end

    # Plot recipes, also for all sub-packages
    #include("plot_recipes/CausalityToolsPlotRecipes.jl")

end # module


using Luxor, ColorTypes

const JULIA_BLUE    = RGB(0.251, 0.388, 0.847);
const JULIA_GREEN   = RGB(0.22 , 0.596, 0.149);
const JULIA_PURPLE  = RGB(0.584, 0.345, 0.698);
const JULIA_RED     = RGB(0.796, 0.235, 0.2  );

const JULIA_COLOURS = [JULIA_BLUE, JULIA_GREEN, JULIA_PURPLE, JULIA_RED];

const size_x = 2560
const size_y = 1440

const r_min =   2
const r_max = 100

# const n_circles = 50000
const attempts  = 500

mutable struct C
    x::Int
    y::Int
    r::Int
end

function collides(c₁::C, circles)
    for c₂ in circles
        a = c₁.r + c₂.r
        x = c₁.x - c₂.x
        y = c₁.y - c₂.y

        a >= sqrt(x^2 + y^2) && return true
    end

    (c₁.x + c₁.r >= size_x || c₁.x - c₁.r <= 0) && return true
    (c₁.y + c₁.r >= size_y || c₁.y - c₁.r <= 0) && return true

    return false
end

function main(n_circles)
    Drawing(size_x, size_y, "background.svg")
    circles = C[]

    for i in 1:n_circles
        local c
        safe = false

        for j in 1:attempts
            c = C(floor(rand() * size_x), floor(rand() * size_y), r_min)

            if !collides(c, circles)
                safe = true
                break
            end
        end

        safe || continue

        for r in r_min:r_max
            c.r = r
            if collides(c, circles)
                c.r -= 1
                break
            end
        end

        push!(circles, c)

        sethue(rand(JULIA_COLOURS))
        circle(c.x, c.y, c.r, :fill)
    end

    finish()
    preview()
end

main(3) # compile
main(50000) # run


"""
Adds reactions to the model `m`
"""
function add_reactions(
    m::CoreModel,
    s::V1,
    b::V2,
    c::AbstractFloat,
    xl::AbstractFloat,
    xu::AbstractFloat;
    check_consistency = false,
) where {V1<:VecType,V2<:VecType}
    return add_reactions(
        m,
        sparse(reshape(s, (length(s), 1))),
        sparse(b),
        sparse([c]),
        sparse([xl]),
        sparse([xu]),
        check_consistency = check_consistency,
    )
end

function add_reactions(
    m::CoreModel,
    s::V1,
    b::V2,
    c::AbstractFloat,
    xl::AbstractFloat,
    xu::AbstractFloat,
    rxn::String,
    mets::K;
    check_consistency = false,
) where {V1<:VecType,V2<:VecType,K<:StringVecType}
    return add_reactions(
        m,
        sparse(reshape(s, (length(s), 1))),
        sparse(b),
        sparse([c]),
        sparse([xl]),
        sparse([xu]),
        [rxn],
        mets,
        check_consistency = check_consistency,
    )
end

function add_reactions(
    m::CoreModel,
    Sp::M,
    b::V,
    c::V,
    xl::V,
    xu::V;
    check_consistency = false,
) where {M<:MatType,V<:VecType}
    rxns = ["r$x" for x = length(m.rxns)+1:length(m.rxns)+length(xu)]
    mets = ["m$x" for x = length(m.mets)+1:length(m.mets)+size(Sp)[1]]
    return add_reactions(
        m,
        Sp,
        b,
        c,
        xl,
        xu,
        rxns,
        mets,
        check_consistency = check_consistency,
    )
end

function add_reactions(m1::CoreModel, m2::CoreModel; check_consistency = false)
    return add_reactions(
        m1,
        m2.S,
        m2.b,
        m2.c,
        m2.xl,
        m2.xu,
        m2.rxns,
        m2.mets,
        check_consistency = check_consistency,
    )
end

function add_reactions(
    m::CoreModel,
    Sp::M,
    b::V,
    c::V,
    xl::V,
    xu::V,
    rxns::K,
    mets::K;
    check_consistency = false,
) where {M<:MatType,V<:VecType,K<:StringVecType}

    Sp = sparse(Sp)
    b = sparse(b)
    c = sparse(c)
    xl = sparse(xl)
    xu = sparse(xu)

    all([length(b), length(mets)] .== size(Sp, 1)) ||
        throw(DimensionMismatch("inconsistent number of metabolites"))
    all(length.([c, xl, xu, rxns]) .== size(Sp, 2)) ||
        throw(DimensionMismatch("inconsistent number of reactions"))

    new_reactions = findall(Bool[!(rxn in m.rxns) for rxn in rxns])
    new_metabolites = findall(Bool[!(met in m.mets) for met in mets])

    if check_consistency
        (newReactions1, newMetabolites1) = verify_consistency(
            m,
            Sp,
            b,
            c,
            xl,
            xu,
            rxns,
            mets,
            new_reactions,
            new_metabolites,
        )
    end

    new_mets = vcat(m.mets, mets[new_metabolites])

    zero_block = spzeros(length(new_metabolites), n_reactions(m))
    ext_s = vcat(sparse(m.S), zero_block)

    mapping = [findfirst(isequal(met), new_mets) for met in mets]
    (I, J, elements) = findnz(sparse(Sp[:, new_reactions]))
    ext_sp = spzeros(length(new_mets), length(new_reactions))
    for (k, i) in enumerate(I)
        new_i = mapping[i]
        ext_sp[new_i, J[k]] = elements[k]
    end

    new_s = hcat(ext_s, ext_sp)
    newb = vcat(m.b, b[new_metabolites])
    #new_c = hcat(m.C, spzeros(size(m.C, 1), length(new_reactions)))
    newc = vcat(m.c, c[new_reactions])
    newxl = vcat(m.xl, xl[new_reactions])
    newxu = vcat(m.xu, xu[new_reactions])
    new_rxns = vcat(m.rxns, rxns[new_reactions])
    #new_lp = CoreModel(new_s, newb, new_c, m.cl, m.cu, newc, newxl, newxu, new_rxns, new_mets)
    new_lp = CoreModel(new_s, newb, newc, newxl, newxu, new_rxns, new_mets)

    if check_consistency
        return (new_lp, new_reactions, new_metabolites)
    else
        return new_lp
    end
end

"""
Verifies the consistency of a given model
"""
function verify_consistency(
    m::CoreModel,
    Sp::M,
    b::V,
    c::V,
    xl::V,
    xu::V,
    names::K,
    mets::K,
    new_reactions,
    new_metabolites,
) where {M<:MatType,V<:VecType,K<:StringVecType}

    if !isempty(new_reactions)
        statuses = Vector{ReactionStatus}(undef, length(names))
        for (i, name) in enumerate(names)
            rxn_index = findfirst(isequal(name), m.rxns)
            reaction = Sp[:, i]
            stoich_index = findfirst(Bool[reaction == m.S[:, j] for j = 1:size(m.S, 2)])
            if isnothing(rxn_index) & isnothing(stoich_index)
                statuses[i] = ReactionStatus(false, 0, "new")
            end

            if !isnothing(rxn_index) & isnothing(stoich_index)
                statuses[i] = ReactionStatus(true, 0, "same name")
            end

            if isnothing(rxn_index) & !isnothing(stoich_index)
                statuses[i] = ReactionStatus(true, 0, "same stoichiometry")
            end

            if !isnothing(rxn_index) & !isnothing(stoich_index)
                statuses[i] = ReactionStatus(true, 0, "same name, same stoichiometry")
            end
        end
    end

    return (new_reactions, new_metabolites)
end

"""
Removes a set of reactions from a CoreModel.
Also removes the metabolites not involved in any reaction.
"""
function remove_reactions(m::CoreModel, rxns::Vector{Int})
    rxns_to_keep = filter(e -> e ∉ rxns, 1:n_reactions(m))
    temp_s = m.S[:, rxns_to_keep]

    (mets_to_keep, J, val) = findnz(temp_s)
    sort!(mets_to_keep)
    unique!(mets_to_keep)
    new_s = m.S[mets_to_keep, rxns_to_keep]
    newb = m.b[mets_to_keep]
    newc = m.c[rxns_to_keep]
    newxl = m.xl[rxns_to_keep]
    newxu = m.xu[rxns_to_keep]
    new_rxns = m.rxns[rxns_to_keep]
    new_mets = m.mets[mets_to_keep]
    new_model = CoreModel(new_s, newb, newc, newxl, newxu, new_rxns, new_mets)
    return new_model
end

function remove_reactions(m::CoreModel, rxn::Integer)
    return remove_reactions(m, [rxn])
end

function remove_reactions(m::CoreModel, rxn::String)
    return remove_reactions(m, [rxn])
end

function remove_reactions(m::CoreModel, rxns::Vector{String})
    rxn_indices = [findfirst(isequal(name), m.rxns) for name in intersect(rxns, m.rxns)]
    if isempty(rxn_indices)
        return m
    else
        return remove_reactions(m, rxn_indices)
    end
end

"""
Returns indices of exchange reactions.
Exchange reactions are identified based on most commonly used prefixes.
"""
function find_exchange_reactions(
    model::CoreModel;
    exclude_biomass = false,
    biomass_str::String = "biomass",
    exc_prefs = ["EX_"; "Exch_"; "Ex_"],
)
    is_exc = falses(n_reactions(model))
    for pref in exc_prefs
        is_exc = is_exc .| startswith.(model.rxns, pref)
    end
    exc_inds = findall(is_exc)
    if exclude_biomass
        biom_inds = findall(x -> occursin(biomass_str, x), model.rxns)
        exc_inds = setdiff(exc_inds, biom_inds)
    end
    return exc_inds
end

"""
Returns indices of exchanged metabolites, ie, the outermost metabolites in the network
In practice returns the metabolites consumed by the reactions given by `find_exchange_reactions`
and if called with the same arguments, the two outputs correspond.
"""
function find_exchange_metabolites(
    model::CoreModel;
    exclude_biomass = false,
    biomass_str::String = "biomass",
    exc_prefs = ["EX_"; "Exch_"; "Ex_"],
)
    exc_rxn_inds = find_exchange_reactions(
        model,
        exclude_biomass = exclude_biomass,
        biomass_str = biomass_str,
        exc_prefs = exc_prefs,
    )
    exc_met_inds = [findfirst(x -> x == -1, model.S[:, j]) for j in exc_rxn_inds]
    return exc_met_inds
end

"""
Change the lower and/or upper bounds ('xl' and 'xu') for given reactions
"""
function change_bounds!(
    model::CoreModel,
    rxns::Vector{Int};
    xl::V = Float64[],
    xu::V = Float64[],
) where {V<:VecType}
    found = [index ∈ 1:n_reactions(model) for index in rxns]
    length(rxns[found]) == length(unique(rxns[found])) ||
        error("`rxns` appears to contain duplicates")

    if !isempty(xl)
        length(rxns) == length(xl) ||
            throw(DimensionMismatch("`rxns` size doesn't match with `xl`"))
        model.xl[rxns[found]] = xl[found]
    end
    if !isempty(xu)
        length(rxns) == length(xu) ||
            throw(DimensionMismatch("`rxns` size doesn't match with `xu`"))
        model.xu[rxns[found]] = xu[found]
    end
end

function change_bounds!(
    model::CoreModel,
    rxns::Vector{String};
    xl::V = Float64[],
    xu::V = Float64[],
) where {V<:VecType}
    found = [name ∈ model.rxns for name in rxns]
    rxn_indices = zeros(Int, length(rxns))
    rxn_indices[found] = [findfirst(isequal(name), model.rxns) for name in rxns[found]]
    change_bounds!(model, rxn_indices, xl = xl, xu = xu)
end


#=============================#
#== put needed modules here ==#
#=============================#
import JuliaChem

#================================#
#== JuliaChem execution script ==#
#================================#
include("minimal-rhf-repl.jl")

JuliaChem.initialize()
minimal_rhf(ARGS[1])
JuliaChem.finalize()


using Definitions
using DataFrames
using CSVFiles
using Model_Household

function map_person( model_person :: DataFrameRow )

    income = Incomes_Dict()
    for i in instances(Incomes_Type)
        ikey = make_sym_for_frame("income", i)
        if ! ismissing(model_person[ikey])
            if model_person[ikey] != 0.0
                v = model_person[ikey] # this is a hack because in j 1.3 csv is parsing many cols as strings & I don't understand why
                tv = typeof(v)
                if tv == String # FIXME delete not needed
                    v = parse( Float64, v )
                end
                # println( "setting ikey = $ikey to $v of type $tv")
                income[i] = v
            end
        end
    end

    assets = Asset_Dict()
    for i in instances(Asset_Type)
        if i != Missing_Asset_Type
            ikey = make_sym_for_asset( i )
            if ! ismissing(model_person[ikey])
                v = model_person[ikey]
                if typeof(v) == String
                    v = parse( Float64, v )
                end
                if model_person[ikey] != 0.0
                    assets[i] = model_person[ikey]
                end
            end
        end
    end

    disabilities = Disability_Dict()
    for i in instances(Disability_Type)
        ikey = make_sym_for_frame("disability", i)
        if ! ismissing(model_person[ikey])
            if model_person[ikey] == 1
                disabilities[i] = Bool(model_person[ikey])
            end
        end
    end

    relationships = Relationship_Dict()
    for i in 1:14
        relmod = Symbol( "relationship_$(i)") # :relationship_10 or :relationship_2
        irel = model_person[relmod]
        if (! ismissing( irel )) & ( irel >= 0 )
            pid = get_pid(
                FRS,
                model_person.data_year,
                model_person.hid,
                i )
            relationships[pid] = Relationship( irel )
        end
    end

    Person(

        BigInt(model_person.hid),  # BigInt# == sernum
        BigInt(model_person.pid),  # BigInt# == unique id (year * 100000)+
        model_person.pno,  # Integer# person number in household
        model_person.default_benefit_unit,  # Integer
        model_person.age,  # Integer

        Sex(model_person.sex),  # Sex
        Ethnic_Group(safe_assign(model_person.ethnic_group)),  # Ethnic_Group
        Marital_Status(safe_assign(model_person.marital_status)),  # Marital_Status
        Qualification_Type(safe_assign(model_person.highest_qualification)),  # Qualification_Type

        SIC_2007(safe_assign(model_person.sic)),  # SIC_2007
        Standard_Occupational_Classification(safe_assign(model_person.occupational_classification)),  # Standard_Occupational_Classification
        Employment_Sector(safe_assign(model_person.public_or_private)),  #  Employment_Sector
        Employment_Type(safe_assign(model_person.principal_employment_type)),  #  Employment_Type

        Socio_Economic_Group(safe_assign(model_person.socio_economic_grouping)),  # Socio_Economic_Group
        m2z(model_person.age_completed_full_time_education),  # Integer
        m2z(model_person.years_in_full_time_work),  # Integer
        ILO_Employment(safe_assign(model_person.employment_status)),  # ILO_Employment
        m2z(model_person.actual_hours_worked),  # Real
        m2z(model_person.usual_hours_worked),  # Real

        income,
        assets,

        safe_to_bool(model_person.registered_blind),
        safe_to_bool(model_person.registered_partially_sighted),
        safe_to_bool(model_person.registered_deaf),

        disabilities,
        Health_Status(safe_assign(model_person.health_status)),
        relationships,
        Relationship(model_person.relationship_to_hoh),
        safe_to_bool(model_person.is_informal_carer),
        safe_to_bool(model_person.receives_informal_care_from_non_householder),
        m2z(model_person.hours_of_care_received),
        m2z(model_person.hours_of_care_given),
        m2z(model_person.hours_of_childcare),
        m2z(model_person.cost_of_childcare),
        Child_Care_Type(safe_assign(model_person.childcare_type )),
        safe_to_bool( model_person.employer_provides_child_care ),

        Fuel_Type( m2z(model_person.company_car_fuel_type )),
        m2z(model_person.company_car_value),
        m2z(model_person.company_car_contribution),
        m2z(model_person.fuel_supplied)

    )

end

function map_hhld( hno::Integer, frs_hh :: DataFrameRow )

    people = People_Dict()

    Household(
        hno,
        frs_hh.hid,
        frs_hh.interview_year,
        frs_hh.interview_month,
        frs_hh.quarter,
        Tenure_Type(frs_hh.tenure),
        Standard_Region(frs_hh.region),
        CT_Band(frs_hh.ct_band),
        m2z(frs_hh.council_tax),
        m2z(frs_hh.water_and_sewerage ),
        m2z(frs_hh.mortgage_payment),
        m2z(frs_hh.mortgage_interest),
        m2z(frs_hh.years_outstanding_on_mortgage),
        m2z(frs_hh.mortgage_outstanding),
        m2z(frs_hh.year_house_bought),
        m2z(frs_hh.gross_rent),
        safe_to_bool(frs_hh.rent_includes_water_and_sewerage),
        m2z(frs_hh.other_housing_charges),
        m2z(frs_hh.gross_housing_costs),
        m2z(frs_hh.total_income),
        m2z(frs_hh.total_wealth),
        m2z(frs_hh.house_value),
        m2z(frs_hh.weight),
        people )
end

function load_hhld_from_frame( year :: Integer, hid :: Integer; hhld_fr :: DataFrame, pers_fr :: DataFrame ) :: Household
     frs_hh = hhld_fr[((hhld_fr.data_year .== year).& (hhld_fr.hid .== hid)),:]
     nhh = size( frs_hh )[1]
     @assert nhh in [0,1]
     if nhh == 1
         return load_hhld_from_frs( 1, frs_hh[1,:], pers_fr )
     else
        return missing
     end
end

function load_hhld_from_frame( hseq::Integer, hhld_fr :: DataFrameRow, pers_fr :: DataFrame ) :: Household
     hh = map_hhld( hseq, hhld_fr )
     pers_fr_in_this_hh = pers_fr[((pers_fr.data_year .== hhld_fr.data_year).&(pers_fr.hid .== hh.hid)),:]
     npers = size( pers_fr_in_this_hh )[1]
     @assert npers in 1:19
     for p in 1:npers
         pers = map_person( pers_fr_in_this_hh[p,:])
         hh.people[pers.pid] = pers
     end
     hh
 end


"Filters HMM outout and filters out desired matches"
function extract_rRNA_names(names_file::String, r16s::String, t16s::String, minl::Int, input::String)
    namesf = open(names_file, "w")

    # get read limit
    r_data = split(r16s, ":")
    r_star = parse(Int64, r_data[1])
    r_end = parse(Int64, r_data[2])
    println(stderr, "Filtering out matches in reads between $r_star:$r_end")

    # get red limit
    t_data = split(t16s, ":")
    t_star = parse(Int64, t_data[1])
    t_end = parse(Int64, t_data[2])
    t_star, t_end = sort([t_star, t_end])
    r_star, r_end = sort([r_star, r_end])
    println(stderr, "Filtering out matches in template between $t_star:$t_end")
    chosen_reads = Set{String}()
    # read the hmm output
    open(input) do f
        line = 1
        while !eof(f)
            x = readline(f)
            x = lstrip(x)
            if x[1] != '#'
                parts = split(x)
                rname = parts[1]
                ts = parse(Int64, parts[5])
                te = parse(Int64, parts[6])
                rs = parse(Int64, parts[9])
                re = parse(Int64, parts[10])
                ts, te = sort([ts, te])
                rs, re = sort([rs, re])

                if ((re-rs+1) >= minl) && (ts >= t_star) && (ts <= t_end) && (te >= t_star) && (te <= t_end) && (rs >= r_star) && (rs <= r_end) && (re >= r_star) && (re <= r_end)
                    println(namesf, rname)
                    push!(chosen_reads, rname)
                end
            end
            line += 1
        end
    end
    close(namesf)
    return chosen_reads
end


using Dates
using SHA
using DataFrames
using CSV
using JSON

hashlog(ID) = (read(`docker logs $(ID)`) |> sha256 |> bytes2hex)
containerDestroyByID(ID) = read(`docker rm -f $(ID)`)

function servicePort(str)
	a = read(`docker service inspect $(str)`, String) |> JSON.parse
	return a[1]["Spec"]["EndpointSpec"]["Ports"][1]["PublishedPort"]
end

function stats()
	a = read(`docker stats --no-stream`, String)

	b = split(a, '\n')
	c = map(x -> split(x), b)
	containerID = map(x -> String(x[1]), c[2:end-1])
	containerStr = map(x -> String(x[2] |> x -> split(x, '.')[1]), c[2:end-1])
	NetIO = map(x -> x[10], c[2:end-1])

	function mparse(x)
		N = try
			parse(Float64, x)
		catch
			nothing
		end
	end

	function parseIO(x)
		testunit(x, unit) = split(x, unit)[1] |> mparse

		testunit(x, "MB")
		if ~isnothing(testunit(x,"MB"))
			return (testunit(x, "MB"), "MB")
		elseif ~isnothing(testunit(x,"kB"))
			return (testunit(x, "kB"), "kB")
		elseif ~isnothing(testunit(x,"B"))
			return (testunit(x, "B"), "B")
		else
			return (0, "unknown")
		end
	end

	function netIOtoBytes(val)
		if (val[2] == "B")
			return val[1]
		elseif (val[2] == "kB")
			return val[1] * 1024
		elseif (val[2] == "MB")
			return val[1] * 1024 * 1024
		elseif (val[2] == "GiB")
			return val[1] * 1024 * 1024 * 1024
		else
			return 0
		end
	end

	parsedNetIO = map(parseIO, NetIO)
	cleanNetIO = map(netIOtoBytes, parsedNetIO)
	return containerID, containerStr, cleanNetIO
end

function containerAgeByID(ID)
	c = read(`docker ps -f "ID=$(ID)" --format "{{.CreatedAt}}"`, String)
	if c == ""
		return nothing
	end
	d = split(c) |> x -> String.(x)
	t = Date(d[1], dateformat"YYYY-mm-dd") + Time(d[2], dateformat"HH:MM:SS")
	age = round(now() - t, Minute)
end

function updateActivity(containerActivity, ID, netIO)
	try
		netIO - containerActivity.value[ID]
	catch
		netIO
	end
end

function regenerate_service(containerStr::String, port::Int, delay::Int)
	sleep(delay)
	z = String.(split(containerStr, '-'))
    thisPort = servicePort(containerStr)
	if thisPort ∉ resolve_ports[z[1]]
		println("Port was checked out, regenerate")
		push!(resolve_ports[z[1]], port)
	else
		println("Container regenerated, but container was not checked out")
	end
end

function containerManager()
	containerID, containerStr, netIO = stats()
	age = map(containerAgeByID, containerID)
	activity = map((x,y) -> updateActivity(containerActivity,x,y), containerID, netIO)
	map((x,y) -> containerActivity.value[x] = y, containerID, netIO)

	taintedFlag = map(containerID, containerStr) do x,y
		z = String.(split(y, '-'))
		# hash = open(f->read(f, String), z[1])
        thisPort = servicePort(y)
		println(y, "  ", thisPort, "  ", resolve_ports[z[1]])
		# (hashlog(x) != hash) || (thisPort ∉ resolve_ports[z[1]])
		thisPort ∉ resolve_ports[z[1]]
	end

	log = DataFrame(
		t = now(),
		containerID = containerID,
		containerStr = containerStr,
		age = age,
		netIO = netIO,
		activity = activity,
		taintedFlag = taintedFlag
	)

	function destroyContainer(containerID, containerStr, age, taintedFlag, activity)
		if taintedFlag && (activity < 1.0)
			z = String.(split(containerStr, '-'))
			port = port_base[z[1]] + parse(Int, z[2])
			println("Destroy $(containerID)")
			println("Restore Service port: $(port)")
			containerDestroyByID(containerID)
			@async regenerate_service(containerStr, port, 300) # restore to pool w/300s delay
		end

		return nothing
	end

	map(destroyContainer, containerID, containerStr, age, taintedFlag, activity)
	log |> CSV.write("logfile.csv", append = true)
	show(log)
	println()

	return log
end


const UrbanState = Records.Frame{Records.Entity{AutomotiveDrivingModels.VehicleState,AutomotiveDrivingModels.VehicleDef,Int64}}

const UrbanObs = Vector{Float64}

#### Action type

struct UrbanAction
    acc::Float64
end

function Base.copyto!(a::UrbanAction, b::UrbanAction)
    a.acc = b.acc
end

function Base.hash(a::UrbanAction, h::UInt64 = zero(UInt64))
    return hash(a.acc, h)
end

function Base.:(==)(a::UrbanAction, b::UrbanAction)
    return a.acc == b.acc
end

#### POMDP type

@with_kw mutable struct UrbanPOMDP <: POMDP{UrbanState, UrbanAction, UrbanObs}
    env::UrbanEnv = UrbanEnv()
    obs_dist::ObstacleDistribution = ObstacleDistribution(UrbanEnv())
    lidar::Bool = false
    sensor::AbstractSensor = GaussianSensor()
    models::Dict{Int64, DriverModel} = Dict{Int64, DriverModel}(1=>EgoDriver(UrbanAction(0.)))
    ego_type::VehicleDef = VehicleDef()
    car_type::VehicleDef = VehicleDef()
    ped_type::VehicleDef = VehicleDef(AgentClass.PEDESTRIAN, 1.0, 1.0)
    max_cars::Int64 = 10
    max_peds::Int64 = 10
    max_obstacles::Int64 = 3
    max_acc::Float64 = 2.0
    ego_start::Float64 = env.params.stop_line - ego_type.length/2
    ego_goal::LaneTag = LaneTag(2,2)
    off_grid::VecSE2 = VecSE2(UrbanEnv().params.x_min+VehicleDef().length/2, UrbanEnv().params.y_min+VehicleDef().width/2, 0)
    car_models::Dict{SVector{2, LaneTag}, DriverModel} = get_car_models(env, get_ttc_model)
    n_features::Int64 = 4
    ΔT::Float64 = 0.5 # decision frequency
    car_birth::Float64 = 0.3
    ped_birth::Float64 = 0.3
    a_noise::Float64 = 1.0
    v_noise::Float64 = 1.
    pos_obs_noise::Float64 = 0.5
    vel_obs_noise::Float64 = 0.5
    collision_cost::Float64 = -1.
    action_cost::Float64 = 0.
    goal_reward::Float64 = 1.
    γ::Float64 = 0.95 # discount factor
end

## HELPERS

function POMDPs.discount(pomdp::UrbanPOMDP)
    return pomdp.γ
end

POMDPs.actions(pomdp::UrbanPOMDP) = [UrbanAction(-2.0), UrbanAction(0.0), UrbanAction(2.0), UrbanAction(4.0)]

function POMDPs.actionindex(pomdp::UrbanPOMDP, action::UrbanAction)
    if action.acc == -4.0
        return 1
    elseif action.acc == -2.0
        return 2
    elseif action.acc == 0.
        return 3
    else
        return 4
    end
end

# only works for single lane intersection!
function get_car_models(env::UrbanEnv, get_model::Function)
    d = Dict{SVector{2, LaneTag}, DriverModel}()

    r1 = SVector(LaneTag(1,1), LaneTag(2,1))
    d[r1] = get_model(env, r1)

    r2 = SVector(LaneTag(1,1), LaneTag(5,1))
    d[r2] = get_model(env, r2)

    r3 = SVector(LaneTag(3,1), LaneTag(4,1))
    d[r3] = get_model(env, r3)

    r4 = SVector(LaneTag(3,1), LaneTag(5,1))
    d[r4] = get_model(env, r4)
    return d
end


# *********************************************************************************
# REopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this list
# of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
# *********************************************************************************
"""
    ElectricUtility

    Base.@kwdef struct ElectricUtility
        outage_start_time_step::Int=0  # for modeling a single outage, with critical load spliced into the baseline load ...
        outage_end_time_step::Int=0  # ... utiltity production_factor = 0 during the outage
        allow_simultaneous_export_import::Bool=true  # if true the site has two meters (in effect)
        # variables below used for minimax the expected outage cost,
        # with max taken over outage start time, expectation taken over outage duration
        outage_start_timesteps::Array{Int,1}=Int[]  # we minimize the maximum outage cost over outage start times
        outage_durations::Array{Int,1}=Int[]  # one-to-one with outage_probabilities, outage_durations can be a random variable
        outage_probabilities::Array{Real,1}=[1.0]
        outage_timesteps::Union{Missing, UnitRange} = isempty(outage_durations) ? missing : 1:maximum(outage_durations)
        scenarios::Union{Missing, UnitRange} = isempty(outage_durations) ? missing : 1:length(outage_durations)
    end

"""
Base.@kwdef struct ElectricUtility
    outage_start_time_step::Int=0  # for modeling a single outage, with critical load spliced into the baseline load ...
    outage_end_time_step::Int=0  # ... utiltity production_factor = 0 during the outage
    allow_simultaneous_export_import::Bool=true  # if true the site has two meters (in effect)
    # variables below used for minimax the expected outage cost,
    # with max taken over outage start time, expectation taken over outage duration
    outage_start_timesteps::Array{Int,1}=Int[]  # we minimize the maximum outage cost over outage start times
    outage_durations::Array{Int,1}=Int[]  # one-to-one with outage_probabilities, outage_durations can be a random variable
    outage_probabilities::Array{Real,1}=[1.0]
    outage_timesteps::Union{Missing, UnitRange} = isempty(outage_durations) ? missing : 1:maximum(outage_durations)
    scenarios::Union{Missing, UnitRange} = isempty(outage_durations) ? missing : 1:length(outage_durations)
    net_metering_limit_kw::Real = 0
    interconnection_limit_kw::Real = 1.0e9
end


export md_hmac, fp_prime_get

# Module initialization function
function __init__()
    # make sure the native lib was configure correctly
    @assert sizeof(Limb) == LIMB_SIZE
    @assert MD_LEN == 32  # SHA256
    @assert BN_SIZE >= 32  # 256-bits
    @assert BN_ST_SIZE == sizeof(BN)
    @assert FP_ST_SIZE == sizeof(FP)
    @assert FP2_ST_SIZE == sizeof(FP2)
    #@assert FP3_ST_SIZE == sizeof(FP3)
    @assert G1_ST_SIZE == sizeof(EP)
    @assert G2_ST_SIZE == sizeof(EP2)
    Threads.@threads for i = 1:Threads.nthreads()
        iszero(ccall((:core_init, LIB), Cint, ())) || error("core_init failed")
        iszero(ccall((:ep_param_set_any_pairf, LIB), Cint, ())) || error("ep_param_set_any_pairf failed")
        @info "$(basename(LIB)) initialised successfully (on Thread $(Threads.threadid()))!"
    end
end

# the data is always last so this should be safe for both BN and BigInt
unsafe_bnptr(bn::BN) = Ptr{Limb}(pointer_from_objref(bn) + sizeof(bn) - sizeof(bn.dp))

function bn_add_dig!(c::BN, a::BN, b::Limb)
    ccall((:bn_add_dig, LIB), Cvoid, (Ref{BN}, Ref{BN}, Limb), c, a, b)
    return c
end
function bn_add!(c::BN, a::BN, b::BN)
    ccall((:bn_add, LIB), Cvoid, (Ref{BN}, Ref{BN}, Ref{BN}), c, a, b)
    return c
end

function bn_mod_basic!(c::BN, a::BN, m::BN)
    ccall((:bn_mod_basic, LIB), Cvoid, (Ref{BN}, Ref{BN}, Ref{BN}), c, a, m)
    return c
end

function bn_mul_dig!(c::BN, a::BN, b::Limb)
    ccall((:bn_mul_dig, LIB), Cvoid, (Ref{BN}, Ref{BN}, Limb), c, a, b)
    return c
end
function bn_mul_comba!(c::BN, a::BN, b::BN)
    ccall((:bn_mul_comba, LIB), Cvoid, (Ref{BN}, Ref{BN}, Ref{BN}), c, a, b)
    return c
end
function bn_neg!(c::BN, a::BN)
    ccall((:bn_neg, LIB), Cvoid, (Ref{BN}, Ref{BN}), c, a)
    return c
end

function bn_rand!(a::BN, bits::Int)
    if bits < 1 || bits > typemax(Cint)
        error("invalid bits specified: $bits")
    end
    ccall((:bn_rand, LIB), Cvoid, (Ref{BN}, Cint, Cint), a, 0, bits)
    return a
end
function bn_read_bin!(a::BN, bin::Vector{UInt8})
    ccall((:bn_read_bin, LIB), Cvoid, (Ref{BN}, Ptr{UInt8}, Cint), a, bin, length(bin))
    return a
end

bn_size_bin(a::BN) = Int(ccall((:bn_size_bin, LIB), Cint, (Ref{BN},), a))

function bn_sub_dig!(c::BN, a::BN, b::Limb)
    ccall((:bn_sub_dig, LIB), Cvoid, (Ref{BN}, Ref{BN}, Limb), c, a, b)
    return c
end
function bn_sub!(c::BN, a::BN, b::BN)
    ccall((:bn_sub, LIB), Cvoid, (Ref{BN}, Ref{BN}, Ref{BN}), c, a, b)
    return c
end

function bn_write_bin!(bin::Vector{UInt8}, a::BN)
    ccall((:bn_write_bin, LIB), Cvoid, (Ptr{UInt8}, Cint, Ref{BN}), bin, length(bin), a)
    return bin
end

function ep_add_basic!(c::EP, a::EP, b::EP)
    ccall((:ep_add_basic, LIB), Cvoid, (Ref{EP}, Ref{EP}, Ref{EP}), c, a, b)
    return c
end

function ep_dbl_basic!(c::EP, a::EP)
    ccall((:ep_dbl_basic, LIB), Cvoid, (Ref{EP}, Ref{EP}), c, a)
    return c
end

function ep_curve_get_gen!(g::EP)
    ccall((:ep_curve_get_gen, LIB), Cvoid, (Ref{EP},), g)
    return g
end

function ep_curve_get_ord!(n::BN)
    ccall((:ep_curve_get_ord, LIB), Cvoid, (Ref{BN},), n)
    return n
end

function ep_map!(p::EP, msg::Vector{UInt8})
    ccall((:ep_map, LIB), Cvoid, (Ref{EP}, Ptr{UInt8}, Cint), p, msg, length(msg))
    return p
end

function ep_mul_lwnaf!(c::EP, a::EP, b::BN)
    ccall((:ep_mul_lwnaf, LIB), Cvoid, (Ref{EP}, Ref{EP}, Ref{BN}), c, a, b)
    return c
end

function ep_neg_basic!(c::EP, a::EP)
    ccall((:ep_neg_basic, LIB), Cvoid, (Ref{EP}, Ref{EP}), c, a)
    return c
end

function ep_read_bin!(a::EP, bin::Vector{UInt8})
    ccall((:ep_read_bin, LIB), Cvoid, (Ref{EP}, Ptr{UInt8}, Cint), a, bin, length(bin))
    return a
end

function ep_rand!(ep::EP)
    ccall((:ep_rand, LIB), Cvoid, (Ref{EP},), ep)
    return ep
end

ep_size_bin(ep::EP, pack::Bool) = Int(ccall((:ep_size_bin, LIB), Cint, (Ref{EP}, Cint), ep, pack ? one(Cint) : zero(Cint)))

function ep_sub_basic!(c::EP, a::EP, b::EP)
    ccall((:ep_sub_basic, LIB), Cvoid, (Ref{EP}, Ref{EP}, Ref{EP}), c, a, b)
    return c
end

function ep_write_bin!(bin::Vector{UInt8}, a::EP, pack::Bool)
    flags = pack ? one(Cint) : zero(Cint)
    ccall((:ep_write_bin, LIB), Cvoid, (Ptr{UInt8}, Cint, Ref{EP}, Cint), bin, length(bin), a, flags)
    return bin
end

function ep2_add_basic!(c::EP2, a::EP2, b::EP2)
    ccall((:ep2_add_basic, LIB), Cvoid, (Ref{EP2}, Ref{EP2}, Ref{EP2}), c, a, b)
    return c
end

function ep2_dbl_basic!(c::EP2, a::EP2)
    ccall((:ep2_dbl_basic, LIB), Cvoid, (Ref{EP2}, Ref{EP2}), c, a)
    return c
end

function ep2_curve_get_gen!(g::EP2)
    ccall((:ep2_curve_get_gen, LIB), Cvoid, (Ref{EP2},), g)
    return g
end

function ep2_curve_get_ord!(n::BN)
    ccall((:ep2_curve_get_ord, LIB), Cvoid, (Ref{BN},), n)
    return n
end

function ep2_map!(p::EP2, msg::Vector{UInt8})
    ccall((:ep2_map, LIB), Cvoid, (Ref{EP2}, Ptr{UInt8}, Cint), p, msg, length(msg))
    return p
end

function ep2_mul_lwnaf!(c::EP2, a::EP2, b::BN)
    ccall((:ep2_mul_lwnaf, LIB), Cvoid, (Ref{EP2}, Ref{EP2}, Ref{BN}), c, a, b)
    return c
end

function ep2_neg_basic!(c::EP2, a::EP2)
    ccall((:ep2_neg_basic, LIB), Cvoid, (Ref{EP2}, Ref{EP2}), c, a)
    return c
end

function ep2_rand!(ep::EP2)
    ccall((:ep2_rand, LIB), Cvoid, (Ref{EP2},), ep)
    return ep
end

function ep2_read_bin!(a::EP2, bin::Vector{UInt8})
    ccall((:ep2_read_bin, LIB), Cvoid, (Ref{EP2}, Ptr{UInt8}, Cint), a, bin, length(bin))
    return a
end

ep2_size_bin(ep::EP2, pack::Bool) = Int(ccall((:ep2_size_bin, LIB), Cint, (Ref{EP2}, Cint), ep, pack ? one(Cint) : zero(Cint)))

function ep2_sub_basic!(c::EP2, a::EP2, b::EP2)
    ccall((:ep2_sub_basic, LIB), Cvoid, (Ref{EP2}, Ref{EP2}, Ref{EP2}), c, a, b)
    return c
end

function ep2_write_bin!(bin::Vector{UInt8}, a::EP2, pack::Bool)
    flags = pack ? one(Cint) : zero(Cint)
    ccall((:ep2_write_bin, LIB), Cvoid, (Ptr{UInt8}, Cint, Ref{EP2}, Cint), bin, length(bin), a, flags)
    return bin
end

function fp_add_basic!(c::FP, a::FP, b::FP)
    ccall((:fp_add_basic, LIB), Cvoid, (Ref{FP}, Ref{FP}, Ref{FP}), c, a, b)
    return c
end

function fp_exp_slide!(c::FP, a::FP, b::BN)
    ccall((:fp_exp_slide, LIB), Cvoid, (Ref{FP}, Ref{FP}, Ref{BN}), c, a, b)
    return c
end

function fp_inv_lower!(c::FP, a::FP)
    ccall((:fp_inv_lower, LIB), Cvoid, (Ref{FP}, Ref{FP}), c, a)
    return c
end

function fp_mul_comba!(c::FP, a::FP, b::FP)
    ccall((:fp_mul_comba, LIB), Cvoid, (Ref{FP}, Ref{FP}, Ref{FP}), c, a, b)
    return c
end

function fp_neg_basic!(c::FP, a::FP)
    ccall((:fp_neg_basic, LIB), Cvoid, (Ref{FP}, Ref{FP}), c, a)
    return c
end

function fp_prime_get()
    fp = ccall((:fp_prime_get, LIB), Ptr{Limb}, ())
    bn = BN(used=FP_SIZE)
    unsafe_copyto!(unsafe_bnptr(bn), fp, bn.used)
    return bn
end

function fp_rand!(fp::FP)
    ccall((:fp_rand, LIB), Cvoid, (Ref{FP},), fp)
    return fp
end

function fp_sqr_comba!(c::FP, a::FP)
    ccall((:fp_sqr_comba, LIB), Cvoid, (Ref{FP}, Ref{FP}), c, a)
    return c
end

function fp_sub_basic!(c::FP, a::FP, b::FP)
    ccall((:fp_sub_basic, LIB), Cvoid, (Ref{FP}, Ref{FP}, Ref{FP}), c, a, b)
    return c
end

function fp_zero(c::FP)
    ccall((:fp_zero, LIB), Cvoid, (Ref{FP},), c)
    return c
end

function fp2_add_integ!(c::FP2, a::FP2, b::FP2)
    ccall((:fp2_add_integ, LIB), Cvoid, (Ref{FP2}, Ref{FP2}, Ref{FP2}), c, a, b)
    return c
end

function fp2_exp!(c::FP2, a::FP2, b::BN)
    ccall((:fp2_exp, LIB), Cvoid, (Ref{FP2}, Ref{FP2}, Ref{BN}), c, a, b)
    return c
end

function fp2_inv!(c::FP2, a::FP2)
    ccall((:fp2_inv, LIB), Cvoid, (Ref{FP2}, Ref{FP2}), c, a)
    return c
end

function fp2_mul_integ!(c::FP2, a::FP2, b::FP2)
    ccall((:fp2_mul_integ, LIB), Cvoid, (Ref{FP2}, Ref{FP2}, Ref{FP2}), c, a, b)
    return c
end

function fp2_neg!(c::FP2, a::FP2)
    ccall((:fp2_neg, LIB), Cvoid, (Ref{FP2}, Ref{FP2}), c, a)
    return c
end

function fp2_rand!(a::FP2)
    ccall((:fp2_rand, LIB), Cvoid, (Ref{FP2},), a)
    return a
end

function fp2_sub_integ!(c::FP2, a::FP2, b::FP2)
    ccall((:fp2_sub_integ, LIB), Cvoid, (Ref{FP2}, Ref{FP2}, Ref{FP2}), c, a, b)
    return c
end

function fp2_sqr_integ!(c::FP2, a::FP2)
    ccall((:fp2_sqr_integ, LIB), Cvoid, (Ref{FP2}, Ref{FP2}), c, a)
    return c
end

function fp2_zero(c::FP2)
    ccall((:fp2_zero, LIB), Cvoid, (Ref{FP2},), c)
    return c
end

function fp6_add!(c::FP6, a::FP6, b::FP6)
    ccall((:fp6_add, LIB), Cvoid, (Ref{FP6}, Ref{FP6}, Ref{FP6}), c, a, b)
    return c
end

function fp6_exp!(c::FP6, a::FP6, b::BN)
    ccall((:fp6_exp, LIB), Cvoid, (Ref{FP6}, Ref{FP6}, Ref{BN}), c, a, b)
    return c
end

function fp6_inv!(c::FP6, a::FP6)
    ccall((:fp6_inv, LIB), Cvoid, (Ref{FP6}, Ref{FP6}), c, a)
    return c
end

function fp6_mul_lazyr!(c::FP6, a::FP6, b::FP6)
    ccall((:fp6_mul_lazyr, LIB), Cvoid, (Ref{FP6}, Ref{FP6}, Ref{FP6}), c, a, b)
    return c
end

function fp6_neg!(c::FP6, a::FP6)
    ccall((:fp6_neg, LIB), Cvoid, (Ref{FP6}, Ref{FP6}), c, a)
    return c
end

function fp6_rand!(a::FP6)
    ccall((:fp6_rand, LIB), Cvoid, (Ref{FP6},), a)
    return a
end

function fp6_sub!(c::FP6, a::FP6, b::FP6)
    ccall((:fp6_sub, LIB), Cvoid, (Ref{FP6}, Ref{FP6}, Ref{FP6}), c, a, b)
    return c
end

function fp6_sqr_lazyr!(c::FP6, a::FP6)
    ccall((:fp6_sqr_lazyr, LIB), Cvoid, (Ref{FP6}, Ref{FP6}), c, a)
    return c
end

function fp6_zero(c::FP6)
    ccall((:fp6_zero, LIB), Cvoid, (Ref{FP6},), c)
    return c
end

function fp12_add!(c::FP12, a::FP12, b::FP12)
    ccall((:fp12_add, LIB), Cvoid, (Ref{FP12}, Ref{FP12}, Ref{FP12}), c, a, b)
    return c
end

function fp12_exp!(c::FP12, a::FP12, b::BN)
    ccall((:fp12_exp, LIB), Cvoid, (Ref{FP12}, Ref{FP12}, Ref{BN}), c, a, b)
    return c
end

function fp12_inv!(c::FP12, a::FP12)
    ccall((:fp12_inv, LIB), Cvoid, (Ref{FP12}, Ref{FP12}), c, a)
    return c
end

function fp12_mul_lazyr!(c::FP12, a::FP12, b::FP12)
    ccall((:fp12_mul_lazyr, LIB), Cvoid, (Ref{FP12}, Ref{FP12}, Ref{FP12}), c, a, b)
    return c
end

function fp12_neg!(c::FP12, a::FP12)
    ccall((:fp12_neg, LIB), Cvoid, (Ref{FP12}, Ref{FP12}), c, a)
    return c
end

function fp12_rand!(a::FP12)
    ccall((:fp12_rand, LIB), Cvoid, (Ref{FP12},), a)
    return a
end

function fp12_sub!(c::FP12, a::FP12, b::FP12)
    ccall((:fp12_sub, LIB), Cvoid, (Ref{FP12}, Ref{FP12}, Ref{FP12}), c, a, b)
    return c
end

function fp12_sqr_lazyr!(c::FP12, a::FP12)
    ccall((:fp12_sqr_lazyr, LIB), Cvoid, (Ref{FP12}, Ref{FP12}), c, a)
    return c
end

function fp12_zero(c::FP12)
    ccall((:fp12_zero, LIB), Cvoid, (Ref{FP12},), c)
    return c
end

function md_hmac(in::Vector{UInt8}, key::Vector{UInt8})
    mac = Vector{UInt8}(undef, MD_LEN)
    ccall((:md_hmac, LIB), Cvoid, (Ref{UInt8}, Ptr{UInt8}, Cint, Ptr{UInt8}, Cint),
        mac, in, length(in), key, length(key))
    return mac
end

function md_map_sh256(msg::Vector{UInt8})
    hash = Vector{UInt8}(undef, 32)
    ccall((:md_map_sh256, LIB), Cvoid, (Ref{UInt8}, Ptr{UInt8}, Cint), hash, msg, length(msg))
    return hash
end

function pp_exp_k12!(c::FP12, a::FP12)
    ccall((:pp_exp_k12, LIB), Cvoid, (Ref{FP12}, Ref{FP12}), c, a)
    return c
end

function pp_map_oatep_k12!(r::FP12, p::EP, q::EP2)
    ccall((:pp_map_oatep_k12, LIB), Cvoid, (Ref{FP12}, Ref{EP}, Ref{EP2}), r, p, q)
    return r
end


module SodiumSeal

@static if VERSION >= v"1.3"
    using libsodium_jll: libsodium
else
    let deps = joinpath(@__DIR__, "..", "deps", "deps.jl")
        isfile(deps) || error(
            "SodiumSeal is not installed properly. ",
            """Run Pkg.build("SodiumSeal"), restart Julia and try again.""",
        )
        include(deps)
    end
end

export
    KeyPair,
    seal,
    unseal

using Base64: base64encode, base64decode

const SEALBYTES = Ref{Csize_t}()
const PUBLICKEYBYTES = Ref{Csize_t}()
const SECRETKEYBYTES = Ref{Csize_t}()

### Errors ###

struct SodiumError
    fun::Symbol
    code::Int
end

Base.showerror(io::IO, e::SodiumError) = print(io, "$(e.fun) returned error code $(e.code)")

macro check(allowed, ex)
    fun = ex.args[2].args[1]
    return quote
        code = $(esc(ex))
        code in $allowed || throw(SodiumError($fun, code))
        code
    end
end

### Key pairs ###

"""
    KeyPair([public[, secret]])

Construct a new `KeyPair` with existing keys, or generate a new one.
If the keys are `AbstractString`s, they are assumed to be Base64-encoded.
If you are only interested in encrypting, you need not supply the secret key.
"""
struct KeyPair
    public::Vector{Cuchar}
    secret::Vector{Cuchar}
end

KeyPair(public) = KeyPair(public, [])
KeyPair(public::AbstractString) = KeyPair(public, "")
KeyPair(public::AbstractString, secret::AbstractString) =
    KeyPair(base64decode(public), base64decode(secret))
function KeyPair()
    public = Vector{Cuchar}(undef, PUBLICKEYBYTES[])
    secret = Vector{Cuchar}(undef, SECRETKEYBYTES[])

    @check 0 ccall(
        (:crypto_box_keypair, libsodium),
        Cint,
        (Ptr{Cuchar}, Ptr{Cuchar}),
        public, secret,
    )

    return KeyPair(public, secret)
end

Base.show(io::IO, k::KeyPair) = print(io, "KeyPair(...)")

### Encryption ###

"""
    seal(plaintext, keypair::KeyPair) -> Union{String, Vector{UInt8}}

Encrypt some data.

If `plaintext` is an `AbstractString`, it is assumed to be Base64-encoded,
and the output is also a Base64-encoded `String`.
Otherwise, it is a `Vector{UInt8}`.
"""
seal(plaintext::AbstractString, keypair::KeyPair) =
    base64encode(seal(base64decode(plaintext), keypair))
function seal(plaintext, keypair::KeyPair)
    len = length(plaintext)
    dest = Vector{Cuchar}(undef, SEALBYTES[] + len)

    @check 0 ccall(
        (:crypto_box_seal, libsodium),
        Cint,
        (Ptr{Cuchar}, Ptr{Cuchar}, Culonglong, Ptr{Cuchar}),
        dest, plaintext, len, keypair.public,
    )

    return dest
end

### Decryption ###

"""
    unseal(ciphertext, keypair::KeyPair) -> Union{String, Vector{UInt8}}

Decrypt some data.

If `ciphertext` is an `AbstractString`, it is assumed to be Base64-encoded,
and the output is also a Base64-encoded `String`.
Otherwise, it is a `Vector{UInt8}`.
"""
unseal(ciphertext::AbstractString, keypair::KeyPair) =
    base64encode(unseal(base64decode(ciphertext), keypair))
function unseal(ciphertext, keypair)
    len = length(ciphertext)
    dest = Vector{Cuchar}(undef, len - SEALBYTES[])

    @check 0 ccall(
        (:crypto_box_seal_open, libsodium),
        Cint,
        (Ptr{Cuchar}, Ptr{Cuchar}, Culonglong, Ptr{Cuchar}, Ptr{Cuchar}),
        dest, ciphertext, len, keypair.public, keypair.secret,
    )

    return dest
end

### Init ###

function __init__()
    code = @check (0, 1) ccall((:sodium_init, libsodium), Cint, ())
    if code == 0
        SEALBYTES[] = ccall((:crypto_box_sealbytes, libsodium), Csize_t, ())
        PUBLICKEYBYTES[] = ccall((:crypto_box_publickeybytes, libsodium), Csize_t, ())
        SECRETKEYBYTES[] = ccall((:crypto_box_secretkeybytes, libsodium), Csize_t, ())
    end
end

end


## Julia based

using FlashWeave

# Load  metadata files
meta_data_path = "files/FlashWeave_data/metadata_flashweave_friendly.csv"

# Load SPECIES data files
species_data_path = "files/FlashWeave_data/flashweave_friendly_species_relab.csv"
####### Species Model #######
species_results = learn_network(species_data_path, meta_data_path, sensitive=true, heterogeneous=false, verbose=true, feed_forward= true, alpha=0.05, FDR=true)
save_network("data/Co_Abundance_Analysis/species_network_output.edgelist", species_results)
save_network("data/Co_Abundance_Analysis/species_network_output.gml", species_results)

# Load GENUS data files
genus_data_path = "files/FlashWeave_data/flashweave_friendly_genus_relab.csv"
####### Species Model #######
genus_results = learn_network(genus_data_path, meta_data_path, sensitive=true, heterogeneous=false, verbose=true, feed_forward= true, alpha=0.05, FDR=true)
save_network("data/Co_Abundance_Analysis/genus_network_output.edgelist", genus_results)
save_network("data/Co_Abundance_Analysis/genus_network_output.gml", genus_results)

# Load PHYLUM data files
phylum_data_path = "files/FlashWeave_data/flashweave_friendly_phylum_relab.csv"
####### Species Model #######
phylum_results = learn_network(phylum_data_path, meta_data_path, sensitive=true, heterogeneous=false, verbose=true, feed_forward= true, alpha=0.05, FDR=true)
save_network("data/Co_Abundance_Analysis/phylum_network_output.edgelist", phylum_results)
save_network("data/Co_Abundance_Analysis/phylum_network_output.gml", phylum_results)

# Load PATHWAY data files
pathways_data_path = "files/FlashWeave_data/flashweave_friendly_pathways_relab.csv"
####### PATHWAY Model #######
pathways_results = learn_network(pathways_data_path, meta_data_path, sensitive=true, heterogeneous=false, verbose=true, feed_forward= true, alpha=0.05, FDR=true)
save_network("data/Co_Abundance_Analysis/pathways_network_output.edgelist", pathways_results)
save_network("data/Co_Abundance_Analysis/pathways_network_output.gml", pathways_results)

# Load ENZYME data files
enzymes_data_path = "files/FlashWeave_data/flashweave_friendly_enzymes_relab.csv"
####### ENZYME Model #######
enzymes_results = learn_network(enzymes_data_path, meta_data_path, sensitive=true, heterogeneous=true, verbose=true, feed_forward= true, alpha=0.05, FDR=true)
save_network("data/Co_Abundance_Analysis/enzymes_network_output.edgelist", enzymes_results)
save_network("data/Co_Abundance_Analysis/enzymes_network_output.gml", enzymes_results)

# Load Kegg Ortholog data files
kos_data_path = "files/FlashWeave_data/flashweave_friendly_kos_relab.csv"
####### KO Model #######
kos_results = learn_network(kos_data_path, meta_data_path, sensitive=true, heterogeneous=true, verbose=true, feed_forward= true, alpha=0.05, FDR=true)
save_network("data/Co_Abundance_Analysis/kos_network_output.edgelist", kos_results)
save_network("data/Co_Abundance_Analysis/kos_network_output.gml", kos_results)


struct CommissionReport
  execId::String
  commission::Float64
  currency::String
  realizedPNL::Union{Float64,Nothing}
  yield::Union{Float64,Nothing}
  yieldRedemptionDate::Union{Int,Nothing}     # yyyymmdd format TODO: -> String or Date
end

struct ContractDescription
  contract::Contract
  derivativeSecTypes::Vector{String}
end

mutable struct ContractDetails
  contract::Contract
  marketName::String
  minTick::Float64
  orderTypes::String
  validExchanges::String
  priceMagnifier::Int
  underConId::Int
  longName::String
  contractMonth::String
  industry::String
  category::String
  subcategory::String
  timeZoneId::String
  tradingHours::String
  liquidHours::String
  evRule::String
  evMultiplier::Union{Float64,Nothing}
  mdSizeMultiplier::Union{Int,Nothing}
  aggGroup::Union{Int,Nothing}
  underSymbol::String
  underSecType::String
  marketRuleIds::String
  realExpirationDate::String
  lastTradeTime::String
  stockType::String
  secIdList::NamedTuple
  cusip::String
  ratings::String
  descAppend::String
  bondType::String
  couponType::String
  callable::Bool
  putable::Bool
  coupon::Float64
  convertible::Bool
  maturity::String
  issueDate::String
  nextOptionDate::String
  nextOptionType::String
  nextOptionPartial::Bool
  notes::String
end
ContractDetails() = ContractDetails(Contract(), ns, 0.0, ns, ns, 0, 0, fill(ns, 9)..., 0,
                                    nothing, nothing, fill(ns, 6)..., NamedTuple(),
                                    fill(ns, 5)..., false, false, 0, false, ns, ns, ns, ns,
                                    false, ns)

struct Execution
  orderId::Union{Int,Nothing}     # Moved up from position 10 for convenience when constructing
  execId::String
  time::String
  acctNumber::String
  exchange::String
  side::String
  shares::Float64
  price::Float64
  permId::Int
  clientId::Int
  liquidation::Int
  cumQty::Float64
  avgPrice::Float64
  orderRef::String
  evRule::String
  evMultiplier::Union{Float64,Nothing}
  modelCode::String
  lastLiquidity::Int
end

struct FamilyCode
  accountID::String
  familyCodeStr::String
end

struct OrderState
  status::String
  initMarginBefore::String
  maintMarginBefore::String
  equityWithLoanBefore::String
  initMarginChange::String
  maintMarginChange::String
  equityWithLoanChange::String
  initMarginAfter::String
  maintMarginAfter::String
  equityWithLoanAfter::String
  commission::Union{Float64,Nothing}
  minCommission::Union{Float64,Nothing}
  maxCommission::Union{Float64,Nothing}
  commissionCurrency::String
  warningText::String
  completedTime::String
  completedStatus::String
end

TickAttrib =       NamedTuple{(:canAutoExecute, :pastLimit, :preOpen),NTuple{3,Bool}}
TickAttribLast =   NamedTuple{(:pastLimit, :unreported),NTuple{2,Bool}}
TickAttribBidAsk = NamedTuple{(:bidPastLow, :askPastHigh),NTuple{2,Bool}}


using CSV
test = CSV.read("/Users/Matthias/.julia/dev/ExoplanetsSysSim/data/q1q17_dr25_gaia_fgk.csv",allowmissing=:all)

include("clusters.jl")
sim_param = setup_sim_param_model()

##### To generate the underlying systems:

@time cat_phys = generate_kepler_physical_catalog(sim_param)

##### For saving the underlying/true planets/systems:

f = open("physical_catalog_planets.txt", "w")
write_model_params(f, sim_param)
println(f, "target_id star_id planet_mass planet_radius period ecc")
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys) > 1 #this should never happen
        println("There is more than one system for a given target? Check index: ", i)
    end
    if length(targ.sys[1].planet) > 0
        for (j,planet) in enumerate(targ.sys[1].planet)
            println(f, join([i, targ.sys[1].star.id, planet.mass, planet.radius, targ.sys[1].orbit[j].P, targ.sys[1].orbit[j].ecc], " "))
        end
    end
end
close(f)

f = open("physical_catalog_stars.txt", "w")
write_model_params(f, sim_param)
println(f, "target_id star_id star_mass star_radius num_planets")
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys) > 1 #this should never happen
        println("There is more than one system for a given target? Check index: ", i)
    end
    println(f, join([i, targ.sys[1].star.id, targ.sys[1].star.mass, targ.sys[1].star.radius, length(targ.sys[1].planet)], " "))
end
close(f)

f = open("periods_all.out", "w")
write_model_params(f, sim_param)
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys[1].orbit) > 0
        periods_sys = Array{Float64}(undef, length(targ.sys[1].orbit))
        for (j,planet) in enumerate(targ.sys[1].orbit)
            periods_sys[j] = planet.P #days
        end
        println(f, periods_sys)
    end
end
close(f)

f = open("eccentricities_all.out", "w")
write_model_params(f, sim_param)
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys[1].orbit) > 0
        ecc_sys = Array{Float64}(undef, length(targ.sys[1].orbit))
        for (j,planet) in enumerate(targ.sys[1].orbit)
            ecc_sys[j] = planet.ecc
        end
        println(f, ecc_sys)
    end
end
close(f)

f = open("radii_all.out", "w")
write_model_params(f, sim_param)
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys[1].planet) > 0
        radii_sys = Array{Float64}(undef, length(targ.sys[1].planet))
        for (j,planet) in enumerate(targ.sys[1].planet)
            radii_sys[j] = planet.radius #solar radii
        end
        println(f, radii_sys)
    end
end
close(f)

f = open("masses_all.out", "w")
write_model_params(f, sim_param)
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys[1].planet) > 0
        masses_sys = Array{Float64}(undef, length(targ.sys[1].planet))
        for (j,planet) in enumerate(targ.sys[1].planet)
            masses_sys[j] = planet.mass #solar masses
        end
        println(f, masses_sys)
    end
end
close(f)

##### For saving the stellar properties of all the systems:

f = open("stellar_masses_with_planets.out", "w")
write_model_params(f, sim_param)
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys[1].planet) > 0
        star_mass = targ.sys[1].star.mass #solar masses
        println(f, star_mass)
    end
end
close(f)

f = open("stellar_radii_with_planets.out", "w")
write_model_params(f, sim_param)
for (i,targ) in enumerate(cat_phys.target)
    if length(targ.sys[1].planet) > 0
        star_radius = targ.sys[1].star.radius #solar radii
        println(f, star_radius)
    end
end
close(f)

##### To generate the observed systems:

@time begin
    cat_phys_cut = ExoplanetsSysSim.generate_obs_targets(cat_phys,sim_param)
    cat_obs = observe_kepler_targets_single_obs(cat_phys_cut,sim_param)
    summary_stat = calc_summary_stats_model(cat_obs,sim_param)
end

##### For saving the observed planets/systems:

f = open("observed_catalog_planets.txt", "w")
write_model_params(f, sim_param)
println(f, "target_id star_id period depth duration")
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        for k in 1:num_pl_in_sys
            println(f, join([j, cat_phys.target[j].sys[1].star.id, map(ExoplanetsSysSim.period, cat_obs.target[j].obs)[k], map(ExoplanetsSysSim.depth, cat_obs.target[j].obs)[k], map(ExoplanetsSysSim.duration, cat_obs.target[j].obs)[k]], " "))
        end
    end
end
close(f)

f = open("observed_catalog_stars.txt", "w")
write_model_params(f, sim_param)
println(f, "target_id star_id star_mass star_radius num_obs_planets")
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        println(f, join([j, cat_phys.target[j].sys[1].star.id, cat_obs.target[j].star.mass, cat_obs.target[j].star.radius, length(cat_obs.target[j].obs)], " "))
    end
end
close(f)

f = open("periods.out", "w")
write_model_params(f, sim_param)
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    period_array = Array{Float64}(undef, num_pl_in_sys, num_targets)
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        period_array[:,i] = map(ExoplanetsSysSim.period, cat_obs.target[j].obs)[1:num_pl_in_sys]
    end
    println(f,"# Periods of systems with ", num_pl_in_sys, " detected planets.")
    if length(period_array) > 0
        println(f,period_array')
    end
end
close(f)

f = open("depths.out", "w")
write_model_params(f, sim_param)
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    depths_array = Array{Float64}(undef, num_pl_in_sys, num_targets)
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        depths_array[:,i] = map(ExoplanetsSysSim.depth, cat_obs.target[j].obs)[1:num_pl_in_sys]
    end
    println(f,"# Transit depths of systems with ", num_pl_in_sys, " detected planets.")
    if length(depths_array) > 0
        println(f,depths_array')
    end
end
close(f)

f = open("durations.out", "w")
write_model_params(f, sim_param)
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    duration_array = Array{Float64}(undef, num_pl_in_sys, num_targets)
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        duration_array[:,i] = map(ExoplanetsSysSim.duration, cat_obs.target[j].obs)[1:num_pl_in_sys]
    end
    println(f,"# Transit durations of systems with ", num_pl_in_sys, " detected planets.")
    if length(duration_array) > 0
        println(f,duration_array')
    end
end
close(f)

#=
f = open("eccentricities.out", "w")
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    ecc_array = Array{Float64}(num_pl_in_sys,num_targets)
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        ecc_array[:,i] = map(ExoplanetsSysSim.eccentricity, cat_obs.target[j].obs)[1:num_pl_in_sys]
    end
    println(f,"# Eccentricities of systems with ", num_pl_in_sys, " detected planets.")
    if length(ecc_array) > 0
        println(f,ecc_array')
    end
end
close(f)
=#

##### For saving the stellar properties of the systems with observed planets:

f = open("stellar_masses_obs.out", "w")
write_model_params(f, sim_param)
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    stellar_mass_array = Array{Float64}(undef, num_targets)
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        stellar_mass_array[i] = cat_obs.target[j].star.mass
    end
    println(f,"# Stellar masses of systems with ", num_pl_in_sys, " detected planets.")
    if length(stellar_mass_array) > 0
        println(f, stellar_mass_array)
    end
end
close(f)

f = open("stellar_radii_obs.out", "w")
write_model_params(f, sim_param)
for num_pl_in_sys in 1:length(summary_stat.cache["idx_n_tranets"])
    num_targets = length(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
    stellar_radius_array = Array{Float64}(undef, num_targets)
    for (i,j) in enumerate(summary_stat.cache["idx_n_tranets"][num_pl_in_sys])
        stellar_radius_array[i] = cat_obs.target[j].star.radius
    end
    println(f,"# Stellar radii of systems with ", num_pl_in_sys, " detected planets.")
    if length(stellar_radius_array) > 0
        println(f, stellar_radius_array)
    end
end
close(f)


function load_xyz_data(file_path :: String; atom_names = nothing, radii = [3.5], weights = [1.0], masses = [1.0], boundary_type = ["p", "p", "p"], units = "lj")
    # frames = read_frames(file_path)
    # n = length(frames)
    # r = Vector{Configuration}(undef, n)

    # if atom_names == nothing
    #     atom_names = unique(frames[1]["arrays"]["species"])
    # end
    # num_atom_types = length(atom_names)

    # if length(radii) != num_atom_types
    #     radii = radii[1] .* ones(num_atom_types)
    # end
    # if length(weights) != num_atom_types
    #     weights = weights .* ones(num_atom_types)
    # end
    # if length(masses) !=  num_atom_types
    #     masses = masses .* ones(num_atom_types)
    # end

    # for i = 1:n
    #     frame = frames[i]
    #     num_atoms = frame["N_atoms"]
    #     Atoms = Vector{Atom}(undef, num_atoms)
    #     for j = 1:num_atoms
    #         arrays = frame["arrays"]
    #         name = arrays["species"][j]
    #         id = findall(name .== atom_names)[1]
    #         pos = arrays["pos"][:, j]
    #         if "vel" in keys(arrays)
    #             vel = arrays[vel][:, j]
    #         else
    #             vel = zeros(3)
    #         end
    #         a = Atom(masses[id], pos, vel, name)
    #         Atoms[j] = a

    #     c = Configuration(Atoms, num_atom_types,
    #                       Angles, num_angle_types,
    #                       Bonds, num_bond_types,
    #                       Impropers, num_improper_types,
    #                       Dihedrals, num_dihedral_types,
    #                       atom_names, masses, radii, weights,
    #                       domain, units)
    #     r[i] = c
    #     end
    # end

    println("This is not yet implemented")
end

function save_xyz_data(c::Configuration, file_path::String)
    println("This is not yet implemented")
end


insts = OrderedDict()
insts["minimal"] = [
    ((2, 4, 1.0),),
    ((2, 4, 0.5),),
    ]
insts["fast"] = [
    ((15, 20, 1.0),),
    ((15, 20, 0.25),),
    ((100, 100, 1.0),),
    ((100, 100, 0.15),),
    ((500, 100, 1.0),),
    ((500, 100, 0.15),),
    ]
insts["various"] = [
    ((250, 1000, 0.1),),
    ((500, 5000, 0.05),),
    ((500, 2000, 1.0),),
    ]
return (LinearOptNative, insts)


"""
    create_geometry(setup)

Create cells, surfaces and finite element mesh.
This function does the following:
- Check geometry setup consistency
- Create or load cell configuration
- Create or load surface triangulation
- Call TetGen
- Deform domain
- Split mesh into compartments
For custom geometries with more than one compartment, call `split_mesh`
directly instead. This requires facet and element labels.
"""
function create_geometry(setup::Setup)

    @unpack ncell, ecs_shape, refinement = setup

    # File name for saving or loading geometry
    filename = "meshfiles" * "/" * setup.name

    # Make sure that folder exists
    dir = dirname(filename)
    isdir(dir) || mkpath(dir)

    # Check correct input format
    if isa(setup, NeuronSetup) && ncell != 1
        error("Neuron cell type is only available for ncell=1.")
    end
    !(ecs_shape == "no_ecs" && ncell > 1) ||
        @error "Geometry must include ECS if more than one cell"
    @assert ecs_shape ∈ ["no_ecs", "box", "convex_hull", "tight_wrap"]

    # Check if cell description file is already available
    cellfilename = filename * "_cells"
    if isfile(cellfilename)
        cells = read_cells(cellfilename)
    elseif isa(setup, SphereSetup) || isa(setup, CylinderSetup)
        # Create cells
        cells = create_cells(setup)

        # Save cell configuration
        save_cells(cells, cellfilename)
    else
        # No cells
        cells = nothing
    end

    # Make directory for storing finite elements mesh
    is_stl = endswith(filename, ".stl")
    if is_stl
        # ECS is currently only available for surface meshes
        @assert ecs_shape == "no_ecs"
        filename = filename[1:end-4]
    end
    save_meshdir_path = filename * "_dir"
    if !isdir(save_meshdir_path)
        mkpath(save_meshdir_path)
    end

    # Use an existing finite elements mesh or create a new finite
    # elements mesh. The name of the finite elements mesh is stored in the string
    # fname_tetgen_femesh
    refinement_str = isnothing(refinement) ? "" : "_refinement$refinement"
    fname_tetgen =
        save_meshdir_path * "/" * split(filename, "/")[end] * refinement_str * "_mesh"

    # Read or create surface triangulation
    if isfile(fname_tetgen * ".node") && isfile(fname_tetgen * ".poly")
        surfaces = read_surfaces(fname_tetgen)
    else
        if isa(setup, SphereSetup)
            # Create surface geometry of spheres
            surfaces = create_surfaces_sphere(cells, setup)
        elseif isa(setup, CylinderSetup)
            # Create surface geometry of cylinders
            surfaces = create_surfaces_cylinder(cells, setup)
        elseif isa(setup, NeuronSetup)
            if is_stl
                surfaces = nothing
            else
                surfaces = create_surfaces_neuron(filename, setup)
            end
        end

        if !is_stl
            save_surfaces(fname_tetgen, surfaces)
        end
    end

    # Add ".1" suffix to output file name, since this is what Tetgen does
    fname_tetgen_femesh = fname_tetgen * ".1"

    if isfile(fname_tetgen_femesh * ".node")
        # Read global mesh from Tetgen output
        mesh_all = read_tetgen(fname_tetgen_femesh)
    elseif is_stl
        error("Not implemented")
    else
        mesh_all = call_tetgen(surfaces, refinement)
        save_tetgen(mesh_all, fname_tetgen_femesh)
    end

    # # Check that at correct number of compartments and boundaries has been found
    # compartments_new = unique(mesh_all.elementmarkers)
    # boundaries_new = unique(mesh_all.facetmarkers)

    # solution = "use smaller refinement or change surface triangulation."
    # length(compartments_new) == ncompartment ||
    #     @error "Incorrect number of compartments, " * solution
    # length(boundaries_new) == nboundary ||
    #     @error "Incorrect number of boundaries, " * solution

    # Deform domain
    if isa(setup, CylinderSetup) && (setup.bend > 1e-16 || setup.twist > 1e-16)
        println("Deforming domain with bend $(setup.bend) and twist $(setup.twist)")
        deform_domain!(mesh_all.points, setup.bend, setup.twist)
    end

    # Split mesh into compartments
    @time mesh = split_mesh(mesh_all)

    mesh, surfaces, cells
end