export monodromy_solve,
    MonodromyResult,
    real_solutions,
    is_success,
    is_heuristic_stop,
    nreal,
    parameters,
    verify_solution_completeness,
    solution_completeness_witnesses

#####################
# Monodromy Options #
#####################
const monodromy_options_supported_keywords = [
    :distance,
    :identical_tol,
    :done_callback,
    :group_action,
    :group_actions,
    :group_action_on_all_nodes,
    :parameter_sampler,
    :equivalence_classes,
    :complex_conjugation,
    :check_startsolutions,
    :target_solutions_count,
    :timeout,
    :min_solutions,
    :max_loops_no_progress,
    :reuse_loops,
]

struct MonodromyOptions{F<:Function,F1<:Function,F2<:Tuple,F3<:Function}
    distance_function::F
    identical_tol::Float64
    done_callback::F1
    group_actions::GroupActions{F2}
    group_action_on_all_nodes::Bool
    parameter_sampler::F3
    equivalence_classes::Bool
    complex_conjugation::Bool
    check_startsolutions::Bool
    # stopping heuristic
    target_solutions_count::Int
    timeout::Float64
    min_solutions::Int
    max_loops_no_progress::Int
    reuse_loops::Symbol
end

function MonodromyOptions(
    is_real_system::Bool,
    accuracy::Float64,
    distance;
    identical_tol::Float64 = sqrt(accuracy),
    done_callback = always_false,
    group_action = nothing,
    group_actions = group_action === nothing ? nothing : GroupActions(group_action),
    group_action_on_all_nodes = false,
    parameter_sampler = independent_normal,
    equivalence_classes = true,
    complex_conjugation = is_real_system,
    check_startsolutions = true,
    # stopping heuristic
    target_solutions_count = nothing,
    timeout = float(typemax(Int)),
    min_solutions::Int = default_min_solutions(target_solutions_count),
    max_loops_no_progress::Int = 10,
    reuse_loops::Symbol = :all,
)

    if group_actions isa GroupActions
        actions = group_actions
    else
        if group_actions === nothing
            equivalence_classes = false
        end
        actions = GroupActions(group_actions)
    end

    MonodromyOptions(
        distance,
        identical_tol,
        done_callback,
        actions,
        group_action_on_all_nodes,
        parameter_sampler,
        equivalence_classes,
        complex_conjugation,
        check_startsolutions,
        target_solutions_count === nothing ? typemax(Int) : target_solutions_count,
        float(timeout),
        min_solutions,
        max_loops_no_progress,
        reuse_loops,
    )
end

default_min_solutions(::Nothing) = 1
function default_min_solutions(target_solutions_count::Int)
    div(target_solutions_count, 2)
end

always_false(x, sols) = false

has_group_actions(options::MonodromyOptions) =
    !(options.group_actions isa GroupActions{Tuple{}})

"""
    independent_normal(p::AbstractVector{T}) where {T}

Sample a vector where each entries is drawn independently from the univariate normal distribution.
"""
independent_normal(p::SVector{N,T}) where {N,T} = @SVector randn(T, N)
independent_normal(p::AbstractVector{T}) where {T} = randn(T, length(p))

##########################
## Monodromy Statistics ##
##########################

mutable struct MonodromyStatistics
    ntrackedpaths::Int
    ntrackingfailures::Int
    nreal::Int
    nparametergenerations::Int
    nsolutions_development::Vector{Int}
end

MonodromyStatistics(nsolutions::Int) = MonodromyStatistics(0, 0, 0, 1, [nsolutions])
function MonodromyStatistics(solutions)
    stats = MonodromyStatistics(length(solutions))
    for s in solutions
        if is_real_vector(s)
            stats.nreal += 1
        end
    end
    stats
end

Base.show(io::IO, S::MonodromyStatistics) = print_fieldnames(io, S)
Base.show(io::IO, ::MIME"application/prs.juno.inline", S::MonodromyStatistics) = S

# update routines
function trackedpath!(stats::MonodromyStatistics, retcode)
    if is_success(retcode)
        stats.ntrackedpaths += 1
    else
        stats.ntrackingfailures += 1
    end
end

function generated_parameters!(stats::MonodromyStatistics, nsolutions::Int)
    stats.nparametergenerations += 1
    push!(stats.nsolutions_development, nsolutions)
end

function finished!(stats, nsolutions)
    push!(stats.nsolutions_development, nsolutions)
end

function n_completed_loops_without_change(stats, nsolutions)
    k = 0
    for i = length(stats.nsolutions_development):-1:1
        if stats.nsolutions_development[i] != nsolutions
            return max(k - 1, 0)
        end
        k += 1
    end
    return max(k - 1, 0)
end

function n_solutions_current_loop(statistics, nsolutions)
    nsolutions - statistics.nsolutions_development[end]
end

#############################
# Loops and Data Structures #
#############################

export Triangle, Petal

#######################
# Loop data structure
#######################

struct LoopEdge
    p₁::Vector{ComplexF64}
    p₀::Vector{ComplexF64}
    weights::Union{Nothing,NTuple{2,ComplexF64}}
end

function LoopEdge(p₁, p₀; weights = false)
    if weights
        γ = (randn(ComplexF64), randn(ComplexF64))
    else
        γ = nothing
    end
    LoopEdge(convert(Vector{ComplexF64}, p₁), convert(Vector{ComplexF64}, p₀), γ)
end

struct MonodromyLoop
    edges::Vector{LoopEdge}
end

function MonodromyLoop(base_p, nnodes::Int, options::MonodromyOptions; weights = true)
    edges = LoopEdge[]
    p₁ = base_p
    for i = 2:nnodes
        p₀ = options.parameter_sampler(p₁)
        push!(edges, LoopEdge(p₁, p₀; weights = weights))
        p₁ = p₀
    end
    push!(edges, LoopEdge(p₁, base_p; weights = weights))
    MonodromyLoop(edges)
end

##############
# Loop styles
##############
"""
    LoopStyle

Abstract type defining a style of a loop.
"""
abstract type LoopStyle end

"""
    Triangle(;useweights=true)

A triangle is a loop consisting of the main node and two addtional nodes.
If `weights` is true the edges are equipped with additional random weights.
Note that this is usually only necessary for real parameters.
"""
struct Triangle <: LoopStyle
    useweights::Bool
end
Triangle(; useweights = true) = Triangle(useweights)

function MonodromyLoop(strategy::Triangle, p, options::MonodromyOptions)
    MonodromyLoop(p, 3, options, weights = strategy.useweights)
end

"""
    Petal()

A petal is a loop consisting of the main node and one other node connected
by two edges with different random weights.
"""
struct Petal <: LoopStyle end
function MonodromyLoop(strategy::Petal, p, options::MonodromyOptions)
    MonodromyLoop(p, 2, options, weights = true)
end

"""
    regenerate!(loop::MonodromyLoop, options::MonodromyOptions, stats::MonodromyStatistics)

Regenerate all random parameters in the loop in order to introduce a new monodromy action.
"""
function regenerate!(
    loop::MonodromyLoop,
    options::MonodromyOptions,
    stats::MonodromyStatistics,
)
    main = mainnode(loop)

    # The first node is the main node and doesn't get touched
    store_points = options.group_action_on_all_nodes && has_group_actions(options)
    for i ∈ 2:length(loop.nodes)
        loop.nodes[i] = Node(
            options.parameter_sampler(main.p),
            loop.nodes[i],
            options;
            store_points = store_points, is_main_node = false,
        )
    end
    loop.edges .= Edge.(loop.edges)
    generated_parameters!(stats, length(main.points)) # bookkeeping
end

"""
    track(tracker, x, edge::LoopEdge, loop::MonodromyLoop, stats::MonodromyStatistics)

Track `x` along the edge `edge` in the loop `loop` using `tracker`. Record statistics
in `stats`.
"""
function track(
    tracker::CoreTracker,
    x::AbstractVector,
    loop::MonodromyLoop,
    ntrackedpaths::Base.RefValue{Int},
    ntrackingfailures::Base.RefValue{Int},
)
    H = basehomotopy(tracker.homotopy)
    local retcode::CoreTrackerStatus.states
    y = x
    for e in loop.edges
        set_parameters!(H, (e.p₁, e.p₀), e.weights)
        retcode = track!(tracker, y)
        if is_success(retcode)
            ntrackedpaths[] += 1
        else
            ntrackingfailures[] += 1
        end
        is_success(retcode) || break
        y = current_x(tracker)
    end
    retcode
end

#############
## Results ##
#############
"""
    MonodromyResult

The monodromy result contains the result of the `monodromy_solve` computation.
"""
struct MonodromyResult{AV<:AbstractVector,T}
    returncode::Symbol
    solutions::Vector{AV}
    parameters::Vector{T}
    statistics::MonodromyStatistics
    equivalence_classes::Bool
    seed::Int
end

Base.iterate(R::MonodromyResult) = iterate(R.solutions)
Base.iterate(R::MonodromyResult, state) = iterate(R.solutions, state)

Base.show(io::IO, ::MIME"application/prs.juno.inline", x::MonodromyResult) = x
function Base.show(io::IO, result::MonodromyResult{N,T}) where {N,T}
    println(io, "MonodromyResult")
    println(io, "==================================")
    if result.equivalence_classes
        println(
            io,
            "• ",
            nsolutions(result),
            " classes of solutions (modulo group action) (",
            nreal(result),
            ") real)",
        )
    else
        println(io, "• $(nsolutions(result)) solutions ($(nreal(result)) real)")
    end
    println(io, "• return code → $(result.returncode)")
    println(io, "• $(result.statistics.ntrackedpaths) tracked paths")
    println(io, "• seed → $(result.seed)")
end

TreeViews.hastreeview(::MonodromyResult) = true
TreeViews.numberofnodes(::MonodromyResult) = 6
TreeViews.treelabel(io::IO, x::MonodromyResult, ::MIME"application/prs.juno.inline") =
    print(
        io,
        "MonodromyResult",
    )

function TreeViews.nodelabel(
    io::IO,
    x::MonodromyResult,
    i::Int,
    ::MIME"application/prs.juno.inline",
)
    if i == 1
        if x.equivalence_classes
            print(io, "$(nsolutions(x)) classes of solutions (modulo group action)")
        else
            print(io, "$(nsolutions(x)) solutions")
        end
    elseif i == 2
        if x.equivalence_classes
            print(io, "$(nreal(x)) classes of real solutions")
        else
            print(io, "$(nreal(x)) real solutions")
        end
    elseif i == 3
        print(io, "Return code")
    elseif i == 4
        print(io, "Statistics")
    elseif i == 5
        print(io, "Parameters")
    elseif i == 6
        print(io, "Seed")
    end
end
function TreeViews.treenode(r::MonodromyResult, i::Integer)
    if i == 1
        return r.solutions
    elseif i == 2
        return real_solutions(r)
    elseif i == 3
        return r.returncode
    elseif i == 4
        return r.statistics
    elseif i == 5
        return r.parameters
    elseif i == 6
        return r.seed
    end
    missing
end

"""
    is_success(result::MonodromyResult)

Returns true if the monodromy computation achieved its target solution count.
"""
is_success(result::MonodromyResult) = result.returncode == :success

"""
    is_heuristic_stop(result::MonodromyResult)

Returns true if the monodromy computation stopped due to the heuristic.
"""
is_heuristic_stop(result::MonodromyResult) = result.returncode == :heuristic_stop

"""
    mapresults(f, result::MonodromyResult; only_real=false, real_tol=1e-6)

Apply the function `f` to all entries of `MonodromyResult` for which the given conditions apply.

## Example
```julia
# This gives us all solutions considered real (but still as a complex vector).
real_solutions = mapresults(solution, R, only_real=true)
```
"""
function mapresults(f, R::MonodromyResult; only_real = false, real_tol = 1e-6)
    [f(r) for r in R.solutions if (!only_real || is_real_vector(r, real_tol))]
end

"""
    solutions(result::MonodromyResult; only_real=false, real_tol=1e-6)

Return all solutions (as `SVector`s) for which the given conditions apply.

## Example
```julia
real_solutions = solutions(R, only_real=true)
```
"""
function solutions(R::MonodromyResult; kwargs...)
    mapresults(identity, R; kwargs...)
end

"""
    nsolutions(result::MonodromyResult)

Returns the number solutions of the `result`.
"""
nsolutions(res::MonodromyResult) = length(res.solutions)

"""
    real_solutions(res::MonodromyResult; tol=1e-6)

Returns the solutions of `res` whose imaginary part has norm less than 1e-6.
"""
function real_solutions(res::MonodromyResult; tol = 1e-6)
    map(r -> real_vector(r), filter(r -> is_real_vector(r, tol), res.solutions))
end

"""
    nreal(res::MonodromyResult; tol=1e-6)

Counts how many solutions of `res` have imaginary part norm less than 1e-6.
"""
function nreal(res::MonodromyResult; tol = 1e-6)
    count(r -> is_real_vector(r, tol), res.solutions)
end

"""
    parameters(r::MonodromyResult)

Return the parameters corresponding to the given result `r`.
"""
parameters(r::MonodromyResult) = r.parameters

#####################
## monodromy solve ##
#####################
struct MonodromySolver{
    T<:Number,
    UP<:UniquePoints,
    MO<:MonodromyOptions,
    Tracker<:CoreTracker,
    Prob<:Problem,
}
    parameters::Vector{T}
    solutions::UP
    solutions_lock::ReentrantLock
    loops::Vector{MonodromyLoop}
    options::MO
    statistics::MonodromyStatistics
    trackers::Vector{Tracker}
    problem::Prob
end

function MonodromySolver(
    F,
    solution::AbstractVector{<:Number},
    p₀::AbstractVector{TP};
    kwargs...,
) where {TP}
    MonodromySolver(F, [solution], p₀; kwargs...)
end
function MonodromySolver(
    F,
    startsolutions::Vector{<:AbstractVector{<:Number}},
    p::AbstractVector{TP};
    distance = default_distance(eltype(startsolutions)),
    parameters = nothing,
    strategy = nothing,
    show_progress = true,
    showprogress = nothing, #deprecated
    accuracy = 1e-6, # set a higher refinement_accuracy than the default
    precision = PRECISION_ADAPTIVE,
    kwargs...,
) where {TP,NVars}

    @deprecatekwarg showprogress show_progress

    if parameters !== nothing && length(p) ≠ length(parameters)
        throw(ArgumentError("Number of provided parameters doesn't match the length of initially provided parameter `p₀`."))
    end

    x₀ = complex.(first(startsolutions))
    p₀ = Vector{promote_type(Float64, TP)}(p)

    optionskwargs, restkwargs = splitkwargs(kwargs, monodromy_options_supported_keywords)

    supported, rest = splitkwargs(restkwargs, problem_startsolutions_supported_keywords)
    prob, _ = problem_startsolutions(
        F,
        [x₀];
        parameters = parameters, generic_parameters = p₀, γ₁ = 1, γ₀ = 1, supported...,
    )
    tracker =
        CoreTracker(prob, x₀; accuracy = accuracy, precision = PRECISION_ADAPTIVE, rest...)
    # Check whether homotopy is real
    is_real_system = numerically_check_real(tracker.homotopy, x₀)
    options = MonodromyOptions(is_real_system, accuracy, distance; optionskwargs...)
    # construct UniquePoints

    uniquepoints = UniquePoints(
        typeof(x₀),
        distance;
        group_actions = options.equivalence_classes ? options.group_actions : nothing,
        check_real = true,
    )

    # add only unique points that are true solutions
    for s in startsolutions
        if !options.check_startsolutions || is_valid_start_value(tracker, s, 0.0)
            add!(uniquepoints, s; tol = options.identical_tol)
        end
    end

    statistics = MonodromyStatistics(uniquepoints)

    if strategy === nothing
        strategy = default_strategy(F, parameters, p; is_real_system = is_real_system)
    end

    # construct Loop
    loops = [MonodromyLoop(strategy, p₀, options)]

    MonodromySolver(
        p₀,
        uniquepoints,
        ReentrantLock(),
        loops,
        options,
        statistics,
        [tracker],
        prob,
    )
end

"""
    solutions(MS::MonodromySolver)

Get the solutions of the loop.
"""
solutions(MS::MonodromySolver) = MS.solutions.points

"""
    nsolutions(loop::MonodromyLoop)

Get the number solutions of the loop.
"""
nsolutions(MS::MonodromySolver) = length(solutions(MS))

"""
    monodromy_solve(F, [sols, p]; parameters=..., options..., pathtrackerkwargs...)

Solve a polynomial system `F(x;p)` with specified parameters and initial solutions `sols`
by monodromy techniques. This makes loops in the parameter space of `F` to find new solutions.
If `F` the parameters `p` only occur *linearly* in `F` it is eventually possible to compute
a *start pair* ``(x₀, p₀)`` automatically. In this case `sols` and `p` can be omitted and
the automatically generated parameters can be obtained with the [`parameters`](@ref) function
from the [`MonodromyResult`](@ref).

## Options

* `check_startsolutions=true`: If `true`, we do a Newton step for each entry of `sols`for
  checking if it is a valid startsolutions. Solutions which are not valid are sorted out.
* `distance_function=euclidean_distance`: The distance function used for
  [`UniquePoints`](@ref).
* `done_callback=always_false`: A callback to end the computation early. This function takes
  2 arguments. The first one is the new solution `x` and the second one are all current
  solutions (including `x`). Return `true` if the compuation is done.
* `equivalence_classes=true`: This only applies if there is at least one group action
  supplied. We then consider two solutions in the same equivalence class if we can transform
  one to the other by the supplied group actions. We only track one solution per equivalence
  class.
* `identical_tol::Float64=1e-6`: The tolerance with which it is decided whether two
  solutions are identical.
* `group_action=nothing`: A function taking one solution and returning other solutions if
  there is a constructive way to obtain them, e.g. by symmetry.
* `group_actions=nothing`: If there is more than one group action you can use this to chain
  the application of them. For example if you have two group actions `foo` and `bar` you can
  set `group_actions=[foo, bar]`. See [`GroupActions`](@ref) for details regarding the
  application rules.
* `max_loops_no_progress::Int=10`: The maximal number of iterations (i.e. loops generated)
  without any progress.
* `min_solutions`: The minimal number of solutions before a stopping heuristic is applied.
  By default this is half of `target_solutions_count` if applicable otherwise 1.
* `parameter_sampler=independent_normal`: A function taking the parameter `p` and returning
  a new random parameter `q`. By default each entry of the parameter vector is drawn
  independently from the univariate normal distribution.
* `resuse_loops::Symbol=:all`: Strategy to reuse other loops for new found solutions. `:all`
  propagates a new solution through all other loops, `:random` picks a random loop, `:none`
  doesn't reuse a loop.
* `show_progress=true`: Enable a progress meter.
* `strategy`: The strategy used to create loops. If `F` only depends linearly on `p` this
  will be [`Petal`](@ref). Otherwise this will be [`Triangle`](@ref) with weights if `F`
  is a real system.
* `target_solutions_count=nothing`: The computations are stopped if this number of
  solutions is reached.
* `threading = true`: Enable multithreading of the path tracking.
* `timeout=float(typemax(Int))`: The maximal number of *seconds* the computation is allowed
  to run.
"""
function monodromy_solve(
    F::Union{MPPolys,System};
    parameters = F isa MPPolys ?
                 throw(ArgumentError("Necessary to set `parameters = ...`.")) : nothing,
    variable_ordering = nothing,
    seed = randseed(),
    show_progress = true,
    threading::Bool = true,
    kwargs...,
)
    Random.seed!(seed)
    x₀ = p₀ = nothing
    try
        x₀, p₀ = find_start_pair(F, parameters; variable_ordering = variable_ordering)
    catch
        throw(ArgumentError("Cannot compute a start pair (x, p) automatically."))
    end
    monodromy_solve!(
        MonodromySolver(
            F,
            [x₀],
            p₀;
            parameters = parameters,
            variable_ordering = variable_ordering,
            kwargs...,
        ),
        seed;
        show_progress = show_progress,
        threading = threading,
    )
end

function monodromy_solve(
    F::Vector{<:ModelKit.Expression};
    parameters = throw(ArgumentError("Necessary to set `parameters = ...`.")),
    variable_ordering = throw(ArgumentError("Necessary to set `variable_ordering = ...`.")),
    kwargs...,
)
    monodromy_solve(System(F, variable_ordering, parameters))
end
function monodromy_solve(
    F::Vector{<:ModelKit.Expression},
    S,
    p;
    parameters = throw(ArgumentError("Necessary to set `parameters = ...`.")),
    variable_ordering = throw(ArgumentError("Necessary to set `variable_ordering = ...`.")),
    kwargs...,
)
    monodromy_solve(System(F, variable_ordering, parameters), S, p)
end

function find_start_pair(F::System, ::Nothing; variable_ordering = nothing)
    vars = polyvar.(F.variables)
    params = polyvar.(F.parameters)
    P = evaluate(F.expressions, F.variables => vars, F.parameters => params)
    find_start_pair(P, params; variable_ordering = vars)
end

function find_start_pair(F::MPPolys, parameters; variable_ordering = nothing)
    if variable_ordering !== nothing
        vars = variable_ordering
    else
        vars = variables(F, parameters)
    end
    x₀ = randn(ComplexF64, length(vars))
    if F isa Composition
        F = expand(F)
    end
    F_x₀ = [subs(f, vars => x₀) for f in F]
    # let's use the helper function above to get the matrix form
    A, b = linear_system(F_x₀)
    if iszero(b)
        p_end = randn(ComplexF64)
        p₀ = [LA.qr(A[:, 1:end-1], Val(true)) \ ((-p_end) .* A[:, end]); p_end]
    else
        p₀ = LA.qr(A, Val(true)) \ b
    end

    all(
        f -> abs(f(parameters => p₀)) < 1e-8,
        F_x₀,
    ) || throw(ArgumentError("Cannot compute start pair."))

    (x₀, p₀)
end

function monodromy_solve(
    args...;
    seed = randseed(),
    show_progress = true,
    threading::Bool = true,
    kwargs...,
)
    Random.seed!(seed)
    monodromy_solve!(
        MonodromySolver(args...; kwargs...),
        seed;
        show_progress = show_progress,
        threading = threading,
    )
end

function default_strategy(
    F::Union{System,MPPolyInputs},
    parameters,
    p₀::AbstractVector{TP};
    is_real_system = false,
) where {TC,TP}
    # If F depends only linearly on the parameters a petal is sufficient
    vars = variables(F, parameters)
    if all(d -> d ≤ 1, maxdegrees(F; parameters = vars))
        Petal()
    # For a real system we should introduce some weights to avoid the discriminant
    elseif is_real_system
        Triangle(useweights = true)
    else
        Triangle(useweights = false)
    end
end
function default_strategy(F::AbstractSystem, parameters, p₀; is_real_system = false)
    # For a real system we should introduce some weights to avoid the discriminant
    if is_real_system
        Triangle(useweights = true)
    else
        Triangle(useweights = false)
    end
end

function numerically_check_real(H::AbstractHomotopy, x)
    y = copy(x)
    Random.randn!(y)
    for i in eachindex(y)
        y[i] = real(y[i]) + 0.0im
    end
    t = rand()
    r = evaluate(H, y, t)
    isapprox(LinearAlgebra.norm(imag.(r), Inf), 0.0, atol = 1e-14)
end

#################
## Actual work ##
#################

"""
    MonodromyJob

A `MonodromyJob` is consisting of a solution id and a loop id.
"""
struct MonodromyJob
    id::Int
    loop_id::Int
end

function monodromy_solve!(
    MS::MonodromySolver,
    seed;
    show_progress::Bool = true,
    threading::Bool = true,
)
    if nsolutions(MS) == 0
        @warn "None of the provided solutions is a valid start solution."
        return MonodromyResult(
            :invalid_startvalue,
            similar(MS.solutions.points, 0),
            MS.parameters,
            MS.statistics,
            MS.options.equivalence_classes,
            seed,
        )
    end
    # solve
    retcode = :not_assigned
    if show_progress
        if !MS.options.equivalence_classes
            desc = "Solutions found:"
        else
            desc = "Classes of solutions (modulo group action) found:"
        end
        progress =
            ProgressMeter.ProgressUnknown(desc; delay = 0.5, clear_output_ijulia = true)
    else
        progress = nothing
    end

    n_blas_threads = single_thread_blas()

    retcode = monodromy_solve!(MS, seed, progress; threading = threading)

    n_blas_threads > 1 && set_num_BLAS_threads(n_blas_threads)
    finished!(MS.statistics, nsolutions(MS))
    MonodromyResult(
        retcode,
        solutions(MS),
        MS.parameters,
        MS.statistics,
        MS.options.equivalence_classes,
        seed,
    )
end

function monodromy_solve!(
    MS::MonodromySolver,
    seed,
    progress::Union{Nothing,ProgressMeter.ProgressUnknown};
    threading::Bool = true,
)
    Threads.resize_nthreads!(MS.trackers)
    t₀ = time_ns()
    iterations_without_progress = 0 # stopping heuristic
    # intialize job queue
    queue = MonodromyJob.(1:nsolutions(MS), 1)
    thread_queues = [MonodromyJob[] for _ = 1:Threads.nthreads()]

    n = nsolutions(MS)
    retcode = :none
    while n < MS.options.target_solutions_count
        retcode =
            empty_queue!(queue, thread_queues, MS, t₀, progress; threading = threading)

        if retcode == :done
            retcode = :success
            break
        elseif retcode == :timeout ||
               retcode == :invalid_startvalue || retcode == :interrupt
            break
        end

        for q in thread_queues
            append!(queue, q)
            empty!(q)
        end
        isempty(queue) || continue

        # Iterations heuristic
        n_new = nsolutions(MS)
        if n == n_new
            iterations_without_progress += 1
        else
            iterations_without_progress = 0
            n = n_new
        end
        if iterations_without_progress == MS.options.max_loops_no_progress &&
           n_new ≥ MS.options.min_solutions
            retcode = :heuristic_stop
            break
        end

        regenerate_loop_and_schedule_jobs!(queue, MS)
    end
    update_progress!(progress, nsolutions(MS), MS.statistics; finish = true)

    retcode
end

function empty_queue!(
    queue,
    thread_queues,
    MS::MonodromySolver,
    t₀::UInt,
    progress;
    threading::Bool = true,
)
    if !threading
        ntracked = Ref(0)
        nfailures = Ref(0)
        n_since_update = 0
        retcode = :incomplete
        try
            while !isempty(queue)
                job = pop!(queue)
                status = process!(queue, job, MS, progress, ntracked, nfailures, 1)
                if status == :done || status == :invalid_startvalue || status == :interrupt
                    retcode = status
                    break
                end
                n_since_update += 1

                MS.statistics.ntrackedpaths += ntracked[]
                MS.statistics.ntrackingfailures += nfailures[]
                ntracked[] = nfailures[] = 0
                if n_since_update ≥ 4
                    n_since_update = 0
                    update_progress!(progress, nsolutions(MS), MS.statistics)
                end
                # check timeout
                if (time_ns() - t₀) > MS.options.timeout * 1e9 # convert s to ns
                    retcode = :timeout
                    break
                end
            end
        catch e
            if isa(e, InterruptException)
                return :interrupt
            else
                rethrow()
            end
        end
        n_since_update > 0 && update_progress!(progress, nsolutions(MS), MS.statistics)
        retcode
    else
        stop = Ref(false)
        return_status = Ref(:incomplete)
        start_ntrackedpaths = MS.statistics.ntrackedpaths
        start_ntrackingfailures = MS.statistics.ntrackingfailures
        ntrackedpaths = [Ref(0) for _ = 1:Threads.nthreads()]
        ntrackingfailures = [Ref(0) for _ = 1:Threads.nthreads()]
        Threads.@threads for job in queue
            try
                !stop[] || continue
                tid = Threads.threadid()
                status = process!(
                    thread_queues[tid],
                    job,
                    MS,
                    progress,
                    ntrackedpaths[tid],
                    ntrackingfailures[tid],
                    tid,
                )
                if status == :done
                    return_status[] = :done
                    stop[] = true
                elseif status == :invalid_startvalue
                    return_status[] = :invalid_startvalue
                    stop[] = true
                end
                if tid == 1
                    n_tracked = start_ntrackedpaths + sum(getindex, ntrackedpaths)
                    n_failures = start_ntrackingfailures + sum(getindex, ntrackingfailures)
                    MS.statistics.ntrackedpaths = n_tracked
                    MS.statistics.ntrackingfailures = n_failures
                    update_progress!(progress, nsolutions(MS), MS.statistics)
                end
            # check timeout
                if (time_ns() - t₀) > MS.options.timeout * 1e9 # convert s to ns
                    return_status[] = :invalid_startvalue
                    stop[] = true
                end
                if stop[]
                    break
                end
            catch e
                if isa(e, InterruptException)
                    return_status[] = :interrupt
                    stop[] = true
                    break
                else
                    rethrow()
                end
            end
        end
        empty!(queue)
        n_tracked = start_ntrackedpaths + sum(getindex, ntrackedpaths)
        n_failures = start_ntrackingfailures + sum(getindex, ntrackingfailures)
        MS.statistics.ntrackedpaths = n_tracked
        MS.statistics.ntrackingfailures = n_failures
        update_progress!(progress, nsolutions(MS), MS.statistics)

        return_status[]
    end
end

function process!(
    queue,
    job::MonodromyJob,
    MS::MonodromySolver,
    progress,
    ntrackedpaths::Base.RefValue{Int},
    ntrackingfailures::Base.RefValue{Int},
    tid::Integer,
)
    x = solutions(MS)[job.id]
    loop = MS.loops[job.loop_id]
    retcode = track(MS.trackers[tid], x, loop, ntrackedpaths, ntrackingfailures)
    is_success(retcode) || return :incomplete

    y = pull_back(MS.problem, current_x(MS.trackers[tid]))
    add_and_schedule!(MS, queue, y, job) && return :done

    if MS.options.complex_conjugation
        add_and_schedule!(MS, queue, conj.(y), job) && return :done
    end

    if !MS.options.equivalence_classes
        apply_actions(MS.options.group_actions, y) do yᵢ
            add_and_schedule!(MS, queue, yᵢ, job)
        end
    end
    return :incomplete
end

"""
    add_and_schedule!(MS, queue, y, job)

Add `y` to the current `node` (if it not already exists) and schedule a new job to the
`queue`. Returns `true` if we are done. Otherwise `false`.
"""
function add_and_schedule!(MS::MonodromySolver, queue, y, job::MonodromyJob) where {N,T}
    lock(MS.solutions_lock)
    k = add!(MS.solutions, y, Val(true); tol = MS.options.identical_tol)
    unlock(MS.solutions_lock)
    if k == NOT_FOUND || k == NOT_FOUND_AND_REAL
        # Check if we are done
        isdone(MS.solutions, y, MS.options) && return true
        push!(queue, MonodromyJob(nsolutions(MS), job.loop_id))
        # Schedule also on other loops
        if MS.options.reuse_loops == :random && length(MS.loops) > 1
            r_loop_id = rand(2:length(MS.loops))
            if r_loop_id == job.loop_id
                r_loop_id -= 1
            end
            push!(queue, MonodromyJob(nsolutions(MS), r_loop_id))
        elseif MS.options.reuse_loops == :all
            for r_loop_id = 1:length(MS.loops)
                r_loop_id != job.loop_id || continue
                push!(queue, MonodromyJob(nsolutions(MS), r_loop_id))
            end
        end
    end
    MS.statistics.nreal += (k == NOT_FOUND_AND_REAL)
    false
end

function update_progress!(
    ::Nothing,
    nsolutions,
    statistics::MonodromyStatistics;
    finish = false,
)
    nothing
end
function update_progress!(
    progress,
    nsolutions,
    statistics::MonodromyStatistics;
    finish = false,
)
    ProgressMeter.update!(
        progress,
        nsolutions,
        showvalues = (
            ("# paths tracked", statistics.ntrackedpaths),
            ("# loops generated", statistics.nparametergenerations),
            (
                "# completed loops without change",
                n_completed_loops_without_change(statistics, nsolutions),
            ),
            (
                "# solutions in current loop",
                n_solutions_current_loop(statistics, nsolutions),
            ),
            ("# real solutions", statistics.nreal),
        ),
    )
    if finish
        ProgressMeter.finish!(progress)
    end
    nothing
end

function isdone(solutions::UniquePoints, x, options::MonodromyOptions)
    options.done_callback(
        x,
        solutions.points,
    ) || length(solutions) ≥ options.target_solutions_count
end

function regenerate_loop_and_schedule_jobs!(queue, MS::MonodromySolver)
    es = MS.loops[1].edges
    loop = MonodromyLoop(
        MS.parameters,
        length(es),
        MS.options,
        weights = !isnothing(es[1].weights),
    )
    push!(MS.loops, loop)
    for id = 1:nsolutions(MS)
        push!(queue, MonodromyJob(id, length(MS.loops)))
    end
    generated_parameters!(MS.statistics, nsolutions(MS))
    nothing
end

##################
## VERIFICATION ##
##################
"""
    verify_solution_completeness(F, res::MonodromyResult;
                                 parameters=..., trace_tol=1e-6, options...)

Verify that the monodromy computation found all solutions by [`monodromy_solve`](@ref).
This uses a trace test as described in [^LRS18].
The trace is a numerical value which is 0 if all solutions are found, for this the
`trace_tol` keyword argument is used. The function returns `nothing` if some computation
couldn't be carried out. Otherwise returns a boolean. Note that this function requires the
computation of solutions to another polynomial system using monodromy. This routine can
return `false` although all solutions are found if this additional solution set is not
complete. The `options...` arguments can be everything which is accepted by `solve` and
`monodromy_solve`.

### Example

```
julia> @polyvar x y a b c;

julia> f = x^2+y^2-1;

julia> l = a*x+b*y+c;

julia> res = monodromy_solve([f,l], [-0.6-0.8im, -1.2+0.4im], [1,2,3]; parameters=[a,b,c])
MonodromyResult
==================================
• 2 solutions (0 real)
• return code → heuristic_stop
• 44 tracked paths
• seed → 367230

julia> verify_solution_completeness([f,l], res; parameters=[a,b,c], trace_tol = 1e-8)
[ Info: Compute additional witnesses for completeness...
[ Info: Found 2 additional witnesses
[ Info: Compute trace...
[ Info: Norm of trace: 1.035918995391323e-15
true
```

    verify_solution_completeness(F, S, p; parameters=..., kwargs...)

Verify the solution completeness using the computed solutions `S` to the parameter `p`.

    verify_solution_completeness(TTS, S, W₁₀, p₀::Vector{<:Number}, l₀)

Use the already computeded additional witnesses `W₁₀`. You want to obtain
`TTS`, `W₁₀` and `l₀` as the output from [`solution_completeness_witnesses`](@ref).

[^LRS18]:
    Leykin, Anton, Jose Israel Rodriguez, and Frank Sottile. "Trace test."
    Arnold Mathematical Journal 4.1 (2018): 113-125.
"""
function verify_solution_completeness(F, R::MonodromyResult; kwargs...)
    verify_solution_completeness(F, solutions(R), R.parameters; kwargs...)
end

function verify_solution_completeness(
    F,
    W₀₁::AbstractVector{<:AbstractVector},
    p₀::Vector{<:Number};
    show_progress = true,
    parameters = nothing,
    trace_tol = 1e-6,
    kwargs...,
)
    W₁₀, TTS, l₀ = solution_completeness_witnesses(
        F,
        W₀₁,
        p₀;
        show_progress = show_progress, parameters = parameters, kwargs...,
    )
    verify_solution_completeness(TTS, W₀₁, W₁₀, p₀, l₀; show_progress = show_progress)
end

function verify_solution_completeness(
    TTS,
    W₀₁::AbstractVector{<:AbstractVector},
    W₁₀::AbstractVector{<:AbstractVector},
    p₀::Vector{<:Number},
    l₀;
    show_progress = true,
    trace_tol = 1e-6,
    kwargs...,
)
    # Combine W₀₁ and W₁₀.
    S = append!([[x; 0.0] for x in W₀₁], W₁₀)
    # To verify that we found all solutions we need move in the pencil
    if show_progress
        @info("Compute trace...")
    end

    trace = track_and_compute_trace(TTS, S, l₀; kwargs...)
    if isnothing(trace)
        return nothing
    else
        show_progress && @info("Norm of trace: $(LinearAlgebra.norm(trace))")
        LinearAlgebra.norm(trace) < trace_tol
    end
end

"""
    solution_completeness_witnesses(F, S, p; parameters=..., kwargs...)

Compute the additional necessary witnesses. Returns a triple `(W₁₀, TTS, l)`
containing the additional witnesses `W₁₀`, a trace test system `TTS` and
the parameters `l` for `TTS`.
"""
function solution_completeness_witnesses(
    F,
    W₀₁,
    p₀::Vector{<:Number};
    parameters = nothing,
    show_progress = true,
    kwargs...,
)
    # generate another start pair
    q₀ = randn(ComplexF64, length(p₀))
    # Construct the trace test system
    TTS = TraceTestSystem(SPSystem(F; parameters = parameters), p₀, q₀ - p₀)

    y₁ = solution(solve(F, W₀₁[1]; p₁ = p₀, p₀ = q₀, parameters = parameters, kwargs...)[1])
    # construct an affine hyperplane l(x) going through y₀
    l₁ = cis.(2π .* rand(length(y₁)))
    push!(l₁, -sum(l₁ .* y₁))
    # This is numerically sometimes not so nice. Let's move to a truly generic one.
    l₀ = randn(ComplexF64, length(l₁))
    y₀ = solution(solve(TTS, [y₁; 1]; p₁ = l₁, p₀ = l₀)[1])

    if show_progress
        @info("Compute additional witnesses for completeness...")
    end

    R₁₀ = monodromy_solve(TTS, y₀, l₀; max_loops_no_progress = 5, kwargs...)
    best_result = R₁₀
    best_params = l₀
    result_agreed = 0
    for i = 1:10
        k₀ = randn(ComplexF64, length(l₀))
        S_k₀ = solutions(solve(
            TTS,
            solutions(R₁₀);
            start_parameters = l₀, target_parameters = k₀,
        ))
        new_result = monodromy_solve(TTS, S_k₀, k₀; max_loops_no_progress = 5)
        if nsolutions(new_result) == nsolutions(best_result)
            result_agreed += 1
        elseif nsolutions(new_result) > nsolutions(best_result)
            best_result = new_result
            best_params = k₀
        end
        if result_agreed > 2
            break
        end
    end

    W₁₀ = solutions(best_result)

    if show_progress
        @info("Found $(length(W₁₀)) additional witnesses")
    end
    W₁₀, TTS, best_params
end

function track_and_compute_trace(TTS::TraceTestSystem, S, l₀; kwargs...)
    for i = 1:3
        TTP = TraceTestPencil(TTS, l₀)
        R₁ = solve(TTP, S, start_parameters = [0.0], target_parameters = [0.1], kwargs...)
        R₂ = solve(TTP, S, start_parameters = [0.0], target_parameters = [-.1], kwargs...)
        if nsolutions(R₁) ≠ length(S) || nsolutions(R₂) ≠ length(S)
            if i == 3
                printstyled("Lost solutions $i times. Abort.\n", color = :red, bold = true)
                return nothing
            end
            printstyled(
                "Lost solutions, need to recompute trace...\n",
                color = :yellow,
                bold = true,
            )
        end
        s₁ = sum(solutions(R₁))
        s = sum(S)
        s₂ = sum(solutions(R₂))
        # Due to floating point errors, we compute the RELATIVE trace
        trace = ((s₁ .- s) .- (s .- s₂)) ./ max.(abs.(s₁ .- s), 1.0)
        return trace
    end
    nothing
end


export PlayerOptions,
    CheckRaiseFold,
    CallRaiseFold,
    CallAllInFold,
    CallFold

abstract type PlayerOptions end
struct CheckRaiseFold <: PlayerOptions end
struct CallRaiseFold <: PlayerOptions end
struct CallAllInFold <: PlayerOptions end
struct CallFold <: PlayerOptions end

validate_action(::Fold, ::CheckRaiseFold) = nothing
validate_action(::Fold, ::CallRaiseFold) = nothing
validate_action(::Fold, ::CallAllInFold) = nothing
validate_action(::Fold, ::CallFold) = nothing

validate_action(::Check, ::CheckRaiseFold) = nothing
validate_action(::Check, ::CallRaiseFold) = error("Cannot check. Available options: CallRaiseFold")
validate_action(::Check, ::CallAllInFold) = error("Cannot check. Available options: CallAllInFold")
validate_action(::Check, ::CallFold) = error("Cannot check. Available options: CallFold")

# We catch this error before `call!` completes, so we don't need to catch here.
# validate_action(::Call, ::CheckRaiseFold) = error("Cannot call. Available options: CheckRaiseFold")
validate_action(::Call, ::CallRaiseFold) = nothing
validate_action(::Call, ::CallAllInFold) = nothing
validate_action(::Call, ::CallFold) = nothing

validate_action(::Raise, ::CheckRaiseFold) = nothing
validate_action(::Raise, ::CallRaiseFold) = nothing
validate_action(::Raise, ::CallAllInFold) = nothing
validate_action(::Raise, ::CallFold) = error("Cannot Raise. Available options: CallFold")

function player_option!(game::Game, player::Player)
    table = game.table
    call_amt = call_amount(table, player)
    if !(call_amt ≈ 0) # must call to stay in
        cond_1 = bank_roll(player) > call_amt
        cond_2 = an_opponent_can_call_a_raise(table, player)
        raise_possible = cond_1 && cond_2
        if raise_possible # raise possible
            vrb = valid_raise_bounds(table, player)
            if first(vrb) ≈ last(vrb) # only all-in raise possible
                option = CallAllInFold()
            else
                option = CallRaiseFold()
            end
        else # only all-in possible
            option = CallFold()
        end
    else
        option = CheckRaiseFold()
    end
    @debug "option = $option"

    n_actions = length(action_history(player))

    player_option!(game, player, option)

    # exactly 1 action must be taken
    player_action = action_history(player)
    @assert n_actions + 1 == length(player_action) "Must take exactly 1 action."
    validate_action(last(player_action), option)
end

# By default, forward to `player_option!` with
# game state:
player_option!(game::Game, player::Player, option) =
    player_option!(game, player, state(game.table), option)

#####
##### Human player options (ask via prompts)
#####

function player_option!(game::Game, player::Player{Human}, ::CheckRaiseFold, io::IO=stdin)
    table = game.table
    vrb = valid_raise_bounds(table, player)
    options = ["Check", "Raise [\$$(minimum(vrb)), \$$(maximum(vrb))]", "Fold"]
    menu = RadioMenu(options, pagesize=4)
    choice = request("$(name(player))'s turn to act:", menu)
    choice == -1 && error("Uncaught case")
    choice == 1 && check!(game, player)
    choice == 2 && raise_to!(game, player, input_raise_amt(table, player, io))
    choice == 3 && fold!(game, player)
end
function player_option!(game::Game, player::Player{Human}, ::CallRaiseFold, io::IO=stdin)
    table = game.table
    vrb = valid_raise_bounds(table, player)
    call_amt = call_amount(table, player)
    blind_str = is_blind_call(table, player) ? " (blind)" : ""
    options = ["Call \$$(call_amt)$blind_str", "Raise [\$$(minimum(vrb)), \$$(maximum(vrb))]", "Fold"]
    menu = RadioMenu(options, pagesize=4)
    choice = request("$(name(player))'s turn to act:", menu)
    choice == -1 && error("Uncaught case")
    choice == 1 && call!(game, player)
    choice == 2 && raise_to!(game, player, input_raise_amt(game.table, player, io))
    choice == 3 && fold!(game, player)
end
function player_option!(game::Game, player::Player{Human}, ::CallAllInFold)
    table = game.table
    call_amt = call_amount(table, player)
    all_in_amt = round_bank_roll(player)
    blind_str = is_blind_call(table, player) ? " (blind)" : ""
    options = ["Call \$$(call_amt)$blind_str", "Raise all-in (\$$(all_in_amt))", "Fold"]
    menu = RadioMenu(options, pagesize=4)
    choice = request("$(name(player))'s turn to act:", menu)
    choice == -1 && error("Uncaught case")
    choice == 1 && call!(game, player)
    choice == 2 && raise_all_in!(game, player)
    choice == 3 && fold!(game, player)
end
function player_option!(game::Game, player::Player{Human}, ::CallFold)
    table = game.table
    call_amt = call_amount(table, player)
    blind_str = is_blind_call(table, player) ? " (blind)" : ""
    options = ["Call \$$(call_amt)$blind_str", "Fold"]
    menu = RadioMenu(options, pagesize=4)
    choice = request("$(name(player))'s turn to act:", menu)
    choice == -1 && error("Uncaught case")
    choice == 1 && call!(game, player)
    choice == 2 && fold!(game, player)
end

# io only works for tests, but does not for user input
# so we have a switch for the test suite
use_input_io() = false
println_io(io::IO, msg) = use_input_io() ? println(io, msg) : println(msg)

function input_raise_amt(table, player::Player{Human}, io::IO=stdin)
    raise_amt = nothing
    while true
        println_io(io, "Enter raise amt:")
        raise_amt = readline(io)
        try
            raise_amt = parse(Float64, raise_amt)
            is_valid, msg = is_valid_raise_amount(table, player, raise_amt)
            is_valid && break
            println_io(io, msg)
        catch
            println_io(io, "Raise must be a Float64")
        end
    end
    @assert raise_amt ≠ nothing
    amt = valid_raise_amount(table, player, raise_amt)
    return amt
end

#####
##### AbstractAI
#####

##### Bot5050

function player_option!(game::Game, player::Player{Bot5050}, ::CheckRaiseFold)
    if rand() < 0.5
        check!(game, player)
    else
        amt = Int(round(rand()*bank_roll(player), digits=0))
        amt = bound_raise(game.table, player, amt)
        raise_to!(game, player, amt)
    end
end
function player_option!(game::Game, player::Player{Bot5050}, ::CallRaiseFold)
    if rand() < 0.5
        if rand() < 0.5 # Call
            call!(game, player)
        else # re-raise
            amt = Int(round(rand()*bank_roll(player), digits=0))
            amt = bound_raise(game.table, player, amt)
            raise_to!(game, player, amt)
        end
    else
        fold!(game, player)
    end
end
function player_option!(game::Game, player::Player{Bot5050}, ::CallAllInFold)
    if rand() < 0.5
        if rand() < 0.5 # Call
            call!(game, player)
        else # re-raise
            raise_all_in!(game, player)
        end
    else
        fold!(game, player)
    end
end
function player_option!(game::Game, player::Player{Bot5050}, ::CallFold)
    if rand() < 0.5
        call!(game, player)
    else
        fold!(game, player)
    end
end


#=
Copyright (c) 2018-2022 Chris Coey, Lea Kapelevich, and contributors

This Julia package Hypatia.jl is released under the MIT license; see LICENSE
file in the root directory or at https://github.com/chriscoey/Hypatia.jl
=#

insts = OrderedDict()
insts["minimal"] = [
    ((2, 2, true),),
    ((2, 2, false),),
    ]
insts["fast"] = [
    ((2, 3, true),),
    ((2, 3, false),),
    ((5, 6, true),),
    ((5, 6, false),),
    ((10, 20, true),),
    ((10, 20, false),),
    ((20, 40, true),),
    ((20, 40, false),),
    ]
insts["various"] = [
    ((15, 20, true),),
    ((15, 20, false),),
    ((30, 40, true),),
    ((30, 40, false),),
    ((60, 80, true),),
    ((60, 80, false),),
    ((30, 160, true),),
    ((30, 160, false),),
    ]
return (MatrixQuadraticJuMP, insts)


## Problems

function build_max_import(pm::_PM.AbstractBFModel)
    _FP.post_flex_tnep(pm; objective = false, intertemporal_constraints = false)
    constraint_same_investments(pm)
    objective_max_import(pm)
end

function build_max_export(pm::_PM.AbstractBFModel)
    _FP.post_flex_tnep(pm; objective = false, intertemporal_constraints = false)
    for n in _PM.nw_ids(pm)
        constraint_load_pcurt_zero(pm, n)
    end
    constraint_same_investments(pm)
    objective_max_export(pm)
end

function build_min_cost_with_same_investments_in_all_sub_nws(pm::_PM.AbstractBFModel)
    _FP.post_flex_tnep(pm; objective = false, intertemporal_constraints = false)
    for n in _PM.nw_ids(pm)
        constraint_load_pcurt_zero(pm, n)
    end
    constraint_same_investments(pm)
    objective_min_investment_cost(pm)
end

function build_max_import_with_current_investments(pm::_PM.AbstractBFModel)
    _FP.post_flex_tnep(pm; objective = false, intertemporal_constraints = false)
    for n in _PM.nw_ids(pm)
        constraint_ne_branch_indicator_fix(pm, n)
        constraint_ne_storage_indicator_fix(pm, n)
    end
    objective_max_import(pm)
end

function build_max_export_with_current_investments(pm::_PM.AbstractBFModel)
    _FP.post_flex_tnep(pm; objective = false, intertemporal_constraints = false)
    for n in _PM.nw_ids(pm)
        constraint_ne_branch_indicator_fix(pm, n)
        constraint_ne_storage_indicator_fix(pm, n)
        constraint_load_pcurt_zero(pm, n)
    end
    objective_max_export(pm)
end

function build_min_cost_with_fixed_investments(pm::_PM.AbstractBFModel)
    _FP.post_flex_tnep(pm; objective = true, intertemporal_constraints = true)
    for n in _PM.nw_ids(pm)
        constraint_ne_branch_indicator_fix(pm, n)
        constraint_ne_storage_indicator_fix(pm, n)
    end
end

function build_max_flex_band_with_bounded_cost(pm::_PM.AbstractBFModel)
    _FP.post_flex_tnep(pm; objective = false, intertemporal_constraints = false)
    for n in _PM.nw_ids(pm)
        constraint_load_pcurt_zero(pm, n)
    end
    constraint_same_investments(pm)
    constraint_investment_cost_max(pm; sub_nw = 1)
    objective_max_flex_band_2sub(pm)
end

## Constraints

"Disable involuntary curtailment of loads"
function constraint_load_pcurt_zero(pm::_PM.AbstractPowerModel, n::Int)
    for i in _PM.ids(pm, n, :load)
        pcurt = _PM.var(pm, n, :pcurt, i)
        JuMP.@constraint(pm.model, pcurt == 0.0)
    end
end

"Fix investment decisions on candidate branches according to values in data structure"
function constraint_ne_branch_indicator_fix(pm::_PM.AbstractPowerModel, n::Int)
    for i in _PM.ids(pm, n, :ne_branch)
        indicator = _PM.var(pm, n, :branch_ne, i)
        value = _PM.ref(pm, n, :ne_branch, i, "built")
        JuMP.@constraint(pm.model, indicator == value)
    end
end

"Fix investment decisions on candidate storage according to values in data structure"
function constraint_ne_storage_indicator_fix(pm::_PM.AbstractPowerModel, n::Int)
    for i in _PM.ids(pm, n, :ne_storage)
        indicator = _PM.var(pm, n, :z_strg_ne, i)
        value = _PM.ref(pm, n, :ne_storage, i, "isbuilt")
        JuMP.@constraint(pm.model, indicator == value)
    end
end

"Put an upper bound on investment cost related to a single subnetwork"
function constraint_investment_cost_max(pm::_PM.AbstractPowerModel; sub_nw = 1)
    inv_cost = sum(
            _FP.calc_ne_branch_cost(pm, n)
            + _FP.calc_ne_storage_cost(pm, n)
            + _FP.calc_load_investment_cost(pm, n)
        for n in _FP.nw_ids(pm; hour=1, scenario=1, sub_nw))
    max_cost = pm.ref[:it][_PM.pm_it_sym][:max_cost]

    JuMP.@constraint(pm.model, inv_cost <= max_cost)
end

"Ensure that investment decisions are the same in each subnetwork"
function constraint_same_investments(pm::_PM.AbstractPowerModel)
    for y in 1:_FP.dim_length(pm, :year)
        n_1, rest = Iterators.peel(_FP.nw_ids(pm; hour=1, scenario=1, year=y))
        for n_2 in rest
            for i in _PM.ids(pm, :ne_branch, nw = n_2)
                _FP.constraint_ne_branch_investment_same(pm, n_1, n_2, i)
            end
            for i in _PM.ids(pm, :ne_storage, nw = n_2)
                _FP.constraint_ne_storage_investment_same(pm, n_1, n_2, i)
            end
            n_1 = n_2
        end
    end
end

## Objectives

function objective_max_import(pm::_PM.AbstractPowerModel)
    return JuMP.@objective(pm.model, Max,
        sum( calc_td_coupling_power_active(pm, n) for (n, nw_ref) in _PM.nws(pm) )
    )
end

function objective_max_export(pm::_PM.AbstractPowerModel)
    return JuMP.@objective(pm.model, Min,
        sum( calc_td_coupling_power_active(pm, n) for (n, nw_ref) in _PM.nws(pm) )
    )
end

function calc_td_coupling_power_active(pm::_PM.AbstractPowerModel, n::Int)
    pcc_gen = _FP.dim_prop(pm, n, :sub_nw, "d_gen")
    p = _PM.var(pm, n, :pg, pcc_gen)
    return p
end

function objective_max_flex_band_2sub(pm::_PM.AbstractPowerModel)
    return JuMP.@objective(pm.model, Max,
        sum( calc_td_coupling_power_active(pm, n) for n in _FP.nw_ids(pm; hour=1, scenario=1, sub_nw=1) )
        - sum( calc_td_coupling_power_active(pm, n) for n in _FP.nw_ids(pm; hour=1, scenario=1, sub_nw=2) )
    )
end

function objective_min_investment_cost(pm::_PM.AbstractPowerModel)
    investment = sum(
        _FP.calc_ne_branch_cost(pm, n)
        + _FP.calc_ne_storage_cost(pm, n)
        + _FP.calc_load_investment_cost(pm, n)
        for n in _FP.nw_ids(pm; hour=1, scenario=1)
    )
    JuMP.@objective(pm.model, Min, investment)
end


abstract type WordVectorsEmbedder{T,S} <: AbstractEmbedder{T,S} end

"""
Constant that represents embeddings libraries used in text embedding.
"""
const EmbeddingsLibrary{T,S} = Union{
    ConceptNet{<:Languages.Language,S,T},
    Word2Vec.WordVectors{S,T,<:Integer},
    Glowe.WordVectors{S,T,<:Integer},
    EmbeddingsAnalysis.CompressedWordVectors{<:QuantizedArrays.AbstractQuantization,
                                             <:Unsigned,<:Distances.PreMetric,T,S,<:Integer}
}

"""
    document2vec(embedder, document)

Word-embeddings approach to document embedding. It embeds documents
using word embeddings libraries and some algorithm for combining
these (depending on the type of `embedder`).

# Arguments
  * `embedder::WordVectorsEmbedder` is the embedder
  * `document::Vector{String}` the document to be embedded,
     where each vector element corresponds to a sentence
"""
function __document2vec(embedder::WordVectorsEmbedder{T,S},
                        document::Vector{String};
                        kwargs...  # for the unused arguments
                       ) where {T,S}
    # Initializations
    n = length(document)
    m = size(embedder.embeddings)[1]  # number of vector components
    embedded_words = Vector{Vector{String}}(undef, n)
    doc_word_embeddings = Vector{Matrix{T}}(undef, n)

    # Get word embeddings for each sentence
    @inbounds for i in 1:n
        words = tokenize(document[i], method=:stringanalysis)
        _embs, _mtoks = word_embeddings(embedder.embeddings,
                                        words;
                                        keep_size=false,
                                        max_compound_word_length=1,
                                        wildcard_matching=true,
                                        print_matched_words=false)
        doc_word_embeddings[i] = _embs
        embedded_words[i] = words[setdiff(1:length(words), _mtoks)]
    end

    # Remove empty embeddings
    filter!(!isempty, doc_word_embeddings)
    filter!(!isempty, embedded_words)

    # Create sentence embeddings
    sntembs = sentences2vec(embedder,
                            doc_word_embeddings;
                            embedded_words=embedded_words)
    # If no words were embedded, sntembs is a `D`×0 Matrix
    # which, when squashed, becomes a `D`-element zero Vector
    # (`D` is the dimensionality of the document embedding)
    return squash(sntembs)
end

function document2vec(embedder::WordVectorsEmbedder{T,S},
                      document::Vector{String};
                      kwargs...  # for the unused arguments
                     ) where {T,S}
    # An embedded document is a zero vector only if it was not properly embedded
    embedded_document = __document2vec(embedder, document; kwargs...)
    is_embedded = !iszero(embedded_document)
    # Check for OOV (out-of-vocabulary) policy
    if embedder.config.oov_policy == :large_vector && !is_embedded
        embedded_document .= T(DEFAULT_OOV_VAL)
    end
    return embedded_document, is_embedded
end

"""
    word_embeddings(word_vectors, document_tokens [;kwargs])

Returns a matrix corresponding to the word embeddings of `document_tokens`
as well as the indices of missing i.e. not-embedded tokens.

# Arguments
  * `word_vectors::EmbeddingsLibrary` wordvectors object; can be
     a `Word2Vec.WordVectors`, `Glowe.WordVectors` or `ConceptnetNumberbatch.ConceptNet`
  * `document_tokens::Vector{String}` the words to be embedded,
     where each vector element corresponds to a word

# Keyword arguments
  * `keep_size::Bool` a `false` value discards vectors for words not found
     while a `true` value (default) places a zero vector in the embeddings
     matrix
  * `print_matched_words::Bool` if `true`, the words that were and that were
     not embedded are printed (default `false`)
  * `kwargs...` the rest of the keyword arguments are `ConceptNet` specific
     and can be found by inspecting the help of `ConceptnetNumberbatch.embed_document`
"""
function word_embeddings(word_vectors::Union{Word2Vec.WordVectors{S1,T,H},
                                             Glowe.WordVectors{S1,T,H},
                                             EmbeddingsAnalysis.CompressedWordVectors{Q,U,D,T,S1,H}},
                         document_tokens::Vector{S2};
                         keep_size::Bool=true,
                         print_matched_words::Bool=false,
                         kwargs...
                        ) where {S1<:AbstractString, T<:Real,
                                 H<:Integer, S2<:AbstractString,
                                 Q<:QuantizedArrays.AbstractQuantization,
                                 U<:Unsigned, D<:Distances.PreMetric}
    # Initializations
    n = size(word_vectors)[1]
    p = length(document_tokens)
    found_positions = Int[] # column positions in word embedding matrix of found tokens
    found_tokens = Int[]    # positions in the token vector on found tokens

    # Search for matching words (exact match)
    for (i, token) in enumerate(document_tokens)
        if haskey(word_vectors.vocab_hash, token)
            push!(found_positions, word_vectors.vocab_hash[token])
            push!(found_tokens, i)
        end
    end
    missing_tokens = setdiff(1:p, found_tokens)  # positions of tokens that were not found

    # Construct document structure
    m = ifelse(keep_size, p, length(found_tokens))
    embedding = zeros(T, n, m)
    for (pos, tok, j) in zip(found_positions, found_tokens, 1:m)
        embedding[:, ifelse(keep_size, tok, j)] = word_vectors.vectors[:, pos]
    end

    # Return document embedding and missing tokens
    if print_matched_words
        println("Embedded words: $(tokens[found_tokens])")
        println("Mismatched words: $(tokens[missing_tokens])")
    end
    return embedding, missing_tokens
end

# Replicate ConceptnetNumberbatch embed_document function
function word_embeddings(conceptnet::ConceptNet{L,K,E},
                         document_tokens::Vector{S};
                         language=Languages.English(),
                         keep_size::Bool=true,
                         compound_word_separator::String="_",
                         max_compound_word_length::Int=1,
                         wildcard_matching::Bool=false,
                         print_matched_words::Bool=false
                        ) where {L<:Language, K, E<:Real, S<:AbstractString}
    return ConceptnetNumberbatch.embed_document(conceptnet, document_tokens,
                language=language, keep_size=keep_size,
                compound_word_separator=compound_word_separator,
                max_compound_word_length=max_compound_word_length,
                wildcard_matching=wildcard_matching,
                print_matched_words=print_matched_words)
end

"""
    squash(m)

Function that creates a single mean vector from a matrix `m`
and performs some normalization operations as well.
"""
squash(m::Matrix{T}) where T<:AbstractFloat = begin
    # Calculate document embedding
    ##############################
    v = vec(sum(m, dims=2))
    return v./(norm(v,2)+eps(T))
end

"""
    squash(vv, m)

Function that creates a single mean vector from a vector of vectors `vv`
where each vector has a length `m` and performs some normalization
operations as well.
"""
squash(vv::Vector{Vector{T}}, m::Int) where T<:AbstractFloat = begin
    v = zeros(T,m)
    for w in vv
        v+= w
    end
    return v./(norm(v,2)+eps(T))
end


# getfunction.jl
#
# Makro to ease access to C functions; mainly adopted from the GetC-Package
# for Julia. What is desired is e.g. the following:
#
# INPUT:
# void mgl_surf(HMGL gr, HCDT z, char *sch, char *opt);
# mgl_surf(gr::HMGL, z::HCDT, sch::Ptr{Cchar}, opt::Ptr{Cchar})::Void
#
# OUTPUT:
# function mgl_surf(gr, z, sch, opt)
#     ccall(("libmgl", :mgl_surf), Void, (HMGL, HCDT, Ptr{Cchar}, Ptr{Cchar}), gr, z, sch, opt)
# end

macro getf(lib, fun)
    # Name of the function
    cname     = string(fun.args[1].args[1])                # :mgl_surf
    jname     = Symbol(replace(string(cname), "mgl_", "")) # :surf
    arguments = fun.args[1].args[2:end]                    # [:(gr::HMGL), :(z::HCDT), ...]

    # Get info out of arguments of 'fun'
    inputC   = [ arg.args[2] for arg in arguments ]   # [ :HMGL, :HCDT, ...]
    varnames = [ arg.args[1] for arg in arguments ]   # [ :gr, :z, :sch, :opt ]
    rettype  = fun.args[2]                            # :Void

    # Construct the result.
    csym = Expr(:quote, cname)
    body = Expr(:ccall, :($csym, $lib), rettype, Expr(:tuple, inputC...),
                varnames...)
    ret = Expr(:function, Expr(:call, jname, varnames...), body)
    return esc(ret)
end


using DataFrames
using Test

m = page_model()
include("../src/components/NonMarketDamages.jl")

nonmarketdamages = addnonmarketdamages(m)

set_param!(m, :NonMarketDamages, :rtl_realizedtemperature, readpagedata(m, "test/validationdata/rtl_realizedtemperature.csv"))
set_param!(m, :NonMarketDamages, :rcons_per_cap_MarketRemainConsumption, readpagedata(m,
"test/validationdata/rcons_per_cap_MarketRemainConsumption.csv"))
set_param!(m, :NonMarketDamages, :rgdp_per_cap_MarketRemainGDP, readpagedata(m,
"test/validationdata/rgdp_per_cap_MarketRemainGDP.csv"))
set_param!(m, :NonMarketDamages, :atl_adjustedtolerableleveloftemprise, readpagedata(m,
"test/validationdata/atl_adjustedtolerableleveloftemprise_nonmarket.csv"))
set_param!(m, :NonMarketDamages, :imp_actualreduction, readpagedata(m,
"test/validationdata/imp_actualreduction_nonmarket.csv"))
set_param!(m, :NonMarketDamages, :isatg_impactfxnsaturation, 28.333333333333336)

p = load_parameters(m)
p["y_year_0"] = 2008.
p["y_year"] = Mimi.dim_keys(m.md, :time)
set_leftover_params!(m, p)

run(m)

rcons_per_cap = m[:NonMarketDamages, :rcons_per_cap_NonMarketRemainConsumption]
rcons_per_cap_compare = readpagedata(m, "test/validationdata/rcons_per_cap_NonMarketRemainConsumption.csv")
@test rcons_per_cap ≈ rcons_per_cap_compare rtol=1e-2

rgdp_per_cap = m[:NonMarketDamages, :rgdp_per_cap_NonMarketRemainGDP]
rgdp_per_cap_compare = readpagedata(m, "test/validationdata/rgdp_per_cap_NonMarketRemainGDP.csv")
@test rgdp_per_cap ≈ rgdp_per_cap_compare rtol=1e-2


#Defines the Navigation system type, including its discrete time behaviour
#include("DiscreteTimeSystem.jl")
#= NavigationFunction should output the "measured state" of the vehicle, based on the
ODE state (simulated measured state) through some filter.
=#

#Define ControlSystem as a subtype of the DiscreteTimeSystem
abstract type AbstractNavigationSystem <: DiscreteTimeSystem

########### Template function ###########################
mutable struct NavigationSystem{T<:Number} <: AbstractNavigationSystem
  Active::Bool
  Ts::T #sample time
  NavigationRelation::Function
  Tprev::T #storage for previous time it was updated, hence must be a mutable type
end
NavigationSystem() = NavigationSystem(false, 0.0 , x-> nothing, 0.0)
NavigationSystem(Ts::Number) = NavigationSystem(false, Float64(Ts), x -> (), 0.0)
NavigationSystem(Ts::Number, csfunc::Function) = NavigationSystem(true, Float64(Ts), csfunc, 0.0)
NavigationSystem(Active::Bool, Ts::Number, csfunc::Function) = NavigationSystem(Active, Float64(Ts), csfunc, 0.0)

#Set the callback affect condition of the ControlSystem
#placeholder!
function affect!(int,dts::AbstractNavigationSystem)
    for cu in user_cache(int)
        cu.u = [dts.NavigationRelation(int.u)...]
    end
  dts.Tprev = int.t
end


using Currencies
using Test

currencies = ((:USD, 2, 840, "US Dollar"),
              (:EUR, 2, 978, "Euro"),
              (:JPY, 0, 392, "Yen"),
              (:JOD, 3, 400, "Jordanian Dinar"),
              (:CNY, 2, 156, "Yuan Renminbi"))

# This just makes sure that the data was loaded and at least some basic values are set as expected
@testset "Basic currencies" begin
    for (s, u, c, n) in currencies
        ccy = Currency{s}
        @test currency(s) == ccy
        @test symbol(ccy) == s
        @test unit(ccy) == u
        @test name(ccy) == n
        @test code(ccy) == c
    end
end

# This makes sure that the values are within expected ranges
@testset "Validation" begin
    @test length(Currencies.allpairs()) >= 155
    for (s, (ct,u,c,n)) in Currencies.allpairs()
        ccy = Currency(s)
        @test symbol(ct) == s
        @test length(string(s)) == 3
        @test u >= 0
        @test 0 < c < 2000
        @test length(n) < 40
    end
end


# Linear GP circuit chromosome functions
# When Parameters are used, numlevelsback specifies numregisters.
# A circuit is a sequence of instructions on the register array R
#   which has length numregisters + numinputs
# R is normally local to the function where is is used.
# The first numregisters components or R are calculation registers.
# The remaining numinputs components are read-only input registers which are
#   initialized with the components of the current context.
# Even though there is a nodearity keyword value, this file assumes nodearity==2
# An instruction is specified either by a 4-tuple of MyInts or by an integer of OutputType.
# The elements of the (2+nodearity)-tuple are (assuming nodearity==2):
#  1.  The index of the logical operation (such as AND, OR) in the funcs array.
#  2.  The index of the element of R where the output is stored
#  3.  The index of the element of R which is the first operand of the logical operation
#  4.  The index of the element of R which is the second operand of the logical operation
# The functions vect_to_int() and int_to_vect() convert between these representations.
# When parameters are used, numinputs and numoutputs have the same meaning for the cartesian and linear representatons.
# numinteriors means numinstructions for the linear representation
# numlevelsback means numregisters for the linear representation
# There are 3 alternative representations for the circuit_vects component of a  LinCicuit.
#   1.  A list of numinstructions vectors where each vector is  (2+nodearity)-tuple described above.
#   2.  A list of numinstructions Int64s where each of these is a 1-based encoding of the vectors
#   3.  A Int128 integer which is an encoding of the integers of step 3.
# See notes/3_24.txt for outline
# See test/testLinCircuit.jl for tests.
export LinCircuit, output_values
export execute_lcircuit, instruction_vect_to_instruction_int, instruction_int_to_instruction_vect, lincomplexity
export vect_to_int, int_to_vect, rand_ivect, execute_random_circuit, number_active, degeneracy, recover_phenotype
export num_instructions # This is the total number of possible instructions for a parameter setting
export rand_lcircuit, mutate_circuit!, mutate_instruction, mutate_circuit_all, mutate_all, print_circuit
export instruction_vects_to_instruction_ints, instruction_ints_to_instruction_vects
export instruction_ints_to_circuit_int, circuit_int_to_instruction_ints
export circuit_int_to_circuit, circuit_to_circuit_int
#export circuit_int
OutputType = Int64

#=
mutable struct LinCircuit
    # Here, numinteriors represents number of gates, numlevelsback represents number of computational registers
    # the First numoutputs registers are the output registers
    circuit_vects::Vector{Vector{MyInt}}
    params::Parameters
end
=#

function output_values( c::LinCircuit, funcs::Vector{Func}=Func[] )
  if length(funcs) == 0
    funcs = lin_funcs( c.params.numinputs )
  end
  #println("funcs: ",funcs)
  R = execute_lcircuit( c, funcs )
  R[1:c.params.numoutputs]
end

# Not used.  Assumes that R and funcs are in the execution environment
function vect_to_funct( lc::Vector{MyInt} )
  (lc)->R[lc[2]] = funcs[lc[1]].func(R[lc[3]],R[lc[4]])
end

# Executes the circuit, returns the register array R.
# numregisters is the number of calculation registers, not the total number of registers
# The outputs are  R[1:numoutputs]
function execute_lcircuit( circuit_ints::Vector{OutputType}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity::Int64=2 )
  lcv = map( ci->instruction_int_to_instruction_vect( ci, numregisters, numinputs, funcs ), circuit_ints )
  execute_lcircuit( lcv, numregisters, numinputs, funcs )
end

function execute_lcircuit( circuit::LinCircuit, funcs::Vector{Func} )
  p = circuit.params
  R = fill(MyInt(0), p.numlevelsback + p.numinputs ) # numlevelsback is the number of computational registers
  R[(p.numlevelsback+1):end] = construct_context(p.numinputs)
  for lc in circuit.circuit_vects
    #println("lc: ",lc,"  R: ",R)
    R[lc[2]] = funcs[lc[1]].func(R[lc[3]],R[lc[4]])
  end
  R
end

# Executes the circuit, returns the register array R.
# The outputs are  R[1:numoutputs]
function execute_lcircuit( circuit_vects::Vector{Vector{MyInt}}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity::Int64=2 )
  R = fill(MyInt(0), numregisters+numinputs )
  R[numregisters+1:end] = construct_context(numinputs)
  for lc in circuit_vects
    #println("lc: ",lc,"  func: ",funcs[lc[1]],"  R: ",R)
    R[lc[2]] = funcs[lc[1]].func(R[lc[3]],R[lc[4]])
    #println("lc: ",lc,"  func: ",funcs[lc[1]],"  R: ",R)
  end
  R
end

function lincomplexity( circuit_vects::Vector{Vector{MyInt}}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity::Int64=2 )
  p = Parameters( numinputs, 1, length(circuit_vects), numregisters )
  lincomplexity( LinCircuit(circuit_vects,p), funcs, nodearity=nodearity )
end

# Executes the circuit, returns the array X of instruction outputs
# The outputs are  R[1:numoutputs]
function lincomplexity( circuit::LinCircuit, funcs::Vector{Func}; nodearity::Int64=2 )
  p = circuit.params
  R = fill(MyInt(0), p.numlevelsback+p.numinputs )
  X = fill(MyInt(0), length(circuit.circuit_vects) )
  R[p.numlevelsback+1:end] = construct_context(p.numinputs)
  i = 1
  for lc in circuit.circuit_vects
    #println("lc: ",lc,"  func: ",funcs[lc[1]],"  R: ",R)
    R[lc[2]] = X[i] = funcs[lc[1]].func(R[lc[3]],R[lc[4]])
    #println("lc: ",lc,"  func: ",funcs[lc[1]],"  R: ",R)
    i += 1
  end
  complexity5( X, p.numinputs )
end

# The number of possible instructions with these settings.
# This is distinct from numinstructions which is the number of instructions in a circuit.
function num_instructions( numregisters::Int64, numinputs::Int64, funcs::Vector{Func};
     nodearity::Int64=2 )
  length(funcs)*numregisters*(numinputs+numregisters)^nodearity
end

function num_instructions( p::Parameters, funcs::Vector{Func} )
  num_instructions( p.numlevelsback, p.numinputs, funcs, nodearity=p.nodearity )
end

function number_active( circuit::LinCircuit )  # Dummy function for now 10/16/21
  return 0
end

function degeneracy( circuit::LinCircuit )  # Dummy function for now 10/16/21
  return 0
end

function recover_phenotype( circuit::LinCircuit, max_steps_recover, maxtrials_recover, maxtries_recover )  # Dummy function for now 10/16/21
  return (0,0)
end

# Convert a vectors of integers to a unique integer.
# The components of the vector are 1-based rather than 0-based.
# Example 1:
# julia> vect_to_int( [3,4,5,6],[10,10,10,10])
#  2345
# Example 2:
# julia> vect_to_int([3,2,1,4],[7,2,5,8])
#  203
function vect_to_int( ints::Vector{Int64}, max::Vector{Int64} )
  result = Int128(ints[1])-1
  for i = 2:length(ints)
    result *= max[i]
    result += ints[i]-1
  end
  result
end

# julia> int_to_vect( 2345, [10,10,10,10])
# 4-element Array{Int64,1}:
#  Int64[ 3, 4, 5, 6 ]
# julia> int_to_vect(203,[7,2,5,8])
# 4-element Array{Int64,1}:
#  Int64[ 3, 2, 1, 4]
function int_to_vect( int::Integer, max::Vector{Int64} )
  ints = zeros(Int64,length(max))
  for i = length(max):-1:1
    ints[i] = int % max[i] + 1
    int = div(int,max[i])
  end
  ints
end

# Converts an instruction specified by a vector to the instruction specified by an integer.
function instruction_vect_to_instruction_int( inst_vect::Vector{MyInt}, numregisters::Int64, numinputs::Int64,
    funcs::Vector{Func}; nodearity::Int64=2 )
  result = Int64(inst_vect[1]-1)  # Function code
  multiplier = 1
  #println("multiplier: ",multiplier,"  result: ",result)
  multiplier = numregisters
  result *= multiplier
  result += inst_vect[2]-1
  #println("multiplier: ",multiplier,"  result: ",result)
  multiplier = numregisters+numinputs
  for j = 1:nodearity
    result *= multiplier
    result += inst_vect[2+j]-1
    #println("multiplier: ",multiplier,"  result: ",result)
  end
  result + 1    # Convert 1-based from 0-based
end

# Converts an instruction specified by an integer to the instruction specified by a vector.
function instruction_int_to_instruction_vect( inst_int::OutputType, numregisters::Int64, numinputs::Int64,
    funcs::Vector{Func}; nodearity::Int64=2 )
  #println("ii_to_iv inst_int: ",inst_int)
  result = fill(MyInt(0),2+nodearity)
  multiplier = numregisters+numinputs
  inst_int -= 1   # Convert from 1-based to 0-based so that mod and div work
  for j = nodearity:-1:1
    try
      result[j+2] = inst_int % multiplier + 1
    catch
      println("catch inst_int: ",inst_int,"  multiplier: ",multiplier)
      error(" error ")
    end
    inst_int = div(inst_int,multiplier)
    #println("j+2: ",j+2,"  multiplier: ", multiplier,"  inst_int: ",inst_int,"  result[j+2]: ",result[j+2])
  end
  multiplier = numregisters
  result[2] = inst_int % multiplier + 1
  inst_int = div(inst_int,multiplier)
  #println("i: ",2,"  multiplier: ", multiplier,"  inst_int: ",inst_int,"  result[2]: ",result[2])
  multiplier = length(funcs)
  result[1] = inst_int % multiplier + 1
  inst_int = div(inst_int,multiplier)
  #println("i: ",1,"  multiplier: ", multiplier,"  inst_int: ",inst_int,"  result[1]: ",result[1])
  result
end

function instruction_vects_to_instruction_ints( cv::Vector{Vector{MyInt}}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func};
    nodearity::Int64=2 )
  map(x->instruction_vect_to_instruction_int(x, numregisters, numinputs, funcs, nodearity=nodearity ), cv )
end

function instruction_vects_to_instruction_ints( circ::LinCircuit, funcs::Vector{Func}=Func[] )
  if length(funcs) == 0
    funcs = lin_funcs( circ.params.numinputs )
  end
  instruction_vects_to_instruction_ints( circ.circuit_vects, circ.params.numlevelsback, circ.params.numinputs, funcs )
end

function instruction_ints_to_instruction_vects( c_ints::Vector{Int64}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func};
    nodearity::Int64=2 )
  map(x->instruction_int_to_instruction_vect(x, numregisters, numinputs, funcs, nodearity=nodearity ), c_ints )
end

function instruction_ints_to_instruction_vects( c_ints::Vector{Int64}, p::Parameters, funcs::Vector{Func}=Func[];
    nodearity::Int64=2 )
  if length(funcs) == 0
    funcs = lin_funcs( p.numinputs )
  end
  instruction_ints_to_instruction_vects( c_ints, p.numlevelsback, p.numinputs, funcs )
end

#=
function circuit_vect_to_circuit_ints( c::LinCircuit, funcs::Vector{Func}=Func[] )
  if length(funcs) == 0
    funcs = lin_funcs( c.params.numinputs )
  end
  circuit_vect_to_circuit_ints( c.circuit_vects, c.params.numinteriors, c.params.numinputs, funcs )
end
=#

function instruction_ints_to_circuit_int( c_ints::Vector{Int64}, p::Parameters, funcs::Vector{Func} )
  numregisters = p.numlevelsback
  multiplier = num_instructions( p, funcs )
  #println("multiplier: ",multiplier)
  vect_to_int( c_ints, fill(multiplier,length(c_ints)))
end

function circuit_int_to_instruction_ints( c_int::Int128, p::Parameters, funcs::Vector{Func} )
  numinstructions = p.numinteriors
  multiplier = num_instructions( p, funcs )
  #println("multiplier: ",multiplier)
  int_to_vect( c_int, fill( multiplier, numinstructions ))
end

function circuit_to_circuit_int( circ::LinCircuit, funcs::Vector{Func} )
  instruction_ints_to_circuit_int( instruction_vects_to_instruction_ints( circ, funcs ), circ.params, funcs )
end

function circuit_int_to_circuit( c_int::Int128, p::Parameters, funcs::Vector{Func} )
  i_vects = instruction_ints_to_instruction_vects( circuit_int_to_instruction_ints( c_int, p, funcs), p, funcs )
  LinCircuit( i_vects, p )
end

# Random instruction
# Assumes gates are arity 2
function rand_ivect( numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity::Int64=2 )
  MyInt[ rand(1:length(funcs)), rand(1:numregisters),  rand(1:(numregisters+numinputs)), rand(1:(numregisters+numinputs)) ]
end

# Random instruction
# Assumes gates are arity 2 which means that instructions have 2 components
function rand_ivect( p::Parameters, funcs::Vector{Func}; nodearity::Int64=2 )
  MyInt[ rand(1:length(funcs)), rand(1:p.numlevelsback),  rand(1:(p.numlevelsback+p.numinputs)),
    rand(1:(p.numlevelsback+p.numinputs)) ]
end

# Random circuit as a list of instruction integers
# Assumes gates are arity 2
function rand_lcircuit( n_instructions::Int64, numregisters::Int64, numinputs::Int64, funcs::Vector{Func} )
  p = Parameters( numinputs, 1, n_instructions, numregisters )
  LinCircuit( [ rand_ivect( numregisters, numinputs, funcs ) for _ = 1:n_instructions ], p )
end

# Random circuit as a list of instruction integers
# Assumes gates are arity 2
# p.numinteriors is equivalent to numinstructions
# p.numlevelsback is equivalent to numregisters
function rand_lcircuit( p::Parameters, funcs::Vector{Func}=lin_funcs(p.numinputs) )
  LinCircuit( [ rand_ivect( p.numlevelsback, p.numinputs, funcs ) for _ = 1:p.numinteriors], p )
end

function execute_random_circuits( ncircuits::Int64, prog_length::Int64, numregisters::Int64, numinputs::Int64, funcs::Vector{Func} )
  n_instructions = num_instructions( numregisters, numinputs, funcs )
  for i = 1:ncircuits
    circuit_ints = rand(0:(n_instructions-1),prog_length)
    execute_lcircuit( circuit_ints, numregisters, numinputs, funcs )
  end
end

# Mutates the instruction instruction_vect, returns the instruction vector of the mutated instruction
# mut_loc is the mutate location.
# Returns the mutated instruction as a vector of MyInts
# The mutated instruction is guaranteed to be different than instruction_vect.
function mutate_instruction( instruction_vect::Vector{MyInt}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func}, mut_loc::Int64;
      nodearity::Int64=2 )
  lfuncs = length(funcs)
  result_vect = deepcopy(instruction_vect)
  num_mutate_locations = ((lfuncs>1) ? (lfuncs-1) : 0) + (numregisters-1) + nodearity*(numregisters+numinputs-1)
  @assert 1 <= mut_loc && mut_loc <= num_mutate_locations
  if mut_loc <= lfuncs-1
    result_vect[1] = (mut_loc<instruction_vect[1]) ? mut_loc : (mut_loc+1)
    #println("result_vect[1]: ",result_vect[1])
  elseif mut_loc <= (lfuncs-1) + (numregisters-1)
    loc = mut_loc - (lfuncs-1)
    result_vect[2] = (loc<instruction_vect[2]) ? loc : (loc+1)
    #println("result_vect[2]: ",result_vect[2])
  else
    for i = 1:nodearity
      if mut_loc <= (lfuncs-1) + (numregisters-1) + (numregisters+numinputs-1)*i
        loc = mut_loc - (lfuncs-1) - (numregisters-1) - (numregisters+numinputs-1)*(i-1)
        result_vect[2+i] = (loc<instruction_vect[2+i]) ? loc : (loc+1)
        #println("result_vect[",2+i,"]: ",result_vect[2+i])
        break
      end
    end
  end
  result_vect
end

function mutate_instruction( instruction_vect::Vector{MyInt}, p::Parameters, funcs::Vector{Func}, mut_loc::Int64;
      nodearity::Int64=2 )
  mutate_instruction( instruction_vect, p.numlevelsback, p.numinputs, funcs, mut_loc, nodearity=p.nodearity )
end

# Mutates the instruction instruction_vect at a random location, returns the instruction vector of the mutated instruction
# Returns the mutated instruction as a vector of MyInts
function mutate_instruction( instruction_vect::Vector{MyInt}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity::Int64=2 )
  lfuncs = length(funcs)
  num_mutate_locations = ((lfuncs>1) ? (lfuncs-1) : 0) + (numregisters-1) + nodearity*(numregisters+numinputs-1)
  mut_loc = rand(1:num_mutate_locations)
  #println("num_mutate_locations: ",num_mutate_locations,"  mut_loc: ",mut_loc)
  mutate_instruction( instruction_vect, numregisters, numinputs, funcs, mut_loc, nodearity=nodearity )
end

function mutate_instruction( instruction_vect::Vector{MyInt}, p::Parameters, funcs::Vector{Func}; nodearity::Int64=2 )
  mutate_instruction( instruction_vect, p.numlevelsback, p.numinputs, funcs, nodearity=p.nodearity )
end

function mutate_instruction_all( instruction_vect::Vector{MyInt}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func};
      nodearity::Int64=2 )
  lenfuncs = length(funcs)
  num_mutate_locations = ((lenfuncs>1) ? (lenfuncs-1) : 0) + (numregisters-1) + nodearity*(numregisters+numinputs-1)
  instructions = Vector{MyInt}[]
  for mut_loc = 1:num_mutate_locations
    new_instruction = mutate_instruction( instruction_vect, numregisters, numinputs, funcs, mut_loc, nodearity=nodearity )
    push!(instructions,new_instruction)
  end
  instructions
end

function mutate_circuit!( circuit_vect::Vector{Vector{MyInt}}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity::Int64=2 )
  numinstructions = length(circuit_vect)
  ind = rand(1:numinstructions)
  #println("ind: ",ind)
  circuit_vect[ind] = mutate_instruction( circuit_vect[ind], numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity=nodearity )
  circuit_vect
end

function mutate_circuit!( circuit_vect::Vector{Vector{MyInt}}, p::Parameters, funcs::Vector{Func}; nodearity::Int64=2 )
  mutate_circuit!( circuit_vect, p.numlevelsback, p.numinputs, funcs, nodearity=p.nodearity )
end

function mutate_circuit!( circuit::LinCircuit, funcs::Vector{Func}; nodearity::Int64=2 )
  #LinCircuit( params, mutate_circuit!( circuit.circuit_vects, circuit.params, funcs ) )
  LinCircuit( mutate_circuit!( circuit.circuit_vects, circuit.params, funcs ), circuit.params )
end

function mutate_circuit_all( circuit_vect::Vector{Vector{MyInt}}, numregisters::Int64, numinputs::Int64, funcs::Vector{Func}; nodearity::Int64=2 )
  lenfuncs = length(funcs)
  result = Vector{Vector{MyInt}}[]
  for ind = 1:length(circuit_vect)
    mutated_instructions=mutate_instruction_all(circuit_vect[ind],numregisters,numinputs,funcs,nodearity=nodearity)
    for mi in mutated_instructions
      mutated_circuit = deepcopy(circuit_vect)
      mutated_circuit[ind] = mi
      push!(result,mutated_circuit)
    end
  end
  result
end

function mutate_circuit_all( circuit_vect::Vector{Vector{MyInt}}, p::Parameters, funcs::Vector{Func}; nodearity::Int64=2 )
  mutate_circuit_all( circuit_vect, p.numlevelsback, p.numinputs, funcs, nodearity=p.nodearity )
end

function mutate_all( circuit::LinCircuit, lfuncs::Vector{Func}=Func[]; output_outputs::Bool=true, nodearity::Int64=2 )
  mca = mutate_circuit_all( circuit, lfuncs, nodearity=nodearity )
  map( x->output_values(LinCircuit(x,circuit.params)), mca )
end

function mutate_circuit_all( circuit::LinCircuit, funcs::Vector{Func}=Func[]; nodearity::Int64=2 )
  if length(funcs) == 0
    funcs = lin_funcs( circuit.params.numinputs )
  end
  mutate_circuit_all( circuit.circuit_vects, circuit.params, funcs )
end

function print_circuit( f::IO, circuit::LinCircuit, funcs::Vector{Func} )
  print(f,"(")
  i = 1
  for lc in circuit.circuit_vects
    print(f,"R$(lc[1])=$(funcs[lc[2]].name)(R$(lc[3]),R$(lc[4]))")
    if i < length(circuit.circuit_vects) print(f,",") end
    i += 1
  end
  println(f,")")
end

function print_circuit( circuit::LinCircuit, funcs::Vector{Func} )
  print_circuit( Base.stdout, circuit, funcs )
end

function instruction_distance( instruction_vect1::Vector{MyInt}, instruction_vect2::Vector{MyInt} )
  @assert length(instruction_vect1) == length(instruction_vect2)
  count_diffs = 0
  for i in 1:length(instruction_vect1)
    count_diffs += instruction_vect1[i] == instruction_vect2[i] ? 0 : 1
  end
  count_diffs
end

function circuit_distance( circuit_vect1::Vector{Vector{MyInt}},circuit_vect2::Vector{Vector{MyInt}} )
  @assert length(circuit_vect1) == length(circuit_vect2)
  sum_distances = 0
  for i in 1:length(circuit_vect1)
    sum_distances += instruction_distance( circuit_vect1[i], circuit_vect2[i] )
  end
  sum_distances
end


export load_img, save_img, split_rgb

using Images

"""
    load_img(filename::String)

Load an image given file name.
"""
function load_img(filename::String)
    img = FileIO.load(filename)
    convert(Array{UInt8}, rawview(channelview(img)))
end

"""
    load_img(filename::String)

Load a pre-saved Atari image given ROM name and frame number.
"""
function load_img(rom_name::String, frame_number::Int64)
    filename = string(@__DIR__, "/../images/", rom_name, "_frame_30.png")
    load_img(filename)
end

"""
    load_rgb(rom_name::String, frame_number::Int64)

Load a pre-saved Atari image given ROM name and frame number.
Return an array containing the separated chanels.
"""
function load_rgb(rom_name::String, frame_number::Int64)
    filename = string(@__DIR__, "/../images/", rom_name, "_frame_$frame_number.png")
    split_rgb(load_img(filename))
end

"""
    save_img(img::Array{Float64,2}, filename::String)

Save an image to the input path.
"""
function save_img(img::Array{Float64}, filename::String)
    n_dims = ndims(img)
    if n_dims == 2
        FileIO.save(filename, colorview(Gray, img))
    elseif n_dims == 3
        FileIO.save(filename, colorview(RGB, img))
    else
        println("Warning: `save_img` image dimension is $n_dims - not saving")
    end
end

"""
    save_img(img::Array{UInt8,2}, filename::String)

Save an image to the input path.
"""
function save_img(img::Array{UInt8}, filename::String)
    mini, maxi = minimum(img), maximum(img)
    img = Float64.(img)
    if mini != maxi
        img = adjust_histogram(img, LinearStretching())
    end
    save_img(img, filename)
end

"""
    split_rgb(img::Array{UInt8,3})

Split input array in three channels.
"""
function split_rgb(img::Array{UInt8,3})
    @assert size(img)[1] == 3
    [img[i, :, :] for i in 1:3]
end


parseresponse(response) = map(
    x -> parse(Float64, split(x, ' ')[2]),
    split(String(response.body), '\n')[4:end-1],
)

bodykey(::Type{PhotoelectricAbsorption}) = "photoelectric"
bodykey(::Type{Coherent}) = "coherent"
bodykey(::Type{Incoherent}) = "incoherent"
bodykey(::Type{InNuclearField}) = "nuclear"
bodykey(::Type{InElectronField}) = "electron"
bodykey(::Type{WithCoherent}) = "with"
bodykey(::Type{WithoutCoherent}) = "without"

formatenergies(energies::AbstractArray{<:Unitful.Energy}) =
    join(["$(Float64(uconvert(MeV, e).val))" for e in energies], ';')

const basebody = Dict{String,String}(
    "NumAdd" => "1",
    "OutOpt" => "PIC",
    "Output" => "",
    "character" => "space",
    "WindowXmin" => "0.001",
    "WindowXmax" => "100000",
)

const XCOM_URL = "https://www.physics.nist.gov/cgi-bin/Xcom/data.pl"

function XCOM(body)
    merge!(body, basebody)
    r = HTTP.request(
        "POST",
        XCOM_URL,
        [],
        join(["$k=$v" for (k, v) in body], '&'),
    )
    if r.status != 200
        desc = STATUS_CODES[r.status]
        error("XCOM error - try again")
    end
    parseresponse(r)
end


function _unpackdensematrix!(A::GBVecOrMat{T}; desc = nothing) where {T}
    desc = _handledescriptor(desc)
    Csize = Ref{libgb.GrB_Index}(length(A) * sizeof(T))
    values = Ref{Ptr{Cvoid}}(Ptr{T}())
    isiso = Ref{Bool}(false)
    libgb.GxB_Matrix_unpack_FullC(
        A.p,
        values,
        Csize,
        isiso,
        desc
    )
    return unsafe_wrap(Array{T}, Ptr{T}(values[]), size(A))
end

function _unpackcscmatrix!(A::GBVecOrMat{T}; desc = nothing) where {T}
    desc = _handledescriptor(desc)
    colptr = Ref{Ptr{libgb.GrB_Index}}()
    rowidx = Ref{Ptr{libgb.GrB_Index}}()
    values = Ref{Ptr{Cvoid}}(Ptr{T}())
    colptrsize = Ref{libgb.GrB_Index}()
    rowidxsize = Ref{libgb.GrB_Index}()
    valsize = Ref{libgb.GrB_Index}()
    isiso = Ref{Bool}(false)
    isjumbled = Ref{Bool}(false)
    libgb.GxB_Matrix_unpack_CSC(
        A.p,
        colptr,
        rowidx,
        values,
        colptrsize,
        rowidxsize,
        valsize,
        isiso,
        isjumbled,
        desc
    )
    colptr = unsafe_wrap(Array{libgb.GrB_Index}, colptr[], colptrsize[] ÷ sizeof(libgb.GrB_Index))
    rowidx = unsafe_wrap(Array{libgb.GrB_Index}, rowidx[], rowidxsize[] ÷ sizeof(libgb.GrB_Index))
    colptr .+= 1
    rowidx .+= 1
    return colptr,
    rowidx,
    unsafe_wrap(Array{T}, Ptr{T}(values[]), valsize[] ÷ sizeof(T))
end

function _unpackcsrmatrix!(A::GBVecOrMat{T}; desc = nothing) where {T}
    desc = _handledescriptor(desc)
    rowptr = Ref{Ptr{libgb.GrB_Index}}()
    colidx = Ref{Ptr{libgb.GrB_Index}}()
    values = Ref{Ptr{Cvoid}}(Ptr{T}())
    colidxsize = Ref{libgb.GrB_Index}()
    rowptrsize = Ref{libgb.GrB_Index}()
    valsize = Ref{libgb.GrB_Index}()
    isiso = Ref{Bool}(false)
    isjumbled = Ref{Bool}(false)
    libgb.GxB_Matrix_unpack_CSR(
        A.p,
        rowptr,
        colidx,
        values,
        rowptrsize,
        colidxsize,
        valsize,
        isiso,
        isjumbled,
        desc
    )
    rowptr = unsafe_wrap(Array{libgb.GrB_Index}, rowptr[], rowptrsize[] ÷ sizeof(libgb.GrB_Index))
    colidx = unsafe_wrap(Array{libgb.GrB_Index}, colidx[], colidxsize[] ÷ sizeof(libgb.GrB_Index))
    rowptr .+= 1
    colidx .+= 1
    return rowptr,
    colidx,
    unsafe_wrap(Array{T}, Ptr{T}(values[]), valsize[] ÷ sizeof(T))
end