# The following is largely inspired from JuMP/test/JuMPExtension.jl

@enum Level BOTH LOWER_ONLY UPPER_ONLY DUAL_OF_LOWER

abstract type AbstractBilevelModel <: JuMP.AbstractModel end

Base.broadcastable(model::AbstractBilevelModel) = Ref(model)

mutable struct BilevelModel <: AbstractBilevelModel
    # Structured data
    upper::JuMP.AbstractModel
    lower::JuMP.AbstractModel

    # Model data
    nextvaridx::Int                                 # Next variable index is nextvaridx+1
    variables::Dict{Int, JuMP.AbstractVariable}     # Map varidx -> variable
    varnames::Dict{Int, String}                     # Map varidx -> name
    varnames_rev::Dict{String, Int}                 # Map varidx -> name
    var_level::Dict{Int, Level}
    var_upper::Dict{Int, JuMP.AbstractVariableRef}
    var_lower::Dict{Int, JuMP.AbstractVariableRef} #
    var_info::Dict{Int, VariableInfo}

    # upper level decisions that are "parameters" of the second level
    upper_to_lower_link::Dict{JuMP.AbstractVariableRef, JuMP.AbstractVariableRef}
    # lower level decisions that are input to upper
    lower_to_upper_link::Dict{JuMP.AbstractVariableRef, JuMP.AbstractVariableRef}
    # lower level decisions that are input to upper
    upper_var_to_lower_ctr_link::Dict{JuMP.AbstractVariableRef, JuMP.ConstraintRef}
    # joint link
    link::Dict{JuMP.AbstractVariableRef, JuMP.AbstractVariableRef}

    nextconidx::Int                                 # Next constraint index is nextconidx+1
    constraints::Dict{Int, JuMP.AbstractConstraint} # Map conidx -> variable
    connames::Dict{Int, String}                     # Map varidx -> name
    connames_rev::Dict{String, Int}                     # Map varidx -> name
    ctr_level::Dict{Int, Level}
    ctr_upper::Dict{Int, JuMP.ConstraintRef}
    ctr_lower::Dict{Int, JuMP.ConstraintRef}
    ctr_info::Dict{Int, ConstraintInfo}

    upper_objective_sense::MOI.OptimizationSense
    upper_objective_function::JuMP.AbstractJuMPScalar

    lower_objective_sense::MOI.OptimizationSense
    lower_objective_function::JuMP.AbstractJuMPScalar

    # solution data
    need_rebuild::Bool
    need_rebuild_names_ctr::Bool
    need_rebuild_names_var::Bool
    solver#::MOI.ModelLike
    mode
    upper_to_sblm
    lower_to_sblm
    lower_dual_to_sblm
    sblm_to_solver
    lower_primal_dual_map

    solve_time::Float64
    build_time::Float64

    copy_names::Bool
    copy_names_to_solver::Bool
    pass_start::Bool

    objdict::Dict{Symbol, Any}    # Same that JuMP.Model's field `objdict`

    function BilevelModel()

        model = new(
            JuMP.Model(),
            JuMP.Model(),

            # var
            0, Dict{Int, JuMP.AbstractVariable}(),
            Dict{Int, String}(),Dict{String, Int}(),
            Dict{Int, Level}(), Dict{Int, JuMP.AbstractVariable}(), Dict{Int, JuMP.AbstractVariable}(),
            Dict{Int, VariableInfo}(),
            # links
            Dict{JuMP.AbstractVariable, JuMP.AbstractVariable}(), Dict{JuMP.AbstractVariable, JuMP.AbstractVariable}(),
            Dict{JuMP.AbstractVariable, JuMP.ConstraintRef}(),
            Dict{JuMP.AbstractVariable, JuMP.AbstractVariable}(),
            #ctr
            0, Dict{Int, JuMP.AbstractConstraint}(),
            Dict{Int, String}(),Dict{String, Int}(),
            Dict{Int, Level}(), Dict{Int, JuMP.AbstractConstraint}(), Dict{Int, JuMP.AbstractConstraint}(),
            Dict{Int, ConstraintInfo}(),
            #obj
            MOI.FEASIBILITY_SENSE,
            zero(JuMP.GenericAffExpr{Float64, BilevelVariableRef}), # Model objective
            MOI.FEASIBILITY_SENSE,
            zero(JuMP.GenericAffExpr{Float64, BilevelVariableRef}), # Model objective

            true,
            true,
            true,
            nothing,
            NoMode{Float64},
            nothing,
            nothing,
            nothing,
            nothing,
            nothing,

            NaN,
            NaN,
            false,
            false,
            true,
            Dict{Symbol, Any}(),
            )

        return model
    end
end
function BilevelModel(optimizer_constructor;
    mode::AbstractBilevelSolverMode = SOS1Mode(),
    bridge_constraints::Bool=true,
    with_names=true)
    bm = BilevelModel()
    bm.copy_names=with_names
    set_mode(bm, mode)
    JuMP.set_optimizer(bm, optimizer_constructor;
        bridge_constraints=bridge_constraints,
        with_names=with_names)
    return bm
end
function set_mode(bm::BilevelModel, mode::AbstractBilevelSolverMode)
    bm.mode = deepcopy(mode)
    reset!(bm.mode)
    return bm
end

abstract type InnerBilevelModel <: AbstractBilevelModel end
struct UpperModel <: InnerBilevelModel
    m::BilevelModel
end
Upper(m::BilevelModel) = UpperModel(m)
UpperToLower(m::BilevelModel) = UpperModel(m)
struct LowerModel <: InnerBilevelModel
    m::BilevelModel
end
Lower(m::BilevelModel) = LowerModel(m)
LowerToUpper(m::BilevelModel) = LowerModel(m)
bilevel_model(m::InnerBilevelModel) = m.m
mylevel_model(m::UpperModel) = bilevel_model(m).upper
mylevel_model(m::LowerModel) = bilevel_model(m).lower
level(m::LowerModel) = LOWER_ONLY
level(m::UpperModel) = UPPER_ONLY
mylevel_ctr_list(m::LowerModel) = bilevel_model(m).ctr_lower
mylevel_ctr_list(m::UpperModel) = bilevel_model(m).ctr_upper
mylevel_var_list(m::LowerModel) = bilevel_model(m).var_lower
mylevel_var_list(m::UpperModel) = bilevel_model(m).var_upper

# obj

mylevel_obj_sense(m::LowerModel) = bilevel_model(m).lower_objective_sense
mylevel_obj_function(m::LowerModel) = bilevel_model(m).lower_objective_function
mylevel_obj_sense(m::UpperModel) = bilevel_model(m).upper_objective_sense
mylevel_obj_function(m::UpperModel) = bilevel_model(m).upper_objective_function

set_mylevel_obj_sense(m::LowerModel, val) = bilevel_model(m).lower_objective_sense = val
set_mylevel_obj_function(m::LowerModel, val) = bilevel_model(m).lower_objective_function = val
set_mylevel_obj_sense(m::UpperModel, val) = bilevel_model(m).upper_objective_sense = val
set_mylevel_obj_function(m::UpperModel, val) = bilevel_model(m).upper_objective_function = val

function set_link!(m::UpperModel, upper::JuMP.AbstractVariableRef, lower::JuMP.AbstractVariableRef)
    bilevel_model(m).upper_to_lower_link[upper] = lower
    bilevel_model(m).link[upper] = lower
    nothing
end
function set_link!(m::LowerModel, upper::JuMP.AbstractVariableRef, lower::JuMP.AbstractVariableRef)
    bilevel_model(m).lower_to_upper_link[lower] = upper
    bilevel_model(m).link[upper] = lower
    nothing
end

# Models to deal with variables that are not exchanged between models
abstract type SingleBilevelModel <: AbstractBilevelModel end
struct UpperOnlyModel <: SingleBilevelModel
    m::BilevelModel
end
UpperOnly(m::BilevelModel) = UpperOnlyModel(m)
struct LowerOnlyModel <: SingleBilevelModel
    m::BilevelModel
end
LowerOnly(m::BilevelModel) = LowerOnlyModel(m)

bilevel_model(m::SingleBilevelModel) = m.m
mylevel_model(m::UpperOnlyModel) = bilevel_model(m).upper
mylevel_model(m::LowerOnlyModel) = bilevel_model(m).lower
level(::LowerOnlyModel) = LOWER_ONLY
level(::UpperOnlyModel) = UPPER_ONLY
mylevel_var_list(m::LowerOnlyModel) = bilevel_model(m).var_lower
mylevel_var_list(m::UpperOnlyModel) = bilevel_model(m).var_upper

in_upper(l::Level) = l == BOTH || l == UPPER_ONLY || l == DUAL_OF_LOWER
in_lower(l::Level) = l == BOTH || l == LOWER_ONLY

#### Model ####

# Variables
struct BilevelVariableRef <: JuMP.AbstractVariableRef
    model::BilevelModel # `model` owning the variable
    idx::Int       # Index in `model.variables`
    level::Level
end
function BilevelVariableRef(model, idx)
    return BilevelVariableRef(model, idx, model.var_level[idx])
end

# Constraints
const BilevelConstraintRef = JuMP.ConstraintRef{BilevelModel, Int}#, Shape <: AbstractShape

# Objective

# Etc

JuMP.object_dictionary(m::BilevelModel) = m.objdict
JuMP.object_dictionary(m::AbstractBilevelModel) = JuMP.object_dictionary(bilevel_model(m))

function JuMP.index(d::Dict)
    ret = Dict{VI,VI}()
    # sizehint!(ret, length(d))
    for (k,v) in d
        ret[JuMP.index(k)] = JuMP.index(v)
    end
    return ret
end
function index2(d::Dict)
    ret = Dict{VI,CI}()
    # sizehint!(ret, length(d))
    for (k,v) in d
        ret[JuMP.index(k)] = JuMP.index(v)
    end
    return ret
end

# Names
JuMP.name(vref::BilevelVariableRef) = vref.model.varnames[vref.idx]
function JuMP.set_name(vref::BilevelVariableRef, name::String)
    vref.model.varnames[vref.idx] = name
end
JuMP.name(cref::BilevelConstraintRef) = cref.model.connames[cref.index]
function JuMP.set_name(cref::BilevelConstraintRef, name::String)
    cref.model.connames[cref.index] = name
end
function JuMP.variable_by_name(model::BilevelModel, name::String)
    if model.need_rebuild_names_var
        model.varnames_rev = Dict(v => k for (k,v) in model.varnames)
    end
    idx = model.varnames_rev[name]
    return BilevelVariableRef(model, idx)
end
function JuMP.constraint_by_name(model::BilevelModel, name::String)
    if model.need_rebuild_names_ctr
        model.connames_rev = Dict(v => k for (k,v) in model.connames)
    end
    idx = model.connames_rev[name]
    return BilevelConstraintRef(model, idx)
end

function JuMP.primal_status(model::BilevelModel)
    _check_solver(model)
    return MOI.get(model.solver, MOI.PrimalStatus())
end
function JuMP.primal_status(model::InnerBilevelModel)
    return JuMP.primal_status(model.m)
end

# Statuses

JuMP.dual_status(::BilevelModel) = error(
    "Dual status cant be queried for BilevelModel, but you can query for Upper and Lower models.")
function JuMP.dual_status(model::UpperModel)
    _check_solver(model.m)
    return MOI.get(model.m.solver, MOI.DualStatus())
end
function JuMP.dual_status(model::LowerModel)
    _check_solver(model.m)
    return MOI.get(model.m.solver, MOI.PrimalStatus())
end

function JuMP.termination_status(model::BilevelModel)
    _check_solver(model)
    return MOI.get(model.solver, MOI.TerminationStatus())
end
function JuMP.raw_status(model::BilevelModel)
    _check_solver(model)
    return MOI.get(model.solver, MOI.RawStatusString())
end

# Replace variables

function replace_variables(var::BilevelVariableRef,
    model::BilevelModel,
    inner::JuMP.AbstractModel,
    variable_map::Dict{Int, V},
    level::Level) where {V<:JuMP.AbstractVariableRef}
    if var.model === model && in_level(var, level)
        return variable_map[var.idx]
    elseif var.model === model
        error("Variable $(var) belonging Only to $(var.level) level, was added in the $(level) level.")
    else
        error("A BilevelModel cannot have expression using variables of a BilevelModel different from itself")
    end
end
function replace_variables(aff::JuMP.GenericAffExpr{C, BilevelVariableRef},
    model::BilevelModel,
    inner::JuMP.AbstractModel,
    variable_map::Dict{Int, V},
    level::Level) where {C,V<:JuMP.AbstractVariableRef}
    result = JuMP.GenericAffExpr{C, JuMP.VariableRef}(0.0)#zero(aff)
    result.constant = aff.constant
    for (coef, var) in JuMP.linear_terms(aff)
        JuMP.add_to_expression!(result,
        coef,
        replace_variables(var, model, model, variable_map, level))
    end
    return result
end
function replace_variables(quad::JuMP.GenericQuadExpr{C, BilevelVariableRef},
    model::BilevelModel,
    inner::JuMP.AbstractModel,
    variable_map::Dict{Int, V},
    level::Level) where {C,V<:JuMP.AbstractVariableRef}
    aff = replace_variables(quad.aff, model, model, variable_map, level)
    quadv = JuMP.GenericQuadExpr{C, JuMP.VariableRef}(aff)
    for (coef, var1, var2) in JuMP.quad_terms(quad)
        JuMP.add_to_expression!(quadv,
        coef,
        replace_variables(var1, model, model, variable_map, level),
        replace_variables(var2, model, model, variable_map, level))
    end
    return quadv
end
replace_variables(funcs::Vector, args...) = map(f -> replace_variables(f, args...), funcs)

function print_lp(m, name, file_format = MOI.FileFormats.FORMAT_AUTOMATIC)
    dest = MOI.FileFormats.Model(format = file_format, filename = name)
    MOI.copy_to(dest, m)
    MOI.write_to_file(dest, name)
end

# Optimize

JuMP.optimize!(::T) where {T<:AbstractBilevelModel} =
    error("Can't solve a model of type: $T ")
function JuMP.optimize!(model::BilevelModel;
    lower_prob = "", upper_prob = "", bilevel_prob = "", solver_prob = "", file_format = MOI.FileFormats.FORMAT_AUTOMATIC)

    if model.mode === nothing
        error("No solution mode selected, use `set_mode(model, mode)` or initialize with `BilevelModel(optimizer_constructor, mode = some_mode)`")
    else
        mode = model.mode
    end

    _check_solver(model)

    solver = model.solver #optimizer#MOI.Bridges.full_bridge_optimizer(optimizer, Float64)
    if true#!MOI.is_empty(solver)
        MOI.empty!(solver)
    end

    upper = JuMP.backend(model.upper)
    lower = JuMP.backend(model.lower)

    if length(lower_prob) > 0
        print_lp(lower, lower_prob, file_format)
    end
    if length(upper_prob) > 0
        print_lp(upper, upper_prob, file_format)
    end

    t0 = time()

    moi_upper = JuMP.index.(
        collect(values(model.upper_to_lower_link)))
    moi_link = JuMP.index(model.link)
    moi_link2 = index2(model.upper_var_to_lower_ctr_link)

    # build bound for FortunyAmatMcCarlMode
    build_bounds!(model, mode)

    single_blm, upper_to_sblm, lower_to_sblm, lower_primal_dual_map, lower_dual_to_sblm =
        build_bilevel(upper, lower, moi_link, moi_upper, mode, moi_link2,
            copy_names = model.copy_names, pass_start = model.pass_start)

    # pass lower level dual variables info (start, upper, lower)
    for (idx, info) in model.ctr_info
        if haskey(model.ctr_lower, idx)
            ctr = model.ctr_lower[idx]
            pre_duals = lower_primal_dual_map.primal_con_dual_var[JuMP.index(ctr)] # vector
            duals = map(x->lower_dual_to_sblm[x], pre_duals)
            pass_dual_info(single_blm, duals, info)
        end
    end
    # pass lower & upper level primal variables info (upper, lower)
    for (idx, info) in model.var_info
        if haskey(model.var_lower, idx)
            var = lower_to_sblm[JuMP.index(model.var_lower[idx])]
        elseif haskey(model.var_upper, idx)
            var = upper_to_sblm[JuMP.index(model.var_upper[idx])]
        else
            continue
        end
        pass_primal_info(single_blm, var, info)
    end
    if length(bilevel_prob) > 0
        print_lp(single_blm, bilevel_prob, file_format)
    end

    sblm_to_solver = MOI.copy_to(solver, single_blm, copy_names = model.copy_names_to_solver)

    if length(solver_prob) > 0
        print_lp(solver, solver_prob, file_format)
    end

    model.upper_to_sblm = upper_to_sblm
    model.lower_to_sblm = lower_to_sblm
    model.lower_dual_to_sblm = lower_dual_to_sblm
    model.lower_primal_dual_map = lower_primal_dual_map
    model.sblm_to_solver = sblm_to_solver

    t1 = time()
    model.build_time = t1 - t0

    MOI.optimize!(solver)

    model.solve_time = time() - t1

    return nothing
end

# Extra info

function pass_primal_info(single_blm, primal, info::VariableInfo)
    if !isnan(info.upper) &&
        !MOI.is_valid(single_blm, CI{SVF,LT{Float64}}(primal.value))
        MOI.add_constraint(single_blm,
            SVF(primal), LT{Float64}(info.upper))
    end
    if !isnan(info.lower) &&
        !MOI.is_valid(single_blm, CI{SVF,GT{Float64}}(primal.value))
        MOI.add_constraint(single_blm,
            SVF(primal), GT{Float64}(info.lower))
    end
    return
end

function pass_dual_info(single_blm, dual, info::ConstraintInfo{Float64})
    if !isnan(info.start)
        MOI.set(single_blm, MOI.VariablePrimalStart(), dual[], info.start)
    end
    if !isnan(info.upper) &&
        !MOI.is_valid(single_blm, CI{SVF,LT{Float64}}(dual[].value))
        MOI.add_constraint(single_blm,
            SVF(dual[]), LT{Float64}(info.upper))
    end
    if !isnan(info.lower) &&
        !MOI.is_valid(single_blm, CI{SVF,GT{Float64}}(dual[].value))
        MOI.add_constraint(single_blm,
            SVF(dual[]), GT{Float64}(info.lower))
    end
    return
end
function pass_dual_info(single_blm, dual, info::ConstraintInfo{Vector{Float64}})
    for i in eachindex(dual)
        if !isnan(info.start[i])
            MOI.set(single_blm, MOI.VariablePrimalStart(), dual[i], info.start[i])
        end
        if !isnan(info.upper[i]) &&
            !MOI.is_valid(single_blm, CI{SVF,LT{Float64}}(dual[i].value))
            MOI.add_constraint(single_blm,
                SVF(dual[i]), LT{Float64}(info.upper[i]))
        end
        if !isnan(info.lower[i]) &&
            !MOI.is_valid(single_blm, CI{SVF,GT{Float64}}(dual[i].value))
            MOI.add_constraint(single_blm,
                SVF(dual[i]), MOI.GreaterThan{Float64}(info.lower[i]))
        end
    end
    return
end

# Bounds

function build_bounds!(::BilevelModel, ::AbstractBilevelSolverMode{T}) where T
    return nothing
end
function build_bounds!(model::BilevelModel, mode::AbstractBoundedMode{T}) where T
    return _build_bounds!(model, mode.cache)
end
function _build_bounds!(model::BilevelModel, mode::ComplementBoundCache)
    # compute variable bounds for FA mode
    fa_vi_up = mode.upper
    fa_vi_lo = mode.lower
    fa_vi_ld = mode.ldual
    empty!(fa_vi_up)
    empty!(fa_vi_lo)
    empty!(fa_vi_ld)
    for (idx, _info) in model.var_info
        if haskey(model.var_lower, idx)
            var = JuMP.index(model.var_lower[idx])
            is_lower = true
        elseif haskey(model.var_upper, idx)
            var = JuMP.index(model.var_upper[idx])
            is_lower = false
        else
            continue
        end
        vref = BilevelVariableRef(model, idx)

        is_dual = vref.level == DUAL_OF_LOWER

        info = deepcopy(_info)

        lb = JuMP.has_lower_bound(vref) ? JuMP.lower_bound(vref) : -Inf
        ub = JuMP.has_upper_bound(vref) ? JuMP.upper_bound(vref) : +Inf
        info.upper = min(ub, inf_if_nan(+, info.upper))
        info.lower = max(lb, inf_if_nan(-, info.lower))
        if is_lower
            fa_vi_lo[var] = info
        else
            fa_vi_up[var] = info
        end
        @assert info.lower <= info.upper
    end
    for (idx, _info) in model.ctr_info
        if haskey(model.ctr_lower, idx)
            ctr = JuMP.index(model.ctr_lower[idx])
            info = deepcopy(_info)
            cref = BilevelConstraintRef(model, idx)
            # scalar
            ub = dual_upper_bound(ctr)
            lb = dual_lower_bound(ctr)
            info.upper = min.(ub, inf_if_nan.(+, info.upper))
            info.lower = max.(lb, inf_if_nan.(-, info.lower))
            @assert sum(info.lower .<= info.upper) == length(info.upper)
            # TODO vector
            fa_vi_ld[ctr] = info
        end
    end
    return nothing
end

inf_if_nan(::typeof(+), val) = ifelse(isnan(val), Inf, val)
inf_if_nan(::typeof(-), val) = ifelse(isnan(val), -Inf, val)

dual_lower_bound(::CI{F,LT{T}}) where {F,T} = -Inf
dual_upper_bound(::CI{F,LT{T}}) where {F,T} =  0.0

dual_lower_bound(::CI{F,GT{T}}) where {F,T} =  0.0
dual_upper_bound(::CI{F,GT{T}}) where {F,T} = +Inf

dual_lower_bound(::CI{F,ET{T}}) where {F,T} =  0.0
dual_upper_bound(::CI{F,ET{T}}) where {F,T} =  0.0

dual_lower_bound(::CI{F,S}) where {F,S} = -Inf
dual_upper_bound(::CI{F,S}) where {F,S} = +Inf

# Initialize

function _check_solver(bm::BilevelModel)
    if bm.solver === nothing
        error("No solver attached, use `set_optimizer(model, optimizer_constructor)` or initialize with `BilevelModel(optimizer_constructor)`")
    end
end

function JuMP.set_optimizer(bm::BilevelModel, optimizer_constructor;
    bridge_constraints::Bool=true, with_names=true)
    # error_if_direct_mode(model, :set_optimizer)
    if bridge_constraints
        # We set `with_names=false` because the names are handled by the first
        # caching optimizer. If `default_copy_to` without names is supported,
        # no need for a second cache.
        optimizer = MOI.instantiate(optimizer_constructor, with_bridge_type=Float64, with_names=with_names)
        # for bridge_type in model.bridge_types
        #     _moi_add_bridge(optimizer, bridge_type)
        # end
    else
        optimizer = MOI.instantiate(optimizer_constructor)
    end

    bm.solver = optimizer
    if !MOI.is_empty(bm.solver)
        error("Calling the `optimizer_constructor` must return an empty optimizer")
    end
    return bm
end

function pass_cache(bm::BilevelModel, mode::FortunyAmatMcCarlMode{T}) where T
    mode.cache = bm.mode.cache
    return nothing
end
function pass_cache(bm::BilevelModel, mode::AbstractBilevelSolverMode{T}) where T
    return nothing
end

function check_mixed_mode(::MixedMode{T}) where T
end
function check_mixed_mode(mode)
    error("Cant set/get mode on a specific object because the base mode is $mode while it should be MixedMode in this case. Run `set_mode(model, BilevelJuMP.MixedMode())`")
end
function set_mode(ci::BilevelConstraintRef, mode::AbstractBilevelSolverMode{T}) where T
    bm = ci.model
    check_mixed_mode(bm.mode)
    _mode = deepcopy(mode)
    pass_cache(bm, _mode)
    ctr = JuMP.index(bm.ctr_lower[ci.index])
    bm.mode.constraint_mode_map_c[ctr] = _mode
    return nothing
end
function unset_mode(ci::BilevelConstraintRef)
    bm = ci.model
    check_mixed_mode(bm.mode)
    ctr = JuMP.index(bm.ctr_lower[ci.index])
    delete!(bm.mode.constraint_mode_map_c, ctr)
    return nothing
end

function get_mode(ci::BilevelConstraintRef)
    bm = ci.model
    check_mixed_mode(bm.mode)
    ctr = JuMP.index(bm.ctr_lower[ci.index])
    if haskey(bm.mode.constraint_mode_map_c, ctr)
        return bm.mode.constraint_mode_map_c[ctr]
    else
        return nothing
    end
end

function set_mode(::BilevelConstraintRef, ::MixedMode{T}) where T
    error("Cant set MixedMode in a specific constraint")
end
function set_mode(::BilevelConstraintRef, ::StrongDualityMode{T}) where T
    error("Cant set StrongDualityMode in a specific constraint")
end

function set_mode(vi::BilevelVariableRef, mode::AbstractBilevelSolverMode{T}) where T
    bm = vi.model
    check_mixed_mode(bm.mode)
    _mode = deepcopy(mode)
    pass_cache(bm, _mode)
    var = JuMP.index(bm.var_lower[vi.idx])
    # var is the MOI backend index in jump
    bm.mode.constraint_mode_map_v[var] = _mode
    return nothing
end
function unset_mode(vi::BilevelVariableRef)
    bm = vi.model
    check_mixed_mode(bm.mode)
    var = JuMP.index(bm.var_lower[vi.idx])
    delete!(bm.mode.constraint_mode_map_v, var)
    return nothing
end

function get_mode(vi::BilevelVariableRef)
    bm = vi.model
    check_mixed_mode(bm.mode)
    var = JuMP.index(bm.var_lower[vi.idx])
    if haskey(bm.mode.constraint_mode_map_v, var)
        return bm.mode.constraint_mode_map_v[var]
    else
        return nothing
    end
end

function set_mode(::BilevelVariableRef, ::MixedMode{T}) where T
    error("Cant set MixedMode in a specific variable")
end
function set_mode(::BilevelVariableRef, ::StrongDualityMode{T}) where T
    error("Cant set StrongDualityMode in a specific variable")
end


##############################################################################
##
## Fit is the only exported function
##
##############################################################################
function regife(df, m::FixedEffectModels.ModelTerm; kwargs...)
    regife(df, m.f; m.dict..., kwargs...)
end

function regife(df, f::FormulaTerm, vcov::CovarianceEstimator = Vcov.simple();
            weights::Union{Symbol, Nothing} = nothing,
            subset::Union{AbstractVector, Nothing} = nothing,
             method::Symbol = :dogleg,
             lambda::Number = 0.0,
             maxiter::Integer = 10_000,
             tol::Real = 1e-9,
             save::Union{Bool, Nothing} = nothing,
             contrasts::Dict = Dict{Symbol, Any}(),
             feformula::Union{Symbol, Expr, Nothing} = nothing,
            ifeformula::Union{Symbol, Expr, Nothing} = nothing,
            vcovformula::Union{Symbol, Expr, Nothing} = nothing,
            subsetformula::Union{Symbol, Expr, Nothing} = nothing)

    ##############################################################################
    ##
    ## Transform DataFrame -> Matrix
    ##
    ##############################################################################
    df = DataFrame(df; copycols = false)

    # to deprecate
    if vcovformula != nothing
        if (vcovformula == :simple) | (vcovformula == :(simple()))
            vcov = Vcov.Simple()
        elseif (vcovformula == :robust) | (vcovformula == :(robust()))
            vcov = Vcov.Robust()
        else
            vcov = Vcov.cluster(StatsModels.termvars(@eval(@formula(0 ~ $(vcovformula.args[2]))))...)
        end
    end
    if subsetformula != nothing
        subset = eval(evaluate_subset(df, subsetformula))
    end

    if  (ConstantTerm(0) ∉ FixedEffectModels.eachterm(f.rhs)) & (ConstantTerm(1) ∉ FixedEffectModels.eachterm(f.rhs))
        formula = FormulaTerm(f.lhs, tuple(ConstantTerm(1), FixedEffectModels.eachterm(f.rhs)...))
    end

    formula, formula_endo, formula_iv = FixedEffectModels.parse_iv(f)

    m, formula = parse_interactivefixedeffect(df, formula)
    if ifeformula != nothing # remove after depreciation
        m = OldInteractiveFixedEffectFormula(ifeformula)
    end

    ## parse formula
    if formula_iv != nothing
        error("partial_out does not support instrumental variables")
    end
    has_weights = (weights != nothing)

    ## create a dataframe without missing values & negative weightss
    vars = StatsModels.termvars(formula)
    if feformula != nothing # remove after depreciation
        vars = vcat(vars, StatsModels.termvars(@eval(@formula(0 ~ $(feformula)))))
    end
    factor_vars = [m.id, m.time]
    all_vars = unique(vcat(vars, factor_vars))
    esample = completecases(df[!, all_vars])
    esample .&= Vcov.completecases(df, vcov)
    if has_weights
        esample .&= BitArray(!ismissing(x) & (x > 0) for x in df[!, weights])
        all_vars = unique(vcat(all_vars, weights))
    end
    if subset != nothing
        if length(subset) != size(df, 1)
            error("df has $(size(df, 1)) rows but the subset vector has $(length(subset)) elements")
        end
        esample .&= BitArray(!ismissing(x) && x for x in subset)
    end
    main_vars = unique(vcat(vars, factor_vars))

    # Compute data needed for errors
    vcov_method_data = Vcov.materialize(view(df, esample,:), vcov)

     # Compute weights
     # Compute weights
     if has_weights
         weights = Weights(convert(Vector{Float64}, view(df, esample, weights)))
     else
         weights = Weights(Ones{Float64}(sum(esample)))
     end
     sqrtw = sqrt.(values(weights))
    for a in FixedEffectModels.eachterm(formula.rhs)
       if has_fe(a)
           isa(a, InteractionTerm) && error("Fixed effects cannot be interacted")
           Symbol(FixedEffectModels.fesymbol(a)) ∉ factor_vars && error("FixedEffect should correspond to id or time dimension of the factor model")
       end
    end
    fes, ids, formula = FixedEffectModels.parse_fixedeffect(df, formula)
    if feformula != nothing # remove after depreciation
        feformula = @eval(@formula(0 ~ $(feformula)))
        fes, ids = FixedEffectModels.oldparse_fixedeffect(df, feformula)
    end
    has_fes = !isempty(fes)
    has_fes_intercept = false
    ## Compute factors, an array of AbtractFixedEffects
    if has_fes
        if any([isa(fe.interaction, Ones) for fe in fes])
                formula = FormulaTerm(formula.lhs, tuple(ConstantTerm(0), (t for t in FixedEffectModels.eachterm(formula.rhs) if t!= ConstantTerm(1))...))
                has_fes_intercept = true
        end
        fes = FixedEffect[FixedEffectModels._subset(fe, esample) for fe in fes]
        feM = FixedEffectModels.AbstractFixedEffectSolver{Float64}(fes, weights, Val{:cpu})
    end

    has_intercept = ConstantTerm(1) ∈ FixedEffectModels.eachterm(formula.rhs)

    iterations = 0
    converged = false
    # get two dimensions

    id = FixedEffects.group(df[esample, m.id])
    time = FixedEffects.group(df[esample, m.time])

    ##############################################################################
    ##
    ## Construict vector y and matrix X
    ##
    ##############################################################################
    subdf = columntable(df[esample, unique(vcat(vars))])

    formula_schema = apply_schema(formula, schema(formula, subdf, contrasts), StatisticalModel)

    y = convert(Vector{Float64}, response(formula_schema, subdf))
    tss_total = FixedEffectModels.tss(y, has_intercept || has_fes_intercept, weights)

    X = convert(Matrix{Float64}, modelmatrix(formula_schema, subdf))

    # change default if has_regressors
    has_regressors = size(X, 2) > 0
    if save == nothing
        save = !has_regressors
    end

    # Compute demeaned X
    yname, coef_names = coefnames(formula_schema)
    if !isa(coef_names, Vector)
        coef_names = [coef_names]
    end
    yname = Symbol(yname)
    coef_names = Symbol.(coef_names)

    # demean variables
    if has_fes
        FixedEffectModels.solve_residuals!(y, feM)
        FixedEffectModels.solve_residuals!(X, feM)
    end

    ##############################################################################
    ##
    ## Estimate Model on Matrix
    ##
    ##############################################################################

    # initialize factor models at 0.1
    idpool = fill(0.1, length(id.pool), m.rank)
    timepool = fill(0.1, length(time.pool), m.rank)

    y .= y .* sqrtw
    X .= X .* sqrtw
    if !has_regressors
        # factor model
        fp = FactorModel(y, sqrtw, id.refs, time.refs, m.rank)
        fs = FactorSolution(idpool, timepool)
        (fs, iterations, converged) =
            fit!(Val{method}, fp, fs; maxiter = maxiter, tol = tol, lambda = lambda)
    else
        # interactive fixed effect
        coef = X \ y
        fp = FactorModel(y - X * coef, sqrtw, id.refs, time.refs, m.rank)
        fs = FactorSolution(idpool, timepool)
        fit!(Val{:levenberg_marquardt}, fp, fs; maxiter = 100, tol = 1e-3, lambda = lambda)

        fs = InteractiveFixedEffectsSolution(coef, fs.idpool, fs.timepool)
        fp = InteractiveFixedEffectsModel(y, sqrtw, X, id.refs, time.refs, m.rank)

        ym = copy(y)
        Xm = copy(X)

        while true
            # estimate the model
           (fs, iterations, converged) =
                fit!(Val{method}, fp, fs; maxiter = maxiter, tol = tol, lambda = lambda)
            # check that I obtain the same coefficient if I solve
            # y ~ x + γ1 x factors + γ2 x loadings
            # if not, this means fit! ended up on a a local minimum.
            # restart with randomized coefficients, factors, loadings
            newfeM = FixedEffectModels.AbstractFixedEffectSolver{Float64}(getfactors(fp, fs), weights, Val{:cpu})
            ym .= ym ./ sqrtw
            FixedEffectModels.solve_residuals!(ym, newfeM, tol = tol, maxiter = maxiter)
            ym .= ym .* sqrtw
            Xm .= Xm ./ sqrtw
            FixedEffectModels.solve_residuals!(Xm, newfeM, tol = tol, maxiter = maxiter)
            Xm .= Xm .* sqrtw
            ydiff = Xm  * (fs.b - Xm \ ym)
            if iterations >= maxiter || norm(ydiff)  <= 0.01 * norm(y)
                break
            end
            @info "Algorithm ended up on a local minimum. Restarting from a new, random, x0."
            map!(x -> randn() * x, fs, fs)
            copyto!(ym, y)
            copyto!(Xm, X)
        end
    end

    ##############################################################################
    ##
    ## Compute residuals
    ##
    ##############################################################################

    # compute residuals
    fp = FactorModel(copy(y), sqrtw, id.refs, time.refs, m.rank)
    if has_regressors
        LinearAlgebra.BLAS.gemm!('N', 'N', -1.0, X, fs.b, 1.0, fp.y)
    end
    subtract_factor!(fp, fs)
    fp.y .= fp.y ./ sqrtw
    residuals = fp.y
    ##############################################################################
    ##
    ## Compute errors
    ##
    ##############################################################################
    if !has_regressors
        rss = sum(abs2, residuals)
    else
        residualsm = ym .- Xm * fs.b
        crossxm = cholesky!(Symmetric(Xm' * Xm))
        ## compute the right degree of freedom
        df_absorb_fe = 0
        if has_fes
            for fe in fes
                df_absorb_fe += length(unique(fe.refs))
            end
        end
        dof_residual = max(size(X, 1) - size(X, 2) - df_absorb_fe, 1)

        ## estimate vcov matrix
        vcov_data = FixedEffectModels.VcovData(Xm, crossxm, residualsm, dof_residual)
        matrix_vcov = StatsBase.vcov(vcov_data, vcov_method_data)
        # compute various r2
        nobs = sum(esample)
        rss = sum(abs2, residualsm)
        _tss = FixedEffectModels.tss(ym ./ sqrtw, has_intercept || has_fes_intercept, weights)
        r2_within = 1 - rss / _tss

        rss = sum(abs2, residuals)
        r2 = 1 - rss / tss_total
        r2_a = 1 - rss / tss_total * (nobs - has_intercept) / dof_residual
    end

    ##############################################################################
    ##
    ## Save factors and loadings in a dataframe
    ##
    ##############################################################################

    if !save
        augmentdf = DataFrame()
    else
        augmentdf = DataFrame(fp, fs, esample)
        # save residuals in a dataframe
        if all(esample)
            augmentdf[!, :residuals] = residuals
        else
            augmentdf[!, :residuals] =  Vector{Union{Float64, Missing}}(missing, size(augmentdf, 1))
            augmentdf[esample, :residuals] = residuals
        end

        # save fixed effects in a dataframe
        if has_fes
            # residual before demeaning
             oldresiduals = convert(Vector{Float64}, response(formula_schema, subdf))
             oldresiduals .= oldresiduals .* sqrtw
             oldX = convert(Matrix{Float64}, modelmatrix(formula_schema, subdf))
             oldX .= oldX .* sqrtw
            if has_regressors
                LinearAlgebra.BLAS.gemm!('N', 'N', -1.0, oldX, coef, 1.0, oldresiduals)
            end
            fp = FactorModel(oldresiduals, sqrtw, id.refs, time.refs, m.rank)
            subtract_factor!(fp, fs)
            axpy!(-1.0, residuals, oldresiduals)
            # get fixed effect
            newfes, b, c = FixedEffectModels.solve_coefficients!(oldresiduals, feM; tol = tol, maxiter = maxiter)
            for j in 1:length(fes)
                augmentdf[!, ids[j]] = Vector{Union{Float64, Missing}}(missing, length(esample))
                augmentdf[esample, ids[j]] = newfes[j]
            end
        end
    end

    if !has_regressors
        return FactorResult(esample, augmentdf, rss, iterations, converged)
    else
        return InteractiveFixedEffectModel(fs.b, matrix_vcov, vcov, esample, augmentdf,
            coef_names, yname, f, nobs, dof_residual, r2, r2_a, r2_within,
            rss, sum(iterations), all(converged))
    end
end


module ERFA

using LinearAlgebra: I, cross, dot, norm, normalize

using ERFA_jll

export ERFAException
export DPI, D2PI, DR2D, DD2R, DR2AS, DAS2R, DS2R, TURNAS, DMAS2R, DTY, DAYSEC, DJY, DJC
export DJ00, DJM0, DJM00, DJM77, TTMTAI, DAU, CMPS, AULT, DC, ELG, ELB, TDB0, SRS
export WGS84, GRS80, WGS72

include("exports.jl")
include("erfa_common.jl")

include("a.jl")
include("b.jl")
include("c.jl")
include("d.jl")
include("e.jl")
include("f.jl")
include("g.jl")
include("h.jl")
include("i.jl")
include("j.jl")
include("l.jl")
include("m.jl")
include("n.jl")
include("o.jl")
include("p.jl")
include("r.jl")
include("s.jl")
include("t.jl")
include("u.jl")
include("x.jl")
include("z.jl")

end # module


topoint(x::AbstractVector{Point{N,Float32}}) where {N} = x

# GRRR STUPID SubArray, with eltype different from getindex(x, 1)
topoint(x::SubArray) = topoint([el for el in x])

function topoint(x::AbstractArray{<:Point{N,T}}) where {T,N}
    return topoint(Point{N,Float32}.(x))
end

function topoint(x::AbstractArray{<:Tuple{P,P}}) where {P<:Point}
    return topoint(reinterpret(P, x))
end

function create_shader(scene::Scene, plot::Union{Lines,LineSegments})
    # Potentially per instance attributes
    positions = lift(plot[1], transform_func_obs(plot)) do points, trans
        points = apply_transform(trans, topoint(points))
        if plot isa LineSegments
            return points
        else
            # Repeat every second point to connect the lines !
            return topoint(TupleView{2, 1}(points))
        end
        trans
    end
    startr = lift(p -> 1:2:(length(p) - 1), positions)
    endr = lift(p -> 2:2:length(p), positions)
    p_start_end = lift(positions) do positions
        return (positions[startr[]], positions[endr[]])
    end

    per_instance = Dict{Symbol,Any}(:segment_start => Buffer(lift(first, p_start_end)),
                                    :segment_end => Buffer(lift(last, p_start_end)))
    uniforms = Dict{Symbol,Any}()
    for k in (:linewidth, :color)
        attribute = lift(plot[k]) do x
            x = convert_attribute(x, Key{k}(), key"lines"())
            if plot isa LineSegments
                return x
            else
                # Repeat every second point to connect the lines!
                return isscalar(x) ? x : reinterpret(eltype(x), TupleView{2, 1}(x))
            end
        end
        if isscalar(attribute)
            uniforms[k] = attribute
            uniforms[Symbol("$(k)_start")] = attribute
            uniforms[Symbol("$(k)_end")] = attribute
        else
            if attribute[] isa AbstractVector{<:Number} && haskey(plot, :colorrange)
                attribute = lift(array2color, attribute, plot.colormap, plot.colorrange)
            end
            per_instance[Symbol("$(k)_start")] = Buffer(lift(x -> x[startr[]], attribute))
            per_instance[Symbol("$(k)_end")] = Buffer(lift(x -> x[endr[]], attribute))
        end
    end

    uniforms[:resolution] = scene.camera.resolution
    uniforms[:model] = plot.model
    positions = meta(Point2f0[(0, -1), (0, 1), (1, -1), (1, 1)],
                     uv=Vec2f0[(0, 0), (0, 0), (0, 0), (0, 0)])
    instance = GeometryBasics.Mesh(positions, GLTriangleFace[(1, 2, 3), (2, 4, 3)])
    return InstancedProgram(WebGL(), lasset("line_segments.vert"),
                            lasset("line_segments.frag"), instance,
                            VertexArray(; per_instance...); uniforms...)
end


using Celeste:
    Model, DeterministicVI, CelesteEDA, DeterministicVIImagePSF, SDSSIO,
    Infer, PSF

# NB: PyPlot is not in REQUIRE.
using PyPlot

##########
# Load data.

const datadir = joinpath(Pkg.dir("Celeste"), "test", "data")
rcf = SDSSIO.RunCamcolField(4263, 5,119)
images = SDSSIO.load_field_images(rcf, datadir);
catalog = SDSSIO.read_photoobj_files([rcf], datadir, duplicate_policy=:first);

# Pick an object.
for cat in catalog
    if minimum(cat.star_fluxes) > 300
        print(cat.objid)
        print("\n")
    end
end
objid = "1237663784734490677"
objids = [ce.objid for ce in catalog];
sa = findfirst(objids, objid);
neighbors = Infer.find_neighbors([sa], catalog, images)[1];
cat_local = vcat(catalog[sa], catalog[neighbors]);
vp = Vector{Float64}[Infer.init_source(ce) for ce in cat_local];
patches = Infer.get_sky_patches(images, cat_local);

###########################
# Visualize a source at initial parameters using both classic and FFT Celeste.

ea = ElboArgs(images, deepcopy(vp), patches, [1]);
Celeste.Infer.load_active_pixels!(ea.images, ea.patches);

ea_fft = ElboArgs(images, deepcopy(vp), patches, [1], psf_K=1);
Celeste.Infer.load_active_pixels!(ea_fft.images, ea.patches; exclude_nan=false);

psf_image_mat = Matrix{Float64}[
    PSF.get_psf_at_point(ea.patches[s, b].psf) for s in 1:ea_fft.S, b in 1:ea_fft.N];
fsm_mat = DeterministicVIImagePSF.FSMSensitiveFloatMatrices[
    DeterministicVIImagePSF.FSMSensitiveFloatMatrices() for
    s in 1:ea_fft.S, b in 1:ea_fft.N];
DeterministicVIImagePSF.initialize_fsm_sf_matrices!(fsm_mat, ea_fft, psf_image_mat);

s = 1
n = 3

image_fft = CelesteEDA.render_source_fft(ea, fsm_mat, s, n, include_iota=false, field=:E_G);
image_orig = CelesteEDA.render_source(ea, s, n, include_iota=false, field=:E_G);

PyPlot.close("all")
matshow(image_fft); colorbar(); title("fft Celeste")
matshow(image_orig); colorbar(); title("original Celeste")


# All the types native to ogl, wgl and vulkan shaders

const number_types = (Float32, Cint, Cuint, Cdouble)
const small_vecs = (((StaticVector{N, T} for T in number_types, N in (2, 3, 4)))...,)
const small_mats = (((StaticArray{Tuple{i, j}, T, 2} for T in ShaderAbstractions.number_types, i in (2, 3, 4), j in (2, 3, 4)))...,)
const small_arrays = (small_vecs..., small_mats...)
const native_types = (number_types..., small_arrays...)

const NativeNumbers = Union{number_types...}
const SmallVecs = Union{small_vecs...}
const SmallMats = Union{small_mats...}
const SmallArrays = Union{small_arrays...}
const NativeTypes = Union{native_types...}

"""
    native_type(context, Int128)
Returns a native type for a non native type.
E.g. native_type(context, Int128) -> Cint
"""
native_type(context::AbstractContext, ::Type{T}) where T <: NativeTypes = T

native_type(context::AbstractContext, x::Type{T}) where {T <: Integer} = Cint
native_type(context::AbstractContext, x::Type{Union{Int16, Int8}}) = x

native_type(context::AbstractContext, x::Type{T}) where {T <: Unsigned} = Cuint
native_type(context::AbstractContext, x::Type{Union{UInt16, UInt8}})  = x

native_type(context::AbstractContext, x::Type{T}) where {T <: AbstractFloat} = Float32
native_type(context::AbstractContext, x::Type{Float16})               = x

native_type(context::AbstractContext, x::Type{T}) where T <: Normed = N0f32
native_type(context::AbstractContext, x::Type{N0f16}) = x
native_type(context::AbstractContext, x::Type{N0f8}) = x

native_type(context::AbstractContext, x::Type{<: StaticArray{S, T, N}}) where {S, T, N} = similar_type(x, native_type(context, T))

map_t(f, tuple) = map_t(f, (), tuple)
map_t(f, result, ::Type{Tuple{}}) = Tuple{result...}
function map_t(f, result, T::Type{<: Tuple})
    map_t(
        f,
        (result..., f(Base.tuple_type_head(T))),
        Base.tuple_type_tail(T)
    )
end

native_type(context::AbstractContext) = x-> native_type(context, x)

function native_type(context::AbstractContext, ::Type{NamedTuple{Names, Types}}) where {Names, Types}
    return NamedTuple{Names, map_t(native_type(context), Types)}
end

"""
Curry form to easier pass this via e.g. map
"""
convert_uniform(context::AbstractContext) = x-> convert_uniform(context, x)

"""
All native types don't need conversion
"""
convert_uniform(context::AbstractContext, x::NativeTypes) = x

"""
Vector of native types, e.g `vec3 [4]`
"""
convert_uniform(context::AbstractContext, x::StaticVector{N, T}) where {N, T <: NativeTypes} = x

"""
Static Array with non native uniform type
"""
function convert_uniform(context::AbstractContext, x::StaticVector{N, T}) where {N, T}
    convert(similar_type(x, native_type(context, T)), x)
end

"""
Colors get special treatment to get them as vecs in shaders
"""
convert_uniform(context::AbstractContext, x::Colorant{T}) where T <: NativeNumbers = x

convert_uniform(context::AbstractContext, x::Colorant{T}) where T = mapc(native_type(context, T), x)

convert_uniform(context::AbstractContext, x::AbstractSampler{T, N}) where {T, N} = x

function convert_uniform(context::AbstractContext, x::AbstractVector{T}) where T
    return convert(Vector{native_type(context, T)}, x)
end

function convert_uniform(context::AbstractContext, x::NamedTuple{Names, Types}) where {Names, Types}
    return map(convert_uniform(context), x)
end

function convert_uniform(context::AbstractContext, x::Observable)
    return map(convert_uniform(context), x)
end
function convert_uniform(context::AbstractContext, x::T) where T
    all(t-> isbits(t), fieldtypes(T)) || error("All field types need to be isbits. Found: $(T) with $(fieldtypes(T))")
    return x
end

type_prefix(context::AbstractContext, x::Type{T}) where {T <: Union{FixedPoint, Float32, Float16}} = ""
type_prefix(context::AbstractContext, x::Type{T}) where {T <: Float64} = "d"
type_prefix(context::AbstractContext, x::Type{Cint}) = "i"
type_prefix(context::AbstractContext, x::Type{T}) where {T <: Union{Cuint, UInt8, UInt16}} = "u"

type_postfix(context::AbstractContext, x::Type{Float64}) = "dv"
type_postfix(context::AbstractContext, x::Type{Float32}) = "fv"
type_postfix(context::AbstractContext, x::Type{Cint})    = "iv"
type_postfix(context::AbstractContext, x::Type{Cuint})   = "uiv"

type_string(context::AbstractContext, x::T) where {T} = type_string(context, T)
type_string(context::AbstractContext, t::Type{Float32}) = "float"
type_string(context::AbstractContext, t::Type{Float64}) = "double"
type_string(context::AbstractContext, t::Type{Cuint}) = "uint"
type_string(context::AbstractContext, t::Type{Cint}) = "int"

function type_string(context::AbstractContext, t::Type{T}) where {T <: Union{StaticVector, Colorant}}
    return string(type_prefix(context, eltype(T)), "vec", length(T))
end

function type_string(context::AbstractContext, t::Type{<: AbstractSamplerBuffer{T}}) where T
    string(type_prefix(context, eltype(T)), "samplerBuffer")
end
is_arraysampler(t) = false
function type_string(context::AbstractContext, t::Type{<: AbstractSampler{T, D}}) where {T, D}
    str = string(type_prefix(context, eltype(T)), "sampler", D == 1 ? 2 : D, "D")
    is_arraysampler(t) && (str *= "Array")
    return str
end

function type_string(context::AbstractContext, t::Type{<: StaticMatrix})
    M, N = size(t)
    string(type_prefix(context, eltype(t)), "mat", M == N ? M : string(M, "x", N))
end
type_string(context::AbstractContext, t::Bool) = "bool"

type_string(context::AbstractContext, t::Observable) = type_string(context, t[])

type_string(context::AbstractContext, t::Type) = error("Type $t not supported")


# TODO: Add tests for came configurations

using REPL.TerminalMenus
export configure_game

function cofigure_n_players()
    n_players = nothing
    while true
        println("Enter number of players")
        n_players = readline()
        try
            n_players = parse(Int, n_players)
            2 ≤ n_players ≤ 10 && break
            println("Number of players must be between 2 and 10 (inclusive)")
        catch
            println("Number of players must be an integer between 2 and 10 (inclusive)")
        end
    end
    @assert n_players ≠ nothing
    @info "$n_players playing"
    return n_players
end

function cofigure_bank_roll(blinds)
    bank_roll = nothing
    while true
        println("Enter bank roll:")
        bank_roll = readline()
        try
            bank_roll = parse(Float64, bank_roll)
            bank_roll ≥ blinds.big && break
            println("Bank roll must be larger than or equal to the big blind")
        catch
            println("Bank roll must be a Float64 larger than or equal to the big blind")
        end
    end
    @assert bank_roll ≠ nothing
    @info "Players starting with a bank roll of \$$bank_roll"
    return bank_roll
end

function cofigure_blinds()
    big_blinds = nothing
    while true
        println("Enter big blind amount:")
        big_blinds = readline()
        try
            big_blinds = parse(Float64, big_blinds)
            big_blinds > sqrt(eps(Float64)) && break
            println("Blinds must be larger than `sqrt(eps(Float64))`")
        catch
            println("Blinds must be a float larger than `sqrt(eps(Float64))`")
        end
    end
    @assert big_blinds ≠ nothing
    @info "Blinds: $(big_blinds/2), $(big_blinds)"
    return Blinds(big_blinds/2, big_blinds)
end

function configure_basic_heads_up_game()
    bank_roll = 100
    blinds = Blinds(1, 2)
    players = (
        Player(Human(), 1; bank_roll=bank_roll),
        Player(ai_to_use(), 2; bank_roll=bank_roll)
    )
    return Game(players; blinds=blinds)
end

function configure_basic_1v4_game()
    bank_roll = 100
    blinds = Blinds(1, 2)
    players = (
        Player(Human(), 1; bank_roll=bank_roll),
        Player(ai_to_use(), 2; bank_roll=bank_roll),
        Player(ai_to_use(), 3; bank_roll=bank_roll),
        Player(ai_to_use(), 4; bank_roll=bank_roll),
        Player(ai_to_use(), 5; bank_roll=bank_roll),
    )
    return Game(players; blinds=blinds)
end

function configure_basic_2_bots_game()
    bank_roll = 100
    blinds = Blinds(1, 2)
    players = (
        Player(ai_to_use(), 1; bank_roll=bank_roll),
        Player(ai_to_use(), 2; bank_roll=bank_roll),
    )
    return Game(players; blinds=blinds)
end

function configure_basic_4_bots_game()
    bank_roll = 100
    blinds = Blinds(1, 2)
    players = (
        Player(ai_to_use(), 1; bank_roll=bank_roll),
        Player(ai_to_use(), 2; bank_roll=bank_roll),
        Player(ai_to_use(), 3; bank_roll=bank_roll),
        Player(ai_to_use(), 4; bank_roll=bank_roll),
    )
    return Game(players; blinds=blinds)
end

function configure_human_players(n_players)
    options = string.(1:n_players)
    menu = MultiSelectMenu(options) # charset=:unicode is not supported in earlier Julia versions
    choices = request("Select which players are human:", menu)
    println("$(length(choices)) human players ($(join(sort(collect(choices)), ", ")))")
    length(choices) > 0 || println("No human players.")
    human_seat_numbers = collect(choices)
    return human_seat_numbers
end

function configure_custom_game()

    n_players = cofigure_n_players()
    blinds = cofigure_blinds()
    human_seat_numbers = configure_human_players(n_players)
    bank_roll = cofigure_bank_roll(blinds)

    players = ntuple(n_players) do i
        if i in human_seat_numbers
            Player(Human(), i; bank_roll=bank_roll)
        else
            Player(ai_to_use(), i; bank_roll=bank_roll)
        end
    end

    return Game(players; blinds=blinds)
end

function configure_game()
    options = [
        "1) Heads up-- you vs AI, \$1-\$2 blinds, \$100 bank roll",
        "2) 1v4-- you vs 4 AI, \$1-\$2 blinds, \$100 bank roll",
        "3) 2 bots!, \$1-\$2 blinds, \$100 bank roll",
        "4) 4 bots!, \$1-\$2 blinds, \$100 bank roll",
        "5) custom game"
    ]
    menu = RadioMenu(options, pagesize=4)
    choice = request("Configure game:", menu)
    if choice != -1
        choice == 1 && return configure_basic_heads_up_game()
        choice == 2 && return configure_basic_1v4_game()
        choice == 3 && return configure_basic_2_bots_game()
        choice == 4 && return configure_basic_4_bots_game()
        choice == 5 && return configure_custom_game()
        error("Uncaught case")
    else
        println("Menu canceled.")
    end
end


"""
QuasiGeostrophy

# Description

- A series of functions for testing
"""
module QuasiGeostrophy
# Using
using LinearAlgebra, CUDA
# Spatial Discretization Includes
include("spatial_discretization/nodal_chebyshev.jl")
include("spatial_discretization/fourier_core.jl")
# Spatial Discretization Includes
export cheb
export fourier_nodes, fourier_wavenumbers

# Utility Includes
include("utils/utils.jl")
# Utility Exports
export appropriate_dims, plot

# Symbolic Includes
include("symbolics/abstract_core.jl")

# Grid Includes
include("grid/grid_core.jl")

# Include Hooks to other packages
include("utils/fourier_fields_hooks.jl")
include("utils/fourier_operators_hooks.jl")
include("utils/plot_hooks.jl")
include("utils/convenience_functions.jl")

# Most Important functions
function print_colors()
    for i in 0:255
        printstyled("color(" * string(i) * ")", color = i)
    end
    return nothing
end
export print_colors

end # module


""" Visualize raw and semantic flow graphs.
"""
module Visualization

using Compat

using Catlab.WiringDiagrams, Catlab.Graphics
import Catlab.Graphics.WiringDiagramLayouts: box_label, wire_label
using ..Doctrine
using ..RawFlowGraphs
import ..Serialization: text_label

# Raw flow graphs
#################

# FIXME: These methods use language-specific attributes. Perhaps there should
# be some standardization across languages.

function box_label(::MIME"text/plain", node::RawNode)
  lang = node.language
  get_first(lang, ["slot", "qual_name", "function", "kind"], "")
end

function wire_label(::MIME"text/plain", port::RawPort)
  lang = port.language
  get_first(lang, ["qual_name", "class"], "")
end

function get_first(collection, keys, default)
  if isempty(keys); return default end
  get(collection, splice!(keys, 1)) do
    get_first(collection, keys, default)
  end
end

# Semantic flow graphs
######################

box_label(::MIME"text/plain", f::Monocl.Hom) = text_label(f)
wire_label(::MIME"text/plain", A::Monocl.Ob) = text_label(A)

end


export TabularDeterministicPolicy

"""
    TabularDeterministicPolicy(table::Vector{Int}, nactions::Int) -> TabularDeterministicPolicy

Always select the specific action according to the state defined in `table`.
`nactions` is used to define the number of possible actions.
"""
Base.@kwdef struct TabularDeterministicPolicy <: AbstractPolicy
    table::Vector{Int}
    nactions::Int
end

(p::TabularDeterministicPolicy)(s::Int) = p.table[s]
(p::TabularDeterministicPolicy)(obs::Observation) = p(get_state(obs))
get_prob(p::TabularDeterministicPolicy, s::Int, a::Int) = p.table[s] == a ? 1.0 : 0.0

function update!(p::TabularDeterministicPolicy, correction)
    s, a = correction
    p.table[s] = a
end


module ExtendableSparse
using DocStringExtensions
using SparseArrays
using LinearAlgebra

include("sparsematrixcsc.jl")
include("sparsematrixlnk.jl")
include("extendable.jl")
include("preconditioners.jl")
include("sprand.jl")

export SparseMatrixLNK,ExtendableSparseMatrix,flush!,nnz, sprand!,sprand_sdd!, fdrand,fdrand!
export JacobiPreconditioner, ILU0Preconditioner, updateindex!

export colptrs
end # module


import Base.Broadcast: _broadcast_getindex, preprocess, preprocess_args, Broadcasted, broadcast_unalias, combine_axes, broadcast_axes, broadcast_shape, check_broadcast_axes, check_broadcast_shape, throwdm, broadcastable, AbstractArrayStyle, DefaultArrayStyle
import Base: copyto!, tail, axes, length, ndims
struct DiffEqBC{T}
    x::T
end
@inline axes(b::DiffEqBC) = axes(b.x)
@inline length(b::DiffEqBC) = length(b.x)
@inline broadcastable(b::DiffEqBC) = b
@inline Base.ndims(b::Type{DiffEqBC{T}}) where T = ndims(T)
Base.@propagate_inbounds _broadcast_getindex(b::DiffEqBC, i) = _broadcast_getindex(b.x, i)
Base.@propagate_inbounds _broadcast_getindex(b::DiffEqBC{<:AbstractArray{<:Any,0}}, i) = b.x[]
Base.@propagate_inbounds _broadcast_getindex(b::DiffEqBC{<:AbstractVector}, i) = b.x[i[1]]
Base.@propagate_inbounds _broadcast_getindex(b::DiffEqBC{<:AbstractArray}, i) = b.x[i]
diffeqbc(x::Array) = DiffEqBC(x)
diffeqbc(x) = x

# Ensure inlining
@inline combine_axes(A, B) = broadcast_shape(broadcast_axes(A), broadcast_axes(B)) # Julia 1.0 compatible
@inline check_broadcast_axes(shp, A::Union{Number, Array, Broadcasted}) = check_broadcast_shape(shp, axes(A))

@inline preprocess(f, dest, bc::Broadcasted{Style}) where {Style} = Broadcasted{Style}(bc.f, preprocess_args(f, dest, bc.args), bc.axes)
preprocess(f, dest, x) = f(broadcast_unalias(dest, x))

@inline preprocess_args(f, dest, args::Tuple) = (preprocess(f, dest, args[1]), preprocess_args(f, dest, tail(args))...)
@inline preprocess_args(f, dest, args::Tuple{Any}) = (preprocess(f, dest, args[1]),)
preprocess_args(f, dest, args::Tuple{}) = ()

# Performance optimization for the common identity scalar case: dest .= val
@inline copyto!(dest::DiffEqBC, bc::Broadcasted{<:AbstractArrayStyle{0}}) = copyto!(dest.x, bc)
@inline function copyto!(dest::DiffEqBC, bc::Broadcasted)
    axes(dest) == axes(bc) || throwdm(axes(dest), axes(bc))
    dest′ = dest.x
    # Performance optimization: broadcast!(identity, dest, A) is equivalent to copyto!(dest, A) if indices match
    if bc.f === identity && bc.args isa Tuple{AbstractArray} # only a single input argument to broadcast!
        A = bc.args[1]
        if axes(dest) == axes(A)
            return copyto!(dest′, A)
        end
    end
    bcs′ = preprocess(diffeqbc, dest, bc)
    @simd ivdep for I in eachindex(bcs′)
        @inbounds dest′[I] = bcs′[I]
    end
    return dest′ # return the original array without the wrapper
end

# Forcing `broadcasted` to inline is not necessary, since `Vern9` plays well
# with the Base implementation, and `Feagin`s do not use broadcasting.
#
#import Base.Broadcast: broadcasted, combine_styles
#map_nostop(f, t::Tuple{})              = ()
#map_nostop(f, t::Tuple{Any,})          = (f(t[1]),)
#map_nostop(f, t::Tuple{Any, Any})      = (f(t[1]), f(t[2]))
#map_nostop(f, t::Tuple{Any, Any, Any}) = (f(t[1]), f(t[2]), f(t[3]))
#map_nostop(f, t::Tuple)                = (Base.@_inline_meta; (f(t[1]), map_nostop(f,tail(t))...))
#@inline function broadcasted(f::Union{typeof(*), typeof(+), typeof(muladd)}, arg1, arg2, args...)
#    arg1′ = broadcastable(arg1)
#    arg2′ = broadcastable(arg2)
#    args′ = map_nostop(broadcastable, args)
#    broadcasted(combine_styles(arg1′, arg2′, args′...), f, arg1′, arg2′, args′...)
#end

macro ..(x)
    expr = Base.Broadcast.__dot__(x)
    if expr.head in (:(.=), :(.+=), :(.-=), :(.*=), :(./=), :(.\=), :(.^=)) # we exclude `÷=` `%=` `&=` `|=` `⊻=` `>>>=` `>>=` `<<=` because they are for integers
      name = gensym()
      dest = :(DiffEqBase.diffeqbc($(expr.args[1])))
      expr.args[1] = name
      return esc(quote
                  $name = $dest
                  $expr
                 end)
    else
      return esc(expr)
    end
end


import GeoStatsBase: fit, predict, status

# ----------------
# DUMMY ESTIMATOR
# ----------------

struct DummyEstimator end
struct FittedDummyEstimator end

fit(::DummyEstimator, data) = FittedDummyEstimator()
predict(::FittedDummyEstimator, var, pₒ) = (0, 1)
status(::FittedDummyEstimator) = true

# ------------------------
# DUMMY SIMULATION SOLVER
# ------------------------

import GeoStatsBase: solvesingle

@simsolver DummySimSolver begin end
function solvesingle(problem::SimulationProblem, covars::NamedTuple,
                     ::DummySimSolver, preproc)
  npts = nelements(domain(problem))
  mactypeof = Dict(name(v) => mactype(v) for v in variables(problem))
  reals = map(covars.names) do var
    V    = mactypeof[var]
    real = vcat(fill(zero(V), npts÷2), fill(one(V), npts÷2))
    var => real
  end
  Dict(reals)
end


function instantiate(model_type::DataType, data, pools, param::Dict{Symbol, <:Any})
    debug(LOGGER, "Instantiating model of type $(model_type) with params $(param)")
    m = model_type(data, pools)
    set_param!(m, param)
    return m
end

function set_param!(model, param::Dict{Symbol, <:Any})
    has_changed = false
    for (k, v) in param
        k in keys(get_model_params(model)) || throw(ArgumentError("Model $(typeof(model)) has no parameter $k. Valid parameters are $(get_model_params(model))."))
        fieldtype = typeof(getfield(model, k))
        typeof(v) == fieldtype || throw(ArgumentError("Cannot set parameter $k for $(typeof(model)), because the parameter type is $(typeof(v)) but the model expects $(fieldtype)."))
        is_valid_param_value(model, Val{k}, v) || throw(ArgumentError("Parameter $k has value $v which is outside the valid range for model $(typeof(model))."))
        if getfield(model,k) != v
            has_changed = true
            setfield!(model, k, v)
        end
    end
    has_changed && invalidate_solution!(model)
    return nothing
end

function set_pools!(model, pools::Vector{Symbol})
    islabelenc(pools, learning_pool_enc) || throw(ArgumentError("Pools must be a LabelEncoding of $(learning_pool_enc.label)."))
    set_pools!(model, labelmap(pools))
    return nothing
end

function set_pools!(model, pools::Dict{Symbol, Vector{Int}})
    if !isdefined(model, :pools) ||model.pools != pools
        model.pools = pools
        invalidate_solution!(model)
    end
    return nothing
end

function set_adjust_K!(model, adjust_K::Bool)
    if adjust_K && !model.adjust_K
        model.adjust_K = adjust_K
        update_K!(model)
        invalidate_solution!(model)
    elseif !adjust_K && model.adjust_K
        model.adjust_K = adjust_K
        invalidate_solution!(model)
    end
    return nothing
end

is_K_adjusted(model)::Bool = model.adjust_K

function update_K!(model)
    updated_K = MLKernels.kernelmatrix(Val(:col), model.kernel_fct, model.data)
    if !isdefined(model, :K) || updated_K != model.K || (!isdefined(model, :K_adjusted) && model.adjust_K)
        debug(LOGGER, "[UPDATE_K] Updating Kernel matrix.")
        invalidate_solution!(model)
        model.K = updated_K
        if model.adjust_K
            model.K_adjusted = adjust_kernel_matrix(updated_K)
        end
    else
        debug(LOGGER, "[UPDATE_K] Nothing to update, kernel matrix has not changed.")
    end
    return nothing
end

function set_data!(model, data::Array{T, 2}) where T <: Real
    if model.data != data
        model.data = data
        update_K!(model)
        invalidate_solution!(model)
    end
    return nothing
end

function set_kernel!(model::OCClassifier, kernel_fct::Kernel)
    if !isdefined(model, :kernel_fct) || model.kernel_fct != kernel_fct
        model.kernel_fct = kernel_fct
        update_K!(model)
        invalidate_solution!(model)
    end
    return nothing
end

function initialize!(model::OCClassifier, strategy::InitializationStrategy)
    debug(LOGGER, "[INITIALIZE] Initializing $(typeof(model)) with strategy: $(typeof(strategy))")
    C, kernel_fct = get_parameters(model, strategy)
    debug(LOGGER, "[INITIALIZE] Fitted parameters are: C = $(C) and kernel_fct = $(kernel_fct)")
    set_C!(model, C)
    set_kernel!(model, kernel_fct)
    model.state = model_initialized
    return nothing
end

get_kernel(model::OCClassifier) = model.kernel_fct


# Copyright 2018 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

module TestRFlowGraphs
using Test

using Catlab.WiringDiagrams
using SemanticFlowGraphs

using ..IntegrationTest: db, read_semantic_graph

# Semantic flow graph
#####################

# K-means clustering on the Iris dataset using base R.
# FIXME: The encapsulated box should only have one input port.
semantic = read_semantic_graph(joinpath("r", "clustering_kmeans"))
d = WiringDiagram([], [])
file = add_box!(d, construct(pair(concepts(db, ["tabular-file", "filename"])...)))
read_file = add_box!(d, concept(db, "read-tabular-file"))
transform = add_box!(d, Box(concepts(db, ["table"]), concepts(db, ["table"])))
kmeans = add_box!(d, construct(pair(concepts(db,
  ["k-means", "clustering-model-n-clusters"])...)))
fit = add_box!(d, Hom("fit",
  otimes(concept(db, "k-means"), concept(db, "data")),
  concept(db, "k-means")))
centroids = add_box!(d, concept(db, "k-means-centroids"))
clusters = add_box!(d, concept(db, "clustering-model-clusters"))
add_wires!(d, [
  (file, 1) => (read_file, 1),
  (read_file, 1) => (transform, 1),
  (kmeans, 1) => (fit, 1),
  (transform, 1) => (fit, 2),
  (fit, 1) => (clusters, 1),
  (fit, 1) => (centroids, 1),
])
@test semantic == d

end


using FastGroupBy, DataBench
svec = DataBench.gen_string_vec_var_len(1_000_000, 24);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(1_000_000, 24);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 24);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 24);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 24);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 24);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(1_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(1_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(1_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd32!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd32!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 32);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd32!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 64);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 64);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(10_000_000, 64);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd32!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 64);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd16!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 64);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd24!(svec)
@assert issorted(svec)

svec = DataBench.gen_string_vec_var_len(100_000_000, 64);
julia_lsd_radixsort_elapsed = @elapsed FastGroupBy.radixsort_lsd32!(svec)
@assert issorted(svec)


#===============================================================================

    McInput.jl

    defines struct of data and all analysis parameters
    to be passed to mc_analysis() in melting_curve.jl

    the constructor is intended as the only interface
    to the melting curve analysis and the only place
    where argument defaults are applied

    Author: Tom Price
    Date:   July 2019

===============================================================================#

import DataStructures.OrderedDict
import StaticArrays: SVector
import DataFrames.DataFrame

#===============================================================================
    defaults >>
===============================================================================#

## defaults for mc_analysis()
const DEFAULT_MC_AUTO_SPAN_SMOOTH               = false
const DEFAULT_MC_SPAN_SMOOTH_DEFAULT            = Float_T(0.015)
const DEFAULT_MC_SPAN_SMOOTH_FACTOR             = Float_T(7.2)
const DEFAULT_MC_MAX_TEMPERATURE                = Int_T(1000)

## defaults for mc_peak_analysis()
const DEFAULT_MC_TEMP_TOO_CLOSE_FRAC            = Float_T(0.05)
const DEFAULT_MC_TEMPERATURE_BANDWIDTH          = Float_T(1.0)
const DEFAULT_MC_DENSER_FACTOR                  = Int_T(10)    ## Integer
const DEFAULT_MC_SMOOTH_FLUO_SPLINE             = false
const DEFAULT_MC_PEAK_SPAN_TEMPERATURE          = Float_T(2.0)
const DEFAULT_MC_NEGDERIV_RANGE_LOW_QUANTILE    = Float_T(0.21)
const DEFAULT_MC_MAX_NUM_CROSS_POINTS           = Int_T(10)    ## Integer
const DEFAULT_MC_NOISE_FACTOR                   = Float_T(0.2)
const DEFAULT_MC_NORM_NEGDERIV_QUANTILE         = Float_T(0.64)
const DEFAULT_MC_MAX_NORM_NEGDERIV              = Float_T(0.8)
# const DEFAULT_MC_TOP1_FROM_MAX_UB               = 1.0
const DEFAULT_MC_MAX_NUM_PEAKS                  = Int_T(4)     ## Integer
const DEFAULT_MC_MIN_NORMALIZED_AREA            = Float_T(0.1)

## default for mutate_dups()
const DEFAULT_MC_JITTER_CONSTANT                = Float_T(0.01)

## defaults for mc_peak_analysis() overridden by mc_analysis()
# const OVERRIDDEN_DEFAULT_AUTO_SPAN_SMOOTH       = true
# const OVERRIDDEN_DEFAULT_MC_SPAN_SMOOTH_DEFAULT = 0.05
# const OVERRIDDEN_DEFAULT_MC_SPAN_SMOOTH_FACTOR  = 7.2

#===============================================================================
    Field definitions >>
===============================================================================#

## name, DataType, default value
const MC_FIELD_DEFS = [
    Field(:raw_df,                      DataFrame),
    Field(:num_wells,                   Int_T),
    Field(:num_channels,                Int_T),
    Field(:wells,                       SVector{W,Symbol} where {W}),
    Field(:channels,                    SVector{C,Int_T} where {C}),
    Field(:calibration_data,            CalibrationData{<: NumberOfChannels, <: Union{Int_T,Float_T}}),
    Field(:calibration_args,            CalibrationParameters, DEFAULT_CAL_ARGS),
    Field(:max_temperature,             Union{Int_T,Float_T},   DEFAULT_MC_MAX_TEMPERATURE),
    Field(:temp_too_close_frac,         Float_T,        	 	DEFAULT_MC_TEMP_TOO_CLOSE_FRAC),
    Field(:temperature_bandwidth,       Float_T,         		DEFAULT_MC_TEMPERATURE_BANDWIDTH),
    Field(:auto_span_smooth,            Bool,                	DEFAULT_MC_AUTO_SPAN_SMOOTH),
    Field(:span_smooth_default,         Float_T,         		DEFAULT_MC_SPAN_SMOOTH_DEFAULT),
    Field(:span_smooth_factor,          Float_T,         		DEFAULT_MC_SPAN_SMOOTH_FACTOR),
    Field(:denser_factor,               Int_T,               	DEFAULT_MC_DENSER_FACTOR),
    Field(:smooth_fluo_spline,          Bool,                	DEFAULT_MC_SMOOTH_FLUO_SPLINE),
    Field(:peak_span_temperature,       Float_T,         		DEFAULT_MC_PEAK_SPAN_TEMPERATURE),
    Field(:negderiv_range_low_quantile, Float_T,         		DEFAULT_MC_NEGDERIV_RANGE_LOW_QUANTILE),
    Field(:max_num_cross_points,        Int_T,               	DEFAULT_MC_MAX_NUM_CROSS_POINTS),
    Field(:noise_factor,                Float_T,         		DEFAULT_MC_NOISE_FACTOR),
    Field(:norm_negderiv_quantile,      Float_T,         		DEFAULT_MC_NORM_NEGDERIV_QUANTILE, "qt_prob"),
    Field(:max_norm_negderiv,           Float_T,         		DEFAULT_MC_MAX_NORM_NEGDERIV, "max_normd_qtv"),
    Field(:max_num_peaks,               Int_T,               	DEFAULT_MC_MAX_NUM_PEAKS, "top_N"),
    Field(:min_normalized_area,         Float_T,         		DEFAULT_MC_MIN_NORMALIZED_AREA),
    Field(:jitter_constant,             Float_T,         		DEFAULT_MC_JITTER_CONSTANT),
    Field(:out_format,                  OutputFormat,        	pre_json_output),
    Field(:reporting,                   Function,            	roundoff(JSON_DIGITS))]

#===============================================================================
    macro calls >>
===============================================================================#

## generate struct and constructor
SCHEMA = MC_FIELD_DEFS
@make_struct_from_SCHEMA McInput Input
@make_constructor_from_SCHEMA McInput


module Bonobo

using DataStructures
using NamedTupleTools

"""
    AbstractNode

The abstract type for a tree node. Your own type for `Node` given to [`initialize`](@ref) needs to subtype it.
The default if you don't provide your own is [`DefaultNode`](@ref).
"""
abstract type AbstractNode end

"""
    AbstractSolution{Node<:AbstractNode, Value}

The abstract type for a `Solution` object. The default is [`DefaultSolution`](@ref).
It is parameterized by `Node` and `Value` where `Value` is the value which describes the full solution i.e the value for every variable.
"""
abstract type AbstractSolution{Node<:AbstractNode, Value} end

"""
    BnBNode

Holds the necessary information of every node.
This needs to be added by every `AbstractNode` as `std::BnBNode`

```julia
id :: Int
lb :: Float64
ub :: Float64
```
"""
mutable struct BnBNode
    id :: Int
    lb :: Float64
    ub :: Float64
end

"""
    DefaultNode <: AbstractNode

The default structure for saving node information.
Currently this includes only the necessary `std::BnBNode` which needs to be part of every [`AbstractNode`](@ref).
"""
mutable struct DefaultNode <: AbstractNode
    std :: BnBNode
end

"""
    DefaultSolution{Node<:AbstractNode,Value} <: AbstractSolution{Node, Value}

The default struct to save a solution of the branch and bound run.
It holds
```julia
objective :: Float64
solution  :: Value
node      :: Node
```
Both the `Value` and the `Node` type are determined by the [`initialize`](@ref) method.

`solution` holds the information to obtain the solution for example the values of all variables.
"""
mutable struct DefaultSolution{Node<:AbstractNode,Value} <: AbstractSolution{Node, Value}
    objective :: Float64
    solution  :: Value
    node      :: Node
end

"""
    AbstractTraverseStrategy

The abstract type for a traverse strategy.
If you implement a new traverse strategy this must be the supertype.

If you want to implement your own strategy the [`get_next_node`](@ref) function needs a new method
which dispatches on the `traverse_strategy` argument.
"""
abstract type AbstractTraverseStrategy end

"""
    AbstractBranchStrategy

The abstract type for a branching strategy.
If you implement a new branching strategy, this must be the supertype.

If you want to implement your own strategy, you must implement a new method for [`get_branching_variable`](@ref)
which dispatches on the `branch_strategy` argument.
"""
abstract type AbstractBranchStrategy end

"""
    BFS <: AbstractTraverseStrategy

The BFS traverse strategy always picks the node with the lowest bound first.
If there is a tie then the smallest node id is used as a tie breaker.
"""
struct BFS <: AbstractTraverseStrategy end

"""
    FIRST <: AbstractBranchStrategy

The `FIRST` strategy always picks the first variable which isn't fixed yet and can be branched on.
"""
struct FIRST <: AbstractBranchStrategy end

"""
    MOST_INFEASIBLE <: AbstractBranchStrategy

The `MOST_INFEASIBLE` strategy always picks the variable which is furthest away from being "fixed" and can be branched on.
"""
struct MOST_INFEASIBLE <: AbstractBranchStrategy end

mutable struct Options
    traverse_strategy   :: AbstractTraverseStrategy
    branch_strategy     :: AbstractBranchStrategy
    atol                :: Float64
    rtol                :: Float64
end

"""
    BnBTree{Node<:AbstractNode,Root,Value,Solution<:AbstractSolution{Node,Value}}

Holds all the information of the branch and bound tree.
"""
mutable struct BnBTree{Node<:AbstractNode,Root,Value,Solution<:AbstractSolution{Node,Value}}
    incumbent::Float64
    lb::Float64
    solutions::Vector{Solution}
    nodes::PriorityQueue{Int,Node}
    root::Root
    branching_indices::Vector{Int}
    num_nodes::Int
    sense::Symbol
    options::Options
end
Base.broadcastable(x::BnBTree) = Ref(x)

include("util.jl")
include("node.jl")
include("branching.jl")

"""
    initialize(; kwargs...)

Initialize the branch and bound framework with the the following arguments.
Later it can be dispatched on `BnBTree{Node, Root, Solution}` for various methods.

# Keyword arguments
- `traverse_strategy` [`BFS`] currently the only supported traverse strategy is [`BFS`](@ref). Should be an [`AbstractTraverseStrategy`](@ref)
- `branch_strategy` [`FIRST`] currently the only supported branching strategies are [`FIRST`](@ref) and [`MOST_INFEASIBLE`](@ref). Should be an [`AbstractBranchStrategy`](@ref)
- `atol` [1e-6] the absolute tolerance to check whether a value is discrete
- `rtol` [1e-6] the relative tolerance to check whether a value is discrete
- `Node` [`DefaultNode`](@ref) can be special structure which is used to store all information about a node.
    - needs to have `AbstractNode` as the super type
    - needs to have `std :: BnBNode` as a field (see [`BnBNode`](@ref))
- `Solution` [`DefaultSolution`](@ref) stores the node and several other information about a solution
- `root` [`nothing`] the information about the root problem. The type can be used for dispatching on types
- `sense` [`:Min`] can be `:Min` or `:Max` depending on the objective sense
- `Value` [`Vector{Float64}`] the type of a solution

Return a [`BnBTree`](@ref) object which is the input for [`optimize!`](@ref).
"""
function initialize(;
    traverse_strategy = BFS,
    branch_strategy = FIRST,
    atol = 1e-6,
    rtol = 1e-6,
    Node = DefaultNode,
    Value = Vector{Float64},
    Solution = DefaultSolution{Node,Value},
    root = nothing,
    sense = :Min,
)
    return BnBTree{Node,typeof(root),Value,Solution}(
        NaN,
        NaN,
        Vector{Solution}(),
        PriorityQueue{Int,Node}(),
        root,
        get_branching_indices(root),
        0,
        sense,
        Options(traverse_strategy(), branch_strategy(), atol, rtol)
    )
end

function get_branching_indices end

"""
    optimize!(tree::BnBTree)

Optimize the problem using a branch and bound approach.

The steps are the following:
```julia
while !terminated(tree) # as long as there are open nodes
    # get the next open node depending on the traverse strategy
    node = get_next_node(tree, tree.options.traverse_strategy)
    # needs to be implemented by you
    # Should evaluate the current node and return the lower and upper bound
    # if the problem is infeasible both values should be set to NaN
    lb, ub = evaluate_node!(tree, node)
    # updates the upper and lower bound of the node struct
    set_node_bound!(tree.sense, node, lb, ub)

    # update the best solution
    updated = update_best_solution!(tree, node)
    updated && bound!(tree, node.id)

    # remove the current node
    close_node!(tree, node)
    # needs to be implemented by you
    # calls get_branching_variable and branch_on_variable! the latter must be implemented by you
    branch!(tree, node)
end
```

every function of the above can be overriden by your own method.
"""
function optimize!(tree::BnBTree)
    while !terminated(tree)
        node = get_next_node(tree, tree.options.traverse_strategy)
        lb, ub = evaluate_node!(tree, node)
        # if the problem was infeasible we simply close the node and continue
        if isnan(lb) && isnan(ub)
            close_node!(tree, node)
            continue
        end

        set_node_bound!(tree.sense, node, lb, ub)

        updated = update_best_solution!(tree, node)
        updated && bound!(tree, node.id)

        close_node!(tree, node)
        branch!(tree, node)
    end
end

"""
    terminated(tree::BnBTree)

Return true when the branch and bound loop in [`optimize!`](@ref) should be terminated.
Default behavior is to terminate the loop only when no nodes exist in the priority queue.
"""
terminated(tree::BnBTree) = isempty(tree.nodes)

"""
    set_node_bound!(objective_sense::Symbol, node::AbstractNode, lb, ub)

Set the bounds of the `node` object to the lower and upper bound given.
Internally everything is stored as a minimization problem. Therefore the objective_sense `:Min`/`:Max` is needed.
"""
function set_node_bound!(objective_sense::Symbol, node::AbstractNode, lb, ub)
    if isnan(ub)
        ub = Inf
    end
    if objective_sense == :Min
        node.lb = lb
        node.ub = ub
    else
        node.lb = -lb
        node.ub = -ub
    end
end

"""
    bound!(tree::BnBTree, current_node_id)

Close all nodes which have a lower bound higher or equal to the incumbent
"""
function bound!(tree::BnBTree, current_node_id)
    for (_,node) in tree.nodes
        if node.id != current_node_id && node.lb >= tree.incumbent
            close_node!(tree, node)
        end
    end
end

"""
    close_node!(tree::BnBTree, node::AbstractNode)

Delete the node from the priority queue from the nodes
"""
close_node!(tree::BnBTree, node::AbstractNode) = delete!(tree.nodes, node.id)

"""
    update_best_solution!(tree::BnBTree, node::AbstractNode)

Update the best solution when we found a better incumbent.
Calls [`add_new_solution!`] if this is the case
"""
function update_best_solution!(tree::BnBTree, node::AbstractNode)
    isinf(node.ub) && return false
    node.ub >= tree.incumbent && return false

    tree.incumbent = node.ub

    add_new_solution!(tree, node)
    return true
end

"""
    add_new_solution!(tree::BnBTree{N,R,V,S}, node::AbstractNode) where {N,R,V,S<:DefaultSolution{N,V}}

Currently it changes the general solution itself by calling [`get_relaxed_values`](@ref) which needs to be implemented by you.
# Todo: Add a possibility to store several solutions based on some options.

This function needs to be implemented by you if you have a different type of Solution object than [`DefaultSolution`](@ref).
"""
function add_new_solution!(tree::BnBTree{N,R,V,S}, node::AbstractNode) where {N,R,V,S<:DefaultSolution{N,V}}
    sol = DefaultSolution(node.ub, get_relaxed_values(tree, node), node)
    if isempty(tree.solutions)
        push!(tree.solutions, sol)
    else
        tree.solutions[1] = sol
    end
end

"""
    get_relaxed_values(tree::BnBTree, node::AbstractNode)

Get the values of the current node. This is always called only after [`evaluate_node!`](@ref) is called.
It is used to store a `Solution` object.
Return the type of `Value` given to the [`initialize`](@ref) method.
"""
function get_relaxed_values end

"""
    get_solution(tree::BnBTree; result=1)

Return the solution values of the problem.
See [`get_objective_value`](@ref) to obtain the objective value instead.
"""
function get_solution(tree::BnBTree{N,R,V,S}; result=1) where {N,R,V,S<:DefaultSolution{N,V}}
    return tree.solutions[result].solution
end

"""
    get_objective_value(tree::BnBTree; result=1)

Return the objective value
"""
function get_objective_value(tree::BnBTree{N,R,V,S}; result=1) where {N,R,V,S<:DefaultSolution{N,V}}
    if tree.sense == :Max
        return -tree.solutions[result].objective
    else
        return tree.solutions[result].objective
    end
end

export BnBTree, BnBNode, AbstractNode, AbstractSolution

export AbstractTraverseStrategy

end


using Flipping

test = "/home/beatriz/mainen.flipping.5ht@gmail.com/Flipping/run_task_photo/raw_data/DN1_170807a.csv"
poke = process_pokes(test)
first(poke[:,[:Streak,:Poke,:LastPoke]],10)
union(poke[:,:Protocol])
##


function a_computer_vision_toolbox()
    mat"""$i = license('test','Video_and_Image_Blockset')"""
    @assert i == 1 "the Matlab Computer Vision System Toolbox is not available"
end
a_csvfile(file) = @assert isfile(file) "missing csv file"
function a_table(t)
    @assert length(t) ≠ 0 "csv file has no lines"
    for x in keys(csvfile_columns)
        @assert x ∈ propertynames(t) "missing column $x from csv file"
    end
    for x in propertynames(t)
        @assert haskey(csvfile_columns, x) "unkown column, $x, in csv file"
    end
end

c_resfile(io, ::Missing) = (println(io, "- res file missing in csv file"); return missing)
function c_resfile(io, resfile)
    if isfile(resfile)
        resfile
    else
        println(io, "- res file does not exist")
        missing
    end
end

c_poi_names(io, ::Missing) = (println(io, "- POI names missing in csv file"); return missing)
function c_poi_names(io, poi_names)
    good = true
    if !all(x -> !occursin(' ', String(x)), poi_names)
        println(io, "- POI name/s contains space/s")
        good = false
    end
    if !allunique(poi_names)
        println(io, "- POI names must be unique")
        good = false
    end
    if all(!occursin("track", String(name)) for name in poi_names)
        println(io, "- a track POI is missing")
        good = false
    end
    good ? poi_names : missing
end

c_expected_locations(io, ::Missing) = nothing
function c_expected_locations(io, expected_locations)
    if !allunique(values(expected_locations))
        println(io, "- at least two of the POI's expected locations are identical")
        return missing
    end
    expected_locations
end

a_expected_locations(io, ::Missing, poi_names, npoints) = nothing
function a_expected_locations(io, expected_locations, poi_names, npoints)
    for (k, v) in expected_locations
        i = findfirst(==(k), poi_names)
        n = npoints[i]
        n == 1 || println(io, "- you expected $k to be at $v, yet there are $n lines in its column (column #$i) in the res-file (instead of just one)")
    end
end

a_singular_pois(io, poi_names, npoints) = map(poi_names, npoints) do name, n
    if n ≠ 1 && name ∈ (:feeder, :pickup, :dropoff, :nest, :north, :south)
        println(io, "- $name is expected to have exactly one row (point) in its res file, yet it has $n rows")
    end
end

a_coords(io, x, y, d, e) = nothing
function a_coords(io, resfile::SystemPath, poi_names::Vector{Symbol}, duration::Float64, expected_locations)
    matopen(string(resfile)) do mio
        for field in ("xdata", "ydata", "status")
            if !MAT.exists(mio, field)
                println(io, "- resfile missing $field")
                return nothing
            end
        end
        haskey(read(mio, "status"), "FrameRate") || println(io,  "- resfile missing frame rate")
        xdata = read(mio, "xdata")
        n = size(xdata, 2)
        npoints = [length(nzrange(xdata, j)) for j in 1:n]
        a_expected_locations(io, expected_locations, poi_names, npoints)
        a_singular_pois(io, poi_names, npoints)
        !all(iszero, npoints) || println(io, "- res file is empty")
        nmax = maximum(npoints)
        nmax > 5 || println(io, "- res file missing a POI with more than 5 data points (e.g. a track)")
        npois = length(poi_names)
        npois == n || println(io, "- number of POIs, $npois, doesn't match the number of res columns, $n")
        rows = rowvals(xdata)
        fr = read(mio, "status")["FrameRate"]
        haskey(read(mio, "status"), "nFrames") || println(io,  "- resfile missing number of total frames")
        nf = read(mio, "status")["nFrames"]
        duration2 = nf/fr
        isapprox(duration2, duration, atol = 1) || println(io, "- the duration of the POI video file, $duration s, and the one reported in the res file, $duration2 s, are not the same")
        if npois > n
            println(io, "- there are more POIs ($npois) than there are columns in the res file ($n)")
            return nothing
        end
        for (j, name) in enumerate(poi_names)
            i = nzrange(xdata, j)
            if !isempty(i)
                t = rows[i[end]]/fr
                t ≤ duration || println(io, "- the time-stamp of the $name POI is not in the video")
            end
        end
    end
end

c_videofile(io, ::Missing, what) = (println(io, "- ", what, " video file missing in csv file"); return missing)
function c_videofile(io, videofile, what)
    if !isfile(videofile)
        println(io, "- ", what, " video file does not exist")
        return missing
    else
        return videofile
    end
end

a_turning_point(io, x, y, d) = nothing
function a_turning_point(io, poi_videofile::AbstractString, t::Float64, duration::Float64)
    0 ≤ t ≤ duration || println(io, "- turning point time-stamp is not in the video")
end

a_checker_size(io, ::Missing) = println(io, "- checker-size is missing in the csv file")
a_checker_size(io, checker_size) = checker_size > 0 || println(io, "- checkers must be larger than zero")

a_extrinsic(io, ::Missing, duration) = println(io, "- extrinsic time-stamp is missing in the csv file")
a_extrinsic(io, extrinsic, duration) = 0 ≤ extrinsic ≤ duration || println(io, "- extrinsic time-stamp is not in the video")

function a_intrinsic(io, intrinsic, duration)
    all(x -> 0 ≤ x ≤ duration, intrinsic) || println(io, "- intrinsic time-stamp is not in the video")
    issorted(intrinsic) || println(io, "- the intrinsic starting time cannot come after the ending time")
end
a_intrinsic(io, ::Missing, _) = nothing

a_calibration(io, ::Missing, extrinsic, intrinsic, checker_size) = nothing
function a_calibration(io, calib_videofile, extrinsic, intrinsic, checker_size)
    a_checker_size(io, checker_size)
    duration = get_duration(calib_videofile)
    a_extrinsic(io, extrinsic, duration)
    a_intrinsic(io, intrinsic, duration)
end

c_nest2feeder(io, ::Missing) = missing
function c_nest2feeder(io, nest2feeder)
    if nest2feeder < 0
        println(io, "- nest to feeder distance must be larger than zero")
        return missing
    else
        return nest2feeder
    end
end
a_azimuth(io, ::Missing) = nothing
a_azimuth(io, azimuth) = 0 < azimuth < 360 || println(io, "- azimuth must be between 0° and 360°")

__getfeeder(x) = get(x, :initialfeeder, get(x, :pickup, get(x, :feeder, missing)))
_get_expected_nest2feeder(::Missing) = missing
_get_expected_nest2feeder(x) = _get_expected_nest2feeder(get(x, :nest, missing), __getfeeder(x))
_get_expected_nest2feeder(nest, feeder) = norm(nest - feeder)
_get_expected_nest2feeder(::Missing, feeder) = missing
_get_expected_nest2feeder(nest, ::Missing) = missing
_get_expected_nest2feeder(::Missing, ::Missing) = missing
a_expected_nest2feeder(io, ::Missing, nest2feeder::Missing) = nothing
a_expected_nest2feeder(io, ::Real, nest2feeder::Missing) = println(io, "- it seems like you have expectations on the distance between the nest and feeder, but nest2feeder is missing")
a_expected_nest2feeder(io, ::Missing, nest2feeder::Real) = nothing#println(io, "- it seems like you should have expectations on the distance between the nest and feeder since nest2feeder is not missing")
a_expected_nest2feeder(io, expected_nest2feeder::Real, nest2feeder::Real) = expected_nest2feeder ≈ nest2feeder || println(io, "- nest2feeder, $nest2feeder, is not equal to the distance between the expected nest and feeder locations, $expected_nest2feeder")

a_extra_correction(io, ::Bool) = nothing
a_extra_correction(io, _) = println(io, "- extra correction is missing/wrong (use true/false)")

function check4errors(x)
    io = IOBuffer()
    resfile = c_resfile(io, x.resfile)
    poi_names = c_poi_names(io, x.poi_names)
    expected_locations = c_expected_locations(io, x.expected_locations)
    poi_videofile = c_videofile(io, x.poi_videofile, "POI")
    duration = get_duration(poi_videofile)
    a_coords(io, resfile, poi_names, duration, expected_locations)
    a_turning_point(io, poi_videofile, x.turning_point, duration)
    calib_videofile = c_videofile(io, x.calib_videofile, "Calibration")
    a_calibration(io, calib_videofile, x.extrinsic, x.intrinsic, x.checker_size)
    nest2feeder = c_nest2feeder(io, x.nest2feeder)
    a_azimuth(io, x.azimuth)
    expected_nest2feeder = _get_expected_nest2feeder(expected_locations)
    a_expected_nest2feeder(io, expected_nest2feeder, nest2feeder)
    a_extra_correction(io, x.extra_correction)
    String(take!(io))
end

get_duration(::Missing) = missing
function get_duration(file)
    p = FFMPEG.exe(`-i $file -show_entries format=duration -v quiet -of csv="p=0"`, command=FFMPEG.ffprobe, collect=true)
    parse(Float64, only(p))
end

# function get_dimentions(file)
#     p = FFMPEG.exe(`-i $file -show_entries stream=width,height -of csv=p=0:s=x`, command=FFMPEG.ffprobe, collect=true)
#     txt = split(only(p), 'x')
#     parse.(Int, txt)
# end