#verilog-ranges.jl..
#creates ranges that help with making the julia verilog and the verilog much
#more interpretable.

doc"""
  v

  is an appendage that reverses the order of julia's ranges so that they have
  the cosmetic appearance of a verilog range.  This can be used both in setters
  and getters for wire collections.

  For example:
  `my_wire[5:0v]`  references a wire over indices 0-5
  `my_wire[6:2v]`  references a wire over indices 2-6

  For dereferencing (getting) you can reverse this:
  `my_wire[2:6v]`  references the wire over indices 2-6 with the msb order reversed.

  if you are using a variable, be sure to put the value in paretheses.

  For example:

  `my_wire[bits:(bits-4)v]`
"""
immutable v
  value
end

doc"""
  msb

  is a special keyword which indicates that the value should go to the most
  significant bit.  There is no equivalent "lsb", since in most cases this will
  be a fairly easy-to-reference value.  Plan your code accordingly.

  For example:
  my_wire[16:0v] = Wire(0xF000, 16)
  my_wire[msb:12]              # ==> Wire{3:0v}(0xF)

  my_wire_2[16:0v] = Wire(0x0F00, 16)
  my_wire[(msb-4):(msb-8)]     # ==> Wire{3:0v}(0xF)
"""
immutable msb;
  value::Integer
end

doc"""
  VerilogRange

  is a range object that represents a verilog range.  These are declared by
  using the v ranges, so 5:0v is a VerilogRange from 0 to 5.
"""
immutable VerilogRange <: AbstractUnitRange{Int64}
  start::Int64
  stop::Int64
end

doc"""
  RelativeRange

  is a range object that can take values which can be relative to the most
  significant bit.
"""
immutable RelativeRange
  start::Union{Int64, msb}
  stop::Union{Int64, msb}
end

function Base.:*(i, ::Type{v}); v(i); end
#allows

function Base.colon(i::Integer, vv::v)
  #first possibility, vv.value is actually the type msb.
  if vv.value == msb
    RelativeRange(msb(0), i)
  elseif isa(vv.value, msb)
    RelativeRange(vv.value, i)
  else
    VerilogRange(vv.value, i)
  end
end

function Base.colon(m::Union{Type{msb}, msb}, vv::v)
  left = (m == msb) ? msb(0) : m
  (vv.value == msb) ? RelativeRange(msb(0), left) : RelativeRange(vv.value, left)
end

function Base.:-(::Type{msb}, n::Integer)
  msb(n)
end
function Base.:-(m::msb, n::Integer)
  msb(m.value + n)
end
function Base.:+(m::msb, n::Integer)
  (n > m.value) && throw(ArgumentError("attempt to dereference beyond msb"))
  msb(m.value - n)
end

function parse_msb(rr::RelativeRange, vr::VerilogRange)
  true_start = isa(rr.start, msb) ? vr.stop - rr.start.value : rr.start
  true_stop  = isa(rr.stop, msb)  ? vr.stop - rr.stop.value  : rr.stop
  true_stop:(true_start)v
end

export v, msb

#because verilog often uses zero-indexing, a python-style range() operator
#is helpful for creating for loop generators.

Base.range(i::Integer) = (i-1):0v

#making a VerilogRange a well-defined iterable.
Base.start(v::VerilogRange) = v.start
Base.next(v::VerilogRange, state) = (state, state + ((v.stop >= v.start) ? 1 : - 1))
Base.done(v::VerilogRange, state) = state == v.stop + ((v.stop >= v.start) ? 1 : -1)
Base.eltype(v::VerilogRange) = Int64
Base.length(v::VerilogRange) = ((v.stop > v.start) ? (v.stop - v.start) : (v.start - v.stop)) + 1

Base.show(io::IO, v::VerilogRange) = print(io, v.stop, ":", v.start, "v")

function Base.reverse(r::VerilogRange)
  if r.start < r.stop
    r.stop:-1:r.start
  else
    r.stop:r.start
  end
end


"""
Calculates the exact boundingbox of a Scene/Plot
"""
function boundingbox(x)
    data_limits(x)
end

boundingbox(x::Combined) = boundingbox(x.plots)

boundingbox(scene::Scene) = boundingbox(plots_from_camera(scene))

function boundingbox(plots::Vector)
    isempty(plots) && return FRect3D(Vec3f0(0), Vec3f0(0))
    plot_idx = iterate(plots)
    bb = FRect3D()
    while plot_idx !== nothing
        plot, idx = plot_idx
        plot_idx = iterate(plots, idx)
        bb2 = boundingbox(plot)
        isfinite(bb) || (bb = bb2)
        isfinite(bb2) || continue
        bb = union(bb, bb2)
    end
    bb
end

function boundingbox(x::Text, text::String)
    position = value(x[:position])
    @get_attribute x (textsize, font, align, rotation, model)
    boundingbox(text, position, textsize, font, align, rotation, model)
end
boundingbox(x::Text) = boundingbox(x, value(x[1]))

function boundingbox(
        text::String, position, textsize;
        font = "default", align = (:left, :bottom), rotation = 0.0,
        model::Mat4f0 = Mat4f0(I)
    )
    boundingbox(
        text, position, textsize,
        to_font(font), to_align(align), to_rotation(rotation), model
    )

end

function boundingbox(text::String, position, textsize, font, align, rotation, model)
    atlas = get_texture_atlas()
    N = length(text)
    ctext_state = iterate(text)
    ctext_state === nothing && return AABB(Vec3f0(0), Vec3f0(0))
    pos_per_char = !isa(position, VecTypes)
    start_pos = Vec(pos_per_char ? first(position) : position)
    start_pos2D = to_ndim(Point2f0, start_pos, 0.0)
    last_pos = Point2f0(0, 0)
    aoffsetn = to_ndim(Vec3f0, align, 0f0)
    start_pos3d = to_ndim(Vec3f0, start_pos, 0.0)
    bb = AABB(start_pos3d, Vec3f0(0))
    broadcast_foreach(1:N, rotation, font, textsize) do i, rotation, font, scale
        c, text_state = ctext_state
        ctext_state = iterate(text, text_state)
        if c != '\r'
            pos = if pos_per_char
                to_ndim(Vec3f0, position[i], 0.0)
            else
                last_pos = calc_position(last_pos, Point2f0(0, 0), atlas, c, font, scale)
                rotation * (start_pos3d .+ to_ndim(Vec3f0, last_pos, 0.0))
            end
            s = glyph_scale!(atlas, c, font, scale)
            srot = rotation * to_ndim(Vec3f0, s, 0.0)
            bb = GeometryTypes.update(bb, pos)
            bb = GeometryTypes.update(bb, pos .+ srot)
        end
    end
    FRect3D(minimum(bb) .- (widths(bb) .* aoffsetn), widths(bb))
end


"""
    parse_difflist(data, offset, bytes_per_sample_id, has_genotypes)

Parses a single `DiffList` from `data`, starting with `offset + 1`-th byte.
"""
function parse_difflist(data::AbstractVector{UInt8},
    offset::Integer,
    bytes_per_sample_id::Integer,
    has_genotype::Bool)
    # length
    len, offset = decode_single(pointer(data); offset=offset)
    # length-zero list
    sample_id_dtype = bytes_to_UInt[bytes_per_sample_id]
    if len == 0
        return DiffList{sample_id_dtype, Nothing}(
            0, C_NULL, C_NULL, has_genotype, nothing, C_NULL), offset
    end
    # sample id bases
    n_groups = ceil_int(len, 0x000040) # 64 in decimal
    sample_id_bases = reinterpret(sample_id_dtype, view(data,
        Int(offset + 1) : Int(offset + n_groups * bytes_per_sample_id)))

    offset += n_groups * bytes_per_sample_id

    # sizes of the final components
    final_component_sizes = view(data, Int(offset + 1): Int(offset + n_groups - 1))
    offset += n_groups - 1

    # genotypes
    if has_genotype
        genotype_bytes = ceil_int(len, 4)
        # genotype_data = view(data, offset + 1 : offset + genotype_bytes)
        genotypes = BitsVector(pointer(data, offset + 1), 2, len)
        # BitsVector{typeof(genotype_data)}(Ref(genotype_data), 2, len)
        offset += genotype_bytes
    else
        genotypes = nothing
    end

    # final component: differences of indices
    final_component_size = sum(final_component_sizes) + 63 * length(final_component_sizes)
    last_group_size = len % 64
    last_group_size = last_group_size == 0 ? 64 : last_group_size
    # increment has one less value
    last_group_size -= 1
    last_incr_size = size_n(data, last_group_size, offset + final_component_size)
    final_component_size += last_incr_size
    final_component = pointer(data, offset + 1)#view(data, Int(offset + 1) : Int(offset + final_component_size))

    # TODO: defer computation of offset of the last block to parse_difflist_sampleids!
    offset += final_component_size

    DiffList{typeof(sample_id_bases),typeof(genotypes)}(
        len, sample_id_bases, pointer(final_component_sizes),
        has_genotype,
        genotypes, final_component), offset
end

"""
    parse_difflist_sampleids!(idx, idx_incr, dl, gid, sid_incr_offset=nothing)

Parses a `gid`-th group of 64 sample ids in a `DiffList` `dl` into `idx`.
"""
function parse_difflist_sampleids!(idx::AbstractArray, idx_incr::AbstractArray,
        dl::DiffList, gid::Integer, sid_incr_offset::Union{Nothing, UInt} = nothing)
    # TODO: compute offset of the last block in this function.
    n_groups = ceil_int(dl.len, 64)
    @assert gid <= n_groups
    if sid_incr_offset === nothing
        sid_incr_offset = zero(UInt)
        @inbounds for i in 1:(gid - 1)
            sid_incr_offset += unsafe_load(dl.last_component_sizes, i) + 0x3f
            #dl.last_component_sizes[][i] + 0x3f # offset by 63 (0x3f)
        end
    end
    idx_incr[1] = 0
    # offset by one to make it 1-based.
    baseidx = dl.sample_id_bases[gid] + 1
    if gid != n_groups
        decode_multiple!(pointer(idx_incr, 2), dl.sample_id_increments;
            count = 63, offset = UInt(sid_incr_offset))
        cumsum!(idx_incr, idx_incr)
        idx .= baseidx .+ idx_incr
    else
        count = dl.len % 64
        count = count == 0 ? 0x00000040 : count
        decode_multiple!(pointer(idx_incr, 2), dl.sample_id_increments;
            count = count - 1, offset = UInt(sid_incr_offset))
        idx[1] = baseidx
        @inbounds for i in 2:count
            idx[i] = idx[i-1] + idx_incr[i]
        end
        @inbounds for i in (count + 1):length(idx)
            idx[i] = 0
        end
        # cumsum!(view(idx_incr, 1:count), view(idx_incr, 1:count))
        # idx[1:count] .= baseidx .+ view(idx_incr, 1:count)
        # fill!(@view(idx[(count + 1):end]), 0)
    end
    idx
end


"""
    MatShell{T}(obj, m, n)

Create a `m×n` PETSc shell matrix object wrapping `obj`.

If `obj` is a `Function`, then the multiply action `obj(y,x)`; otherwise it calls `mul!(y, obj, x)`.
This can be changed by `PETSc._mul!`.

"""
mutable struct MatShell{T,A} <: AbstractMat{T}
  ptr::CMat
  comm::MPI.Comm
  obj::A
end

struct MatOp{T,Op}
end

function _mul!(y,mat::MatShell{T,F},x) where {T, F<:Function}
  mat.obj(y, x)
end

function _mul!(y,mat::MatShell{T},x) where {T}
  LinearAlgebra.mul!(y, mat.obj, x)
end

MatShell{T}(obj, m, n) where {T} = MatShell{T}(obj, MPI.COMM_SELF, m, n, m, n)

@for_libpetsc begin
  function MatShell{$PetscScalar}(obj::A, comm::MPI.Comm, m, n, M, N) where {A}
    mat = MatShell{$PetscScalar,A}(C_NULL, comm, obj)
    # we use the MatShell object itsel
    ctx = pointer_from_objref(mat)
    @chk ccall((:MatCreateShell, $libpetsc), PetscErrorCode,
      (MPI.MPI_Comm,$PetscInt,$PetscInt,$PetscInt,$PetscInt,Ptr{Cvoid},Ptr{CMat}),
      comm, m, n, M, N, ctx, mat)

    mulptr = @cfunction(MatOp{$PetscScalar, MATOP_MULT}(), $PetscInt, (CMat, CVec, CVec))
    @chk ccall((:MatShellSetOperation, $libpetsc), PetscErrorCode, (CMat, MatOperation, Ptr{Cvoid}), mat, MATOP_MULT, mulptr)
    return mat
  end

  function (::MatOp{$PetscScalar, MATOP_MULT})(M::CMat,cx::CVec,cy::CVec)::$PetscInt
    #try
      r_ctx = Ref{Ptr{Cvoid}}()
      @chk ccall((:MatShellGetContext, $libpetsc), PetscErrorCode, (CMat, Ptr{Ptr{Cvoid}}), M, r_ctx)
      ptr = r_ctx[]
      mat = unsafe_pointer_to_objref(ptr)

      x = unsafe_localarray($PetscScalar, cx)
      y = unsafe_localarray($PetscScalar, cy)

      _mul!(y,mat,x)

      Base.finalize(y)
      Base.finalize(x)
      return $PetscInt(0)
    # catch e
    #   if e isa PetscErrorCode
    #     return e.code
    #   else
    #     return
    #   end
    # end
  end

end


compartment = SBML.MathIdent("compartment")
k1 = SBML.MathIdent("k1")
k2 = SBML.MathIdent("k2")
reaction1_k = SBML.MathIdent("reaction1_k")
S1 = SBML.MathIdent("S1")

sbmlfiles = [
    # sbml_test_suite case 00001
    (
        joinpath(@__DIR__, "data", "00001-sbml-l3v2.xml"),
        "https://raw.githubusercontent.com/sbmlteam/sbml-test-suite/master/cases/semantic/00001/00001-sbml-l3v2.xml",
        "3e9a8bfe27343459d7a1e462fc3fd4eb2b01dc7b32af1db06a98f366287da01a",
        x -> nothing,
        1,
        ("reaction1", SBML.MathApply("*", [SBML.MathApply("*", [compartment, k1]), S1])),
        ("S1", 0.00015),
    ),
    # case 00001 in older level and version
    (
        joinpath(@__DIR__, "data", "00001-sbml-l2v1.xml"),
        "https://raw.githubusercontent.com/sbmlteam/sbml-test-suite/master/cases/semantic/00001/00001-sbml-l2v1.xml",
        "71a145c58b08e475d76bdec644589b2a55b5c5c2fee218274c91677c0f30b508",
        SBML.set_level_and_version(3, 2),
        1,
        ("reaction1", SBML.MathApply("*", [SBML.MathApply("*", [compartment, k1]), S1])),
        ("S1", 0.00015),
    ),
    # case 00057 with localParameters
    (
        joinpath(@__DIR__, "data", "00057-sbml-l3v2.xml"),
        "https://raw.githubusercontent.com/sbmlteam/sbml-test-suite/master/cases/semantic/00057/00057-sbml-l3v2.xml",
        "3e84e19cebbb79eea879847f541b1d22db6eb239f1f070ef4609f04c77688659",
        SBML.libsbml_convert("promoteLocalParameters"),
        2,
        (
            "reaction1",
            SBML.MathApply("*", [SBML.MathApply("*", [compartment, reaction1_k]), S1]),
        ),
        ("S1", 0.0003),
    ),
    # case 00025 with functionDefinition
    (
        joinpath(@__DIR__, "data", "00025-sbml-l3v2.xml"),
        "https://raw.githubusercontent.com/sbmlteam/sbml-test-suite/master/cases/semantic/00025/00025-sbml-l3v2.xml",
        "d3231ae3858d9e5dca1b106aa7b106a0caee9e6967ef8413de6b9acde9171c3e",
        SBML.libsbml_convert("expandFunctionDefinitions"),
        1,
        ("reaction1", SBML.MathApply("*", [compartment, SBML.MathApply("*", [k1, S1])])),
        ("S1", 0.0015),
    ),
    # case 00036 with initialAssignment
    (
        joinpath(@__DIR__, "data", "00037-sbml-l3v2.xml"),
        "https://raw.githubusercontent.com/sbmlteam/sbml-test-suite/master/cases/semantic/00037/00037-sbml-l3v2.xml",
        "074f0caeaf2cdc390967bfdf06d80ccad519c648df7f421dc4e9c69e71551dff",
        SBML.libsbml_convert("expandInitialAssignments"),
        2,
        ("reaction1", SBML.MathApply("*", [SBML.MathApply("*", [compartment, k2]), S1])),
        ("S1", 0.01125),
    ),
]

@testset "Loading of models from sbml_test_suite" begin
    for (sbmlfile, url, hash, converter, expected_par, expected_rxn, expected_u0) in
        sbmlfiles
        if !isfile(sbmlfile)
            Downloads.download(url, sbmlfile)
        end

        cksum = bytes2hex(sha256(open(sbmlfile)))
        if cksum != hash
            @warn "The downloaded model `$sbmlfile' seems to be different from the expected one. Tests will likely fail." cksum
        end

        @testset "Loading of $sbmlfile" begin
            mdl = readSBML(sbmlfile, converter)

            @test typeof(mdl) == Model

            @test length(mdl.parameters) == expected_par

            id, math = expected_rxn
            @test isequal(repr(mdl.reactions[id].kinetic_math), repr(math))

            id, ic = expected_u0
            if basename(sbmlfile) == "00037-sbml-l3v2.xml"
                # When expanding initial assignments with libsbml, the initial amount
                # becomes empty.
                @test_broken mdl.species[id].initial_amount[1] == ic
            else
                @test mdl.species[id].initial_amount[1] == ic
            end
        end
    end
end


# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/JuliaFEM.jl/blob/master/LICENSE.md
using JuliaFEM
using JuliaFEM.Core
using JuliaFEM.API: Model

"""
Function for creating solver and all the necessary components
for the calculation
"""
function get_solver(model::Model, case_name::String)
    case = model.load_cases[case_name]

    # Create core elements
    core_elements = create_core_elements(model)

    # Add Neumann boundary conditions to elements
    add_neumann_bcs!(model, case, core_elements)

    # Create Dirichlet boundary conditions
    dirichlet_arr = create_dirichlet_bcs(model, case, core_elements)

    # Create solver
    solver = create_solver(model, case,
                            core_elements, dirichlet_arr)
	return solver
end

"""
"""
function create_solver(model, case, core_elements, dirichlet_arr)
    field_problem = JuliaFEM.Core.(case.problem)()
    all_elsets = model.elsets

    # Pushing all the defined element sets into the calculation
    for element_set in case.sets
        el_ids = all_elsets[element_set].elements
        for each in el_ids
            push!(field_problem, core_elements[each])
        end
    end

    # Creating the solver and pushing problems and
	solver = JuliaFEM.Core.(case.solver)()
	push!(solver, field_problem)
	push!(solver, dirichlet_arr...)
    return solver
end

"""
"""
function create_dirichlet_bcs(model, case, core_elements)
    dirichlet_arr = Any[]
    dirichlet_bcs = case.dirichlet_boundary_conditions
    elsets = model.elsets
    field_problem = JuliaFEM.Core.(case.problem)()
    for each in dirichlet_bcs
        set_name = each.set_name
        value = each.value
        problem = JuliaFEM.Core.DirichletProblem(
           JuliaFEM.Core.get_unknown_field_name(field_problem),
           JuliaFEM.Core.get_unknown_field_dimension(field_problem))
        set_for_bc = each.set_name
        set_ids = elsets[set_for_bc]
        bc = each.value
        for el_id in set_ids.elements
            core_element = core_elements[el_id]
            core_element[bc[1]] = bc[2]
            push!(problem, core_element)
        end
        push!(dirichlet_arr, problem)
    end
    return dirichlet_arr
end

"""
"""
function create_core_elements(model)
    core_elements = Dict()
    nodes = model.nodes
    all_elements = model.elements
    element_ids = keys(all_elements)
    for el_id in element_ids
        element = all_elements[el_id]
        el_type = element.element_type
        el_id = element.id
        mat = element.material
        conn = element.connectivity
        core_element = JuliaFEM.Core.(el_type)(conn)
        core_element["geometry"] = map(x->nodes[x].coords, conn)
        for each in keys(mat.scalar_data)
            core_element[each] = mat.scalar_data[each]
        end
        core_elements[el_id] = core_element
        model.elements[el_id].results = core_element
    end
    return core_elements
end

"""
"""
function add_neumann_bcs!(model, case, core_elements)
    neumann_bcs = case.neumann_boundary_conditions
    elsets = model.elsets
    for each in neumann_bcs
        set_for_bc = each.set_name
        set_ids = elsets[set_for_bc]
        bc = each.value
        for el_id in set_ids.elements
            core_element = core_elements[el_id]
            core_element[bc[1]] = bc[2]
        end
        if !(set_for_bc in case.sets)
            push!(case.sets, set_for_bc)
        end
    end
end

"""
"""
function solve!(model::Model, case_name::String, time::Float64)
    # Create solver
	solver = get_solver(model, case_name)

    # Solve problem at given time
    solver(time)
end


export to_device, to_host

using CUDAapi
using Flux
using Knet
using Adapt
import Adapt: adapt, adapt_storage

to_host(x) = to_device(Val(:cpu), x)
to_device(::Val{:cpu}, x) = x  # cpu(x) is not very efficient! So by default we do nothing here.

if has_cuda()
    using CuArrays

    to_device(::Val{:Zygote_gpu}, x) = Flux.fmap(a -> adapt(CuArray{Float32}, a), x)
    to_device(::Val{:cpu}, x::Union{KnetArray, CuArray}) = adapt(Array, x)

    to_device(::Val{:Knet_gpu}, x) = adapt(KnetArray, x)
    to_device(::Val{:Knet_gpu}, x::SubArray) = KnetArray(x)
end

adapt_storage(T::Type{<:KnetArray}, x::AbstractArray{<:Real}) = T(x)
adapt_storage(T::Type{<:KnetArray}, x::Knet.Param{<:AbstractArray}) = Knet.Param(T(value(x)))


# Helper structs to dispatch known decision variable construction
struct NoSpecifiedConstraint <: MOI.AbstractSet end
Base.copy(set::NoSpecifiedConstraint) = set

struct DecisionSet <: MOI.AbstractScalarSet
    stage::Int
    constraint::Union{MOI.AbstractSet, JuMP.AbstractVectorSet}
    is_recourse::Bool

    function DecisionSet(stage::Integer; constraint::Union{MOI.AbstractSet, JuMP.AbstractVectorSet} = NoSpecifiedConstraint(), is_recourse::Bool = false)
        return new(stage, constraint, is_recourse)
    end
end

struct KnownSet <: MOI.AbstractScalarSet
    stage::Int
end

struct SingleDecisionSet{T} <: MOI.AbstractScalarSet
    stage::Int
    decision::Decision{T}
    constraint::MOI.AbstractSet
    is_recourse::Bool
end
Base.copy(set::SingleDecisionSet) = reuse(set, set.decision)

set_constraint(set::SingleDecisionSet) = set.constraint

struct MultipleDecisionSet{T} <: MOI.AbstractVectorSet
    stage::Int
    decisions::Vector{Decision{T}}
    constraint::MOI.AbstractSet
    is_recourse::Bool
end
MOI.dimension(set::MultipleDecisionSet) = length(set.decisions)
Base.copy(set::MultipleDecisionSet) = reuse(set, set.decisions)

MOIU.variable_function_type(::Type{<:SingleDecisionSet}) = SingleDecision
MOIU.variable_function_type(::Type{<:MultipleDecisionSet}) = VectorOfDecisions

is_decision_type(::Type{SingleDecisionSet}) = true
is_decision_type(::Type{MultipleDecisionSet}) = false

function JuMP.in_set_string(print_mode, set::SingleDecisionSet)
    decision_str(set) = set.is_recourse ? "RecourseDecisions" : "Decisions"
    if state(set.decision) == Taken
        return string(JuMP._math_symbol(print_mode, :in), " $(decision_str(set))(value = $(set.decision.value))")
    end
    if state(set.decision) == Known
        return string(JuMP._math_symbol(print_mode, :in), " Known(value = $(set.decision.value))")
    end
    if set.constraint isa NoSpecifiedConstraint
        return string(JuMP._math_symbol(print_mode, :in), " $(decision_str(set))")
    else
        return string(JuMP._math_symbol(print_mode, :in), " $(decision_str(set))($(JuMP.in_set_string(print_mode, set.constraint)))")
    end
end

function JuMP.in_set_string(print_mode, set::MultipleDecisionSet)
    decision_str(set) = set.is_recourse ? "RecourseDecisions" : "Decisions"
    if all(d -> state(d) == Taken, set.decisions)
        return string(JuMP._math_symbol(print_mode, :in), " $(decision_str(set))(values = $([d.value for d in set.decisions]))")
    end
    if all(d -> state(d) == Known, set.decisions)
        return string(JuMP._math_symbol(print_mode, :in), " Known(values = $([d.value for d in set.decisions]))")
    end
    if set.constraint isa NoSpecifiedConstraint
        return string(JuMP._math_symbol(print_mode, :in), " $(decision_str(set))")
    else
        return string(JuMP._math_symbol(print_mode, :in), " $(decision_str(set))($(JuMP.in_set_string(print_mode, set.constraint)))")
    end
end

function reuse(set::SingleDecisionSet, decision::Decision)
    return SingleDecisionSet(set.stage, decision, copy(set.constraint), set.is_recourse)
end

function reuse(set::MultipleDecisionSet, decisions::Vector{<:Decision})
    return MultipleDecisionSet(set.stage, decisions, copy(set.constraint), set.is_recourse)
end

function decision_set(variable::JuMP.ScalarVariable, set::DecisionSet)
    return SingleDecisionSet(set.stage, Decision(variable.info, Float64), set.constraint, set.is_recourse)
end

function decision_set(variables::Vector{<:JuMP.ScalarVariable}, set::DecisionSet)
    decisions = map(variables) do variable
        Decision(variable.info, Float64)
    end
    if set.constraint isa JuMP.AbstractVectorSet
        return MultipleDecisionSet(set.stage,
                                   decisions,
                                   JuMP.moi_set(set.constraint, length(variables)),
                                   set.is_recourse)
    else
        return MultipleDecisionSet(set.stage,
                                   decisions,
                                   set.constraint,
                                   set.is_recourse)
    end
end

function decision_set(variable::JuMP.ScalarVariable, set::KnownSet)
    return SingleDecisionSet(set.stage, KnownDecision(variable.info, Float64), NoSpecifiedConstraint(), false)
end

function decision_set(variables, set::KnownSet)
    decisions = map(variables) do variable
        KnownDecision(variable.info, Float64)
    end
    return MultipleDecisionSet(set.stage,
                               decisions,
                               NoSpecifiedConstraint(),
                               false)
end


include("utils.jl")

abstract type Optimizer end

abstract type GlobalOptimizer <: Optimizer end

abstract type LocalOptimizer <: Optimizer end

isglobal(::GlobalOptimizer) = true
isglobal(::LocalOptimizer) = false

minimize(o::Optimizer, f, g, lb, ub, x) =
    minimize(o, wrap_function(f, g), lb, ub, x)
maximize(o::Optimizer, f, g, lb, ub, x) =
    maximize(o, wrap_function(f, g), lb, ub, x)
isglobal(o::Optimizer) = error("not implemented")
need_gradient(o::Optimizer) = error("not implemented")

include("stochastic_opt.jl")
include("gradient_opt.jl")
include("gradfree_opt.jl")


#=
# Format specific functions for writing data output files.
=#

function output_values_raw(vars, file)
    # do one variable/component at a time
    if typeof(vars) <: Array
        for vi=1:length(vars) # each variable
            output_values_raw(vars[vi], file);
        end
    else
        # Here vars is just one variable
        N = size(vars.values, 2); # number of points
        for ci=1:size(vars.values,1) # each component for that variable
            file.write(vars.values[ci,:]); # write the values
        end
    end

end

function output_values_csv(vars, file)
    # columns will be like x, y, z, u1,...
    if typeof(vars) <: Array
        N = size(vars[1].values, 2); # number of points (must be same for all vars)
        if vars[1].location == CELL
            x = fv_info.cellCenters;
        else
            x = grid_data.allnodes;
        end

        # header
        line = "x";
        if dim > 1
            line *= ", y";
        end
        if dim > 2
            line *= ", z";
        end
        for vi=1:length(vars) # each variable
            if size(vars[vi].values,1) > 1
                for ci=1:size(vars[vi].values,1)
                    line *= ", "*string(vars[vi].symbol)*"_"*string(ci);
                end
            else
                line *= ", "*string(vars[vi].symbol);
            end
        end
        println(file, line);

        # data
        for i=1:N
            # x, y, z
            line = string(x[1,i]);
            if dim > 1
                line *= ", "*string(x[2,i]);
            end
            if dim > 2
                line *= ", "*string(x[3,i]);
            end

            # u1, u2, ...
            for vi=1:length(vars) # each variable
                for ci=1:size(vars[vi].values,1)
                    line *= ", "*string(vars[vi].values[ci,i]);
                end
            end
            println(file, line);
        end

    else
        N = size(vars.values, 2); # number of points
        if vars.location == CELL
            x = fv_info.cellCenters;
        else
            x = grid_data.allnodes;
        end
        dim = size(x,1);

        # header
        line = "x";
        if dim > 1
            line *= ", y";
        end
        if dim > 2
            line *= ", z";
        end
        if size(vars.values,1) > 1
            for ci=1:size(vars.values,1)
                line *= ", "*string(vars.symbol)*"_"*string(ci);
            end
        else
            line *= ", "*string(vars.symbol);
        end
        println(file, line);

        # data
        for i=1:N
            # x, y, z
            line = string(x[1,i]);
            if dim > 1
                line *= ", "*string(x[2,i]);
            end
            if dim > 2
                line *= ", "*string(x[3,i]);
            end

            # u1, u2, ...
            for ci=1:size(vars.values,1)
                line *= ", "*string(vars.values[ci,i]);
            end
            println(file, line);
        end
    end
end

function output_values_vtk(vars, file, ascii)
    # Use the WriteVTK package.
    # First build MeshCell array
    nel = size(grid_data.loc2glb,2);
    if config.dimension == 1
        np_to_type = Dict([(2,3)]); # line
    elseif config.dimension == 2
        np_to_type = Dict([(3,5),(4,9)]); # triangle, quad
    else
        np_to_type = Dict([(4,10),(8,12)]); # tet, hex
    end

    cells = Array{WriteVTK.MeshCell,1}(undef, nel);
    for ei=1:nel
        nvertex = 0;
        for vi=1:length(grid_data.glbvertex[:,ei])
            if grid_data.glbvertex[vi,ei] > 0
                nvertex = nvertex+1;
            else
                break;
            end
        end
        if nvertex < 2
            printerr("Found element with too few vertices when writing VTK output: el="*string(ei)*", vertexmap="*string(grid_data.glbvertex[:,ei]));
            return;
        end
        cell_type = np_to_type[nvertex];
        cells[ei] = WriteVTK.MeshCell(VTKCellType(cell_type), grid_data.glbvertex[:,ei]);
    end

    # Initialize the file
    vtkfile = vtk_grid(file, grid_data.allnodes, cells, ascii=ascii);

    # Add values
    if typeof(vars) <: Array
        for vi=1:length(vars)
            if length(vars[vi].symvar) > 1
                for ci=1:length(vars[vi].symvar)
                    compname = string(vars[vi].symbol) * "_" * string(ci);
                    vtkfile[compname] = vars[vi].values[ci,:];
                end
            else
                vtkfile[string(vars[vi].symbol)] = vars[vi].values;
            end
        end
    else
        if length(vars.symvar) > 1
            for ci=1:length(vars.symvar)
                compname = string(vars.symbol) * "_" * string(ci);
                vtkfile[compname] = vars.values[ci,:];
            end
        else
            vtkfile[string(vars.symbol)] = vars.values;
        end
    end

    return vtk_save(vtkfile);
end


"""Performs a single step of the EnsembleKalmanProcess."""

# Import modules to all processes
using ArgParse
using CalibrateEDMF
using CalibrateEDMF.DistributionUtils
using CalibrateEDMF.Pipeline
using CalibrateEDMF.HelperFuncs
# Import EKP modules
using EnsembleKalmanProcesses
using EnsembleKalmanProcesses.ParameterDistributions
using JLD2

# Read iteration number of ensemble to be recovered
s = ArgParseSettings()
@add_arg_table s begin
    "--iteration"
    help = "Calibration iteration number"
    arg_type = Int
    "--job_dir"
    help = "Job output directory"
    arg_type = String
    default = "output"
end
parsed_args = parse_args(ARGS, s)
# Recover inputs for update
iteration = parsed_args["iteration"]
outdir_path = parsed_args["job_dir"]

include(joinpath(outdir_path, "config.jl"))

versions = readlines(joinpath(outdir_path, "versions_$(iteration).txt"))
priors = deserialize_prior(load(joinpath(outdir_path, "prior.jld2")))
ekobj = load(ekobj_path(outdir_path, iteration))["ekp"]
config = get_config()
ek_update(ekobj, priors, iteration, config, versions, outdir_path)


function axdoubling!(ax::Axis)
    #WRONG: sort!(append!(ax.ticks, ax.ticks[1:end - 1] + diff(ax.ticks) / 2); alg = QuickSort)
    #the axis could be in a non-sorted form

    doubleindex=(2:2*length(ax.ticks)).÷2
    doubleindex[2:2:end] .+=length(ax.ticks)
    append!(ax.ticks, ax.ticks[1:end - 1] + diff(ax.ticks) / 2)
    ax.ticks[1:end].=ax.ticks[doubleindex];
end

"""
    axesextend!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, axisnumber::Integer; prepend::AbstractArray=[], append::AbstractArray=[])

Extend parameter space of the selected Axis by prepend and append the grid.

Note, that the resolution and the monotonically is not checked.

# Examples
extending the second parameter of the mdbm problem:
```jldoctest
julia> axesextend!(mymdbm,2,prepend=-6.2:0.05:-2.2,append=2.2:0.2:3);
```
"""
function axesextend!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, axisnumber::Integer; prepend::AbstractArray = [], append::AbstractArray = []) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    preplength = length(prepend);
    if preplength > 0
        prepend!(mdbm.axes[axisnumber].ticks, prepend)
        for nc in mdbm.ncubes
            nc.corner[axisnumber] += preplength;
        end
    end
    if length(append) > 0
        append!(mdbm.axes[axisnumber].ticks, append)
    end
end

# function corner(nc::NCube{IT,FT,N}, T01)::Vector{MVector} where IT where FT where N
function corner(nc::NCube{IT,FT,N}, T01) where IT where FT where N
    [nc.corner .+ nc.size .* T for T in T01]
end

# function corner(ncubes::Vector{NCube{IT,FT,N}}, T01)::Vector{Vector{MVector}} where IT where FT where N
function corner(ncubes::Vector{NCube{IT,FT,N}}, T01) where IT where FT where N
    [corner(nc, T01) for nc in ncubes]
    # [nc.corner .+ nc.size .* T for nc in ncubes for T in T01]
end

# function corner(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})::Vector{Vector{SVector}}  where fcT where N where Nf where Nc
function corner(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    [corner(nc, mdbm.T01) for nc in mdbm.ncubes]
end

function getcornerval(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}#get it for all
    #getcornerval.(mdbm.ncubes,Ref(mdbm))
    map(x->((mdbm.fc) ∘ (mdbm.axes))(x...), Base.Iterators.flatten(corner(mdbm)))
end

function getcornerval(ncubes::NCube{IT,FT,N}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    #PostionsVectors=(mdbm.axes.(corner(nc,mdbm.T01)))
    # (mdbm.fc).( (mdbm.axes).(corner(nc,mdbm.T01)) )
    map(x->((mdbm.fc) ∘ (mdbm.axes))(x...), corner(ncubes, mdbm.T01))
end

function T01maker(valk::Val{kdim}) where {kdim}
    SVector{2^kdim}([
        SVector{kdim}([isodd(x ÷ (2^y)) for y in 0:(kdim - 1)])
         for  x in 0:(2^kdim - 1)])
end

function T11pinvmaker(valk::Val{kdim}) where {kdim}
    T11pinv = ([isodd(x ÷ (2^y)) for y in 0:kdim , x in 0:(2^kdim - 1)] * 2.0 .- 1.0) / (2^kdim)
    SMatrix{kdim + 1,2^kdim}(T11pinv)
end

function issingchange(FunTupleVector::AbstractVector, Nf::Integer, Nc::Integer)::Bool
    all([
        [
        any((c)->!isless(c[1][fi], zero(c[1][fi])), FunTupleVector)
        for fi in 1:Nf#length(FunTupleVector[1][1])
        ]#any positive (or zeros) value (!isless)

        [
        any((c)->!isless(zero(c[1][fi]), c[1][fi]), FunTupleVector)
        for fi in 1:Nf#length(FunTupleVector[1][1])
        ]#anany negative (or zero) value

        [
        any((c)->!isless(c[2][fi], zero(c[2][fi])), FunTupleVector)
        for fi in 1:Nc#length(FunTupleVector[1][2])
        ]#any positive (or zeros) value (!isless)
    ])#check for all condition
end

function _interpolate!(ncubes::Vector{NCube{IT,FT,N}}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, ::Type{Val{0}})  where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    isbracketing = map(nc->issingchange(getcornerval(nc, mdbm), Nf, Nc), ncubes) #TODO: parallelize
    deleteat!(ncubes,.!isbracketing)#removeing the non-bracketing ncubes

    for nc in ncubes
        nc.posinterp[:] .= zero(FT)
    end
    return nothing
end

function _interpolate!(ncubes::Vector{NCube{IT,FT,N}}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, ::Type{Val{1}})  where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    for nc in ncubes
        FunTupleVector = getcornerval(nc, mdbm)
        if Nc==0 || all(
            any((c)->!isless(c[2][fi], zero(c[2][fi])), FunTupleVector)
            for fi in 1:length(FunTupleVector[1][2])
            )# check the constraint: do wh have to compute at all?!?

            As = zero(MVector{Nf + Nc,FT})
            ns = zero(MMatrix{N,Nf + Nc,FT})

            #for f---------------------
            for kf = 1:Nf#length(FunTupleVector[1][1])
                solloc = mdbm.T11pinv * [FunTupleVector[kcubecorner][1][kf] for kcubecorner = 1:length(FunTupleVector)]
                As[kf] = solloc[end];
                ns[:,kf] .= solloc[1:end - 1];
            end

            #for c---if needed: that is, it is close to the boundary--------
            activeCostraint = 0;
            for kf = 1:Nc#length(FunTupleVector[1][2])
                #TODO: mivan a többszörös C teljesülése esetén!?!??!# ISSUE
                if any((c)->!isless(zero(c[2][kf]), c[2][kf]), FunTupleVector) && length(As) < N  # use a constraint till it reduces the dimension to zero (point) and no further
                    solloc = mdbm.T11pinv * [FunTupleVector[kcubecorner][2][kf] for kcubecorner = 1:length(FunTupleVector)]
                    activeCostraint += 1;
                    As[Nf + activeCostraint] = solloc[end];
                    ns[:,Nf + activeCostraint] .= solloc[1:end - 1];
                end
            end

            if (Nf + activeCostraint) == 0
                nc.posinterp[:] .= zero(FT)
            else
                # first two leads to error in the fucntion of constant due to the behaviour of pinv (eg. [1 0;0 1;0 0]/[1 0; 0 0; 0 0])
                # nc.posinterp[:] .= transpose(view(ns, :, 1:(Nf + activeCostraint))) \ view(As, 1:(Nf + activeCostraint));#TEST
                # nc.posinterp[:] .= transpose(ns[:, 1:(Nf + activeCostraint)]) \ As[1:(Nf + activeCostraint)];#TEST
                # nc.posinterp[:] .= ns[:, 1:(Nf + activeCostraint)] * ((transpose(ns[:, 1:(Nf + activeCostraint)]) * ns[:, 1:(Nf + activeCostraint)]) \ view(As, 1:(Nf + activeCostraint)));

                a = ns[:, 1:(Nf + activeCostraint)]
                d = view(As, 1:(Nf + activeCostraint))
                A = transpose(a) * a
                if rank(A) == minimum(size(A))
                    nc.posinterp[:] .= a * (A \ d)
                else
                    nc.posinterp[:] .= 1000.0
                end
            end
            #for c---------------------
        else
            nc.posinterp[:] .= 1000.0;#put it outside the cube!
        end
    end

    #TODO: what if it falls outside of the n-cube, it should be removed ->what shall I do with the bracketing cubes?
    # Let the user define it
     # filter!((nc)->sum((abs.(nc.posinterp)).^10.0)<(1.5 ^10.0),mdbm.ncubes)#1e6~=(2.0 ^20.0)
    normp = 5.0
    ncubetolerance = 0.7
    filter!((nc)->norm(nc.posinterp, normp) < 1.0 + ncubetolerance, mdbm.ncubes)
     #filter!((nc)->!any(isnan.(nc.posinterp)),mdbm.ncubes)

    return nothing
end

function _interpolate!(ncubes::Vector{NCube{IT,FT,N}}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, ::Type{Val{Ninterp}}) where {fcT, IT, FT, N, Ninterp, Nf, Nc, t01T, t11T, aT}
    error("order $(Ninterp) interpolation is not supperted (yet)")
end

"""
    interpolate!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}; interpolationorder::Int=1)

Interpolate the solution within the n-cube with order `interpolationorder`.
 - `interpolationorder=0` provide the midpoint of the n-cube
 - `interpolationorder=1` preform a linear fit and provide closest solution point to the mindpoint of the n-cube
"""
function interpolate!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}; interpolationorder::Int = 1) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    _interpolate!(mdbm.ncubes, mdbm, Val{interpolationorder})
end

"""
    refine!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}; directions::Vector{T}=collect(Int64,1:N)) where N where Nf where Nc where T<:Integer

Double the resolution of the axes along the provided `directions` then halving the `ncubes` accordingly.

# Examples
```jldoctest
julia> refine!(mymdbm)
julia> refine!(mymdbm,[1,1,2])
```
"""
function refine!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}; directions::Vector{T} = collect(Int64, 1:N))  where {T<:Integer, fcT, IT, FT, N, Ninterp, Nf, Nc, t01T, t11T, aT}
    doubling!(mdbm, directions)
    refinencubes!(mdbm.ncubes, directions)
    return nothing
end

"""
    doubling!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, directions::Vector{T}) where T<:Integer  where N where Nf where Nc

Double the resolution of the axes along the provided `directions`, then set the new size of the n-cubes.

# Examples
```jldoctest
julia> doubling!(mymdbm)
julia> doubling!(mymdbm,[1,2])
```
"""
function doubling!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, directions::Vector{T}) where T <: Integer where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    axdoubling!.(mdbm.axes[directions])
    for nc in mdbm.ncubes
        for dir in directions
            nc.corner[dir] = (nc.corner[dir] - 1) * 2 + 1
            nc.size[dir] *= 2
        end
    end
end

"""
     refinencubes!(ncubes::Vector{NCube{IT,FT,N}}, directions::Vector{T})

Halving the `ncubes` along the provided `directions`.

# Examples
```jldoctest
julia> refinencubes!(mymdbm.ncubes)
julia> refinencubes!(mymdbm.ncubes,mymdbm,[1,2])
```
"""
function refinencubes!(ncubes::Vector{NCube{IT,FT,N}}, directions::Vector{T}) where IT where FT where N where T <: Integer where t01T #{IT,FT,N} where IT where FT where N
    for dir in directions
        for nc in ncubes
            nc.size[dir] /= 2
        end
        NumofNCubes = length(ncubes)
        append!(ncubes, deepcopy(ncubes))
        for nc in ncubes[1:NumofNCubes]
            nc.corner[dir] = nc.corner[dir] + nc.size[dir]
        end
    end
    sort!(ncubes; alg = QuickSort)
    return nothing
end

"""
    getinterpolatedsolution(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})

Provide the interpolated coordinates of the all the detected solution (approximately where foo(x,y) == 0 and c(x,y)>0).
"""
function getinterpolatedsolution(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
[
    [
        (typeof(mdbm.axes[i].ticks).parameters[1])((mdbm.axes[i].ticks[nc.corner[i]] * (1.0 - (nc.posinterp[i] + 1.0) / 2.0) +
        mdbm.axes[i].ticks[nc.corner[i] + 1] * ((nc.posinterp[i] + 1.0) / 2.0)))
    for nc in mdbm.ncubes]#::Vector{typeof(mdbm.axes[i].ticks).parameters[1]}
for i in 1:length(mdbm.axes)]
end

"""
    getinterpolatedsolution(ncubes::Vector{NCube{IT,FT,N}},mdbm::MDBM_Problem{N,Nf,Nc})

Provide the interpolated coordinates of the detected solution for the selected n-cubes (approximately where foo(x,y) == 0 and c(x,y)>0).
"""
function getinterpolatedsolution(ncubes::Vector{NCube{IT,FT,N}}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
[
    [
        (typeof(mdbm.axes[i].ticks).parameters[1])(
        (mdbm.axes[i].ticks[nc.corner[i]]*(1.0-(nc.posinterp[i]+1.0)/2.0)+
        mdbm.axes[i].ticks[nc.corner[i]+1]*((nc.posinterp[i]+1.0)/2.0))
        )
    for nc in ncubes]#::Vector{typeof(mdbm.axes[i].ticks).parameters[1]}
for i in 1:length(mdbm.axes)]
end

"""
    getinterpolatedsolution(nc::NCube{IT,FT,N}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})

Provide the interpolated coordinates of the solution inside the provided n-cube `nc` (approximately where foo(x,y) == 0 and c(x,y)>0).
"""
function getinterpolatedsolution(nc::NCube{IT,FT,N}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    [
        (typeof(mdbm.axes[i].ticks).parameters[1])((mdbm.axes[i].ticks[nc.corner[i]] * (1.0 - (nc.posinterp[i] + 1.0) / 2.0) +
        mdbm.axes[i].ticks[nc.corner[i] + 1] * ((nc.posinterp[i] + 1.0) / 2.0)))#::Vector{typeof(mdbm.axes[i].ticks).parameters[1]}
for i in 1:length(mdbm.axes)]
end

"""
    getevaluatedpoints(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})

Provide all the coordinates where the function is evaluated.
"""
function getevaluatedpoints(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    [[x.callargs[i] for x in mdbm.fc.fvalarg] for i in 1:N]
end

"""
    getevaluatedpoints(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})

Provide the functionvalues for all the coordinates where the function is evaluated.
(see: `getevaluatedpoints`)
"""
function getevaluatedfunctionvalues(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    [x.funval for x in mdbm.fc.fvalarg]
end

"""
    getevaluatedconstraintvalues(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})

Provide the constraints values for all the coordinates where the function is evaluated.
(see: `getevaluatedpoints`)
"""
function getevaluatedconstraintvalues(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    [x.cval for x in mdbm.fc.fvalarg]
end

# function generateneighbours(ncubes::Vector{NCube},mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where N  where Nf where Nc
function generateneighbours(ncubes::Vector{NCube{IT,FT,N}}, mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    #TODO: Let the user select the method
    #-------all faceing/cornering neightbours----------------
    # #neighbourind=1:2^length(mdbm.axes) #cornering neightbours also - unnecessary
    # neighbourind=2 .^(0:(length(mdbm.axes)-1)) .+ 1 #neighbour only on the side
    # T101=[-mdbm.T01[neighbourind]...,mdbm.T01[neighbourind]...]
    #
    # nc_neighbour = Array{typeof(mdbm.ncubes[1])}(undef,0)
    # NumofNCubes=length(ncubes)
    # for iT in 1:length(T101)
    #     append!(nc_neighbour,deepcopy(ncubes))
    #     for nci in ((1+NumofNCubes*(iT-1)):(NumofNCubes+NumofNCubes*(iT-1)))
    #         nc_neighbour[nci].corner[:]=nc_neighbour[nci].corner+T101[iT].*nc_neighbour[nci].size
    #     end
    # end
    #-------all faceing/cornering neightbours----------------

    #-------direcational neightbours----------------
    nc_neighbour = Array{typeof(mdbm.ncubes[1])}(undef, 0)
    Nface = 2^N
    indpos = [[mdbm.T01[i][dir] for i in 1:Nface] for dir in 1:N]
    indneg = [[!mdbm.T01[i][dir] for i in 1:Nface] for dir in 1:N]

    for nc in ncubes
        fcvals = getcornerval(nc, mdbm)
        for dir in 1:N
            # indpos=[mdbm.T01[i][dir] for i in 1:Nface]
            # indneg=[!mdbm.T01[i][dir] for i in 1:Nface]
            if issingchange(fcvals[indpos[dir]], Nf, Nc)
                push!(nc_neighbour, deepcopy(nc))
                nc_neighbour[end].corner[dir] += nc_neighbour[end].size[dir]
            end
            if issingchange(fcvals[indneg[dir]], Nf, Nc)
                push!(nc_neighbour, deepcopy(nc))
                nc_neighbour[end].corner[dir] -= nc_neighbour[end].size[dir]
            end
        end
    end
    #-------direcational neightbours----------------
    filter!(nc->!(any(nc.corner .< 1) || any((nc.corner + nc.size) .> [length.(mdbm.axes)...])), nc_neighbour)#remove the overhanging ncubes
    sort!(nc_neighbour; alg = QuickSort)
    unique!(nc_neighbour)
    return nc_neighbour
end

"""
    checkneighbour!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}; interpolationorder::Int=0, maxiteration::Int=0)

Check the face-neighbouring n-cubes to discover the lost (missing) part of the solutions.

It is possible, that the sub-manifold (e.g.: curve) of the solution is not completely detected (it is interrupted).
This way the missing part can be explored similarly to a continuation method (with minimal function evaluation).

It works only if the dimension of the solution object is larger the zero (the maifold is not a set of poins).

    # Arguments
    - `interpolationorder::Int=0`: interpolation order method of the neighbours checked
    - `maxiteration::Int=0~: the max number of steps in the 'continuation-like' exploring. If zero, then infinity steps are allowed
"""
function checkneighbour!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}; interpolationorder::Int = 0, maxiteration::Int = 0) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}#only for unite size cubes

    if isempty(mdbm.ncubes)
        println("There is no bracketing n-cubes to check!")
    else
        ncubes2check = mdbm.ncubes
        numberofiteration = 0;
        while !isempty(ncubes2check) && (maxiteration == 0 ? true : numberofiteration < maxiteration)
            numberofiteration += 1
            ncubes2check = generateneighbours(ncubes2check, mdbm)

            deleteat!(ncubes2check, is_sorted_in_sorted(ncubes2check, mdbm.ncubes))#delete the ones which is already presented
            _interpolate!(ncubes2check, mdbm, Val{interpolationorder})#remove the non-bracketing, only proper new bracketing cubes remained

            append!(mdbm.ncubes, deepcopy(ncubes2check))
            sort!(mdbm.ncubes; alg = QuickSort)
        end
    end
end

"""
    connect(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT})

Provide the edge connection (as a list of point-paris) of the solution point-cloud based on the face-neighbouring n-cubes.
"""
function connect(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    #---------- line connection (no corener neighbour is needed) --------------
    DT1 = Array{Tuple{Int64,Int64}}(undef, 0)
    for inc in 1:length(mdbm.ncubes)
        ncneigh = generateneighbours([mdbm.ncubes[inc]], mdbm)
        indinresults = index_sorted_in_sorted(ncneigh, mdbm.ncubes)#delete the ones which is already presented
        append!(DT1, [(inc, x) for x  in indinresults if x != 0])
    end
    filter!(d->(d[2] > d[1]), DT1)#TODO: eleve csak ez egyik irányban levő szomszédokat kellene keresni!!!
    sort!(DT1; alg = QuickSort)
    unique!(DT1)
    return DT1
end

"""
    triangulation(DT1::Array{Tuple{Int64,Int64}})::Array{Tuple{Int64,Int64,Int64}}

Provide the triangulation (as a list of point-triplets) of the solution point-cloud based on the face-neighbouring n-cubes.
DT1 is the result of the edge connection performed by `connect`.
"""
function triangulation(DT1::Array{Tuple{Int64,Int64}})::Array{Tuple{Int64,Int64,Int64}}

    #DT=sort!()[DT1;[(d[2],d[1]) for d in DT1]]);
    DT = sort!([DT1;[d[[2,1]] for d in DT1]])#both direction of line connection is necessay!

    #L=[filter(d->d[1]==i,DT) for i in 1:length(mdbm.ncubes)]
    L = [
    [dd[2] for dd in filter(d->d[1] == i, DT)] for i in 1:maximum(DT1)[2]]

    DT4 = Array{Tuple{Int64,Int64,Int64,Int64}}(undef, 0)#quadratic patch
    for i in 1:size(L, 1)
        for j in L[i]#filter((L{i}>i) for i in )
            if i > j#i must be the larges value to remove the repetition of the some surface segment
                for k in L[j]#(L{j}>i) it seems to be slower
                    if i > k# there is no backstep, and i must be the largest   (Equivalent: %((k~=i) && (i>k))%back step is not allowed
                        for m in L[k]
                            if ((m != j) && (j > m)) #&& (i>m) #back step is not allowed, and i must be the largest
                                if any(i .== L[m])
                                    push!(DT4, (i, j, k, m))
                                end
                            end
                        end
                    end
                end
            end
        end
    end

    return [[dt[[1,2,3]] for dt in DT4];[dt[[3,1,4]] for dt in DT4]]#DT2 triangular patch from the quadratic patch

end

"""
    solve!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, iteration::Int; interpolationorder::Int=1)

Refine the `MDBM_Problem` `iteration` times, then perform a neighbour check.
`interpolationorder` defines the interpolation method within the n-cubes.

# Examples
```julia
julia> solve!(mymdbm,4)
```
"""
function solve!(mdbm::MDBM_Problem{fcT,N,Nf,Nc,t01T,t11T,IT,FT,aT}, iteration::Int; interpolationorder::Int = 1) where {fcT ,N ,Nf ,Nc ,t01T ,t11T ,IT ,FT,aT}
    interpolate!(mdbm, interpolationorder = interpolationorder)
    for k = 1:iteration
        refine!(mdbm)
        interpolate!(mdbm, interpolationorder = interpolationorder)
    end
    checkneighbour!(mdbm)
    interpolate!(mdbm, interpolationorder = interpolationorder)
    return mdbm
end

function index_sorted_in_sorted(a::AbstractVector, b::AbstractVector)::Array{Int64,1}
    # index of a[i] in b (0 if not present)
    # a and b must be sorted
    containingindex = zeros(Int64, length(a))
    startindex = 1;
    for ind2check in 1:length(a)
        detectedrange = searchsorted(b[startindex:end], a[ind2check])
        startindex = max(detectedrange.stop, detectedrange.start) + startindex - 1
        containingindex[ind2check] = isempty(detectedrange) ? 0 : startindex
        if startindex > length(b)
            break
        end
    end
    return containingindex
end

function is_sorted_in_sorted(a::AbstractVector, b::AbstractVector)::Array{Bool,1}
    # is a[i] in b
    # a and b must be sorted
    iscontained = falses(length(a))
    startindex = 1;
    for ind2check in 1:length(a)
        detectedrange = searchsorted(b[startindex:end], a[ind2check])
        etectedrange = searchsorted(view(b,startindex:length(b)), a[ind2check]) # still: it is the critical line!!!
        iscontained[ind2check] = !isempty(detectedrange)
        startindex = max(detectedrange.stop, detectedrange.start) + startindex - 1
        if startindex > length(b)
            break
        end
    end
    return iscontained
end


notary = Notary()
cypher = CypherSuite(notary)

# Somewhere far far away

mixkey = newsigner(notary)

MIX_PORT = 2000
MIX_ID = id(mixkey)

mix = Mix(MIX_PORT, MIX_ID, cypher)
@async SynchronicBallot.serve(mix, mixkey)

sleep(1.)

# Using the mixer

guardiankey = newsigner(notary)
transactions = Trigger(TransactionVector(), t -> println(typeof(t.document)))
tlog = BraidChain(transactions, Guardian(id(guardiankey)), notary)

registrator = newsigner(notary)
reg = Registrator(id(registrator))
push!(tlog, Transaction(reg, guardiankey))

braider = newsigner(notary)

push!(tlog, Transaction(Braider(id(braider)), guardiankey))

gk = GateKeeper(2001, id(braider), cypher, mix)
sb = BraiderConfig(UInt8(3), UInt8(64), gk)
bo = BraidOfficer(sb, tlog)
botask = @async BraidChains.serve!(bo, braider)
sleep(1.)

### Registration of members ###

member1key = newsigner(notary)
member2key = newsigner(notary)
member3key = newsigner(notary)

member1 = Member(id(member1key))
member2 = Member(id(member2key))
member3 = Member(id(member3key))

# on the server
push!(tlog, Transaction(member1, registrator))
push!(tlog, Transaction(member2, registrator))
push!(tlog, Transaction(member3, registrator))

# back to user

pseudonym1 = newsigner(notary)
pseudonym2 = newsigner(notary)
pseudonym3 = newsigner(notary)

@show istaskstarted(botask)

@sync begin
    @async braid(member1key, pseudonym1, bo)
    @async braid(member2key, pseudonym2, bo)
    @async braid(member3key, pseudonym3, bo)
end

### Some tests

@test pseudonyms(tlog, 5) == Set([id(member1key), id(member2key), id(member3key)]) == tlog.state.members
@test pseudonyms(tlog) == Set([id(pseudonym1), id(pseudonym2), id(pseudonym3)])
@test tlog.state.braiders == Set([id(braider)])
@test tlog.state.registrators == Set([id(registrator)])

# ### TODO #####
# kc = KeyChain(notary, user)
# braid!(braidconf, tlog, kc)
# ###############

# ### TODO ######
# # perhaps kc[n] could select the key for nth row
# sign(data,n,kc) # signs data with coresponding pseudonym tagged by nth row in the TransactionLog
# #########


# cudnnReduceTensorOp_t
const CUDNN_REDUCE_TENSOR_ADD = Cint(0)
const CUDNN_REDUCE_TENSOR_MUL = Cint(1)
const CUDNN_REDUCE_TENSOR_MIN = Cint(2)
const CUDNN_REDUCE_TENSOR_MAX = Cint(3)
const CUDNN_REDUCE_TENSOR_AMAX = Cint(4)
const CUDNN_REDUCE_TENSOR_AVG = Cint(5)
const CUDNN_REDUCE_TENSOR_NORM1 = Cint(6)
const CUDNN_REDUCE_TENSOR_NORM2 = Cint(7)
const CUDNN_REDUCE_TENSOR_MUL_NO_ZEROS = Cint(8)

# cudnnReduceTensorIndices_t
const CUDNN_REDUCE_TENSOR_NO_INDICES = Cint(0)
const CUDNN_REDUCE_TENSOR_FLATTENED_INDICES = Cint(1)

mutable struct ReduceTensorDesc
    ptr::Cptr

    function ReduceTensorDesc(::Type{T}, op::Cint) where T
        ref = Ref{Cptr}()
        @cudnn :cudnnCreateReduceTensorDescriptor (Ptr{Cptr},) ref
        desc = new(ref[])
        finalizer(desc) do x
            @cudnn :cudnnDestroyReduceTensorDescriptor (Cptr,) x.ptr
        end

        ind = op == CUDNN_REDUCE_TENSOR_MIN || op == CUDNN_REDUCE_TENSOR_MAX ?
            CUDNN_REDUCE_TENSOR_FLATTENED_INDICES :
            CUDNN_REDUCE_TENSOR_NO_INDICES
        @cudnn(:cudnnSetReduceTensorDescriptor,
            (Cptr,Cint,Cint,Cint,Cint,Cint),
            desc, op, datatype(T), CUDNN_NOT_PROPAGATE_NAN, ind, CUDNN.CUDNN_32BIT_INDICES)
        desc
    end
end

Base.cconvert(::Type{Cptr}, desc::ReduceTensorDesc) = desc.ptr

const DICT_ReduceTensorDesc = Dict()

function reduce(A::CuArray{T}, dim, op) where T
    if size(A,dim) == 1 # CUDNN_STATUS_BAD_PARAM
        C = A
        indices = zeros(CuArray{Cint}, length(A)÷size(A,dim))
        return C, indices
    end

    h = gethandle()
    reducedesc = get!(DICT_ReduceTensorDesc, (T,op)) do
        ReduceTensorDesc(T, op)
    end
    adesc = TensorDesc(A, 4)
    cdims = Int[size(A)...]
    cdims[dim] = 1
    C = similar(A, cdims...)
    cdesc = TensorDesc(C, 4)

    ref = Ref{Csize_t}()
    @cudnn(:cudnnGetReductionIndicesSize,
        (Cptr,Cptr,Cptr,Cptr,Ptr{Csize_t}),
        h, reducedesc, adesc, cdesc, ref)
    indices = ref[] == 0 ? C_NULL : CuArray{Cint}(Int(ref[])÷sizeof(Cint))

    ref = Ref{Csize_t}()
    @cudnn(:cudnnGetReductionWorkspaceSize,
        (Cptr,Cptr,Cptr,Cptr,Ptr{Csize_t}),
        h, reducedesc, adesc, cdesc, ref)
    workspace = CuArray{UInt8}(Int(ref[]))

    @cudnn(:cudnnReduceTensor,
        (Cptr,Cptr,Cptr,Csize_t,Cptr,Csize_t,
        Cptr,Cptr,Cptr,Cptr,Cptr,Cptr),
        h, reducedesc, indices, length(indices)*sizeof(Cint), workspace, length(workspace),
        T[1], adesc, A, T[0], cdesc, C)
    C, indices
end


# Libraries
using Distributed;
@everywhere include("./../src/ElasticNetVAR.jl");
@everywhere using Main.ElasticNetVAR;
using Random;
using FileIO, XLSX, DataFrames, BSON;

# Load data
data_raw = DataFrame(XLSX.readtable("./data/data.xlsx", "FX")...);

# Data
data = data_raw[3:end, 3:end] |> JArray{Float64,2};

# Store dates
date = data_raw[3:end,2];

# Skip EA19 and New Zealand
data = data[:, [1;2;4;collect(6:size(data,2))]] |> JArray{Float64,2};

# Transpose data
data = permutedims(data);

# End presample
end_ps = 80; # 1979-12-31

# End validation sample
end_vs = 160; # 1999-12-31

# Data used for validation
data_validation = data[:, 1:end_vs];
n, T = size(data_validation);

# Select optimal d
@info "Running optimal_d(n, T)";
d = optimal_d(n, T);

# Set options for the selection problem
p_grid=[1, 5]; λ_grid=[1e-4, 5]; α_grid=[0, 1]; β_grid=[1, 5];
vs = ValidationSettings(4, data_validation, t0=end_ps, subsample=d/(n*T), max_samples=5000, log_folder_path=".");
hg = HyperGrid(p_grid, λ_grid, α_grid, β_grid, 1000);

# Run algorithm
Random.seed!(1);
@info "Running select_hyperparameters(vs, hg)";
candidates, errors = select_hyperparameters(vs, hg);

# Save to file
save("./res.bson", Dict("data" => data,
                        "date" => date,
                        "end_ps" => end_ps,
                        "end_vs" => end_vs,
                        "d" => d,
                        "vs" => vs,
                        "hg" => hg,
                        "candidates" => candidates,
                        "errors" => errors));


module Asap

using Base: StackFrame, Float64, Symbol
using Statistics
using LinearAlgebra
using SparseArrays
using GeometryBasics
using FileIO
using ColorSchemes
using Colors
using GLMakie

# data types used for analysis
include("types.jl")
export Node
export Element
export Load
export Structure
export Geometry

# structural analysis
include("structuralAnalysis.jl")
export analyze
export dofMaker
export pseudoSize
export pseudoSize!
export structureMass

# viz tools
include("visualization.jl")
export axo
export structurePlot

end


# This file is a part of Julia. License is MIT: http://julialang.org/license

mutable struct LaTeX
    formula::String
end

@trigger '$' ->
function tex(stream::IO, md::MD)
    result = parse_inline_wrapper(stream, "\$", rep = true)
    return result === nothing ? nothing : LaTeX(result)
end

function blocktex(stream::IO, md::MD)
    withstream(stream) do
        ex = tex(stream, md)
        if ex ≡ nothing
            return false
        else
            push!(md, ex)
            return true
        end
    end
end

show(io::IO, tex::LaTeX) =
    print(io, '$', tex.formula, '$')

latex(io::IO, tex::LaTeX) =
    println(io, "\$\$", tex.formula, "\$\$")

latexinline(io::IO, tex::LaTeX) =
    print(io, '$', tex.formula, '$')

term(io::IO, tex::LaTeX, cols) = println_with_format(:magenta, io, tex.formula)
terminline(io::IO, tex::LaTeX) = print_with_format(:magenta, io, tex.formula)


export rhino,
       reset_rhino_connection

macro rhino_str(str)
    rpc("RH", str)
end

# We need some additional Encoders
encode_Brep = encode_Guid
decode_Brep = decode_Guid
decode_Brep_array = decode_Guid_array
encode_RhinoObject = encode_Guid
encode_RhinoObject_array = encode_Guid_array
decode_RhinoObject = decode_Guid

encode_Plane(c::IO, v::XYZ) =
  let o = in_world(v)
      vx = in_world(vx(1, c.cs))
      vy = in_world(vy(1, c.cs))
    encode_Point3d(c, o)
    encode_Vector3d(c, vx)
    encode_Vector3d(c, vy)
  end
decode_Plane(c::IO) =
    loc_from_o_vx_vy(decode_Point3d(c), decode_Vector3d(c), decode_Vector3d(c))

encode_TableFamily = encode_int
decode_TableFamily = decode_int
encode_ChairFamily = encode_int
decode_ChairFamily = decode_int
encode_TableChairFamily = encode_int
decode_TableChairFamily = decode_int

rhino"public void SetView(Point3d position, Point3d target, double lens, bool perspective, string style)"
rhino"public void View(Point3d position, Point3d target, double lens)"
rhino"public void ViewTop()"
rhino"public Point3d ViewCamera()"
rhino"public Point3d ViewTarget()"
rhino"public double ViewLens()"
#=
rhino"public byte Sync()"
rhino"public byte Disconnect()"
=#
rhino"public Guid Point(Point3d p)"
rhino"public Point3d PointPosition(Guid ent)"

rhino"public Guid PolyLine(Point3d[] pts)"
rhino"public Point3d[] LineVertices(RhinoObject id)"
rhino"public Guid Spline(Point3d[] pts)"
#=
rhino"public Entity InterpSpline(Point3d[] pts, Vector3d tan0, Vector3d tan1)"
=#
rhino"public Guid ClosedPolyLine(Point3d[] pts)"
rhino"public Guid ClosedSpline(Point3d[] pts)"
#=
rhino"public Entity InterpClosedSpline(Point3d[] pts)"
=#
rhino"public Guid Circle(Point3d c, Vector3d n, double r)"
rhino"public Point3d CircleCenter(RhinoObject obj)"
rhino"public Vector3d CircleNormal(RhinoObject obj)"
rhino"public double CircleRadius(RhinoObject obj)"
rhino"public Guid Ellipse(Point3d c, Vector3d n, double radiusX, double radiusY)"
rhino"public Guid Arc(Point3d c, Vector3d n, double radius, double startAngle, double endAngle)"
rhino"public Guid JoinCurves(Guid[] objs)"
#=

rhino"public Entity Text(string str, Point3d corner, Vector3d vx, Vector3d vy, double height)"

rhino"public Guid SurfaceFromCurve(Entity curve)"
=#
rhino"public Guid SurfaceCircle(Point3d c, Vector3d n, double r)"
rhino"public Guid SurfaceEllipse(Point3d c, Vector3d n, double radiusX, double radiusY)"
rhino"public Guid SurfaceArc(Point3d c, Vector3d n, double radius, double startAngle, double endAngle)"
rhino"public Guid SurfaceClosedPolyLine(Point3d[] pts)"
rhino"public Guid SurfaceFrom(Guid[] objs)"
rhino"public Guid Sphere(Point3d c, double r)"
rhino"public Guid Torus(Point3d c, Vector3d vz, double majorRadius, double minorRadius)"
rhino"public Brep Cylinder(Point3d bottom, double radius, Point3d top)"
rhino"public Brep Cone(Point3d bottom, double radius, Point3d top)"
rhino"public Brep ConeFrustum(Point3d bottom, double bottom_radius, Point3d top, double top_radius)"
rhino"public Brep Box(Point3d corner, Vector3d vx, Vector3d vy, double dx, double dy, double dz)"
rhino"public Brep XYCenteredBox(Point3d corner, Vector3d vx, Vector3d vy, double dx, double dy, double dz)"
#=
rhino"public Guid IrregularPyramidMesh(Point3d[] pts, Point3d apex)"
=#
rhino"public Brep IrregularPyramid(Point3d[] pts, Point3d apex)"
rhino"public Brep IrregularPyramidFrustum(Point3d[] bpts, Point3d[] tpts)"
#rhino"public Entity MeshFromGrid(int m, int n, Point3d[] pts, bool closedM, bool closedN)"
rhino"public Guid SurfaceFromGrid(int nU, int nV, Point3d[] pts, bool closedU, bool closedV, int degreeU, int degreeV)"

#=
rhino"public Entity SolidFromGrid(int m, int n, Point3d[] pts, bool closedM, bool closedN, int level, double thickness)"
=#
rhino"public Brep[] Thicken(RhinoObject obj, double thickness)"
rhino"public double[] CurveDomain(RhinoObject obj)"
rhino"public double CurveLength(RhinoObject obj)"
rhino"public Plane CurveFrameAt(RhinoObject obj, double t)"
rhino"public Plane CurveFrameAtLength(RhinoObject obj, double l)"
#=
rhino"public Guid NurbSurfaceFrom(ObjectId id)"
rhino"public double[] SurfaceDomain(Entity ent)"
rhino"public Frame3d SurfaceFrameAt(Entity ent, double u, double v)"
=#
rhino"public Brep Extrusion(RhinoObject obj, Vector3d dir)"
rhino"public Brep[] SweepPathProfile(RhinoObject path, RhinoObject profile, double rotation, double scale)"
#=
rhino"public Guid Sweep(ObjectId pathId, ObjectId profileId, double rotation, double scale)"
rhino"public Guid Loft(ObjectId[] profilesIds, ObjectId[] guidesIds, bool ruled, bool closed)"
rhino"public void Unite(ObjectId objId0, ObjectId objId1)"
=#
rhino"public Guid[] Intersect(RhinoObject obj0, RhinoObject obj1)"
rhino"public Guid[] Subtract(RhinoObject obj0, RhinoObject obj1)"
#=
rhino"public Guid Revolve(ObjectId profileId, Point3d p, Vector3d n, double startAngle, double amplitude)"
=#
rhino"public Guid Move(Guid id, Vector3d v)"
rhino"public Guid Scale(Guid id, Point3d p, double s)"
rhino"public Guid Rotate(Guid id, Point3d p, Vector3d n, double a)"
rhino"public Guid Mirror(Guid id, Point3d p, Vector3d n, bool copy)"
#=
rhino"public Point3d[] GetPoint(string prompt)"
rhino"public Point3d[] BoundingBox(ObjectId[] ids)"
rhino"public void ZoomExtents()"
rhino"public void SetSystemVariableInt(string name, int value)"
rhino"public int Render(int width, int height, string path)"
rhino"public int Command(string cmd)"
rhino"public void DisableUpdate()"
rhino"public void EnableUpdate()"

=#
rhino"public bool IsPoint(RhinoObject e)"
rhino"public bool IsCircle(RhinoObject e)"
rhino"public bool IsPolyLine(RhinoObject e)"
rhino"public bool IsSpline(RhinoObject e)"
rhino"public bool IsInterpSpline(RhinoObject e)"
rhino"public bool IsClosedPolyLine(RhinoObject e)"
rhino"public bool IsClosedSpline(RhinoObject e)"
rhino"public bool IsInterpClosedSpline(RhinoObject e)"
rhino"public bool IsEllipse(RhinoObject e)"
rhino"public bool IsArc(RhinoObject e)"
rhino"public bool IsText(RhinoObject e)"
rhino"public byte ShapeCode(RhinoObject id)"
#=
rhino"public BIMLevel FindOrCreateLevelAtElevation(double elevation)"
rhino"public BIMLevel UpperLevel(BIMLevel currentLevel, double addedElevation)"
rhino"public double GetLevelElevation(BIMLevel level)"

rhino"public FloorFamily FloorFamilyInstance(double totalThickness, double coatingThickness)"
rhino"public Entity LightweightPolyLine(Point2d[] pts, double[] angles, double elevation)"
rhino"public Entity SurfaceLightweightPolyLine(Point2d[] pts, double[] angles, double elevation)"
rhino"public ObjectId CreatePathFloor(Point2d[] pts, double[] angles, BIMLevel level, FloorFamily family)"
=#
rhino"public Guid ClosedPathCurveArray(Point3d[] pts, double[] angles)"
rhino"public Brep[] PathWall(RhinoObject obj, double thickness, double height)"
rhino"public Brep RectangularTable(Point3d c, double angle, double length, double width, double height, double top_thickness, double leg_thickness)"

#rhino"public TableFamily FindOrCreateTableFamily(double length, double width, double height, double top_thickness, double leg_thickness)"
#rhino"public ChairFamily FindOrCreateChairFamily(double length, double width, double height, double seat_height, double thickness)"
#rhino"public TableChairFamily FindOrCreateTableChairFamily(TableFamily tableFamily, ChairFamily chairFamily, int chairsOnTop, int chairsOnBottom, int chairsOnRight, int chairsOnLeft, double spacing)"
#rhino"public Brep RectangularTable(Point3d c, double angle, TableFamily family)"
#rhino"public Brep Chair(Point3d c, double angle, ChairFamily family)"
#rhino"public Guid InstanceChair(Point3d c, double angle, ChairFamily family)"
#rhino"public Brep[] RowOfChairs(Point3d c, double angle, int n, double spacing, ChairFamily family)"
#rhino"public Brep[] CenteredRowOfChairs(Point3d c, double angle, int n, double spacing, ChairFamily family)"
#rhino"public Brep[] RectangularTableAndChairs(Point3d c, double angle, TableChairFamily f)"

rhino"public Brep[] BaseRectangularTable(double length, double width, double height, double top_thickness, double leg_thickness)"
rhino"public int CreateRectangularTableFamily(double length, double width, double height, double top_thickness, double leg_thickness)"
rhino"public Guid Table(Point3d c, double angle, int family)"
rhino"public Brep[] BaseChair(double length, double width, double height, double seat_height, double thickness)"
rhino"public int CreateChairFamily(double length, double width, double height, double seat_height, double thickness)"
rhino"public Guid Chair(Point3d c, double angle, int family)"
rhino"public GeometryBase[] BaseRectangularTableAndChairs(int tableFamily, int chairFamily, double tableLength, double tableWidth, int chairsOnTop, int chairsOnBottom, int chairsOnRight, int chairsOnLeft, double spacing)"
rhino"public int CreateRectangularTableAndChairsFamily(int tableFamily, int chairFamily, double tableLength, double tableWidth, int chairsOnTop, int chairsOnBottom, int chairsOnRight, int chairsOnLeft, double spacing)"
rhino"public Guid TableAndChairs(Point3d c, double angle, int family)"

abstract type RHKey end
const RHId = Guid
const RHRef = GenericRef{RHKey, RHId}
const RHRefs = Vector{RHRef}
const RHEmptyRef = EmptyRef{RHKey, RHId}
const RHNativeRef = NativeRef{RHKey, RHId}
const RHUnionRef = UnionRef{RHKey, RHId}
const RHSubtractionRef = SubtractionRef{RHKey, RHId}
const RH = SocketBackend{RHKey, RHId}

void_ref(b::RH) = RHNativeRef(zeros(UInt8, 16))

create_RH_connection() = create_backend_connection("Rhinoceros", 12000)

const rhino = RH(LazyParameter(TCPSocket, create_RH_connection))
reset_rhino_connection() = reset(rhino.connection)

# This should not be done automatically
backend_name(b::RH) = "Rhino"

realize(b::RH, s::EmptyShape) =
  RHEmptyRef()
realize(b::RH, s::UniversalShape) =
  RHUniversalRef()

backend_stroke(b::RH, path::CircularPath) =
    RHCircle(connection(b), path.center, vz(1, path.center.cs), path.radius)
backend_stroke(b::RH, path::RectangularPath) =
    let c = path.corner,
        dx = path.dx,
        dy = path.dy
        RHClosedPolyLine(connection(b), [c, add_x(c, dx), add_xy(c, dx, dy), add_y(c, dy)])
    end
backend_stroke(b::RH, path::OpenPolygonalPath) =
  	RHPolyLine(connection(b), path.vertices)
backend_stroke(b::RH, path::ClosedPolygonalPath) =
    RHClosedPolyLine(connection(b), path.vertices)

backend_stroke(b::RH, path::OpenSplinePath) =
  if (path.v0 == false) && (path.v1 == false)
    #ACADSpline(connection(b), path.vertices)
    RHSpline(connection(b), path.vertices)
  elseif (path.v0 != false) && (path.v1 != false)
    RHInterpSpline(connection(b), path.vertices, path.v0, path.v1)
  else
    RHInterpSpline(connection(b),
                     path.vertices,
                     path.v0 == false ? path.vertices[2]-path.vertices[1] : path.v0,
                     path.v1 == false ? path.vertices[end-1]-path.vertices[end] : path.v1)
  end
backend_stroke(b::RH, path::ClosedSplinePath) =
    RHInterpClosedSpline(connection(b), path.vertices)
backend_fill(b::RH, path::ClosedSplinePath) =
    backend_fill_curves(b, RHInterpClosedSpline(connection(b), path.vertices))

backend_stroke_unite(b::RH, refs) = RHJoinCurves(connection(b), refs)

#=backend_fill(b::RH, path::ClosedPolygonalPath) =
    RHSurfaceClosedPolyLine(connection(b), path.vertices)
    backend_fill(b::RH, path::RectangularPath) =
        let c = path.corner,
            dx = path.dx,
            dy = path.dy
            RHSurfaceClosedPolyLine(connection(b), [c, add_x(c, dx), add_xy(c, dx, dy), add_y(c, dy)])
        end
backend_fill(b::RH, path::RectangularPath) =
    let c = path.corner,
        dx = path.dx,
        dy = path.dy
        SurfaceClosedPolyLine(connection(b), [c, add_x(c, dx), add_xy(c, dx, dy), add_y(c, dy)])
    end
=#

backend_fill_curves(b::RH, gs::Guids) = RHSurfaceFrom(connection(b), gs)
backend_fill_curves(b::RH, g::Guid) = RHSurfaceFrom(connection(b), [g])

backend_stroke_line(b::RH, vs) = RHPolyLine(connection(b), vs)

backend_stroke_arc(b::RH, center::Loc, radius::Real, start_angle::Real, amplitude::Real) =
    let end_angle = start_angle + amplitude
        if end_angle > start_angle
            RHArc(connection(b), center, vz(1, center.cs), radius, start_angle, end_angle)
        else
            RHArc(connection(b), center, vz(1, center.cs), radius, end_angle, start_angle)
        end
    end

realize(b::RH, s::Point) =
  RHPoint(connection(b), s.position)

realize(b::RH, s::Line) =
  RHPolyLine(connection(b), s.vertices)

realize(b::RH, s::Spline) =
  if (s.v0 == false) && (s.v1 == false)
    RHSpline(connection(b), s.points)
  elseif (s.v0 != false) && (s.v1 != false)
    RHInterpSpline(connection(b), s.points, s.v0, s.v1)
  else
    RHInterpSpline(connection(b),
                     s.points,
                     s.v0 == false ? s.points[2]-s.points[1] : s.v0,
                     s.v1 == false ? s.points[end-1]-s.points[end] : s.v1)
  end

realize(b::RH, s::ClosedSpline) =
  RHInterpClosedSpline(connection(b), s.points)

realize(b::RH, s::Circle) =
  RHCircle(connection(b), s.center, vz(1, s.center.cs), s.radius)

realize(b::RH, s::Arc) =
  if s.radius == 0
    RHPoint(connection(b), s.center)
  elseif s.amplitude == 0
    RHPoint(connection(b), s.center + vpol(s.radius, s.start_angle, s.center.cs))
  elseif abs(s.amplitude) >= 2*pi
    RHCircle(connection(b), s.center, vz(1, s.center.cs), s.radius)
  else
    end_angle = s.start_angle + s.amplitude
    if end_angle > s.start_angle
      RHArc(connection(b), s.center, vz(1, s.center.cs), s.radius, s.start_angle, end_angle)
    else
      RHArc(connection(b), s.center, vz(1, s.center.cs), s.radius, end_angle, s.start_angle)
    end
  end

realize(b::RH, s::Ellipse) =
    RHEllipse(connection(b), s.center, vz(1, s.center.cs), s.radius_x, s.radius_y)

realize(b::RH, s::EllipticArc) =
  error("Finish this")

realize(b::RH, s::Polygon) =
  RHClosedPolyLine(connection(b), s.vertices)

realize(b::RH, s::RegularPolygon) =
  RHClosedPolyLine(connection(b), regular_polygon_vertices(s.edges, s.center, s.radius, s.angle, s.inscribed))

realize(b::RH, s::Rectangle) =
  RHClosedPolyLine(connection(b), [s.corner, add_x(s.corner, s.dx), add_xy(s.corner, s.dx, s.dy), add_y(s.corner, s.dy)])
#
rhino"public Point3d[] CurvePointsAt(RhinoObject obj, double[] ts)"
rhino"public Vector3d[] CurveTangentsAt(RhinoObject obj, double[] ts)"
rhino"public Vector3d[] CurveNormalsAt(RhinoObject obj, double[] ts)"

backend_map_division(b::RH, f::Function, s::Shape1D, n::Int) =
  let conn = connection(b),
      r = ref(s).value,
      (t1, t2) = RHCurveDomain(conn, r),
      ti = division(t1, t2, n),
      ps = RHCurvePointsAt(conn, r, ti),
      ts = RHCurveTangentsAt(conn, r, ti),
      #ns = ACADCurveNormalsAt(conn, r, ti),
      frames = rotation_minimizing_frames(RHCurveFrameAt(conn, r, t1), ps, ts)
    map(f, frames)
  end

rhino"public Guid SurfaceCircle(Point3d c, Vector3d n, double r)"

realize(b::RH, s::SurfaceCircle) =
  RHSurfaceCircle(connection(b), s.center, vz(1, s.center.cs), s.radius)

realize(b::RH, s::SurfaceArc) =
  RHSurfaceArc(connection(b), s.center, vz(1, s.center.cs), s.radius, s.start_angle, s.start_angle + s.amplitude)

realize(b::RH, s::SurfaceEllipticArc) =
  error("Finish this")

realize(b::RH, s::SurfaceEllipse) =
    RHSurfaceEllipse(connection(b), s.center, vz(1, s.center.cs), s.radius_x, s.radius_y)

rhino"public Guid SurfaceClosedPolyLine(Point3d[] pts)"

realize(b::RH, s::SurfacePolygon) =
  RHSurfaceClosedPolyLine(connection(b), s.vertices)
realize(b::RH, s::SurfaceRegularPolygon) =
  RHSurfaceClosedPolyLine(connection(b), regular_polygon_vertices(s.edges, s.center, s.radius, s.angle, s.inscribed))
realize(b::RH, s::SurfaceRectangle) =
  let c = s.corner
    RHSurfaceClosedPolyLine(connection(b), [c, add_x(c, s.dx), add_xy(c, s.dx, s.dy), add_y(c, s.dy)])
  end
#=
realize(b::RH, s::Surface) =
  let ids = RHSurfaceFromCurves(connection(b), collect_ref(s.frontier))
    foreach(mark_deleted, s.frontier)
    ids
  end
realize(b::RH, s::Text) =
  RHText(connection(b), s.str, s.c, vx(1, s.c.cs), vy(1, s.c.cs), s.h)
=#

rhino"public double[] SurfaceDomain(RhinoObject obj)"
rhino"public Plane SurfaceFrameAt(RhinoObject obj, double u, double v)"

backend_surface_domain(b::RH, s::Shape2D) =
    tuple(RHSurfaceDomain(connection(b), ref(s).value)...)

backend_map_division(b::RH, f::Function, s::Shape2D, nu::Int, nv::Int) =
    let conn = connection(b)
        r = ref(s).value
        (u1, u2, v1, v2) = RHSurfaceDomain(conn, r)
        map_division(u1, u2, nu) do u
            map_division(v1, v2, nv) do v
                f(RHSurfaceFrameAt(conn, r, u, v))
            end
        end
    end

realize(b::RH, s::Sphere) =
    RHSphere(connection(b), s.center, s.radius)
realize(b::RH, s::Torus) =
    RHTorus(connection(b), s.center, vz(1, s.center.cs), s.re, s.ri)
realize(b::RH, s::Cuboid) =
    RHIrregularPyramidFrustum(connection(b), [s.b0, s.b1, s.b2, s.b3], [s.t0, s.t1, s.t2, s.t3])
realize(b::RH, s::RegularPyramidFrustum) =
    RHIrregularPyramidFrustum(connection(b),
                              regular_polygon_vertices(s.edges, s.cb, s.rb, s.angle, s.inscribed),
                              regular_polygon_vertices(s.edges, add_z(s.cb, s.h), s.rt, s.angle, s.inscribed))
realize(b::RH, s::RegularPyramid) =
    RHIrregularPyramid(connection(b),
                       regular_polygon_vertices(s.edges, s.cb, s.rb, s.angle, s.inscribed),
                       add_z(s.cb, s.h))
realize(b::RH, s::IrregularPyramid) =
    RHIrregularPyramid(connection(b), s.bs, s.t)
realize(b::RH, s::RegularPrism) =
    let cbs = regular_polygon_vertices(s.edges, s.cb, s.r, s.angle, s.inscribed)
        RHIrregularPyramidFrustum(connection(b),
                                  cbs,
                                  map(p -> add_z(p, s.h), cbs))
    end
realize(b::RH, s::IrregularPrism) =
    RHIrregularPyramidFrustum(connection(b),
                              s.bs,
                              map(p -> (p + s.v), s.bs))

realize(b::RH, s::RightCuboid) =
    RHXYCenteredBox(connection(b), s.cb, vx(1, s.cb.cs), vy(1, s.cb.cs), s.width, s.height, s.h)

realize(b::RH, s::Box) =
    RHBox(connection(b), s.c, vx(1,s.c.cs), vy(1,s.c.cs), s.dx, s.dy, s.dz)

realize(b::RH, s::Cone) =
  RHCone(connection(b), add_z(s.cb, s.h), s.r, s.cb)

realize(b::RH, s::ConeFrustum) =
  RHConeFrustum(connection(b), s.cb, s.rb, s.cb + vz(s.h, s.cb.cs), s.rt)

realize(b::RH, s::Cylinder) =
  RHCylinder(connection(b), s.cb, s.r, s.cb + vz(s.h, s.cb.cs))

#realize(b::RH, s::Circle) = RHCircle(connection(b),

backend_extrusion(b::RH, s::Shape, v::Vec) =
    and_mark_deleted(
        map_ref(s) do r
            RHExtrusion(connection(b), r, v)
        end,
        s)

backend_sweep(b::RH, path::Shape, profile::Shape, rotation::Real, scale::Real) =
  map_ref(profile) do profile_r
    map_ref(path) do path_r
      RHSweepPathProfile(connection(b), path_r, profile_r, rotation, scale)
    end
  end

#=
realize(b::RH, s::Revolve) =
  and_delete_shape(
    map_ref(s.profile) do r
      RHRevolve(connection(b), r, s.p, s.n, s.start_angle, s.amplitude)
    end,
    s.profile)

backend_loft_points(b::Backend, profiles::Shapes, rails::Shapes, ruled::Bool, closed::Bool) =
  let f = (ruled ? (closed ? polygon : line) : (closed ? closed_spline : spline))
    and_delete_shapes(ref(f(map(point_position, profiles), backend=b)),
                      vcat(profiles, rails))
  end

backend_loft_curves(b::RH, profiles::Shapes, rails::Shapes, ruled::Bool, closed::Bool) =
  and_delete_shapes(RHLoft(connection(b),
                             collect_ref(profiles),
                             collect_ref(rails),
                             ruled, closed),
                    vcat(profiles, rails))

backend_loft_surfaces(b::RH, profiles::Shapes, rails::Shapes, ruled::Bool, closed::Bool) =
  backend_loft_curves(b, profiles, rails, ruled, closed)

backend_loft_curve_point(b::RH, profile::Shape, point::Shape) =
  and_delete_shapes(RHLoft(connection(b),
                             vcat(collect_ref(profile), collect_ref(point)),
                             [],
                             true, false),
                    [profile, point])

backend_loft_surface_point(b::RH, profile::Shape, point::Shape) =
  backend_loft_curve_point(b, profile, point)
=#
unite_refs(b::RH, refs::Vector{<:RHRef}) =
    RHUnionRef(tuple(refs...))

unite_refs(b::RH, r::RHUnionRef) =
  r

unite_ref(b::RH, r0::RHNativeRef, r1::RHNativeRef) =
  (RHUnite(connection(b), r0.value, r1.value); r0)

intersect_ref(b::RH, r0::RHNativeRef, r1::RHNativeRef) =
    let refs = RHIntersect(connection(b), r0.value, r1.value)
        n = length(refs)
        if n == 0
            RHEmptyRef()
        elseif n == 1
            RHNativeRef(refs[1])
        else
            RHUnionRef(map(RHNativeRef, tuple(refs...)))
        end
    end

subtract_ref(b::RH, r0::RHNativeRef, r1::RHNativeRef) =
    let refs = RHSubtract(connection(b), r0.value, r1.value)
        n = length(refs)
        if n == 0
            RHEmptyRef()
        elseif n == 1
            if is_empty_guid(refs[1]) # failed
                RHSubtractionRef(r0, tuple(r1))
            else
                RHNativeRef(refs[1])
            end
        else
            RHUnionRef(map(RHNativeRef, tuple(refs...)))
        end
    end

subtract_ref(b::RH, r0::RHRef, r1::RHUnionRef) =
  foldl((r0,r1)->subtract_ref(b,r0,r1), r1.values, init=r0)

#=
slice_ref(b::RH, r::RHNativeRef, p::Loc, v::Vec) =
  (RHSlice(connection(b), r.value, p, v); r)

slice_ref(b::RH, r::RHUnionRef, p::Loc, v::Vec) =
  map(r->slice_ref(b, r, p, v), r.values)

=#

#=
realize(b::RH, s::IntersectionShape) =
  foldl((r0,r1)->intersect_ref(b,r0,r1), UniversalRef{RHId}(), map(ref, s.shapes))
  =#

realize(b::RH, s::Slice) =
  slice_ref(b, ref(s.shape), s.p, s.n)

realize(b::RH, s::Move) =
  let r = map_ref(s.shape) do r
            RHMove(connection(b), r, s.v)
            r
          end
    mark_deleted(s.shape)
    r
  end

realize(b::RH, s::Scale) =
  let r = map_ref(s.shape) do r
            RHScale(connection(b), r, s.p, s.s)
            r
          end
    mark_deleted(s.shape)
    r
  end

realize(b::RH, s::Rotate) =
  let r = map_ref(s.shape) do r
            RHRotate(connection(b), r, s.p, s.v, s.angle)
            r
          end
    mark_deleted(s.shape)
    r
  end

realize(b::RH, s::Mirror) =
  and_delete_shape(map_ref(s.shape) do r
                    RHMirror(connection(b), r, s.p, s.n, false)
                   end,
                   s.shape)

realize(b::RH, s::UnionMirror) =
  let r0 = ref(s.shape),
      r1 = map_ref(r0) do r
            RHMirror(connection(b), r, s.p, s.n, true)
          end
    UnionRef((r0,r1))
  end

realize(b::RH, s::SurfaceGrid) =
  let (nu, nv) = size(s.points)
    RHSurfaceFromGrid(connection(b), nu, nv, reshape(s.points,:), s.closed_u, s.closed_v,
      max(2*floor(Int,nu/30)+1, 2), max(2*floor(Int, nv/30)+1, 2))
  end
#=
realize(b::RH, s::Thicken) =
  and_delete_shape(
    map_ref(s.shape) do r
      RHThicken(connection(b), r, s.thickness)
    end,
    s.shape)

=#
# BIM

# Families

backend_get_family(b::RH, f::TableFamily) =
    RHCreateRectangularTableFamily(connection(b), f.length, f.width, f.height, f.top_thickness, f.leg_thickness)
backend_get_family(b::RH, f::ChairFamily) =
    RHCreateChairFamily(connection(b), f.length, f.width, f.height, f.seat_height, f.thickness)
backend_get_family(b::RH, f::TableChairFamily) =
    RHCreateRectangularTableAndChairsFamily(connection(b),
        ref(f.table_family), ref(f.chair_family),
        f.table_family.length, f.table_family.width,
        f.chairs_top, f.chairs_bottom, f.chairs_right, f.chairs_left,
        f.spacing)

backend_rectangular_table(b::RH, c, angle, family) =
    RHTable(connection(b), c, angle, ref(family))

backend_chair(b::RH, c, angle, family) =
    RHChair(connection(b), c, angle, ref(family))

backend_rectangular_table_and_chairs(b::RH, c, angle, family) =
    RHTableAndChairs(connection(b), c, angle, ref(family))

backend_slab(b::RH, profile, holes, thickness, family) =
    map_ref(b,
            r->RHExtrusion(connection(b), r, vz(thickness)),
            ensure_ref(b, backend_fill(b, profile)))

#Beams are aligned along the top axis.
realize(b::RH, s::Beam) =
  let profile = s.family.profile
      profile_u0 = profile.corner
      c = add_xy(s.cb, profile_u0.x + profile.dx/2, profile_u0.y + profile.dy/2)
      # need to test whether it is rotation on center or on axis
      o = add_y(loc_from_o_phi(s.cb, s.angle), -profile.dy/2)
    RHXYCenteredBox(connection(b), o, vx(1, o.cs), vy(1, o.cs), profile.dx, profile.dy, s.h)
  end

#Columns are aligned along the center axis.
realize(b::RH, s::FreeColumn) =
  let profile = s.family.profile
      profile_u0 = profile.corner
      c = add_xy(s.cb, profile_u0.x + profile.dx/2, profile_u0.y + profile.dy/2)
      # need to test whether it is rotation on center or on axis
      o = loc_from_o_phi(c, s.angle)
    RHXYCenteredBox(connection(b), o, vx(1, o.cs), vy(1, o.cs), profile.dx, profile.dy, s.h)
  end

realize(b::RH, s::Column) =
  let profile = s.family.profile,
      profile_u0 = profile.corner,
      c = add_xy(s.cb, profile_u0.x + profile.dx/2, profile_u0.y + profile.dy/2),
      base_height = s.bottom_level.height,
      height = s.top_level.height - base_height,
      # need to test whether it is rotation on center or on axis
      o = loc_from_o_phi(s.cb + vz(base_height), s.angle)
    RHXYCenteredBox(connection(b), o, vx(1, o.cs), vy(1, o.cs), profile.dx, profile.dy, height)
  end

backend_wall(b::RH, path, height, thickness, family) =
    RHPathWall(connection(b), backend_stroke(b, path), thickness, height)

############################################

backend_bounding_box(b::RH, shapes::Shapes) =
  RHBoundingBox(connection(b), collect_ref(shapes))

import Base.view
set_view(camera::XYZ, target::XYZ, lens::Real, b::RH) =
  RHView(connection(b), camera, target, lens)

get_view(b::RH) =
  let c = connection(b)
    RHViewCamera(c), RHViewTarget(c), RHViewLens(c)
  end

zoom_extents(b::RH) =
  RHZoomExtents(connection(b))

view_top(b::RH) =
    RHViewTop(connection(b))

rhino"public int DeleteAll()"
rhino"public int DeleteAllInLayer(String name)"
rhino"public void Delete(Guid id)"
rhino"public void DeleteMany(Guid[] ids)"

backend_delete_shapes(b::RH, shapes::Shapes) =
  RHDeleteMany(connection(b), collect_ref(shapes))

delete_all_shapes(b::RH) =
  RHDeleteAll(connection(b))

# Layers
rhino"public String CreateLayer(string name)"
rhino"public void SetLayerColor(ObjectId id, byte r, byte g, byte b)"
rhino"public void SetShapeColor(ObjectId id, byte r, byte g, byte b)"
rhino"public String CurrentLayer()"
rhino"public void SetCurrentLayer(String id)"
rhino"public String ShapeLayer(RhinoObject objId)"
rhino"public void SetShapeLayer(RhinoObject objId, String layerId)"

RHLayer = String

current_layer(b::RH)::RHLayer =
  RHCurrentLayer(connection(b))

current_layer(layer::RHLayer, b::RH) =
  RHSetCurrentLayer(connection(b), layer)

create_layer(name::String, b::RH) =
  RHCreateLayer(connection(b), name)

create_layer(name::String, color::RGB, b::RH) =
  let layer = RHCreateLayer(connection(b), name)
    RHSetLayerColor(connection(b), layer, color.r, color.g, color.b)
    layer
  end

delete_all_shapes_in_layer(layer::RHLayer, b::RH) =
  RHDeleteAllInLayer(connection(b), layer)

rhino"public Guid Clone(RhinoObject e)"

shape_from_ref(r, b::RH) =
  let c = connection(b)
    let code = RHShapeCode(c, r),
        ref = LazyRef(b, RHNativeRef(r))
              #ref = LazyRef(b, RHNativeRef(RHClone(connection(b), r))) # HACK CLONING!!!!!
      if code == 1
        point(RHPointPosition(c, r),
              backend=b, ref=ref)
      elseif code == 2
        circle(maybe_loc_from_o_vz(RHCircleCenter(c, r), RHCircleNormal(c, r)),
               RHCircleRadius(c, r),
               backend=b, ref=ref)
      elseif 3 <= code <= 6
        line(RHLineVertices(c, r),
             backend=b, ref=ref)
      elseif code == 7
        spline([xy(0,0)], false, false, #HACK obtain interpolation points
               backend=b, ref=ref)
      elseif 103 <= code <= 106
        polygon(RHLineVertices(c, r),
                backend=b, ref=ref)
      elseif code == 40
        # FIXME: frontier is missing
        surface(frontier=[], backend=b, ref=ref)
      elseif code == 41
        surface(frontier=[], backend=b, ref=ref)
      elseif code == 81
        sphere(backend=b, ref=ref)
      elseif code == 82
        cylinder(backend=b, ref=ref)
      elseif code == 83
        cone(backend=b, ref=ref)
      elseif code == 84
        torus(backend=b, ref=ref)
      else
        error("Unknown shape")
      end
    end
  end

"""            obj = _geometry_from_id(r)
            ref = native_ref(r)
            if isinstance(obj, geo.Point):
                return point.new_ref(ref, fromPt(obj.Location))
            elif isinstance(obj, geo.Curve):
                if rh.IsLine(r) or rh.IsPolyline(r):
                    if rh.IsCurveClosed(r):
                        return polygon.new_ref(ref, [fromPt(p) for p in rh.CurvePoints(r)[:-1]])
                    else:
                        return line.new_ref(ref, [fromPt(p) for p in rh.CurvePoints(r)])
                elif obj.IsCircle(Rhino.RhinoMath.ZeroTolerance):
                    return circle.new_ref(ref, fromPt(rh.CircleCenterPoint(r)), rh.CircleRadius(r))
                elif rh.IsCurveClosed(r):
                    return closed_spline.new_ref(ref, [fromPt(p) for p in rh.CurvePoints(r)])
                else:
                    return spline.new_ref(ref, [fromPt(p) for p in rh.CurvePoints(r)])
            elif rh.IsObject(r) and rh.IsObjectSolid(r):
                return solid(native_ref(r))
            elif rh.IsSurface(r) or rh.IsPolysurface(r):
                return surface(native_ref(r))
            else:
                raise RuntimeError("{0}: Unknown Rhino object {1}".format('shape_from_ref', r))
                """

#
rhino"public Point3d[] GetPosition(string prompt)"

select_position(prompt::String, b::RH) =
  begin
    @info "$(prompt) on the $(b) backend."
    let ans = RHGetPosition(connection(b), prompt)
      length(ans) > 0 ? ans[1] : nothing
    end
  end

select_positions(prompt::String, b::RH) =
  let ps = Loc[]
      p = nothing
    @info "$(prompt) on the $(b) backend."
    while ((p = RHGetPosition(connection(b), prompt)) |> length) > 0
        push!(ps, p...)
    end
    ps
  end

rhino"public Guid[] GetPoint(string prompt)"
rhino"public Guid[] GetPoints(string prompt)"

# HACK: The next operations should receive a set of shapes to avoid re-creating already existing shapes

select_point(prompt::String, b::RH) =
  select_one_with_prompt(prompt, b, RHGetPoint)

select_points(prompt::String, b::RH) =
  select_many_with_prompt(prompt, b, RHGetPoints)

rhino"public Guid[] GetCurve(string prompt)"
rhino"public Guid[] GetCurves(string prompt)"

select_curve(prompt::String, b::RH) =
  select_one_with_prompt(prompt, b, RHGetCurve)

select_curves(prompt::String, b::RH) =
  select_many_with_prompt(prompt, b, RHGetCurves)

rhino"public Guid[] GetSurface(string prompt)"
rhino"public Guid[] GetSurfaces(string prompt)"

select_surface(prompt::String, b::RH) =
  select_one_with_prompt(prompt, b, RHGetSurface)

select_surfaces(prompt::String, b::RH) =
  select_many_with_prompt(prompt, b, RHGetSurfaces)

rhino"public Guid[] GetSolid(string prompt)"
rhino"public Guid[] GetSolids(string prompt)"

select_solid(prompt::String, b::RH) =
  select_one_with_prompt(prompt, b, RHGetSolid)

select_solids(prompt::String, b::RH) =
  select_many_with_prompt(prompt, b, RHGetSolids)

rhino"public Guid[] GetShape(string prompt)"
rhino"public Guid[] GetShapes(string prompt)"

select_shape(prompt::String, b::RH) =
  select_one_with_prompt(prompt, b, RHGetShape)

select_shapes(prompt::String, b::RH) =
  select_many_with_prompt(prompt, b, RHGetShapes)

captured_shape(b::RH, handle) =
  shape_from_ref(handle, b)
#
captured_shapes(b::RH, handles) =
  map(handles) do handle
      shape_from_ref(handle, b)
  end

generate_captured_shape(s::Shape, b::RH) =
    println("captured_shape(rhino, $(ref(s).value))")

generate_captured_shapes(ss::Shapes, b::RH) =
  begin
    print("captured_shapes(autocad, [")
    for s in ss
      print(ref(s).value)
      print(", ")
    end
    println("])")
  end

rhino"public Guid[] GetAllShapes()"
rhino"public Guid[] GetAllShapesInLayer(String name)"

all_shapes(b::RH) =
  [shape_from_ref(r) for r in RHGetAllShapes(connection(b))]

all_shapes_in_layer(layer, b::RH) =
  [shape_from_ref(r) for r in RHGetAllShapesInLayer(connection(b), layer)]


using CSV, DataFrames, StatsPlots, StatsBase

categories = String[
    "food", "animal", "clothing", "tool", "sports equipment", "vegetable", "vehicle", "musical instrument",
    "fruit", "body part", "dessert", "toy", "container", "part of car", "weapon", "bird", "furniture",
    "kitchen tool", "office supply", "clothing accessory", "kitchen appliance", "plant", "insect",
    "home décor", "medical equipment", "electronic device", "drink"
]

function parse_categories(datafile::String)
    df = DataFrame(CSV.File(datafile, header = true))

    # check that all categories are defined
    for i in skipmissing(df.top_down)
        if i ∉ categories
            error("There is no category called \"$(i)\".")
        end
    end

    frequencies = countmap(skipmissing(df.category))

    theme(:solarized)

    plt = bar(
        categories,
        Int[get(frequencies, c, 0) for c in categories],
        legend = false,
        xticks = (0.5:length(categories), categories),
        xrotation = 45,
        fontfamily = font("Times"),
        xlabel = "Object Category",
        ylabel = "Frequency",
        title = "Frequency of Object Categories in chosen ~200 Distractors"
    )

    # plt = bar(
    #     collect(keys(frequencies)),
    #     collect(values(frequencies));
    #     legend = false,
    #     xticks = (0.5:length(keys(frequencies)), collect(keys(frequencies))),
    #     xrotation = 45,
    #     fontfamily = font("Times"),
    #     xlabel = "Category",
    #     ylabel = "Frequency",
    #     title = "Frequency of Categories in chosen 200 Distractors"
    # )

    savefig(plt, "/Users/jakeireland/Desktop/categories.pdf")
end

parse_categories("/Users/jakeireland/projects/GorillaSCExperiment/data/distractor_categories.csv")