""" Data structure for (directed) wiring diagrams, aka string diagrams.

A (directed) wiring diagram consists of a collection of boxes with input and
output ports connected by wires. A box can be atomic (possessing no internal
structure) or can itself be a wiring diagram. Thus, wiring diagrams can be
nested recursively. Wiring diagrams are closely related to what the CS
literature calls "directed graphs with ports" or more simply "port graphs". The
main difference is that a wiring diagram has an "outer box": a wiring diagram
has its own ports that can be connected to the ports of its boxes.

This module provides a generic data structure for wiring diagrams. Arbitrary
data can be attached to the boxes, ports, and wires of a wiring diagram. The
diagrams are "abstract" in the sense that they cannot be directly rendered as
raster or vector graphics. However, they form a useful intermediate
representation that can be serialized to and from GraphML or translated into
Graphviz or other declarative diagram languages.
"""
module DirectedWiringDiagrams
export AbstractBox, Box, WiringDiagram, Wire, Port, PortKind,
  InputPort, OutputPort, input_ports, output_ports, input_id, output_id,
  outer_ids, boxes, box_ids, nboxes, nwires, box, wires, has_wire, graph,
  add_box!, add_boxes!, add_wire!, add_wires!, rem_box!, rem_boxes!, rem_wire!,
  rem_wires!, port_value, validate_ports, is_permuted_equal,
  all_neighbors, neighbors, outneighbors, inneighbors, in_wires, out_wires,
  singleton_diagram, induced_subdiagram, encapsulated_subdiagram,
  substitute, encapsulate

using Compat
using AutoHashEquals
using LightGraphs, MetaGraphs
import LightGraphs: all_neighbors, neighbors, outneighbors, inneighbors

# Data types
############

""" Kind of port: input or output.
"""
@enum PortKind InputPort OutputPort

""" A port on a box to which wires can be connected.
"""
@auto_hash_equals struct Port
  box::Int
  kind::PortKind
  port::Int
end
set_box(port::Port, box::Int) = Port(box, port.kind, port.port)

function Base.isless(p1::Port, p2::Port)::Bool
  # Lexicographic order.
  p1.box < p2.box ||
    (p1.box == p2.box &&
      (p1.kind < p2.kind || (p1.kind == p2.kind && p1.port < p2.port)))
end

""" A wire connecting one port to another.
"""
@auto_hash_equals struct Wire{Value}
  value::Value
  source::Port
  target::Port
end

Wire(value, src::Tuple{Int,PortKind,Int}, tgt::Tuple{Int,PortKind,Int}) =
  Wire(value, Port(src[1],src[2],src[3]), Port(tgt[1],tgt[2],tgt[3]))
Wire(value, src::Tuple{Int,Int}, tgt::Tuple{Int,Int}) =
  Wire(value, Port(src[1],OutputPort,src[2]), Port(tgt[1],InputPort,tgt[2]))
Wire(value, pair::Pair) = Wire(value, first(pair), last(pair))

Wire(src::Port, tgt::Port) = Wire(nothing, src, tgt)
Wire(src::Tuple, tgt::Tuple) = Wire(nothing, src, tgt)
Wire(pair::Pair) = Wire(nothing, first(pair), last(pair))

function Base.show(io::IO, wire::Wire)
  skip_kind = wire.source.kind == OutputPort && wire.target.kind == InputPort
  show_port = (io::IO, port::Port) -> begin
    if skip_kind
      print(io, "($(port.box),$(port.port))")
    else
      print(io, "($(port.box),$(string(port.kind)),$(port.port)")
    end
  end
  print(io, "Wire(")
  if !isnothing(wire.value)
    show(io, wire.value)
    print(io, ", ")
  end
  show_port(io, wire.source)
  print(io, " => ")
  show_port(io, wire.target)
  print(io, ")")
end

function Base.isless(w1::Wire, w2::Wire)::Bool
  # Lexicographic order.
  isless(w1.source, w2.source) ||
    (w1.source == w2.source && isless(w1.target, w2.target))
end

""" Internal wiring diagram data corresponding to `Port`. Do not use directly.
"""
@auto_hash_equals struct PortData
  kind::PortKind
  port::Int
end
to_port_data(port::Port) = PortData(port.kind, port.port)
from_port_data(port::PortData, v::Int) = Port(v, port.kind, port.port)

""" Internal wiring diagram data corresponding to `Wire`. Do not use directly.
"""
@auto_hash_equals struct WireData{Value}
  value::Value
  source::PortData
  target::PortData
end
function to_wire_data(wire::Wire)
  WireData(wire.value, to_port_data(wire.source), to_port_data(wire.target))
end
function from_wire_data(wire::WireData, edge::Edge)
  Wire(wire.value,
       from_port_data(wire.source, src(edge)),
       from_port_data(wire.target, dst(edge)))
end

""" Base type for any box (node) in a wiring diagram.

This type represents an arbitrary black box with inputs and outputs.
"""
abstract type AbstractBox end

input_ports(box::AbstractBox)::Vector = box.input_ports
output_ports(box::AbstractBox)::Vector = box.output_ports

""" An atomic box in a wiring diagram.

These boxes have no internal structure.
"""
@auto_hash_equals struct Box{Value} <: AbstractBox
  value::Value
  input_ports::Vector
  output_ports::Vector
end

Box(inputs::Vector, outputs::Vector) = Box(nothing, inputs, outputs)

function Base.show(io::IO, box::Box)
  print(io, "Box(")
  if !isnothing(box.value)
    show(io, box.value)
    print(io, ", ")
  end
  print(io, "[")
  join(io, [sprint(show, port) for port in box.input_ports], ",")
  print(io, "], [")
  join(io, [sprint(show, port) for port in box.output_ports], ",")
  print(io, "])")
end

""" A directed wiring diagram, aka a string diagram.

The wiring diagram is implemented using the following internal data structures.
A LightGraphs `DiGraph` stores the "skeleton" of the diagram: a simple directed
graph with the boxes as vertices and with an edge between two vertices iff there
is at least one wire between the corresponding boxes. There are two special
vertices, accessible via `input_id` and `output_id`, representing the input and
output ports, respectively.

The `DiGraph` is wrapped inside a `MetaDiGraph` to attach properties to the
vertices and edges. For each edge, an edge property stores the list of wires
between the source and target boxes.
"""
mutable struct WiringDiagram{Theory} <: AbstractBox
  graph::MetaDiGraph
  value::Any
  input_ports::Vector
  output_ports::Vector

  function WiringDiagram{T}(value, inputs::Vector, outputs::Vector) where T
    graph = MetaDiGraph()
    diagram = new{T}(graph, value, inputs, outputs)
    add_vertices!(graph, 2)
    return diagram
  end
  function WiringDiagram(d::WiringDiagram{T}) where T
    # Copy constructor for shallow copy.
    new{T}(copy(d.graph), d.value, d.input_ports, d.output_ports)
  end
end

function WiringDiagram{T}(inputs::Vector, outputs::Vector) where T
  WiringDiagram{T}(nothing, inputs, outputs)
end
WiringDiagram(args...) = WiringDiagram{Any}(args...)

input_id(::WiringDiagram) = 1
output_id(::WiringDiagram) = 2
outer_ids(::WiringDiagram) = (1,2)

""" Check equality of wiring diagrams.

Warning: This method checks exact equality of the underlying graph
representation, not mathematical equality which involves graph isomorphism.

See also: `is_permuted_equal`
"""
function Base.:(==)(d1::WiringDiagram, d2::WiringDiagram)
  (input_ports(d1) == input_ports(d2) && output_ports(d1) == output_ports(d2) &&
   d1.value == d2.value && graph(d1) == graph(d2) &&
   boxes(d1) == boxes(d2) && sort!(wires(d1)) == sort!(wires(d2)))
end

""" Check equality of wiring diagram under permutation of boxes.

When the boxes in the first diagram `d1` are permuted according to `σ`,
does it become identical to the second diagram `d2`?
"""
function is_permuted_equal(d1::WiringDiagram, d2::WiringDiagram, σ::Vector{Int})
  @assert nboxes(d1) == length(σ) && nboxes(d2) == length(σ)
  d1_ids, d2_ids = box_ids(d1), box_ids(d2)
  box_map = Dict{Int,Int}(d1_ids[σ[i]] => d2_ids[i] for i in eachindex(σ))
  is_induced_equal(d1, d2, box_map)
end
function is_induced_equal(d1::WiringDiagram, d2::WiringDiagram, box_map::Dict{Int,Int})
  box_map[input_id(d1)] = input_id(d2)
  box_map[output_id(d1)] = output_id(d2)
  map_wire = wire::Wire -> Wire(wire.value,
    set_box(wire.source, box_map[wire.source.box]),
    set_box(wire.target, box_map[wire.target.box]))
  (input_ports(d1) == input_ports(d2) && output_ports(d1) == output_ports(d2) &&
   all(box(d1,v) == box(d2,box_map[v]) for v in box_ids(d1)) &&
   sort!(map(map_wire, wires(d1))) == sort!(wires(d2)))
end

Base.copy(diagram::WiringDiagram) = WiringDiagram(diagram)

function Base.show(io::IO, diagram::WiringDiagram{T}) where T
  sshowcompact = x -> sprint(show, x, context=:compact => true)
  print(io, "WiringDiagram")
  if T != Any
    print(io, "{$T}")
  end
  print(io, "(")
  if !isnothing(diagram.value)
    show(io, diagram.value)
    print(io, ", ")
  end
  print(io, "[")
  join(io, map(sshowcompact, input_ports(diagram)), ",")
  print(io, "], [")
  join(io, map(sshowcompact, output_ports(diagram)), ",")
  print(io, "], ")
  if get(io, :compact, false)
    print(io, "{$(nboxes(diagram)) boxes}, {$(nwires(diagram)) wires}")
  else
    print(io, "\n[ $(input_id(diagram)) => {inputs},\n  ")
    print(io, "$(output_id(diagram)) => {outputs},\n  ")
    join(io, [ "$v => $(sshowcompact(box(diagram, v)))"
               for v in box_ids(diagram) ], ",\n  ")
    print(io, " ],\n[ ")
    join(io, map(sshowcompact, wires(diagram)), ",\n  ")
    print(io, " ]")
  end
  print(io, ")")
end

# Imperative interface
######################

# Basic accessors.

box(f::WiringDiagram, v::Int) = get_prop(f.graph, v, :box)
boxes(f::WiringDiagram) = AbstractBox[ box(f,v) for v in box_ids(f) ]
nboxes(f::WiringDiagram) = nv(graph(f)) - 2

function box_ids(f::WiringDiagram)
  Int[ v for v in 1:nv(graph(f)) if !(v in outer_ids(f)) ]
end

function wires(f::WiringDiagram, edge::Edge)
  if has_edge(f.graph, edge)
    Wire[ from_wire_data(data, edge) for data in get_prop(f.graph,edge,:wires) ]
  else
    Wire[]
  end
end
wires(f::WiringDiagram, src::Int, tgt::Int) = wires(f, Edge(src,tgt))
wires(f::WiringDiagram) = vcat((wires(f,e) for e in edges(graph(f)))...)
nwires(f::WiringDiagram) =
  sum(Int[ length(get_prop(f.graph,e,:wires)) for e in edges(graph(f)) ])

function has_wire(f::WiringDiagram, src::Int, tgt::Int)
  has_edge(graph(f), Edge(src, tgt))
end
function has_wire(f::WiringDiagram, wire::Wire)
  wire in wires(f, wire.source.box, wire.target.box)
end
has_wire(f::WiringDiagram, pair::Pair) = has_wire(f, Wire(pair))

function input_ports(f::WiringDiagram, v::Int)
  if v == input_id(f)
    error("Input vertex does not have input ports within wiring diagram")
  elseif v == output_id(f)
    output_ports(f)
  else
    input_ports(box(f, v))
  end
end

function output_ports(f::WiringDiagram, v::Int)
  if v == input_id(f)
    input_ports(f)
  elseif v == output_id(f)
    error("Output vertex does not have output ports within wiring diagram")
  else
    output_ports(box(f, v))
  end
end

function port_value(f::WiringDiagram, port::Port)
  get_ports = port.kind == InputPort ? input_ports : output_ports
  get_ports(f, port.box)[port.port]
end

# Graph mutation.

function add_box!(f::WiringDiagram, box::AbstractBox)
  @assert add_vertex!(f.graph, :box, box)
  return nv(f.graph)
end

function add_boxes!(f::WiringDiagram, boxes)
  for box in boxes
    add_box!(f, box)
  end
end

function rem_box!(f::WiringDiagram, v::Int)
  @assert !(v in outer_ids(f))
  rem_vertex!(f.graph, v)
end

function rem_boxes!(f::WiringDiagram, vs)
  # Remove boxes in descending order of vertex ID to maintain ID stability.
  for v in sort!(collect(vs), rev=true)
    rem_box!(f, v)
  end
end

function add_wire!(f::WiringDiagram, wire::Wire)
  # Check compatibility of port types.
  validate_ports(port_value(f, wire.source), port_value(f, wire.target))

  # Add edge and edge properties.
  edge = Edge(wire.source.box, wire.target.box)
  if !has_edge(f.graph, edge)
    add_edge!(f.graph, src(edge), dst(edge), :wires, WireData[])
  end
  push!(get_prop(f.graph, edge, :wires), to_wire_data(wire))
end
add_wire!(f::WiringDiagram, pair::Pair) = add_wire!(f, Wire(pair))

function add_wires!(f::WiringDiagram, wires)
  for wire in wires
    add_wire!(f, wire)
  end
end

function rem_wire!(f::WiringDiagram, wire::Wire)
  edge = Edge(wire.source.box, wire.target.box)
  edge_data = to_wire_data(wire)
  wires = get_prop(f.graph, edge, :wires)
  deleteat!(wires, findlast(d -> d == edge_data, wires))
  if isempty(wires)
    rem_edge!(f.graph, edge)
  end
end
rem_wire!(f::WiringDiagram, pair::Pair) = rem_wire!(f, Wire(pair))

function rem_wires!(f::WiringDiagram, wires)
  for wire in wires
    rem_wire!(f, wire)
  end
end

function rem_wires!(f::WiringDiagram, src::Int, tgt::Int)
  rem_edge!(f.graph, Edge(src, tgt))
end

""" Check compatibility of source and target ports.

The default implementation is a no-op.
"""
function validate_ports(source_port, target_port) end

# Graph properties.

""" Retrieve the underlying LightGraphs graph.

Do not mutate it! All mutations should pass through the `WiringDiagram` methods:
`add_box!`, `rem_box!`, etc.
"""
graph(diagram::WiringDiagram) = diagram.graph.graph

# Convenience methods delegated to LightGraphs.
all_neighbors(d::WiringDiagram, v::Int) = all_neighbors(graph(d), v)
neighbors(d::WiringDiagram, v::Int) = neighbors(graph(d), v)
outneighbors(d::WiringDiagram, v::Int) = outneighbors(graph(d), v)
inneighbors(d::WiringDiagram, v::Int) = inneighbors(graph(d), v)

""" Get all wires coming into or out of the box.
"""
function wires(d::WiringDiagram, v::Int)
  result = wires(d, v, v)
  for u in inneighbors(d, v)
    if u != v
      append!(result, wires(d, u, v))
    end
  end
  for u in outneighbors(d, v)
    if u != v
      append!(result, wires(d, v, u))
    end
  end
  result
end

""" Get all wires coming into the box.
"""
function in_wires(d::WiringDiagram, v::Int)
  result = Wire[]
  for u in inneighbors(d, v)
    append!(result, wires(d, u, v))
  end
  result
end

""" Get all wires coming into the port.
"""
function in_wires(d::WiringDiagram, port::Port)
  result = Wire[]
  for v in inneighbors(d, port.box)
    for wire in wires(d, v, port.box)
      if wire.target == port
        push!(result, wire)
      end
    end
  end
  result
end
function in_wires(d::WiringDiagram, v::Int, port::Int)
  in_wires(d, Port(v, InputPort, port))
end

""" Get all wires coming out of the box.
"""
function out_wires(d::WiringDiagram, v::Int)
  result = Wire[]
  for u in outneighbors(d, v)
    append!(result, wires(d, v, u))
  end
  result
end

""" Get all wires coming out of the port.
"""
function out_wires(d::WiringDiagram, port::Port)
  result = Wire[]
  for v in outneighbors(d, port.box)
    for wire in wires(d, port.box, v)
      if wire.source == port
        push!(result, wire)
      end
    end
  end
  result
end
function out_wires(d::WiringDiagram, v::Int, port::Int)
  out_wires(d, Port(v, OutputPort, port))
end

# Other constructors.

""" A wiring diagram with a single box connected to its outer ports.
"""
function singleton_diagram(T::Type, box::AbstractBox)
  inputs, outputs = input_ports(box), output_ports(box)
  d = WiringDiagram{T}(inputs, outputs)
  v = add_box!(d, box)
  add_wires!(d, ((input_id(d),i) => (v,i) for i in eachindex(inputs)))
  add_wires!(d, ((v,i) => (output_id(d),i) for i in eachindex(outputs)))
  return d
end
singleton_diagram(box::AbstractBox) = singleton_diagram(Any, box)

""" The wiring diagram induced by a subset of its boxes.

See also `encapsulated_subdiagram`.
"""
function induced_subdiagram(d::WiringDiagram{T}, vs::Vector{Int}) where T
  sub = WiringDiagram{T}(input_ports(d), output_ports(d))
  vmap = Dict(input_id(d) => input_id(sub), output_id(d) => output_id(sub))
  for v in vs
    vmap[v] = add_box!(sub, box(d, v))
  end
  for wire in wires(d)
    src, tgt = wire.source, wire.target
    if haskey(vmap, src.box) && haskey(vmap, tgt.box)
      add_wire!(sub,
        Wire(set_box(src, vmap[src.box]), set_box(tgt, vmap[tgt.box])))
    end
  end
  return sub
end

# Substitution
##############

""" Substitute wiring diagrams for boxes.

Performs one or more substitutions. When performing multiple substitutions, the
substitutions are simultaneous.

This operation implements the operadic composition of wiring diagrams
(`ocompose`).
"""
function substitute(d::WiringDiagram; kw...)
  substitute(d, filter(v -> box(d,v) isa WiringDiagram, box_ids(d)); kw...)
end
function substitute(d::WiringDiagram, v::Int; kw...)
  substitute(d, v, box(d,v)::WiringDiagram; kw...)
end
function substitute(d::WiringDiagram, vs::Vector{Int}; kw...)
  substitute(d, vs, WiringDiagram[ box(d,v) for v in vs ]; kw...)
end
function substitute(d::WiringDiagram, v::Int, sub::WiringDiagram; kw...)
  substitute(d, [v], [sub]; kw...)
end

function substitute(d::WiringDiagram{T}, vs::Vector{Int}, subs::Vector{<:WiringDiagram};
                    merge_wire_values=default_merge_wire_values) where T
  # In outline, the algorithm is:
  #
  # 1. Create an empty wiring diagram.
  # 2. Add *all* boxes of original diagram and the diagrams to be substituted
  #    (in the appropriate order).
  # 3. Add *all* wires of original diagram and the diagrams to be substituted.
  # 4. Remove the boxes that were substituted (hence also removing extraneous
  #    wires from step 3).
  #
  # This may seem convoluted, but it is the simplest way I know to handle the
  # problem of *instantaneous wires*. Some authors ban instantaneous wires, but
  # we need them to represent identities, braidings, etc.
  @assert length(vs) == length(subs)
  result = WiringDiagram{T}(d.value, input_ports(d), output_ports(d))

  # Add boxes by interleaving, in the correct order, the non-substituted boxes
  # of the original diagram and the internal boxes of the substituted diagrams.
  # At the very end, add the substituted boxes too.
  vmap = Dict(input_id(d) => input_id(result), output_id(d) => output_id(result))
  sub_maps = Dict(zip(vs, ((sub, Dict{Int,Int}()) for sub in subs)))
  for v in box_ids(d)
    if haskey(sub_maps, v)
      sub, sub_map = sub_maps[v]
      for u in box_ids(sub)
        sub_map[u] = add_box!(result, box(sub, u))
      end
    else
      vmap[v] = add_box!(result, box(d, v))
    end
  end
  for v in vs
    vmap[v] = add_box!(result, box(d, v))
  end

  # Add the wires of the original diagram, then add the internal wires of the
  # substituted diagrams.
  for wire in wires(d)
    add_wire!(result, Wire(wire.value,
      set_box(wire.source, vmap[wire.source.box]),
      set_box(wire.target, vmap[wire.target.box])))
  end
  for v in vs
    substitute_wires!(result, vmap[v], sub_maps[v]...;
      merge_wire_values=merge_wire_values)
  end

  # Finally, remove the substituted boxes. Because they were added last, this
  # will not change the IDs of the other boxes.
  rem_boxes!(result, (vmap[v] for v in vs))
  result
end

""" Substitute wires of sub-diagram into containing wiring diagram.
"""
function substitute_wires!(d::WiringDiagram, v::Int,
                           sub::WiringDiagram, sub_map::Dict{Int,Int};
                           merge_wire_values=default_merge_wire_values)
  for wire in wires(sub)
    src = get(sub_map, wire.source.box, 0)
    tgt = get(sub_map, wire.target.box, 0)
    # Special case: wire from input port to output port.
    if wire.source.box == input_id(sub) && wire.target.box == output_id(sub)
      for in_wire in in_wires(d, v, wire.source.port)
        for out_wire in out_wires(d, v, wire.target.port)
          add_wire!(d, Wire(
            merge_wire_values(in_wire.value, wire.value, out_wire.value),
            in_wire.source, out_wire.target))
        end
      end
    # Special case: wire from input port to internal box.
    elseif wire.source.box == input_id(sub)
      for in_wire in in_wires(d, v, wire.source.port)
        add_wire!(d, Wire(
          merge_wire_values(in_wire.value, wire.value, nothing),
          in_wire.source, set_box(wire.target, tgt)))
      end
    # Special case: wire from internal box to output port.
    elseif wire.target.box == output_id(sub)
      for out_wire in out_wires(d, v, wire.target.port)
        add_wire!(d, Wire(
          merge_wire_values(nothing, wire.value, out_wire.value),
          set_box(wire.source, src), out_wire.target))
      end
    # Default case: wire between two internal boxes.
    else
      add_wire!(d, Wire(
        merge_wire_values(nothing, wire.value, nothing),
        set_box(wire.source, src), set_box(wire.target, tgt)))
    end
  end
end

default_merge_wire_values(::Any, middle::Any, ::Any) = middle

# Encapsulation
###############

""" Encapsulate multiple boxes within a single sub-diagram.

This operation is a (one-sided) inverse to subsitution (see `substitute`).
"""
function encapsulate(d::WiringDiagram, vs::Vector{Int}; value=nothing, kw...)
  encapsulate(d, [vs]; values=[value], kw...)
end

function encapsulate(d::WiringDiagram{T}, vss::Vector{Vector{Int}};
    discard_boxes::Bool=false, make_box=Box, values=nothing) where T
  if isempty(vss); return d end
  if any(isempty(vs) for vs in vss)
    error("Cannot encapsulate an empty set of boxes")
  end
  if !allunique(reduce(vcat, vss))
    error("Cannot encapsulate overlapping sets of boxes")
  end
  if isnothing(values)
    values = repeat([nothing], length(vss))
  end
  result = WiringDiagram{T}(d.value, input_ports(d), output_ports(d))

  # Add boxes, both encapsulated and non-encapsulated, to new wiring diagram.
  encapsulated_representatives = Dict(
    minimum(vs) => (vs, value) for (vs, value) in zip(vss, values))
  all_encapsulated = Set(v for vs in vss for v in vs)
  vmap = Dict(input_id(d) => input_id(result), output_id(d) => output_id(result))
  port_map = Dict{Port,Port}()
  for v in box_ids(d)
    if haskey(encapsulated_representatives, v)
      vs, value = encapsulated_representatives[v]
      sub, sub_map = encapsulated_subdiagram(d, vs;
        discard_boxes=discard_boxes, make_box=make_box, value=value)
      subv = add_box!(result, sub)
      merge!(port_map, Dict(
        port => from_port_data(data, subv) for (port, data) in sub_map))
    elseif v ∉ all_encapsulated
      vmap[v] = add_box!(result, box(d, v))
    end
  end

  # Add wires to new wiring diagram.
  for wire in wires(d)
    src, tgt = wire.source, wire.target
    new_src = if haskey(vmap, src.box); set_box(src, vmap[src.box])
      elseif haskey(port_map, src); port_map[src]
      else; continue end
    new_tgt = if haskey(vmap, tgt.box); set_box(tgt, vmap[tgt.box])
      elseif haskey(port_map, tgt); port_map[tgt]
      else; continue end
    add_wire!(result, Wire(new_src, new_tgt))
  end
  result
end

""" Create an encapsulating box for a set of boxes in a wiring diagram.

To a first approximation, the union of input ports of the given boxes will
become the inputs ports of the encapsulating box and likewise for the output
ports. However, when copies or merges occur, as in a cartesian or cocartesian
category, a simplification procedure may reduce the number of ports on the
encapsulating box.

Specifically:

1. Each input port of an encapsulated box will have at most one incoming wire
from the encapsulating outer box, and each output port of an encapsulated box
will have at most one outgoing wire to the encapsulating outer box.

2. A set of ports connected to the same outside (non-encapsulated) ports will be
simplified into a single port of the encapsulating box.

See also `induced_subdiagram`.
"""
function encapsulated_subdiagram(d::WiringDiagram{T}, vs::Vector{Int};
    discard_boxes::Bool=false, make_box=Box, value=nothing) where T
  # Add encapsulated box to new wiring diagram.
  inputs, outputs = [], []
  result = discard_boxes ? nothing : WiringDiagram{T}(value, inputs, outputs)
  vmap = if discard_boxes
    Dict(v => nothing for v in vs)
  else
    Dict(v => add_box!(result, box(d, v)) for v in vs)
  end

  # Process wires into, or out of, encapsulated boxes.
  port_map = Dict{Port,PortData}()
  inner_port_map = Dict{Tuple{Vector{Port},Any},Int}()
  for v in vs
    # Add input ports to encapsulating box and corresponding wire.
    for (port, value) in enumerate(input_ports(d, v))
      tgt = Port(v, InputPort, port)
      srcs = sort!([ wire.source for wire in in_wires(d, tgt)
                     if !haskey(vmap, wire.source.box) ])
      if isempty(srcs) continue end
      src = get!(inner_port_map, (srcs, value)) do
        push!(inputs, value)
        port_data = port_map[tgt] = PortData(InputPort, length(inputs))
        port_data.port
      end
      if discard_boxes; continue end
      add_wire!(result,
        Wire(Port(input_id(result), OutputPort, src), set_box(tgt, vmap[v])))
    end

    # Add output ports to encapsulating box and corresponding wire.
    for (port, value) in enumerate(output_ports(d, v))
      src = Port(v, OutputPort, port)
      tgts = sort([ wire.target for wire in out_wires(d, src)
                    if !haskey(vmap, wire.target.box) ])
      if isempty(tgts) continue end
      tgt = get!(inner_port_map, (tgts, value)) do
        push!(outputs, value)
        port_data = port_map[src] = PortData(OutputPort, length(outputs))
        port_data.port
      end
      if discard_boxes; continue end
      add_wire!(result,
        Wire(set_box(src, vmap[v]), Port(output_id(result), InputPort, tgt)))
    end

    # Add wires between two encapsulated boxes.
    if discard_boxes; continue end
    for wire in out_wires(d, v)
      src, tgt = wire.source, wire.target
      if haskey(vmap, src.box) && haskey(vmap, tgt.box) # Clause #1 always true.
        add_wire!(result,
          Wire(set_box(src, vmap[src.box]), set_box(tgt, vmap[tgt.box])))
      end
    end
  end

  # Yield input and output port lists with the tightest possible types.
  inputs, outputs = [ x for x in inputs ], [ x for x in outputs ]
  if discard_boxes
    result = make_box(value, inputs, outputs)
  else
    result.input_ports, result.output_ports = inputs, outputs
  end
  (result, port_map)
end

end


struct Vern6Cache{uType,uArrayType,rateType,uEltypeNoUnits,TabType} <: OrdinaryDiffEqMutableCache
  u::uType
  uprev::uType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  k5::rateType
  k6::rateType
  k7::rateType
  k8::rateType
  k9::rateType
  utilde::uArrayType
  tmp::uType
  atmp::uEltypeNoUnits
  tab::TabType
end

u_cache(c::Vern6Cache) = (c.utilde,c.atmp)
du_cache(c::Vern6Cache) = (c.k1,c.k2,c.k3,c.k4,c.k5,c.k6,c.k7,c.k8,c.k9)

function alg_cache(alg::Vern6,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{true}})
  tab = Vern6ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))
  k1 = zero(rate_prototype)
  k2 = zero(rate_prototype); k3 = zero(rate_prototype); k4 = zero(rate_prototype);
  k5 = zero(rate_prototype); k6 = zero(rate_prototype); k7 = zero(rate_prototype);
  k8 = zero(rate_prototype); k9 = zero(rate_prototype);
  utilde = similar(u,axes(u)); tmp = similar(u); atmp = similar(u,uEltypeNoUnits,axes(u));
  Vern6Cache(u,uprev,k1,k2,k3,k4,k5,k6,k7,k8,k9,utilde,tmp,atmp,tab)
end

alg_cache(alg::Vern6,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{false}}) = Vern6ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))

struct Vern7Cache{uType,uArrayType,rateType,uEltypeNoUnits,TabType} <: OrdinaryDiffEqMutableCache
  u::uType
  uprev::uType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  k5::rateType
  k6::rateType
  k7::rateType
  k8::rateType
  k9::rateType
  k10::rateType
  utilde::uArrayType
  tmp::uType
  atmp::uEltypeNoUnits
  tab::TabType
end

u_cache(c::Vern7Cache) = (c.utilde,c.atmp)
du_cache(c::Vern7Cache) = (c.k1,c.k2,c.k3,c.k4,c.k5,c.k6,c.k7,c.k8,c.k9,c.k10)

function alg_cache(alg::Vern7,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{true}})
  tab = Vern7ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))
  k1 = zero(rate_prototype); k2 = zero(rate_prototype); k3 = zero(rate_prototype); k4 = zero(rate_prototype);
  k5 = zero(rate_prototype); k6 = zero(rate_prototype); k7 = zero(rate_prototype); k8 = zero(rate_prototype);
  k9 = zero(rate_prototype); k10 = zero(rate_prototype); utilde = similar(u,axes(u))
  tmp = similar(u); atmp = similar(u,uEltypeNoUnits,axes(u))
  Vern7Cache(u,uprev,k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,utilde,tmp,atmp,tab)
end

alg_cache(alg::Vern7,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{false}}) = Vern7ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))

struct Vern8Cache{uType,uArrayType,rateType,uEltypeNoUnits,TabType} <: OrdinaryDiffEqMutableCache
  u::uType
  uprev::uType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  k5::rateType
  k6::rateType
  k7::rateType
  k8::rateType
  k9::rateType
  k10::rateType
  k11::rateType
  k12::rateType
  k13::rateType
  utilde::uArrayType
  tmp::uType
  atmp::uEltypeNoUnits
  tab::TabType
end

u_cache(c::Vern8Cache) = (c.utilde,c.atmp)
du_cache(c::Vern8Cache) = (c.k1,c.k2,c.k3,c.k4,c.k5,c.k6,c.k7,c.k8,c.k9,c.k10,c.k11,c.k12,c.k13)

function alg_cache(alg::Vern8,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{true}})
  tab = Vern8ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))
  k1 = zero(rate_prototype); k2 = zero(rate_prototype); k3 = zero(rate_prototype);
  k4 = zero(rate_prototype);
  k5 = zero(rate_prototype); k6 = zero(rate_prototype); k7 = zero(rate_prototype);
  k8 = zero(rate_prototype); tmp = similar(u)
  k9 = zero(rate_prototype); k10 = zero(rate_prototype); k11 = zero(rate_prototype);
  k12 = zero(rate_prototype); k13 = zero(rate_prototype)
  utilde = similar(u,axes(u))
  atmp = similar(u,uEltypeNoUnits,axes(u))
  Vern8Cache(u,uprev,k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,utilde,tmp,atmp,tab)
end

alg_cache(alg::Vern8,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{false}}) = Vern8ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))

struct Vern9Cache{uType,uArrayType,rateType,uEltypeNoUnits,TabType} <: OrdinaryDiffEqMutableCache
  u::uType
  uprev::uType
  k1::rateType
  k2::rateType
  k3::rateType
  k4::rateType
  k5::rateType
  k6::rateType
  k7::rateType
  k8::rateType
  k9::rateType
  k10::rateType
  k11::rateType
  k12::rateType
  k13::rateType
  k14::rateType
  k15::rateType
  k16::rateType
  utilde::uArrayType
  tmp::uType
  atmp::uEltypeNoUnits
  tab::TabType
end

u_cache(c::Vern9Cache) = (c.utilde,c.atmp)
du_cache(c::Vern9Cache) = (c.k1,c.k2,c.k3,c.k4,c.k5,c.k6,c.k7,c.k8,c.k9,c.k10,c.k11,c.k12,c.k13,c.k14,c.k15,c.k16)

function alg_cache(alg::Vern9,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{true}})
  tab = Vern9ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))
  k1 = zero(rate_prototype); k2 = zero(rate_prototype);k3 = zero(rate_prototype);
  k4 = zero(rate_prototype);
  k5 = zero(rate_prototype); k6 = zero(rate_prototype);k7 = zero(rate_prototype);
  k8 = zero(rate_prototype);
  k9 = zero(rate_prototype); k10 = zero(rate_prototype); k11 = zero(rate_prototype);
  k12 = zero(rate_prototype);
  k13 = zero(rate_prototype); k14 = zero(rate_prototype); k15 = zero(rate_prototype);
  k16 =zero(rate_prototype);
  utilde = similar(u,axes(u)); tmp = similar(u); atmp = similar(u,uEltypeNoUnits,axes(u));
  Vern9Cache(u,uprev,k1,k2,k3,k4,k5,k6,k7,k8,k9,k10,k11,k12,k13,k14,k15,k16,utilde,tmp,atmp,tab)
end

alg_cache(alg::Vern9,u,rate_prototype,uEltypeNoUnits,uBottomEltypeNoUnits,tTypeNoUnits,uprev,uprev2,f,t,dt,reltol,p,calck,::Type{Val{false}}) = Vern9ConstantCache(real(uBottomEltypeNoUnits),real(tTypeNoUnits))


# ---------------------------------------------------------------------------- #
#
#   problem.jl
#
#   Type for problem definition
#
#   λυτέος
#   Fall 2017
#
#   Max Opgenoord
#
# ---------------------------------------------------------------------------- #

"""
    Problem

Problem type:
Used as placeholder for information about the problem,
e.g name, source function, and boundary conditions.
"""
abstract type Problem

end

include("CDR.jl")
include("Elas.jl")


module VisitingICF2021
    include("Krav.jl")
    include("Wignerd.jl")
    include("LagKrav.jl")
    include("ImageAcq.jl")
    include("ImgRots.jl")
end


# Test constraint deletion
@testset "JuMP.delete (Constraints)" begin
    # initialize model and references
    m = InfiniteModel()
    @infinite_parameter(m, 0 <= par <= 1)
    @infinite_variable(m, inf(par))
    @point_variable(m, inf(0.5), pt)
    @hold_variable(m, x)
    data = DiscreteMeasureData(par, [1], [1])
    meas = measure(inf + par - x, data)
    rv = ReducedInfiniteVariableRef(m, 42)
    @constraint(m, cref, par - inf + pt + 2x - rv + meas <= 1)
    # test normal deletion
    @test isa(delete(m, cref), Nothing)
    @test !is_valid(m, cref)
    @test !haskey(m.param_to_constrs, JuMP.index(par))
    @test !haskey(m.var_to_constrs, JuMP.index(inf))
    @test !haskey(m.var_to_constrs, JuMP.index(pt))
    @test !haskey(m.var_to_constrs, JuMP.index(x))
    @test !haskey(m.reduced_to_constrs, JuMP.index(rv))
    @test !haskey(m.meas_to_constrs, JuMP.index(meas))
    @test !haskey(m.constrs, JuMP.index(cref))
    @test !haskey(m.constr_to_name, JuMP.index(cref))
    @test !haskey(m.constr_in_var_info, JuMP.index(cref))
    # test assertion error
    @test_throws AssertionError delete(m, cref)
end

# Test deleting variable information
@testset "Variable Information" begin
    # initialize model and references
    m = InfiniteModel()
    @infinite_parameter(m, 0 <= par <= 1)
    @infinite_variable(m, 0 <= inf1(par) <= 1, Bin)
    @infinite_variable(m, inf2(par) == 1, Int)
    @point_variable(m, inf1(0.5), pt1)
    @point_variable(m, inf2(0.5), pt2)
    @hold_variable(m, 0 <= gb1 <= 1, Bin)
    @hold_variable(m, gb2 == 1, Int)
    # test delete_lower_bound
    @testset "JuMP.delete_lower_bound" begin
        # test with infinite variable
        @test has_lower_bound(inf1)
        @test isa(delete_lower_bound(inf1), Nothing)
        @test !has_lower_bound(inf1)
        @test_throws ErrorException delete_lower_bound(inf2)
        # test with point variable
        @test has_lower_bound(pt1)
        @test isa(delete_lower_bound(pt1), Nothing)
        @test !has_lower_bound(pt1)
        @test_throws ErrorException delete_lower_bound(pt2)
        # test with hold variable
        @test has_lower_bound(gb1)
        @test isa(delete_lower_bound(gb1), Nothing)
        @test !has_lower_bound(gb1)
        @test_throws ErrorException delete_lower_bound(gb2)
    end
    # test delete_upper_bound
    @testset "JuMP.delete_upper_bound" begin
        # test with infinite variable
        @test has_upper_bound(inf1)
        @test isa(delete_upper_bound(inf1), Nothing)
        @test !has_upper_bound(inf1)
        @test_throws ErrorException delete_upper_bound(inf2)
        # test with point variable
        @test has_upper_bound(pt1)
        @test isa(delete_upper_bound(pt1), Nothing)
        @test !has_upper_bound(pt1)
        @test_throws ErrorException delete_upper_bound(pt2)
        # test with hold variable
        @test has_upper_bound(gb1)
        @test isa(delete_upper_bound(gb1), Nothing)
        @test !has_upper_bound(gb1)
        @test_throws ErrorException delete_upper_bound(gb2)
    end
    # test unfix
    @testset "JuMP.unfix" begin
        # test with infinite variable
        @test is_fixed(inf2)
        @test isa(unfix(inf2), Nothing)
        @test !is_fixed(inf2)
        @test_throws ErrorException unfix(inf1)
        # test with point variable
        @test is_fixed(pt2)
        @test isa(unfix(pt2), Nothing)
        @test !is_fixed(pt2)
        @test_throws ErrorException unfix(pt1)
        # test with hold variable
        @test is_fixed(gb2)
        @test isa(unfix(gb2), Nothing)
        @test !is_fixed(gb2)
        @test_throws ErrorException unfix(gb1)
    end
    # test unset_binary
    @testset "JuMP.unset_binary" begin
        # test with infinite variable
        @test is_binary(inf1)
        @test isa(unset_binary(inf1), Nothing)
        @test !is_binary(inf1)
        @test_throws ErrorException unset_binary(inf2)
        # test with point variable
        @test is_binary(pt1)
        @test isa(unset_binary(pt1), Nothing)
        @test !is_binary(pt1)
        @test_throws ErrorException unset_binary(pt2)
        # test with hold variable
        @test is_binary(gb1)
        @test isa(unset_binary(gb1), Nothing)
        @test !is_binary(gb1)
        @test_throws ErrorException unset_binary(gb2)
    end
    # test unset_integer
    @testset "JuMP.unset_integer" begin
        # test with infinite variable
        @test is_integer(inf2)
        @test isa(unset_integer(inf2), Nothing)
        @test !is_integer(inf2)
        @test_throws ErrorException unset_integer(inf1)
        # test with point variable
        @test is_integer(pt2)
        @test isa(unset_integer(pt2), Nothing)
        @test !is_integer(pt2)
        @test_throws ErrorException unset_integer(pt1)
        # test with hold variable
        @test is_integer(gb2)
        @test isa(unset_integer(gb2), Nothing)
        @test !is_integer(gb2)
        @test_throws ErrorException unset_integer(gb1)
    end
end

# Test parameter deletion
@testset "Parameters" begin
    # initialize model and references
    m = InfiniteModel()
    @infinite_parameter(m, 0 <= par <= 1)
    @infinite_parameter(m, 0 <= pars[1:2] <= 1, container = SparseAxisArray)
    @infinite_parameter(m, 0 <= par2 <= 1)
    @infinite_parameter(m, 0 <= par3 <= 1)
    @infinite_variable(m, inf(par))
    @infinite_variable(m, inf2(par, par2))
    @infinite_variable(m, inf3(par, pars))
    @infinite_variable(m, inf4(par, par2, pars))
    @point_variable(m, inf(0.5), pt)
    pt2 = @point_variable(m, inf2(0.5, 0.5))
    @point_variable(m, inf3(0, [0, 0]), pt3)
    @hold_variable(m, x)
    m.reduced_info[-1] = ReducedInfiniteInfo(inf4, Dict(2 => 0.5))
    m.infinite_to_reduced[JuMP.index(inf4)] = [-1]
    rv = ReducedInfiniteVariableRef(m, -1)
    data = DiscreteMeasureData(par, [1], [1])
    data2 = DiscreteMeasureData(pars, [1], [[1, 1]])
    meas = measure(inf + par - x + rv, data)
    meas2 = measure(inf3 - x + pt2, data2)
    meas3 = measure(inf2 + par2, data)
    meas4 = add_measure(m, Measure(par3, data))
    meas5 = measure(pars[2] + par, data)
    @constraint(m, con, inf2 + inf4 - par2 <= 0)
    m.constrs[-1] = ScalarConstraint(par3, MOI.GreaterThan(0.))
    m.param_to_constrs[JuMP.index(par3)] = [-1]
    con2 = InfiniteConstraintRef(m, -1, JuMP.shape(m.constrs[-1]))
    set_name(con2, "")

    # test _check_param_in_data (scalar)
    @testset "_check_param_in_data (Scalar)" begin
        @test_throws ErrorException InfiniteOpt._check_param_in_data(par, data)
        @test isa(InfiniteOpt._check_param_in_data(par2, data), Nothing)
    end
    # test _check_param_in_data (array)
    @testset "_check_param_in_data (Array)" begin
        @test_throws ErrorException InfiniteOpt._check_param_in_data(pars[1],
                                                                     data2)
        @test isa(InfiniteOpt._check_param_in_data(par, data2), Nothing)
    end
    # test _check_param_in_data (fallback)
    @testset "_check_param_in_data (Fallback)" begin
        @test_throws ErrorException InfiniteOpt._check_param_in_data(par, BadData())
    end
    # test _contains_pref (scalar)
    @testset "_contains_pref (Scalar)" begin
        @test InfiniteOpt._contains_pref(par, par)
        @test !InfiniteOpt._contains_pref(par, par2)
    end
    # test _contains_pref (array)
    @testset "_contains_pref (Array)" begin
        @test InfiniteOpt._contains_pref(pars, pars[1])
        @test !InfiniteOpt._contains_pref(pars, par)
    end
    # test _remove_parameter
    @testset "_remove_parameter" begin
        # test removing single element
        @test InfiniteOpt._remove_parameter((par,), par) == ((), (1,))
        @test InfiniteOpt._remove_parameter((par2, par), par) == ((par2,), (2,))
        # test removing element from array
        new_pars = JuMP.Containers.SparseAxisArray(Dict((2,) => pars[2]))
        @test InfiniteOpt._remove_parameter((pars,),
                                            pars[1]) == ((new_pars,), (1, (1,)))
        @test InfiniteOpt._remove_parameter((par, pars),
                                        pars[1]) == ((par, new_pars), (2, (1,)))
        # test removing from array that becomes empty
        @test InfiniteOpt._remove_parameter((new_pars,), pars[2]) == ((), (1,))
        @test InfiniteOpt._remove_parameter((par, new_pars),
                                            pars[2]) == ((par,), (2,))
    end
    # test _update_infinite_variable
    @testset "_update_infinite_variable" begin
        # test updating when used by measure
        @test isa(InfiniteOpt._update_infinite_variable(inf3, (pars,)), Nothing)
        @test parameter_refs(inf3) == (pars,)
        @test name(inf3) == "inf3(pars)"
        @test name(meas2) == "measure(inf3(pars) - x + inf2(0.5, 0.5))"
        # Undo the changes
        @test isa(InfiniteOpt._update_infinite_variable(inf3, (par, pars)),
                  Nothing)
        @test parameter_refs(inf3) == (par, pars)
        @test name(inf3) == "inf3(par, pars)"
        @test name(meas2) == "measure(inf3(par, pars) - x + inf2(0.5, 0.5))"
    end
    # test _remove_parameter_values
    @testset "_remove_parameter_values" begin
        # test remove single element
        @test InfiniteOpt._remove_parameter_values((0, 1), (1,)) == (1,)
        @test InfiniteOpt._remove_parameter_values((0, 1), (2,)) == (0,)
        @test InfiniteOpt._remove_parameter_values(([0, 0],), (1,)) == ()
        # test remove array element
        arr = convert(JuMP.Containers.SparseAxisArray, [0, 1, 0])
        arr2 = JuMP.Containers.SparseAxisArray(Dict((1,) => 0, (2,) => 1,
                                                    (3,) => 0))
        filter!(x -> x.first != (2,), arr2.data)
        @test InfiniteOpt._remove_parameter_values((arr, 3), (1, (2,))) == (arr2, 3)
        arr3 = JuMP.Containers.SparseAxisArray(filter(x -> x.first != (1,),
                                                      arr2.data))
        @test InfiniteOpt._remove_parameter_values((arr2,), (1, (1,))) == (arr3,)
    end
    # test _update_point_variable
    @testset "_update_point_variable" begin
        # test point variable with single parameter
        @test isa(InfiniteOpt._update_point_variable(pt, ()), Nothing)
        @test parameter_values(pt) == ()
        @test name(pt) == "pt"
        # Undo changes
        @test isa(InfiniteOpt._update_point_variable(pt, (0.5,)), Nothing)
        @test parameter_values(pt) == (0.5,)
        @test name(pt) == "pt"
        # test point variable with multiple parameters
        @test isa(InfiniteOpt._update_point_variable(pt2, (0.5,)), Nothing)
        @test parameter_values(pt2) == (0.5,)
        @test name(pt2) == "inf2(0.5)"
        @test name(meas2) == "measure(inf3(par, pars) - x + inf2(0.5))"
        # Undo changes
        @test isa(InfiniteOpt._update_point_variable(pt2, (0.5, 0.5)), Nothing)
        @test parameter_values(pt2) == (0.5, 0.5)
        @test name(pt2) == "inf2(0.5, 0.5)"
        @test name(meas2) == "measure(inf3(par, pars) - x + inf2(0.5, 0.5))"
    end
    # test _update_reduced_variable
    @testset "_update_reduced_variable" begin
        # test removing single parameter that is not reduced
        @test isa(InfiniteOpt._update_infinite_variable(inf4, (par2, pars)),
                                                        Nothing)
        @test isa(InfiniteOpt._update_reduced_variable(rv, (1, )), Nothing)
        @test m.reduced_info[JuMP.index(rv)].infinite_variable_ref == inf4
        @test m.reduced_info[JuMP.index(rv)].eval_supports == Dict(1 => 0.5)
        @test name(rv) == "inf4(0.5, pars)"
        @test name(meas) == "measure(inf(par) + par - x + inf4(0.5, pars))"
        # Undo changes
        m.reduced_info[-1] = ReducedInfiniteInfo(inf4, Dict(2 => 0.5))
        @test isa(InfiniteOpt._update_infinite_variable(inf4, (par, par2, pars)),
                                                        Nothing)
        # test removing single parameter that is reduced
        @test isa(InfiniteOpt._update_infinite_variable(inf4, (par, pars)),
                                                        Nothing)
        @test isa(InfiniteOpt._update_reduced_variable(rv, (2, )), Nothing)
        @test m.reduced_info[JuMP.index(rv)].infinite_variable_ref == inf4
        @test m.reduced_info[JuMP.index(rv)].eval_supports == Dict{Int,
                               Union{Number, JuMP.Containers.SparseAxisArray}}()
        @test name(rv) == "inf4(par, pars)"
        @test name(meas) == "measure(inf(par) + par - x + inf4(par, pars))"
        # Undo changes
        @test isa(InfiniteOpt._update_infinite_variable(inf4, (par, par2, pars)),
                                                        Nothing)
        # test removing a different single parameter
        m.reduced_info[-1] = ReducedInfiniteInfo(inf4, Dict(1 => 0.5, 2 => 0.5))
        @test isa(InfiniteOpt._update_infinite_variable(inf4, (par, pars)),
                                                        Nothing)
        @test isa(InfiniteOpt._update_reduced_variable(rv, (2, )), Nothing)
        @test m.reduced_info[JuMP.index(rv)].infinite_variable_ref == inf4
        @test m.reduced_info[JuMP.index(rv)].eval_supports == Dict(1 => 0.5)
        @test name(rv) == "inf4(0.5, pars)"
        @test name(meas) == "measure(inf(par) + par - x + inf4(0.5, pars))"
        # Undo changes
        m.reduced_info[-1] = ReducedInfiniteInfo(inf4, Dict(2 => 0.5))
        @test isa(InfiniteOpt._update_infinite_variable(inf4, (par, par2, pars)),
                                                        Nothing)
        # prepare for removing array element
        supp = convert(JuMP.Containers.SparseAxisArray, [0, 0])
        m.reduced_info[-1] = ReducedInfiniteInfo(inf4, Dict(2 => 0.5, 3 => supp))
        new_supp = JuMP.Containers.SparseAxisArray(Dict((1,) => 0, (2,) => 0))
        filter!(x -> x.first != (1,), new_supp.data)
        # test removing array element
        @test isa(InfiniteOpt._update_reduced_variable(rv, (3, (1,))), Nothing)
        @test m.reduced_info[JuMP.index(rv)].infinite_variable_ref == inf4
        @test m.reduced_info[JuMP.index(rv)].eval_supports == Dict(2 => 0.5, 3 => new_supp)
        @test name(rv) == "inf4(par, 0.5, " * string(new_supp) * ")"
        @test name(meas) == "measure(inf(par) + par - x + " * name(rv) * ")"
        # Undo changes
        m.reduced_info[-1] = ReducedInfiniteInfo(inf4, Dict(2 => 0.5))
        @test isa(InfiniteOpt._update_infinite_variable(inf4, (par, par2, pars)),
                                                        Nothing)
    end
    # test JuMP.delete for parameters
    @testset "JuMP.delete (Parameters)" begin
        # test normal usage
        idx = JuMP.index(par2)
        @test isa(delete(m, par2), Nothing)
        @test name(meas3) == "measure(inf2(par))"
        @test !haskey(m.param_to_meas, idx)
        @test parameter_refs(inf2) == (par,)
        @test parameter_refs(inf4) == (par, pars)
        @test parameter_values(pt2) == (0.5,)
        @test parameter_refs(rv) == (par, pars)
        @test name(inf2) == "inf2(par)"
        @test name(inf4) == "inf4(par, pars)"
        @test name(pt2) == "inf2(0.5)"
        @test name(rv) == "inf4(par, pars)"
        @test !haskey(m.param_to_vars, idx)
        @test string(m.constrs[JuMP.index(con)].func) == "inf2(par) + " *
                                                         "inf4(par, pars)"
        @test !haskey(m.param_to_constrs, idx)
        @test !haskey(m.params, idx)
        @test !haskey(m.param_to_name, idx)
        @test !haskey(m.param_to_group_id, idx)
        # test special measure case with single parameter (possible through
        # multiple deletions) and with single parameter in constraint
        idx = JuMP.index(par3)
        @test isa(delete(m, par3), Nothing)
        @test name(meas4) == "measure(0)"
        @test !haskey(m.param_to_meas, idx)
        @test string(m.constrs[JuMP.index(con2)].func) == "0"
        @test !haskey(m.param_to_constrs, idx)
        @test !haskey(m.params, idx)
        @test !haskey(m.param_to_name, idx)
        @test !haskey(m.param_to_group_id, idx)
        # test invalid parameter
        @test_throws AssertionError delete(m, par2)
        @test_throws AssertionError delete(m, par3)
        # test measure data check
        @test_throws ErrorException delete(m, par)
        @test_throws ErrorException delete(m, pars[1])
        # test array element deletion
        delete!(m.measures, JuMP.index(meas2))
        m.param_to_meas[JuMP.index(pars[2])] = [JuMP.index(meas5)]
        delete!(m.var_to_meas, JuMP.index(inf3))
        idx = JuMP.index(pars[2])
        @test isa(delete(m, pars[2]), Nothing)
        @test name(meas5) == "measure(par)"
        @test !haskey(m.param_to_meas, idx)
        new_pars = convert(JuMP.Containers.SparseAxisArray, [pars[1]])
        @test parameter_refs(inf3) == (par, new_pars)
        @test parameter_refs(inf4) == (par, new_pars)
        new_vals = convert(JuMP.Containers.SparseAxisArray, [0])
        @test parameter_values(pt3) == (0, new_vals)
        @test parameter_refs(rv) == (par, new_pars)
        @test name(inf3) == "inf3(par, pars)"
        @test name(inf4) == "inf4(par, pars)"
        @test name(pt3) == "pt3"
        @test name(rv) == "inf4(par, pars)"
        @test !haskey(m.param_to_vars, idx)
        @test string(m.constrs[JuMP.index(con)].func) == "inf2(par) + " *
                                                         "inf4(par, pars)"
        @test !haskey(m.param_to_constrs, idx)
        @test !haskey(m.params, idx)
        @test !haskey(m.param_to_name, idx)
        @test !haskey(m.param_to_group_id, idx)
    end
 end

 # Test reduced variable deletion
 @testset "JuMP.delete (Reduced Variables)" begin
     # intialize the model
     m = InfiniteModel()
     @infinite_parameter(m, 0 <= par <= 1)
     @infinite_parameter(m, 0 <= par2 <= 1)
     @infinite_variable(m, inf(par, par2))
     @point_variable(m, inf(0.5, 0.5), pt)
     @hold_variable(m, x)
     m.reduced_info[-1] = ReducedInfiniteInfo(inf, Dict(2 => 0.5))
     m.infinite_to_reduced[JuMP.index(inf)] = [-1]
     rv = ReducedInfiniteVariableRef(m, -1)
     m.reduced_info[-2] = ReducedInfiniteInfo(inf, Dict(2 => 0.5))
     m.infinite_to_reduced[JuMP.index(inf)] = [-2]
     rv2 = ReducedInfiniteVariableRef(m, -2)
     data = DiscreteMeasureData(par, [1], [1])
     meas = measure(inf + par - x + rv, data)
     meas2 = measure(rv2, data)
     @constraint(m, con, x + rv <= 0)
     m.constrs[-1] = ScalarConstraint(rv2, MOI.GreaterThan(0.))
     m.reduced_to_constrs[JuMP.index(rv2)] = [-1]
     con2 = InfiniteConstraintRef(m, -1, JuMP.shape(m.constrs[-1]))
     set_name(con2, "")
     # test normal deletion
     @test isa(delete(m, rv), Nothing)
     @test name(meas) == "measure(inf(par, par2) + par - x)"
     @test !haskey(m.reduced_to_meas, JuMP.index(rv))
     @test string(m.constrs[JuMP.index(con)].func) == "x"
     @test !haskey(m.reduced_to_constrs, JuMP.index(rv))
     @test m.infinite_to_reduced[JuMP.index(inf)] == [JuMP.index(rv2)]
     @test !haskey(m.reduced_info, JuMP.index(rv))
     # test deletion of special cases
     @test isa(delete(m, rv2), Nothing)
     @test name(meas2) == "measure(0)"
     @test !haskey(m.reduced_to_meas, JuMP.index(rv2))
     @test string(m.constrs[JuMP.index(con2)].func) == "0"
     @test !haskey(m.reduced_to_constrs, JuMP.index(rv2))
     @test !haskey(m.infinite_to_reduced, JuMP.index(inf))
     @test !haskey(m.reduced_info, JuMP.index(rv2))
     # test error
     @test_throws AssertionError delete(m, rv)
     @test_throws AssertionError delete(m, rv2)
 end

 # Test variable deletion
 @testset "JuMP.delete (Hold Variables)" begin
     # intialize the model
     m = InfiniteModel()
     @infinite_parameter(m, 0 <= par <= 1)
     @hold_variable(m, 0 <= x <= 1, Bin)
     @hold_variable(m, y == 1, Int)
     data = DiscreteMeasureData(par, [1], [1])
     meas1 = measure(x + y + par, data)
     meas2 = add_measure(m, Measure(y, data))
     @constraint(m, con1, x + y + par <= 0)
     con2 = add_constraint(m, ScalarConstraint(y, MOI.LessThan(0.)))
     @objective(m, Min, x + y)
     # test deletion of x
     @test isa(delete(m, x), Nothing)
     @test !haskey(m.var_to_lower_bound, JuMP.index(x))
     @test !haskey(m.var_to_upper_bound, JuMP.index(x))
     @test !haskey(m.var_to_zero_one, JuMP.index(x))
     @test name(meas1) == "measure(y + par)"
     @test !haskey(m.var_to_meas, JuMP.index(x))
     @test string(m.constrs[JuMP.index(con1)].func) == "y + par"
     @test !haskey(m.var_to_constrs, JuMP.index(x))
     @test string(objective_function(m)) == "y"
     @test !haskey(m.var_in_objective, JuMP.index(x))
     @test !haskey(m.vars, JuMP.index(x))
     @test !haskey(m.var_to_name, JuMP.index(x))
     # test deletion of y
     set_objective_function(m, y)
     @test isa(delete(m, y), Nothing)
     @test !haskey(m.var_to_fix, JuMP.index(y))
     @test !haskey(m.var_to_integrality, JuMP.index(y))
     @test name(meas1) == "measure(par)"
     @test name(meas2) == "measure(0)"
     @test !haskey(m.var_to_meas, JuMP.index(y))
     @test string(m.constrs[JuMP.index(con1)].func) == "par"
     @test string(m.constrs[JuMP.index(con2)].func) == "0"
     @test !haskey(m.var_to_constrs, JuMP.index(y))
     @test string(objective_function(m)) == "0"
     @test !haskey(m.var_in_objective, JuMP.index(y))
     @test !haskey(m.vars, JuMP.index(y))
     @test !haskey(m.var_to_name, JuMP.index(y))
     # test errors
     @test_throws AssertionError delete(m, x)
     @test_throws AssertionError delete(m, y)
 end

 # Test variable deletion
 @testset "JuMP.delete (Point Variables)" begin
     # intialize the model
     m = InfiniteModel()
     @infinite_parameter(m, 0 <= par <= 1)
     @infinite_variable(m, inf(par))
     @point_variable(m, inf(0), 0 <= x <= 1, Bin)
     @point_variable(m, inf(1), y == 1, Int)
     data = DiscreteMeasureData(par, [1], [1])
     meas1 = measure(x + y + par, data)
     meas2 = add_measure(m, Measure(y, data))
     @constraint(m, con1, x + y + par <= 0)
     con2 = add_constraint(m, ScalarConstraint(y, MOI.LessThan(0.)))
     @objective(m, Min, x + y)
     # test deletion of x
     @test isa(delete(m, x), Nothing)
     @test !haskey(m.var_to_lower_bound, JuMP.index(x))
     @test !haskey(m.var_to_upper_bound, JuMP.index(x))
     @test !haskey(m.var_to_zero_one, JuMP.index(x))
     @test name(meas1) == "measure(y + par)"
     @test !haskey(m.var_to_meas, JuMP.index(x))
     @test string(m.constrs[JuMP.index(con1)].func) == "y + par"
     @test !haskey(m.var_to_constrs, JuMP.index(x))
     @test string(objective_function(m)) == "y"
     @test m.infinite_to_points[JuMP.index(inf)] == [JuMP.index(y)]
     @test !haskey(m.var_in_objective, JuMP.index(x))
     @test !haskey(m.vars, JuMP.index(x))
     @test !haskey(m.var_to_name, JuMP.index(x))
     # test deletion of y
     set_objective_function(m, y)
     @test isa(delete(m, y), Nothing)
     @test !haskey(m.var_to_fix, JuMP.index(y))
     @test !haskey(m.var_to_integrality, JuMP.index(y))
     @test name(meas1) == "measure(par)"
     @test name(meas2) == "measure(0)"
     @test !haskey(m.var_to_meas, JuMP.index(y))
     @test string(m.constrs[JuMP.index(con1)].func) == "par"
     @test string(m.constrs[JuMP.index(con2)].func) == "0"
     @test !haskey(m.var_to_constrs, JuMP.index(y))
     @test string(objective_function(m)) == "0"
     @test !haskey(m.infinite_to_points, JuMP.index(inf))
     @test !haskey(m.var_in_objective, JuMP.index(y))
     @test !haskey(m.vars, JuMP.index(y))
     @test !haskey(m.var_to_name, JuMP.index(y))
     # test errors
     @test_throws AssertionError delete(m, x)
     @test_throws AssertionError delete(m, y)
 end

 # Test variable deletion
 @testset "JuMP.delete (Infinite Variables)" begin
     # intialize the model
     m = InfiniteModel()
     @infinite_parameter(m, 0 <= par <= 1)
     @infinite_variable(m, 0 <= x(par) <= 1, Bin)
     @infinite_variable(m, y(par) == 1, Int)
     @point_variable(m, x(0), x0)
     m.reduced_info[-1] = ReducedInfiniteInfo(x, Dict(1 => 0.5))
     m.infinite_to_reduced[JuMP.index(x)] = [-1]
     rv = ReducedInfiniteVariableRef(m, -1)
     data = DiscreteMeasureData(par, [1], [1])
     meas1 = measure(x + y + par, data)
     meas2 = add_measure(m, Measure(y, data))
     @constraint(m, con1, x + y + par <= 0)
     con2 = add_constraint(m, ScalarConstraint(y, MOI.LessThan(0.)))
     # test deletion of x
     @test isa(delete(m, x), Nothing)
     @test !haskey(m.var_to_lower_bound, JuMP.index(x))
     @test !haskey(m.var_to_upper_bound, JuMP.index(x))
     @test !haskey(m.var_to_zero_one, JuMP.index(x))
     @test name(meas1) == "measure(y(par) + par)"
     @test !haskey(m.var_to_meas, JuMP.index(x))
     @test string(m.constrs[JuMP.index(con1)].func) == "y(par) + par"
     @test !haskey(m.var_to_constrs, JuMP.index(x))
     @test m.param_to_vars[JuMP.index(par)] == [JuMP.index(y)]
     @test !is_valid(m, x0)
     @test !haskey(m.infinite_to_points, JuMP.index(x))
     @test !is_valid(m, rv)
     @test !haskey(m.infinite_to_reduced, JuMP.index(x))
     @test !haskey(m.var_in_objective, JuMP.index(x))
     @test !haskey(m.vars, JuMP.index(x))
     @test !haskey(m.var_to_name, JuMP.index(x))
     # test deletion of y
     @test isa(delete(m, y), Nothing)
     @test !haskey(m.var_to_fix, JuMP.index(y))
     @test !haskey(m.var_to_integrality, JuMP.index(y))
     @test name(meas1) == "measure(par)"
     @test name(meas2) == "measure(0)"
     @test !haskey(m.var_to_meas, JuMP.index(y))
     @test string(m.constrs[JuMP.index(con1)].func) == "par"
     @test string(m.constrs[JuMP.index(con2)].func) == "0"
     @test !haskey(m.var_to_constrs, JuMP.index(y))
     @test !haskey(m.param_to_vars, JuMP.index(par))
     @test !haskey(m.var_in_objective, JuMP.index(y))
     @test !haskey(m.vars, JuMP.index(y))
     @test !haskey(m.var_to_name, JuMP.index(y))
     # test errors
     @test_throws AssertionError delete(m, x)
     @test_throws AssertionError delete(m, y)
 end

 # Test variable deletion
 @testset "JuMP.delete (Measures)" begin
     # intialize the model
     m = InfiniteModel()
     @infinite_parameter(m, 0 <= par <= 1)
     @infinite_parameter(m, 0 <= par2 <= 1)
     @infinite_parameter(m, 0 <= pars[1:2] <= 1)
     @infinite_variable(m, w(pars))
     @infinite_variable(m, x(par))
     @infinite_variable(m, y(par2))
     @point_variable(m, x(0), x0)
     m.reduced_info[-1] = ReducedInfiniteInfo(x, Dict(1 => 0.5))
     m.infinite_to_reduced[JuMP.index(x)] = [-1]
     rv = ReducedInfiniteVariableRef(m, -1)
     data = DiscreteMeasureData(par, [1], [1])
     data2 = DiscreteMeasureData(pars, [1], [[1, 1]])
     meas = measure(x, data)
     meas1 = measure(x + x0 + rv + par + meas + y + par2, data)
     meas2 = measure(x + x0 + rv + par + y + par2, data)
     meas3 = measure(meas1 + x0, data)
     meas4 = measure(meas2, data)
     meas5 = measure(w, data2)
     @constraint(m, con1, x0 + meas1 <= 0)
     con2 = add_constraint(m, ScalarConstraint(meas2, MOI.LessThan(0.)))
     @objective(m, Min, meas1 + x0)
     # test deletion of meas1
     @test isa(delete(m, meas1), Nothing)
     @test name(meas3) == "measure(x0)"
     @test !haskey(m.meas_to_meas, JuMP.index(meas1))
     @test string(m.constrs[JuMP.index(con1)].func) == "x0"
     @test !haskey(m.meas_to_constrs, JuMP.index(meas1))
     @test string(objective_function(m)) == "x0"
     @test m.var_to_meas[JuMP.index(x)] == [JuMP.index(meas), JuMP.index(meas2)]
     @test m.var_to_meas[JuMP.index(y)] == [JuMP.index(meas2)]
     @test m.param_to_meas[JuMP.index(par)] == [1, 3, 4, 5]
     @test m.param_to_meas[JuMP.index(par2)] == [JuMP.index(meas2)]
     @test !haskey(m.meas_to_meas, JuMP.index(meas))
     @test m.reduced_to_meas[JuMP.index(rv)] == [JuMP.index(meas2)]
     @test !haskey(m.meas_in_objective, JuMP.index(meas1))
     @test !haskey(m.measures, JuMP.index(meas1))
     @test !haskey(m.meas_to_name, JuMP.index(meas1))
     # test deletion of meas2
     set_objective_function(m, meas2)
     @test isa(delete(m, meas2), Nothing)
     @test name(meas4) == "measure(0)"
     @test !haskey(m.meas_to_meas, JuMP.index(meas1))
     @test string(m.constrs[JuMP.index(con2)].func) == "0"
     @test !haskey(m.meas_to_constrs, JuMP.index(meas2))
     @test string(objective_function(m)) == "0"
     @test m.var_to_meas[JuMP.index(x)] == [JuMP.index(meas)]
     @test !haskey(m.var_to_meas, JuMP.index(y))
     @test m.param_to_meas[JuMP.index(par)] == [1, 4, 5]
     @test !haskey(m.param_to_meas, JuMP.index(par2))
     @test !haskey(m.reduced_to_meas, JuMP.index(rv))
     @test !haskey(m.meas_in_objective, JuMP.index(meas2))
     @test !haskey(m.measures, JuMP.index(meas2))
     @test !haskey(m.meas_to_name, JuMP.index(meas2))
     # test deletion of meas5
     @test isa(delete(m, meas5), Nothing)
     @test !haskey(m.param_to_meas, JuMP.index(pars[1]))
     @test !haskey(m.param_to_meas, JuMP.index(pars[2]))
     # test errors
     @test_throws AssertionError delete(m, meas1)
     @test_throws AssertionError delete(m, meas2)
 end


# singleton for indicating if no default arguments are present
struct NoDefault end
const NO_DEFAULT = NoDefault()

"""
    @addlogprob!(ex)

Add the result of the evaluation of `ex` to the joint log probability.

# Examples

This macro allows you to [include arbitrary terms in the likelihood](https://github.com/TuringLang/Turing.jl/issues/1332)

```jldoctest; setup = :(using Distributions)
julia> myloglikelihood(x, μ) = loglikelihood(Normal(μ, 1), x);

julia> @model function demo(x)
           μ ~ Normal()
           @addlogprob! myloglikelihood(x, μ)
       end;

julia> x = [1.3, -2.1];

julia> loglikelihood(demo(x), (μ=0.2,)) ≈ myloglikelihood(x, 0.2)
true
```

and to [reject samples](https://github.com/TuringLang/Turing.jl/issues/1328):

```jldoctest; setup = :(using Distributions, LinearAlgebra)
julia> @model function demo(x)
           m ~ MvNormal(zero(x), I)
           if dot(m, x) < 0
               @addlogprob! -Inf
               # Exit the model evaluation early
               return
           end
           x ~ MvNormal(m, I)
           return
       end;

julia> logjoint(demo([-2.1]), (m=[0.2],)) == -Inf
true
```

!!! note
    The `@addlogprob!` macro increases the accumulated log probability regardless of the evaluation context,
    i.e., regardless of whether you evaluate the log prior, the log likelihood or the log joint density.
    If you would like to avoid this behaviour you should check the evaluation context.
    It can be accessed with the internal variable `__context__`.
    For instance, in the following example the log density is not accumulated when only the log prior is computed:
    ```jldoctest; setup = :(using Distributions)
    julia> myloglikelihood(x, μ) = loglikelihood(Normal(μ, 1), x);

    julia> @model function demo(x)
               μ ~ Normal()
               if DynamicPPL.leafcontext(__context__) !== PriorContext()
                   @addlogprob! myloglikelihood(x, μ)
               end
           end;

    julia> x = [1.3, -2.1];

    julia> logprior(demo(x), (μ=0.2,)) ≈ logpdf(Normal(), 0.2)
    true

    julia> loglikelihood(demo(x), (μ=0.2,)) ≈ myloglikelihood(x, 0.2)
    true
    ```
"""
macro addlogprob!(ex)
    return quote
        $(esc(:(__varinfo__))) = acclogp!!($(esc(:(__varinfo__))), $(esc(ex)))
    end
end

"""
    addargnames!(args)

Adds names to unnamed arguments in `args`.

The names are generated with `gensym(:arg)` to avoid conflicts with other variable names.

# Examples

```jldoctest; filter = r"var\\"##arg#[0-9]+\\""
julia> args = :(f(x::Int, y, ::Type{T}=Float64)).args[2:end]
3-element Vector{Any}:
 :(x::Int)
 :y
 :($(Expr(:kw, :(::Type{T}), :Float64)))

julia> DynamicPPL.addargnames!(args)

julia> args
3-element Vector{Any}:
 :(x::Int)
 :y
 :($(Expr(:kw, :(var"##arg#301"::Type{T}), :Float64)))
```
"""
function addargnames!(args)
    if isempty(args)
        return nothing
    end

    @inbounds for i in eachindex(args)
        arg = args[i]
        if MacroTools.@capture(arg, ::T_)
            args[i] = Expr(:(::), gensym(:arg), T)
        elseif MacroTools.@capture(arg, ::T_ = val_)
            args[i] = Expr(:kw, Expr(:(::), gensym(:arg), T), val)
        end
    end

    return nothing
end

"""
    getargs_dottilde(x)

Return the arguments `L` and `R`, if `x` is an expression of the form `L .~ R` or
`(~).(L, R)`, or `nothing` otherwise.
"""
getargs_dottilde(x) = nothing
function getargs_dottilde(expr::Expr)
    return MacroTools.@match expr begin
        (.~)(L_, R_) => (L, R)
        (~).(L_, R_) => (L, R)
        # Julia 1.6: see https://github.com/TuringLang/Turing.jl/issues/1525
        (L_ .~ R_) => (L, R)
        x_ => nothing
    end
end

"""
    getargs_tilde(x)

Return the arguments `L` and `R`, if `x` is an expression of the form `L ~ R`, or `nothing`
otherwise.
"""
getargs_tilde(x) = nothing
function getargs_tilde(expr::Expr)
    return MacroTools.@match expr begin
        (~)(L_, R_) => (L, R)
        x_ => nothing
    end
end

"""
    getargs_assignment(x)

Return the arguments `L` and `R`, if `x` is an expression of the form `L = R`, or `nothing`
otherwise.
"""
getargs_assignment(x) = nothing
function getargs_assignment(expr::Expr)
    return MacroTools.@match expr begin
        (L_ = R_) => (L, R)
        x_ => nothing
    end
end

function to_namedtuple_expr(syms, vals=syms)
    length(syms) == 0 && return :(NamedTuple())

    names_expr = Expr(:tuple, QuoteNode.(syms)...)
    vals_expr = Expr(:tuple, vals...)
    return :(NamedTuple{$names_expr}($vals_expr))
end

#####################################################
# Helper functions for vectorize/reconstruct values #
#####################################################

vectorize(d::UnivariateDistribution, r::Real) = [r]
vectorize(d::MultivariateDistribution, r::AbstractVector{<:Real}) = copy(r)
vectorize(d::MatrixDistribution, r::AbstractMatrix{<:Real}) = copy(vec(r))

# NOTE:
# We cannot use reconstruct{T} because val is always Vector{Real} then T will be Real.
# However here we would like the result to be specifric type, e.g. Array{Dual{4,Float64}, 2},
# otherwise we will have error for MatrixDistribution.
# Note this is not the case for MultivariateDistribution so I guess this might be lack of
# support for some types related to matrices (like PDMat).
reconstruct(d::Distribution, val::AbstractVector) = reconstruct(size(d), val)
reconstruct(::Tuple{}, val::AbstractVector) = val[1]
reconstruct(s::NTuple{1}, val::AbstractVector) = copy(val)
reconstruct(s::NTuple{2}, val::AbstractVector) = reshape(copy(val), s)
function reconstruct!(r, d::Distribution, val::AbstractVector)
    return reconstruct!(r, d, val)
end
function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector)
    r .= val
    return r
end
function reconstruct(d::Distribution, val::AbstractVector, n::Int)
    return reconstruct(size(d), val, n)
end
function reconstruct(::Tuple{}, val::AbstractVector, n::Int)
    return copy(val)
end
function reconstruct(s::NTuple{1}, val::AbstractVector, n::Int)
    return copy(reshape(val, s[1], n))
end
function reconstruct(s::NTuple{2}, val::AbstractVector, n::Int)
    tmp = reshape(val, s..., n)
    orig = [tmp[:, :, i] for i in 1:n]
    return orig
end
function reconstruct!(r, d::Distribution, val::AbstractVector, n::Int)
    return reconstruct!(r, d, val, n)
end
function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector, n::Int)
    r .= val
    return r
end

# Uniform random numbers with range 4 for robust initializations
# Reference: https://mc-stan.org/docs/2_19/reference-manual/initialization.html
randrealuni(rng::Random.AbstractRNG) = 4 * rand(rng) - 2
randrealuni(rng::Random.AbstractRNG, args...) = 4 .* rand(rng, args...) .- 2

const Transformable = Union{
    PositiveDistribution,
    UnitDistribution,
    TransformDistribution,
    SimplexDistribution,
    PDMatDistribution,
}
istransformable(dist) = false
istransformable(::Transformable) = true

#################################
# Single-sample initialisations #
#################################

inittrans(rng, dist::UnivariateDistribution) = invlink(dist, randrealuni(rng))
function inittrans(rng, dist::MultivariateDistribution)
    return invlink(dist, randrealuni(rng, size(dist)[1]))
end
inittrans(rng, dist::MatrixDistribution) = invlink(dist, randrealuni(rng, size(dist)...))

################################
# Multi-sample initialisations #
################################

inittrans(rng, dist::UnivariateDistribution, n::Int) = invlink(dist, randrealuni(rng, n))
function inittrans(rng, dist::MultivariateDistribution, n::Int)
    return invlink(dist, randrealuni(rng, size(dist)[1], n))
end
function inittrans(rng, dist::MatrixDistribution, n::Int)
    return invlink(dist, [randrealuni(rng, size(dist)...) for _ in 1:n])
end

#######################
# Convenience methods #
#######################
collectmaybe(x) = x
collectmaybe(x::Base.AbstractSet) = collect(x)

#######################
# BangBang.jl related #
#######################
function set!!(obj, lens::Setfield.Lens, value)
    lensmut = BangBang.prefermutation(lens)
    return Setfield.set(obj, lensmut, value)
end
function set!!(obj, vn::VarName{sym}, value) where {sym}
    lens = BangBang.prefermutation(Setfield.PropertyLens{sym}() ∘ AbstractPPL.getlens(vn))
    return Setfield.set(obj, lens, value)
end

#############################
# AbstractPPL.jl extensions #
#############################
# This is preferable to `haskey` because the order of arguments is different, and
# we're more likely to specialize on the key in these settings rather than the container.
# TODO: I'm not sure about this name.
"""
    canview(lens, container)

Return `true` if `lens` can be used to view `container`, and `false` otherwise.

# Examples
```jldoctest; setup=:(using Setfield; using DynamicPPL: canview)
julia> canview(@lens(_.a), (a = 1.0, ))
true

julia> canview(@lens(_.a), (b = 1.0, )) # property `a` does not exist
false

julia> canview(@lens(_.a[1]), (a = [1.0, 2.0], ))
true

julia> canview(@lens(_.a[3]), (a = [1.0, 2.0], )) # out of bounds
false
```
"""
canview(lens, container) = false
canview(::Setfield.IdentityLens, _) = true
function canview(lens::Setfield.PropertyLens{field}, x) where {field}
    return hasproperty(x, field)
end

# `IndexLens`: only relevant if `x` supports indexing.
canview(lens::Setfield.IndexLens, x) = false
canview(lens::Setfield.IndexLens, x::AbstractArray) = checkbounds(Bool, x, lens.indices...)

# `ComposedLens`: check that we can view `.outer` and `.inner`, but using
# value extracted using `.outer`.
function canview(lens::Setfield.ComposedLens, x)
    return canview(lens.outer, x) && canview(lens.inner, get(x, lens.outer))
end

"""
    parent(vn::VarName)

Return the parent `VarName`.

# Examples
```julia-repl; setup=:(using DynamicPPL: parent)
julia> parent(@varname(x.a[1]))
x.a

julia> (parent ∘ parent)(@varname(x.a[1]))
x

julia> (parent ∘ parent ∘ parent)(@varname(x.a[1]))
x
```
"""
function parent(vn::VarName)
    p = parent(getlens(vn))
    return p === nothing ? VarName(vn, Setfield.IdentityLens()) : VarName(vn, p)
end

"""
    parent(lens::Setfield.Lens)

Return the parent lens. If `lens` doesn't have a parent,
`nothing` is returned.

See also: [`parent_and_child`].

# Examples
```jldoctest; setup=:(using Setfield; using DynamicPPL: parent)
julia> parent(@lens(_.a[1]))
(@lens _.a)

julia> # Parent of lens without parents results in `nothing`.
       (parent ∘ parent)(@lens(_.a[1])) === nothing
true
```
"""
parent(lens::Setfield.Lens) = first(parent_and_child(lens))

"""
    parent_and_child(lens::Setfield.Lens)

Return a 2-tuple of lenses `(parent, child)` where `parent` is the
parent lens of `lens` and `child` is the child lens of `lens`.

If `lens` does not have a parent, we return `(nothing, lens)`.

See also: [`parent`].

# Examples
```jldoctest; setup=:(using Setfield; using DynamicPPL: parent_and_child)
julia> parent_and_child(@lens(_.a[1]))
((@lens _.a), (@lens _[1]))

julia> parent_and_child(@lens(_.a))
(nothing, (@lens _.a))
```
"""
parent_and_child(lens::Setfield.Lens) = (nothing, lens)
function parent_and_child(lens::Setfield.ComposedLens)
    p, child = parent_and_child(lens.inner)
    parent = p === nothing ? lens.outer : lens.outer ∘ p
    return parent, child
end

"""
    splitlens(condition, lens)

Return a 3-tuple `(parent, child, issuccess)` where, if `issuccess` is `true`,
`parent` is a lens such that `condition(parent)` is `true` and `parent ∘ child == lens`.

If `issuccess` is `false`, then no such split could be found.

# Examples
```jldoctest; setup=:(using Setfield; using DynamicPPL: splitlens)
julia> p, c, issucesss = splitlens(@lens(_.a[1])) do parent
           # Succeeds!
           parent == @lens(_.a)
       end
((@lens _.a), (@lens _[1]), true)

julia> p ∘ c
(@lens _.a[1])

julia> splitlens(@lens(_.a[1])) do parent
           # Fails!
           parent == @lens(_.b)
       end
(nothing, (@lens _.a[1]), false)
```
"""
function splitlens(condition, lens)
    current_parent, current_child = parent_and_child(lens)
    # We stop if either a) `condition` is satisfied, or b) we reached the root.
    while !condition(current_parent) && current_parent !== nothing
        current_parent, c = parent_and_child(current_parent)
        current_child = c ∘ current_child
    end

    return current_parent, current_child, condition(current_parent)
end


using JuliaPBF.StructPBF

function GenParticles(inAnsDataStruct::AnalysisDataStruct)
    npart = inAnsDataStruct.nMaxParticles
    ar_ParticleData = Vector{ParticleData}(undef,npart)
    for ii in 1:npart
        ar_ParticleData[ii] = ParticleData(
                                inAnsDataStruct.nMaxParticles+1,  # gridID :: Int64
                                Vec2(),                           # vel    :: Vec2
                                Vec2(),                           # pos    :: Vec2
                                Vec2(),                           # temppos:: Vec2
                                0.0,                              # Lambda :: Float64
                                0.0,                              # mass   :: Float64
                                0.0)                              # phase  :: Int64
    end
    FillInitialBox(inAnsDataStruct, ar_ParticleData)
    return ar_ParticleData
end

function GenGrids(inAnsDataStruct::AnalysisDataStruct)
    start_point = inAnsDataStruct.analysisBox.staPoint
    end_point = inAnsDataStruct.analysisBox.endPoint
    grid_size = 2.0 * inAnsDataStruct.kernelRadius
    n_Grid_x = Int(ceil((end_point.x - start_point.x)/grid_size))
    n_Grid_y = Int(ceil((end_point.y - start_point.y)/grid_size))

    ini_nNearGrid      = Vector{Int64}(undef,n_Grid_x*n_Grid_y)
    ini_nGridParticles = Vector{Int64}(undef,n_Grid_x*n_Grid_y)
    ini_NearGridIndex  = Vector{Int64}(undef,9*(n_Grid_x-2)*(n_Grid_y-2) + 6*(2*(n_Grid_x-2)+2*(n_Grid_y-2)) + 4*4)
    nong = 0
    for iy in 1:n_Grid_y
        for ix in 1:n_Grid_x
            # 아랫줄
            if iy > 1
                if ix > 1
                    nong+=1
                    ini_NearGridIndex[nong] = ix-1 + (iy-2)*n_Grid_x
                end
                nong+=1
                ini_NearGridIndex[nong] = ix + (iy-2)*n_Grid_x
                if ix < n_Grid_x
                    nong+=1
                    ini_NearGridIndex[nong] = ix+1 + (iy-2)*n_Grid_x
                end
            end
            # 자기줄
            if ix > 1
                nong+=1
                ini_NearGridIndex[nong] = ix-1 + (iy-1)*n_Grid_x
            end
            nong+=1
            ini_NearGridIndex[nong] = ix + (iy-1)*n_Grid_x
            if ix < n_Grid_x
                nong+=1
                ini_NearGridIndex[nong] = ix+1 + (iy-1)*n_Grid_x
            end
            # 윗줄
            if iy < n_Grid_y
                if ix > 1
                    nong+=1
                    ini_NearGridIndex[nong] = ix-1 + iy*n_Grid_x
                end
                nong+=1
                ini_NearGridIndex[nong] = ix + iy*n_Grid_x
                if ix < n_Grid_x
                    nong+=1
                    ini_NearGridIndex[nong] = ix+1 + iy*n_Grid_x
                end
            end
            ii = ix + (iy-1)*n_Grid_x
            ini_nNearGrid[ii] = nong
            ini_nGridParticles[ii] = 0
        end
    end

    return GridData(ini_nNearGrid,ini_NearGridIndex,ini_nGridParticles)
end

function GenPhases(inAnsDataStruct::AnalysisDataStruct)
    ## CREATE BOUNDARY LINES ##
    # Boundary line is a object which shape never
    # change, but only 회전/병진. at this time, it
    # won't be added since defining the method how
    # treat the boundary condition.
    # line collision? dummy particles?
    return []
end

function GenBoundaries(inAnsDataStruct::AnalysisDataStruct)
    ## CREATE BOUNDARY LINES ##
    # Boundary line is a object which shape never
    # change, but only 회전/병진. at this time, it
    # won't be added since defining the method how
    # treat the boundary condition.
    # line collision? dummy particles?
    return []
end

# private

function FillInitialBox(inAnsDataStruct::AnalysisDataStruct, arParticleData::Vector{ParticleData})
    ID = 1
    for initBox_i in 1:length(inAnsDataStruct.a_initialBox)
        initBox_phaseID = inAnsDataStruct.a_initialBox[initBox_i].phaseID
        initBox_box     = inAnsDataStruct.a_initialBox[initBox_i].box
        initBox_vel     = inAnsDataStruct.a_initialBox[initBox_i].vel
        initBox_type    = inAnsDataStruct.a_phases[initBox_phaseID].type
        if initBox_type == "Fluid"
            initBox_name      = inAnsDataStruct.a_phases[initBox_phaseID].Fluid.name
            initBox_size      = inAnsDataStruct.a_phases[initBox_phaseID].Fluid.size
            initBox_density   = inAnsDataStruct.a_phases[initBox_phaseID].Fluid.density
            initBox_viscosity = inAnsDataStruct.a_phases[initBox_phaseID].Fluid.viscosity
            # initBox_mass      = initBox_density * (π*(initBox_size*0.5)^2.0) # Circle
            # initBox_mass      = initBox_density * (initBox_size^2.0) # Box
            initBox_mass      = initBox_density * ((initBox_size^2.0)*0.85 + (π*(initBox_size*0.5)^2.0)*0.15)
        end
        boxDX = initBox_box.endPoint.x - initBox_box.staPoint.x
        boxDY = initBox_box.endPoint.y - initBox_box.staPoint.y
        for ix in 1:div(boxDX+initBox_size*0.5,initBox_size)
            for iy in 1:div(boxDY+initBox_size*0.5,initBox_size)
                xx = initBox_box.staPoint.x + (ix-0.5)*initBox_size
                yy = initBox_box.staPoint.y + (iy-0.5)*initBox_size
                arParticleData[ID] = ParticleData(
                                        0,              # gridID :: Int64
                                        initBox_vel,    # vel    :: Vec2
                                        Vec2(xx,yy),    # pos    :: Vec2
                                        Vec2(xx,yy),    # temppos:: Vec2
                                        0.0,            # lambda :: Float64
                                        initBox_mass,   # mass   :: Float64
                                        initBox_phaseID)# phase  :: Int64
                ID += 1
            end
        end
    end
end


@testset "Loop utilities" begin

function looper1(N::Int, K::Int)
    x = 0
    for i in 1:N
        if (i < 20) continue end
        for j in 1:K
            x += 1
        end
    end
    return x
end

function looper2(N::Int)
    x = 0
    for i in 1 : N
        if (i > 10)
            x += 1
        else
            x += 2
        end
    end
    return x
end

function looper3(N::Int)
    x = 0
    for i in 1 : N
        for j in 1 : i
            for k in i : j
                for l in 1 : 10
                    x += 1
                end
            end
        end
    end
    return x
end

@testset "Loop detection" begin

ir = IR(typeof(looper1), Int, Int)
l1, l2 = Genify.detectloops!(ir)
@test l2.body ⊆ l1.body
@test looper1(10, 10) == IRTools.evalir(ir, nothing, 10, 10)

ir = IR(typeof(looper2), Int)
loops = Genify.detectloops!(ir)
@test length(loops) == 1
@test looper2(20) == IRTools.evalir(ir, nothing, 20)

ir = IR(typeof(looper3), Int)
l1, l2, l3, l4 = Genify.detectloops!(ir)
@test l4.body ⊆ l3.body ⊆ l2.body ⊆ l1.body
@test looper3(20) == IRTools.evalir(ir, nothing, 20)

end

@testset "Preheader insertion" begin

ir = IR(typeof(looper1), Int, Int)
preheaders = Genify.preheaders!(ir, Genify.detectloops!(ir))
loops = Genify.detectloops(ir)
@test all(IRTools.block(ir, l.header-1) in preheaders for l in loops)
@test looper1(10, 10) == IRTools.evalir(ir, nothing, 10, 10)

ir = IR(typeof(looper2), Int)
preheaders = Genify.preheaders!(ir, Genify.detectloops!(ir))
loops = Genify.detectloops(ir)
@test all(IRTools.block(ir, l.header-1) in preheaders for l in loops)
@test looper2(20) == IRTools.evalir(ir, nothing, 20)

ir = IR(typeof(looper3), Int)
preheaders = Genify.preheaders!(ir, Genify.detectloops!(ir))
loops = Genify.detectloops(ir)
@test all(IRTools.block(ir, l.header-1) in preheaders for l in loops)
@test looper3(20) == IRTools.evalir(ir, nothing, 20)

end

@testset "Loop count insertion" begin

ir = IR(typeof(looper1), Int, Int)
loops, countvars = Genify.loopcounts!(ir)
@test length(loops) == length(countvars) == 2
@test looper1(10, 10) == IRTools.evalir(ir, nothing, 10, 10)

ir = IR(typeof(looper2), Int)
loops, countvars = Genify.loopcounts!(ir)
@test length(loops) == length(countvars) == 1
@test looper2(20) == IRTools.evalir(ir, nothing, 20)

ir = IR(typeof(looper3), Int)
loops, countvars = Genify.loopcounts!(ir)
@test length(loops) == length(countvars) == 4
@test looper3(20) == IRTools.evalir(ir, nothing, 20)

end

end


function estimate_error(obj::GraggBulirschStoerStateInterpolator, arg0::Vector{jdouble})
    return jcall(obj, "estimateError", jdouble, (Vector{jdouble},), arg0)
end


using CovidPOMDPs
using POMCPOW
using Test

macro isinferred(ex)
  quote
    try
      @inferred $(Expr(ex.head, esc.(ex.args)...))
      true
    catch err
      println(err)
      false
    end
  end
end

function validate_history(hist::SimHist)
    S = susceptible(hist)
    I = infected(hist)
    R = recovered(hist)
    N = hist.N
    T = hist.T

    @test all(≥(0), S)
    @test all(≥(0), I)
    @test all(≥(0), R)
    @test all(≥(0), hist.pos_test)

    # Population size preservation
    @test all(==(N), S[i] + I[i] + R[i] for i in 1:T)

    @test all( 0.0 .≤ getfield.(hist.actions,:testing_prop) .≤ 1.0)

    return nothing
end

@testset "Common" begin
    include("common.jl")
end

@testset verbose=true "Single Strain" begin
    include(joinpath("single_strain", "single_strain.jl"))
end

@testset verbose=true "Double Strain" begin
    include(joinpath("double_strain", "double_strain.jl"))
end


using SensorFeatureTracking
using Base: Test

@testset "SensorFeatureTracking" begin
    # Common
    include("sensorgeometry.jl")
    include("blockflow.jl")

end


"""
Wrapper around an objective function (compatible with NLSolversBase).
"""
mutable struct EvolutionaryObjective{TC,TF,TX,TP} <: AbstractObjective
    f::TC
    F::TF
    x_f::TX
    f_calls::Int
end

"""
    EvolutionaryObjective(f, x[, F])

Constructor for an objective function object around the function `f` with initial paramter `x`, and objective value `F`.
"""
function EvolutionaryObjective(f, x::AbstractArray,
                               F::Union{Real, AbstractArray{<:Real}} = zero(f(x));
                               eval::Symbol = :serial)
    defval = default_values(x)
    # convert function into the in-place one
    fn = if funargnum(f) == 2 && F isa AbstractArray
        (Fv,xv) -> (Fv .= f(xv))
    else
        f
    end
    EvolutionaryObjective{typeof(fn),typeof(F),typeof(x),Val{eval}}(fn, F, defval, 0)
end

"""
    EvolutionaryObjective(f, x::Expr[, F])

Constructor for an objective object for a Julia evaluatable expression.
"""
function EvolutionaryObjective(f, x::Expr, F::TF = zero(f(x));
                               eval::Symbol = :serial) where {TF<:Real}
    EvolutionaryObjective{typeof(f),TF,typeof(x),Val{eval}}(f, F, :(), 0)
end

f_calls(obj::EvolutionaryObjective) = obj.f_calls

"""
    ismultiobjective(objfun)

Return `true` if the function is multiobjective objective.
"""
ismultiobjective(obj) = obj.F isa AbstractArray

function value(obj::EvolutionaryObjective{TC,TF,TX,TP}, x::TX) where {TC,TF,TX,TP}
    obj.f_calls += 1
    obj.f(x)::TF
end

function value!(obj::EvolutionaryObjective{TC,TF,TX,TP}, x::TX) where {TC,TF,TX,TP}
    obj.F = value(obj, x)
    obj.F
end

function value!!(obj::EvolutionaryObjective{TC,TF,TX,TP}, x::TX) where {TC,TF,TX,TP}
    copyto!(obj.x_f, x)
    value!(obj, x)
end

function value(obj::EvolutionaryObjective{TC,TF,TX,TP}, F, x::TX) where {TC,TF,TX,TP}
    obj.f_calls += 1
    obj.f(F, x)
end
value(obj::EvolutionaryObjective{TC,TF,TX,TP}, x::TX) where {TC,TF<:AbstractArray,TX,TP} = value(obj, copy(obj.F), x)

function value!(obj::EvolutionaryObjective{TC,TF,TX,TP}, F, x::TX) where {TC,TF,TX,TP}
    obj.F = value(obj, F, x)
end

function value!!(obj::EvolutionaryObjective{TC,TF,TX,TP}, F, x::TX) where {TC,TF,TX,TP}
    copyto!(obj.x_f, x)
    value!(obj, F, x)
end

function value!(obj::EvolutionaryObjective{TC,TF,TX,Val{:serial}},
                F::AbstractVector, xs::AbstractVector{TX}) where {TC,TF<:Real,TX}
    broadcast!(x->value(obj,x), F, xs)
    F
end

function value!(obj::EvolutionaryObjective{TC,TF,TX,Val{:thread}},
                F::AbstractVector, xs::AbstractVector{TX}) where {TC,TF<:Real,TX}
    n = length(xs)
    Threads.@threads for i in 1:n
        F[i] = value(obj, xs[i])
    end
    F
end

function value!(obj::EvolutionaryObjective{TC,TF,TX,Val{:serial}},
                F::AbstractMatrix, xs::AbstractVector{TX}) where {TC,TF,TX}
    n = length(xs)
    for i in 1:n
        fv = view(F, :, i)
        value(obj, fv, xs[i])
    end
    F
end

function value!(obj::EvolutionaryObjective{TC,TF,TX,Val{:thread}},
                F::AbstractMatrix, xs::AbstractVector{TX}) where {TC,TF,TX}
    n = length(xs)
    @Threads.threads for i in 1:n
        fv = view(F, :, i)
        value(obj, fv, xs[i])
    end
    F
end


using ArgParse
using DrWatson
@quickactivate
using CSV
using BSON
using Random
using FileIO
using DataFrames
using PrettyTables
using PrettyTables.Crayons

using GenerativeAD.Evaluation: _prefix_symbol, PAT_METRICS, aggregate_stats_mean_max
using GenerativeAD.Evaluation: rank_table, print_rank_table
using GenerativeAD.Datasets: load_data

s = ArgParseSettings()
@add_arg_table! s begin
	"filename"
		arg_type = String
		default = "evaluation/images_eval.bson"
		help = "Location of cached DataFrame."
	"-c", "--criterion-metric"
		arg_type = String
		default = "val_auc"
		help = "Criterion to sort the models."
	"-r", "--rank-metric"
		arg_type = String
		default = "tst_auc"
		help = "Metric to rank models."
	"-o", "--output-prefix"
		arg_type = String
		default = "evaluation/images_eval"
		help = "Output prefix for storing results."
    "-f", "--force"
    	action = :store_true
		help = "Overwrite all generated files."
end

# args = Dict()
# args["filename"] = "evaluation/tabular_sptn_5seeds_quantiles_eval.bson"
# args["criterion-metric"] = "val_tpr_5"
# args["rank-metric"] = "tst_tpr_5"
# args["output-prefix"] = "evaluation/zkouska_graf_rel_params2.bson"
# args["max-params"] = 100
#
# f = datadir(args["filename"])
# df = load(f)[:df]
# cdf = deepcopy(df)
#
# tdf = deepcopy(cdf)
# for i in 1:1
# 	tdf = deepcopy(cdf)
# 	for d in unique(cdf[:dataset])
# 		mx = maximum(cdf[cdf[:dataset] .== d, :npars])
# 		filter!(row -> (row[:dataset] .!= d) .| ((row[:dataset] .== d) .& (row[:npars] .<= i/100 .* mx)), tdf)
# 	end
# 	println(i)
# end
#
# data_dim(s) = size(load_data(s)[1][1], 1)
# datadim = Dict()
# for d in unique(df[:dataset])
#     println(d)
#     datadim[d] = size(load_data(d)[1][1], 1)
# end
# df[data_dim.(df[:dataset]) .== 10]
#
# cdf.datadim = 1
# cdf.relnpars = 1.0
# for f in eachrow(cdf)
#     @show f.datadim = datadim[f[:dataset]]
#     @show f.relnpars = f[:npars]/f[:datadim]
# end
#
# df_agg = aggregate_stats_mean_max(cdf, "val_auc")
# rt = rank_table(df_agg, "tst_auc")
#
# npar_df = DataFrame(modelname = String[],relnpars = Float64[], rank = Float64[])
# for i in minimum(cdf[:relnpars]):10:maximum(cdf[:relnpars])
#     df_agg = aggregate_stats_mean_max(cdf[cdf[:relnpars] .<= i, :], "val_auc")
#     rt = rank_table(df_agg, "tst_auc")
#     for n in names(rt)[2:end]
#         push!(npar_df, (n, i, rt[end, n]))
#     end
#     println(i)
# end
# npar_df
# using Plots
# sptn_df = npar_df[npar_df[:modelname] .== "sptn", :]
# sptn_5_df = npar_df[npar_df[:modelname] .== "sptn_005_010", :]
# sptn_25_df = npar_df[npar_df[:modelname] .== "sptn_0025_0075", :]
# plot(sptn_df[:relnpars], sptn_df[:rank])
# plot!(sptn_5_df[:relnpars], sptn_5_df[:rank])
# plot!(sptn_25_df[:relnpars], sptn_25_df[:rank])

function main(args)
	f = datadir(args["filename"])
	df = load(f)[:df]
    cdf = deepcopy(df)

    #data dim dict
    data_dim(s) = size(load_data(s)[1][1], 1)
    datadim = Dict()
    for d in unique(df[:dataset])
        datadim[d] = size(load_data(d)[1][1], 1)
    end

    # cdf.datadim = 1
    # cdf.relnpars = 1.0
    # for f in eachrow(cdf)
    #     f.datadim = datadim[f[:dataset]]
    #     #relnpars = numb. of params of model / data dim.
    #     f.relnpars = f[:npars]/f[:datadim]
    # end
	@info "Loaded $(nrow(df)) rows from $f"

    # Dataframe for final ranks for each relnpars
    npar_df = DataFrame(modelname = String[],relnpars = Float64[], rank = Float64[])

    # m = args["max-params"]
    # if m == -1
    #     m = maximum(cdf[:relnpars])
    # end
	#
    # for i in minimum(cdf[:relnpars]):1:m
    #     # evaluation and rank table
    #     df_agg = aggregate_stats_mean_max(cdf[cdf[:relnpars] .<= i, :], Symbol(args["criterion-metric"]))
    #     rt = rank_table(df_agg, args["rank-metric"])
    #     for n in names(rt)[2:end]
    #         # row for each model at relnpars
    #         push!(npar_df, (n, i, rt[end, n]))
    #     end
	# 	@info "Number of relative parameters $i"
    # end

	for i in 1:100
		tdf = deepcopy(cdf)
		for d in unique(cdf[:dataset])
			mx = maximum(cdf[cdf[:dataset] .== d, :npars])
			filter!(row -> (row[:dataset] .!= d) .| ((row[:dataset] .== d) .& (row[:npars] .<= i/100 .* mx)), tdf)
		end
		# display(size(tdf))
		df_agg = aggregate_stats_mean_max(tdf, Symbol(args["criterion-metric"]))
		rt = rank_table(df_agg, args["rank-metric"])

        for n in names(rt)[2:end]
            # row for each model at relnpars
            push!(npar_df, (n, i, rt[end, n]))
        end

		@info "Percent of relative parameters $i %"
	end

	@info "Best models chosen by $(args["criterion-metric"])"
	@info "Ranking by $(args["rank-metric"])"

    ff = datadir(args["output-prefix"])
	if (isfile(ff) && args["force"]) || ~isfile(ff)
		@info "Saving $(nrow(npar_df)) rows to $ff."
		wsave(ff, Dict(:df => npar_df))
	end
	@info "---------------- DONE -----------------"
end

main(parse_args(ARGS, s))


module TestBench

using Reactive, ReactiveBasics, BenchmarkTools

const N = 10^3
const V = rand(N)
function testReactive()
    a = Reactive.Signal(0.0)
    b = Reactive.Signal(1.0)
    c = map(+, a, b)
    d = Reactive.foldp(+, 0.0, a)
    x = 0.0
    for i in 1:N
        push!(a, V[i])
        Reactive.run(1)
        x = Reactive.value(c) + Reactive.value(d)
    end
    x
end

function testReactiveBasics()
    a = ReactiveBasics.Signal(0.0)
    b = ReactiveBasics.Signal(1.0)
    c = map(+, a, b)
    d = ReactiveBasics.foldp(+, 0.0, a)
    x = 0.0
    for i in 1:N
        push!(a, V[i])
        x = ReactiveBasics.value(c) + ReactiveBasics.value(d)
    end
    x
end

@show @benchmark testReactive()
@show @benchmark testReactiveBasics()

end # module