""" Data structure for (directed) wiring diagrams, aka string diagrams.

A (directed) wiring diagram consists of a collection of boxes with input and
output ports connected by wires. A box can be atomic (possessing no internal
structure) or can itself be a wiring diagram. Thus, wiring diagrams can be
nested recursively. Wiring diagrams are closely related to what the CS
literature calls "directed graphs with ports" or more simply "port graphs". The
main difference is that a wiring diagram has an "outer box": a wiring diagram
has its own ports that can be connected to the ports of its boxes.

This module provides a generic data structure for wiring diagrams. Arbitrary
data can be attached to the boxes, ports, and wires of a wiring diagram. The
diagrams are "abstract" in the sense that they cannot be directly rendered as
raster or vector graphics. However, they form a useful intermediate
representation that can be serialized to and from GraphML or translated into
Graphviz or other declarative diagram languages.
"""
module DirectedWiringDiagrams
export AbstractBox, Box, WiringDiagram, Wire, Port, PortKind,
  InputPort, OutputPort, input_ports, output_ports,
  set_input_ports!, set_output_ports!, add_input_ports!, add_output_ports!,
  input_id, output_id, outer_ids, boxes, box_ids, nboxes, nwires, box, wires,
  has_wire, graph, internal_graph,
  add_box!, add_boxes!, add_wire!, add_wires!, rem_box!, rem_boxes!, rem_wire!,
  rem_wires!, port_value, validate_ports, is_isomorphic,
  all_neighbors, neighbors, outneighbors, inneighbors, in_wires, out_wires,
  singleton_diagram, induced_subdiagram, encapsulated_subdiagram,
  ocompose, substitute, encapsulate

using AutoHashEquals

using ...Present, ...Graphs.BasicGraphs, ...CategoricalAlgebra.CSets
import ...CategoricalAlgebra.CSets: is_isomorphic
using ...Graphs.BasicGraphs: TheoryGraph
import ...Graphs: all_neighbors, neighbors, outneighbors, inneighbors

# Data types
############

""" Kind of port: input or output.
"""
@enum PortKind InputPort OutputPort

""" A port on a box to which wires can be connected.
"""
@auto_hash_equals struct Port
  box::Int
  kind::PortKind
  port::Int
end
set_box(port::Port, box::Int) = Port(box, port.kind, port.port)

function Base.isless(p1::Port, p2::Port)::Bool
  # Lexicographic order.
  p1.box < p2.box ||
    (p1.box == p2.box &&
      (p1.kind < p2.kind || (p1.kind == p2.kind && p1.port < p2.port)))
end

""" A wire connecting one port to another.
"""
@auto_hash_equals struct Wire{Value}
  value::Value
  source::Port
  target::Port
end

Wire(value, src::Tuple{Int,PortKind,Int}, tgt::Tuple{Int,PortKind,Int}) =
  Wire(value, Port(src[1],src[2],src[3]), Port(tgt[1],tgt[2],tgt[3]))
Wire(value, src::Tuple{Int,Int}, tgt::Tuple{Int,Int}) =
  Wire(value, Port(src[1],OutputPort,src[2]), Port(tgt[1],InputPort,tgt[2]))
Wire(value, pair::Pair) = Wire(value, first(pair), last(pair))

Wire(src::Port, tgt::Port) = Wire(nothing, src, tgt)
Wire(src::Tuple, tgt::Tuple) = Wire(nothing, src, tgt)
Wire(pair::Pair) = Wire(nothing, first(pair), last(pair))

function Base.show(io::IO, wire::Wire)
  skip_kind = wire.source.kind == OutputPort && wire.target.kind == InputPort
  show_port = (io::IO, port::Port) -> begin
    if skip_kind
      print(io, "($(port.box),$(port.port))")
    else
      print(io, "($(port.box),$(string(port.kind)),$(port.port)")
    end
  end
  print(io, "Wire(")
  if !isnothing(wire.value)
    show(io, wire.value)
    print(io, ", ")
  end
  show_port(io, wire.source)
  print(io, " => ")
  show_port(io, wire.target)
  print(io, ")")
end

function Base.isless(w1::Wire, w2::Wire)::Bool
  # Lexicographic order.
  isless(w1.source, w2.source) ||
    (w1.source == w2.source && isless(w1.target, w2.target))
end

""" Base type for any box (node) in a wiring diagram.

This type represents an arbitrary black box with inputs and outputs.
"""
abstract type AbstractBox end

input_ports(box::AbstractBox) = box.input_ports
output_ports(box::AbstractBox) = box.output_ports
value(b::AbstractBox) = b.value

""" An atomic box in a wiring diagram.

These boxes have no internal structure.
"""
@auto_hash_equals struct Box{Value} <: AbstractBox
  value::Value
  input_ports::Vector
  output_ports::Vector
end

Box(inputs::Vector, outputs::Vector) = Box(nothing, inputs, outputs)

function Base.show(io::IO, box::Box)
  print(io, "Box(")
  if !isnothing(box.value)
    show(io, box.value)
    print(io, ", ")
  end
  print(io, "[")
  join(io, [sprint(show, port) for port in box.input_ports], ",")
  print(io, "], [")
  join(io, [sprint(show, port) for port in box.output_ports], ",")
  print(io, "])")
end

@present TheoryWiringDiagram(FreeSchema) begin
  Box::Ob
  (InPort, OutPort, OuterInPort, OuterOutPort)::Ob
  (Wire, InWire, OutWire, PassWire)::Ob

  src::Hom(Wire, OutPort)
  tgt::Hom(Wire, InPort)
  in_src::Hom(InWire, OuterInPort)
  in_tgt::Hom(InWire, InPort)
  out_src::Hom(OutWire, OutPort)
  out_tgt::Hom(OutWire, OuterOutPort)
  pass_src::Hom(PassWire, OuterInPort)
  pass_tgt::Hom(PassWire, OuterOutPort)

  in_port_box::Hom(InPort, Box)
  out_port_box::Hom(OutPort, Box)
end

const AbstractWiringDiagramACSet = AbstractACSetType(TheoryWiringDiagram)

@present TheoryTypedWiringDiagram <: TheoryWiringDiagram begin
  PortValue::Data
  in_port_type::Attr(InPort, PortValue)
  out_port_type::Attr(OutPort, PortValue)
  outer_in_port_type::Attr(OuterInPort, PortValue)
  outer_out_port_type::Attr(OuterOutPort, PortValue)
end

@present TheoryAttributedWiringDiagram <: TheoryTypedWiringDiagram begin
  WireValue::Data
  BoxValue::Data
  BoxType::Data

  value::Attr(Box, BoxValue)
  box_type::Attr(Box, BoxType)
  wire_value::Attr(Wire, WireValue)
  in_wire_value::Attr(InWire, WireValue)
  out_wire_value::Attr(OutWire, WireValue)
  pass_wire_value::Attr(PassWire, WireValue)
end

const WiringDiagramACSet = ACSetType(TheoryAttributedWiringDiagram,
  index=[:src, :tgt, :in_src, :in_tgt, :out_src, :out_tgt, :pass_src, :pass_tgt])

""" A directed wiring diagram, also known as a string diagram.

The wiring diagram is implemented using the following internal data structure.
The "skeleton" of the diagram is an instance of `Catlab.Graphs.AbstractGraph`: a
directed multigraph whose vertices correspond to boxes and whose edges
correspond to wires. There are two special vertices, accessible via `input_id`
and `output_id`, that represent the input and output ports of the outer box.
"""

mutable struct WiringDiagram{Theory, PortValue, WireValue, BoxValue} <: AbstractBox
  diagram::WiringDiagramACSet{PortValue, WireValue, Union{BoxValue, AbstractBox}, DataType}
  value::Any

  function WiringDiagram{T, PortValue, WireValue, BoxValue}(value, inputs::Vector, outputs::Vector) where {T, PortValue, WireValue, BoxValue}
    diagram = WiringDiagramACSet{PortValue, WireValue, Union{BoxValue, AbstractBox}, DataType}()
    f = new{T, PortValue, WireValue, BoxValue}(diagram, value)
    add_parts!(diagram, :OuterInPort, length(inputs),   outer_in_port_type=inputs)
    add_parts!(diagram, :OuterOutPort, length(outputs), outer_out_port_type=outputs)
    return f
  end
  function WiringDiagram(f::WiringDiagram{T, PortValue, WireValue, BoxValue}) where {T, PortValue, WireValue, BoxValue}
    # Copy constructor for shallow copy
    new{T, PortValue, WireValue, BoxValue}(copy(f.diagram), f.value)
  end
end

function WiringDiagram{T, PortValue, WireValue, BoxValue}(inputs::Vector, outputs::Vector) where {T, PortValue, WireValue, BoxValue}
  WiringDiagram{T, PortValue, WireValue, BoxValue}(nothing, inputs, outputs)
end
WiringDiagram(args...) = WiringDiagram{Any, Any, Any, Any}(args...)
WiringDiagram{T}(args...) where T = WiringDiagram{T, Any, Any, Any}(args...)

input_id(::WiringDiagram) = -2
output_id(::WiringDiagram) = -1
outer_ids(::WiringDiagram) = (-2,-1)

""" Are the two wiring diagrams equal?

**Warning**: This method checks equality of the underlying C-set representation.
Use [`is_isomorphic`](@ref) to check isomorphism of wiring diagrams.
"""
function Base.:(==)(d1::WiringDiagram, d2::WiringDiagram)
  (input_ports(d1) == input_ports(d2) &&
   output_ports(d1) == output_ports(d2) && d1.value == d2.value &&
   boxes(d1) == boxes(d2) && sort!(wires(d1)) == sort!(wires(d2)))
end

function is_isomorphic(d1::WiringDiagram, d2::WiringDiagram)
  d1.value == d2.value && is_isomorphic(d1.diagram, d2.diagram)
end

Base.copy(diagram::WiringDiagram) = WiringDiagram(diagram)

function Base.show(io::IO, diagram::WiringDiagram{T}) where T
  sshowcompact = x -> sprint(show, x, context=:compact => true)
  print(io, "WiringDiagram")
  if T != Any
    print(io, "{$T}")
  end
  print(io, "(")
  if !isnothing(diagram.value)
    show(io, diagram.value)
    print(io, ", ")
  end
  print(io, "[")
  join(io, map(sshowcompact, input_ports(diagram)), ",")
  print(io, "], [")
  join(io, map(sshowcompact, output_ports(diagram)), ",")
  print(io, "], ")
  if get(io, :compact, false)
    print(io, "{$(nboxes(diagram)) boxes}, {$(nwires(diagram)) wires}")
  else
    print(io, "\n[ $(input_id(diagram)) => {inputs},\n  ")
    print(io, "$(output_id(diagram)) => {outputs},\n  ")
    join(io, [ "$v => $(sshowcompact(box(diagram, v)))"
               for v in box_ids(diagram) ], ",\n  ")
    print(io, " ],\n[ ")
    join(io, map(sshowcompact, wires(diagram)), ",\n  ")
    print(io, " ]")
  end
  print(io, ")")
end

@present TheoryWiringDiagramGraph <: TheoryGraph begin
  ID::Data
  box::Attr(V,ID)
  wire::Attr(E,ID)
end

""" Graph underlying a directed wiring diagram.
"""
const WiringDiagramGraph = ACSetType(TheoryWiringDiagramGraph,
  index=[:src, :tgt], unique_index=[:box]){Int}

# Imperative interface
######################

# Basic accessors.

function box(f::WiringDiagram, b::Int)
  b ∉ outer_ids(f) || return nothing
  value = subpart(f.diagram, b, :value)
  if value isa AbstractBox
    value
  else
    BoxType = subpart(f.diagram, b, :box_type)
    BoxType(value, input_ports(f, b), output_ports(f, b))
  end
end

boxes(f::WiringDiagram) = map(b -> box(f, b), box_ids(f))
nboxes(f::WiringDiagram) = nparts(f.diagram, :Box)
box_ids(f::WiringDiagram) = collect(parts(f.diagram, :Box))

src_box(f::WiringDiagram, w::Int) = subpart(f.diagram, w, [:src, :out_port_box])
tgt_box(f::WiringDiagram, w::Int) = subpart(f.diagram, w, [:tgt, :in_port_box])
in_tgt_box(f::WiringDiagram, w::Int) = subpart(f.diagram, w, [:in_tgt, :in_port_box])
out_src_box(f::WiringDiagram, w::Int) = subpart(f.diagram, w, [:out_src, :out_port_box])

local_in_port_id(f::WiringDiagram, b::Int, p::Int) =
  searchsortedfirst(incident(f.diagram, b, :in_port_box), p)
local_out_port_id(f::WiringDiagram, b::Int, p::Int) =
  searchsortedfirst(incident(f.diagram, b, :out_port_box), p)

in_port_id(f::WiringDiagram, b::Int, p::Int) = incident(f.diagram, b, :in_port_box)[p]
out_port_id(f::WiringDiagram, b::Int, p::Int) = incident(f.diagram, b, :out_port_box)[p]
in_port_id(f::WiringDiagram, p::Port) = in_port_id(f, p.box, p.port)
out_port_id(f::WiringDiagram, p::Port) = out_port_id(f, p.box, p.port)

@inline Wire(f::WiringDiagram, type::Symbol, w::Int) = Wire(f, Val{type}, w)

function Wire(f::WiringDiagram, ::Type{Val{:PassWire}}, w::Int)
  Wire(subpart(f.diagram, w, :pass_wire_value),
       (input_id(f), subpart(f.diagram, w, :pass_src)) =>
       (output_id(f), subpart(f.diagram, w, :pass_tgt)))
end

function Wire(f::WiringDiagram, ::Type{Val{:InWire}}, w::Int)
  Wire(subpart(f.diagram, w, :in_wire_value),
       (input_id(f),
        subpart(f.diagram, w, :in_src)) =>
       (in_tgt_box(f,w),
        local_in_port_id(f, in_tgt_box(f,w), subpart(f.diagram, w, :in_tgt))))
end

function Wire(f::WiringDiagram, ::Type{Val{:OutWire}}, w::Int)
  Wire(subpart(f.diagram, w, :out_wire_value),
       (out_src_box(f,w),
        local_out_port_id(f, out_src_box(f,w), subpart(f.diagram, w, :out_src))) =>
       (output_id(f),
        subpart(f.diagram, w, :out_tgt)))
end

function Wire(f::WiringDiagram, ::Type{Val{:Wire}}, w::Int)
  Wire(subpart(f.diagram, w, :wire_value),
       (src_box(f,w),
        local_out_port_id(f, src_box(f,w), subpart(f.diagram, w, :src))) =>
       (tgt_box(f,w),
        local_in_port_id(f, tgt_box(f,w), subpart(f.diagram, w, :tgt))))
end

function wires(f::WiringDiagram, src::Int, tgt::Int)
  if src == input_id(f) && tgt == output_id(f)
    [Wire(f, :PassWire, w) for w in parts(f.diagram, :PassWire)]
  elseif src == input_id(f)
    [Wire(f, :InWire, w)
     for w in incident(f.diagram, tgt, [:in_tgt, :in_port_box])]
  elseif tgt == output_id(f)
    [Wire(f, :OutWire, w)
     for w in incident(f.diagram, src, [:out_src, :out_port_box])]
  else
    [Wire(f, :Wire, w) for w in incident(f.diagram, src, [:src, :out_port_box])
     if tgt_box(f,w) == tgt]
  end
end

wires(f::WiringDiagram, type::Symbol) =
  [ Wire(f, type, w) for w in parts(f.diagram, type) ]
wires(f::WiringDiagram) = vcat(wires(f, :PassWire), wires(f, :InWire),
                               wires(f, :Wire), wires(f, :OutWire))

function nwires(f::WiringDiagram, src::Int, tgt::Int)
  if src == input_id(f) && tgt == output_id(f)
    nparts(f.diagram, :PassWire)
  elseif src == input_id(f)
    length(incident(f.diagram, tgt, [:in_tgt, :in_port_box]))
  elseif tgt == output_id(f)
    length(incident(f.diagram, src, [:out_src, :out_port_box]))
  else
    count(w -> tgt_box(f,w) == tgt,
          incident(f.diagram, src, [:src, :out_port_box]))
  end
end

nwires(f::WiringDiagram, type::Symbol) = nparts(f.diagram, type)
nwires(f::WiringDiagram) = (nwires(f, :PassWire) + nwires(f, :InWire) +
                            nwires(f, :Wire) + nwires(f, :OutWire))

has_wire(f::WiringDiagram, src::Int, tgt::Int) = nwires(f, src, tgt) > 0
has_wire(f::WiringDiagram, wire::Wire) =
  wire in wires(f, wire.source.box, wire.target.box)
has_wire(f::WiringDiagram, pair::Pair) = has_wire(f, Wire(pair))

input_ports(f::WiringDiagram) = subpart(f.diagram, :outer_in_port_type)
output_ports(f::WiringDiagram) = subpart(f.diagram, :outer_out_port_type)

set_input_ports!(f::WiringDiagram, input_ports::AbstractVector) =
  set_subpart!(f.diagram, :outer_in_port_type, input_ports)
set_output_ports!(f::WiringDiagram, output_ports::AbstractVector) =
  set_subpart!(f.diagram, :outer_out_port_type, output_ports)
add_input_ports!(f::WiringDiagram, input_ports::AbstractVector) =
  add_parts!(f.diagram, :OuterInPort, length(input_ports),
             outer_in_port_type = input_ports)
add_output_ports!(f::WiringDiagram, output_ports::AbstractVector) =
  add_parts!(f.diagram, :OuterOutPort, length(output_ports),
             outer_out_port_type = output_ports)

function input_ports(f::WiringDiagram, b::Int)
  if b == input_id(f)
    error("Input vertex does not have input ports within wiring diagram")
  elseif b == output_id(f)
    output_ports(f)
  else
    subpart(f.diagram, incident(f.diagram, b, :in_port_box), :in_port_type) |>
      collect # Copy for backwards compatibilty.
  end
end

function output_ports(f::WiringDiagram, b::Int)
  if b == input_id(f)
    input_ports(f)
  elseif b == output_id(f)
    error("Output vertex does not have output ports within wiring diagram")
  else
    subpart(f.diagram, incident(f.diagram, b, :out_port_box), :out_port_type) |>
      collect # Copy for backwards compatibilty.
  end
end

function port_value(f::WiringDiagram, port::Port)
  if port.box == input_id(f)
    subpart(f.diagram, port.port, :outer_in_port_type)
  elseif port.box == output_id(f)
    subpart(f.diagram, port.port, :outer_out_port_type)
  else
    port_id   = port.kind == InputPort ? in_port_id : out_port_id
    type_expr = port.kind == InputPort ? :in_port_type : :out_port_type
    subpart(f.diagram, port_id(f, port), type_expr)
  end
end

# Graph mutation.

function add_box!(f::WiringDiagram, box::B) where B <: AbstractBox
  b = add_part!(f.diagram, :Box,
                box_type=B, value=B <: WiringDiagram ? box : value(box))
  add_parts!(f.diagram, :InPort, length(input_ports(box)),
             in_port_box=b, in_port_type=input_ports(box))
  add_parts!(f.diagram, :OutPort, length(output_ports(box)),
             out_port_box=b, out_port_type=output_ports(box))
  return b
end

function add_boxes!(f::WiringDiagram, boxes)
  firstbox = nboxes(f) + 1
  for box in boxes
    add_box!(f, box)
  end
  lastbox = nboxes(f)
  return firstbox:lastbox
end

function rem_box!(f::WiringDiagram, b::Int)
  rem_boxes!(f, b:b)
end

function rem_boxes!(f::WiringDiagram, bs::AbstractVector{Int})
  @assert all(1 <= b <= nboxes(f) for b in bs)

  # Remove associated wires.
  rem_parts!(f.diagram, :InWire, incident_reduce(f.diagram, bs, [:in_tgt, :in_port_box]))
  rem_parts!(f.diagram, :OutWire, incident_reduce(f.diagram, bs, [:out_src, :out_port_box]))
  rem_parts!(f.diagram, :Wire, incident_reduce(f.diagram, bs, [:tgt, :in_port_box]))
  rem_parts!(f.diagram, :Wire, incident_reduce(f.diagram, bs, [:src, :out_port_box]))

  # Remove associated ports.
  rem_parts!(f.diagram, :InPort, incident_reduce(f.diagram, bs, [:in_port_box]))
  rem_parts!(f.diagram, :OutPort, incident_reduce(f.diagram, bs, [:out_port_box]))

  rem_parts!(f.diagram, :Box, bs)
end
rem_boxes!(f::WiringDiagram, bs) = rem_boxes!(f, collect(Int, bs))

incident_reduce(args...) = sort(reduce(vcat, incident(args...), init=Int[]))

function add_wire!(f::WiringDiagram, wire::Wire)
  validate_ports(port_value(f, wire.source), port_value(f, wire.target))
  if wire.source.box == input_id(f) && wire.target.box == output_id(f)
    add_part!(f.diagram, :PassWire, pass_wire_value = wire.value,
              pass_src = wire.source.port,
              pass_tgt = wire.target.port)
  elseif wire.source.box == input_id(f)
    add_part!(f.diagram, :InWire, in_wire_value = wire.value,
              in_src = wire.source.port,
              in_tgt = in_port_id(f, wire.target))
  elseif wire.target.box == output_id(f)
    add_part!(f.diagram, :OutWire, out_wire_value=wire.value,
              out_src = out_port_id(f, wire.source),
              out_tgt = wire.target.port)
  else
    add_part!(f.diagram, :Wire, wire_value=wire.value,
              src = out_port_id(f, wire.source),
              tgt = in_port_id(f, wire.target))
  end
end
add_wire!(f::WiringDiagram, pair::Pair) = add_wire!(f, Wire(pair))

function add_wires!(f::WiringDiagram, wires)
  for wire in wires
    add_wire!(f, wire)
  end
end

function rem_wire!(f::WiringDiagram, wire::Wire)
  if wire.source.box == input_id(f) && wire.target.box == output_id(f)
    for w in parts(f.diagram, :PassWire)
      subpart(f.diagram, w, :pass_wire_value) == wire.value &&
        subpart(f.diagram, w, :pass_src) == wire.source.port &&
        subpart(f.diagram, w, :pass_tgt) == wire.target.port &&
        return rem_part!(f.diagram, :PassWire, w)
    end
  elseif wire.source.box == input_id(f)
    for w in incident(f.diagram, in_port_id(f, wire.target), :in_tgt)
      subpart(f.diagram, w, :in_wire_value) == wire.value &&
        return rem_part!(f.diagram, :InWire, w)
    end
  elseif wire.target.box == output_id(f)
    for w in incident(f.diagram, out_port_id(f, wire.source), :out_src)
      subpart(f.diagram, w, :out_wire_value) == wire.value &&
        return rem_part!(f.diagram, :OutWire, w)
    end
  else
    for w in incident(f.diagram, out_port_id(f, wire.source), :src)
      subpart(f.diagram, w, :tgt) == in_port_id(f, wire.target) &&
        subpart(f.diagram, w, :wire_value) == wire.value &&
        return rem_part!(f.diagram, :Wire, w)
    end
  end
  error("Wire $wire does not exist, so cannot be removed")
end
rem_wire!(f::WiringDiagram, pair::Pair) = rem_wire!(f, Wire(pair))

function rem_wires!(f::WiringDiagram, wires)
  for wire in wires
    rem_wire!(f, wire)
  end
end

function rem_wires!(f::WiringDiagram, src::Int, tgt::Int)
  if src == input_id(f) && tgt == output_id(f)
    rem_parts!(f.diagram, :PassWire, parts(f.diagram, :PassWire))
  elseif src == input_id(f)
    rem_parts!(f.diagram, :InWire, incident(f.diagram, tgt, [:in_tgt, :in_port_box]))
  elseif tgt == output_id(f)
    rem_parts!(f.diagram, :OutWire, incident(f.diagram, src, [:out_src, :out_port_box]))
  else
    wires = filter(w -> tgt_box(f,w) == tgt,
                   incident(f.diagram, src, [:src, :out_port_box]))
    rem_parts!(f.diagram, :Wire, wires)
  end
end

function add_outer_in_port!(f::WiringDiagram, value)
  add_part!(f.diagram, :OuterInPort, outer_in_port_type=value)
end

function add_outer_out_port!(f::WiringDiagram, value)
  add_part!(f.diagram, :OuterOutPort, outer_out_port_type=value)
end

""" Check compatibility of source and target ports.

The default implementation is a no-op.
"""
function validate_ports(source_port, target_port) end

# Graph properties.

""" Graph underlying wiring diagram, with edges for internal wires only.
"""
function internal_graph(f::WiringDiagram)
  g = WiringDiagramGraph()
  add_parts!(g, :V, nparts(f.diagram, :Box), box = parts(f.diagram, :Box))
  add_parts!(g, :E, nparts(f.diagram, :Wire),
             src = subpart(f.diagram, [:src, :out_port_box]),
             tgt = subpart(f.diagram, [:tgt, :in_port_box]),
             wire = parts(f.diagram, :Wire))
  return g
end

""" Grapn underlying wiring diagram, including parts for noin-internal wires.

The graph has two special vertices representing the input and output boundaries
of the outer box.
"""
function graph(f::WiringDiagram)
  g = internal_graph(f)
  input, output = add_parts!(g, :V, 2, box = [input_id(f), output_id(f)])
  add_parts!(g, :E, nparts(f.diagram, :InWire),
             src = input,
             tgt = subpart(f.diagram, [:in_tgt, :in_port_box]),
             wire = parts(f.diagram, :InWire))
  add_parts!(g, :E, nparts(f.diagram, :OutWire),
             src = subpart(f.diagram, [:out_src, :out_port_box]),
             tgt = output,
             wire = parts(f.diagram, :OutWire))
  add_parts!(g, :E, nparts(f.diagram, :PassWire),
             src = input, tgt = output,
             wire = parts(f.diagram, :PassWire))
  return g
end

function outneighbors(f::WiringDiagram, b::Int)
  if b == input_id(f)
    vcat(subpart(f.diagram, [:in_tgt, :in_port_box]),
         fill(output_id(f), nwires(f, :PassWire)))
  elseif b == output_id(f)
    Int[]
  else
    vcat(subpart(f.diagram, incident(f.diagram, b, [:src, :out_port_box]),
                 [:tgt, :in_port_box]),
         fill(output_id(f),
              length(incident(f.diagram, b, [:out_src, :out_port_box]))))
  end
end

function inneighbors(f::WiringDiagram, b::Int)
  if b == input_id(f)
    Int[]
  elseif b == output_id(f)
    vcat(fill(input_id(f), nwires(f, :PassWire)),
         subpart(f.diagram, [:out_src, :out_port_box]))
  else
    vcat(fill(input_id(f),
              length(incident(f.diagram, b, [:in_tgt, :in_port_box]))),
         subpart(f.diagram, incident(f.diagram, b, [:tgt, :in_port_box]),
                 [:src, :out_port_box]))
  end
end

neighbors(f::WiringDiagram, b::Int) = outneighbors(f, b)
all_neighbors(f::WiringDiagram, b::Int) =
  Iterators.flatten((inneighbors(f, b), outneighbors(f, b)))

""" Get all wires coming into or out of the box.
"""
function wires(f::WiringDiagram, b::Int)
  vcat(in_wires(f,b), out_wires(f,b))
end

""" Get all wires coming into the box.
"""
function in_wires(f::WiringDiagram, b::Int)
  if b == input_id(f)
    Wire[]
  elseif b == output_id(f)
    vcat(wires(f, :PassWire), wires(f, :OutWire))
  else
    vcat([Wire(f, :Wire, w) for w in incident(f.diagram, b, [:tgt, :in_port_box])],
         [Wire(f, :InWire, w) for w in incident(f.diagram, b, [:in_tgt, :in_port_box])])
  end
end

""" Get all wires coming into the port.
"""
function in_wires(d::WiringDiagram, port::Port)
  filter(wire -> wire.target == port, in_wires(d, port.box))
end
function in_wires(d::WiringDiagram, v::Int, port::Int)
  in_wires(d, Port(v, InputPort, port))
end

""" Get all wires coming out of the box.
"""
function out_wires(f::WiringDiagram, b::Int)
  if b == input_id(f)
    vcat(wires(f, :PassWire), wires(f, :InWire))
  elseif b == output_id(f)
    return Wire[]
  else
    vcat([Wire(f, :Wire, w) for w in incident(f.diagram, b, [:src, :out_port_box])],
         [Wire(f, :OutWire, w) for w in incident(f.diagram, b, [:out_src, :out_port_box])])
  end
end

""" Get all wires coming out of the port.
"""
function out_wires(d::WiringDiagram, port::Port)
  filter(wire -> wire.source == port, out_wires(d, port.box))
end
function out_wires(d::WiringDiagram, v::Int, port::Int)
  out_wires(d, Port(v, OutputPort, port))
end

# Other constructors
#-------------------

""" Wiring diagram with a single box connected to the outer ports.
"""
function singleton_diagram(T::Type, box::AbstractBox)
  inputs, outputs = input_ports(box), output_ports(box)
  d = WiringDiagram{T}(inputs, outputs)
  v = add_box!(d, box)
  add_wires!(d, ((input_id(d),i) => (v,i) for i in eachindex(inputs)))
  add_wires!(d, ((v,i) => (output_id(d),i) for i in eachindex(outputs)))
  return d
end
singleton_diagram(box::AbstractBox) = singleton_diagram(Any, box)

""" The wiring diagram induced by a subset of its boxes.

See also `encapsulated_subdiagram`.
"""
function induced_subdiagram(d::WiringDiagram{T}, vs::Vector{Int}) where T
  sub = WiringDiagram{T}(input_ports(d), output_ports(d))
  vmap = Dict(input_id(d) => input_id(sub), output_id(d) => output_id(sub))
  for v in vs
    vmap[v] = add_box!(sub, box(d, v))
  end
  for wire in wires(d)
    src, tgt = wire.source, wire.target
    if haskey(vmap, src.box) && haskey(vmap, tgt.box)
      add_wire!(sub,
        Wire(set_box(src, vmap[src.box]), set_box(tgt, vmap[tgt.box])))
    end
  end
  return sub
end

# Operadic interface
####################

""" Operadic composition of wiring diagrams.

This generic function has two different signatures, corresponding to the "full"
and "partial" notions of operadic composition (Yau, 2018, *Operads of Wiring
Diagrams*, Definitions 2.3 and 2.10).

This operation is a simple wrapper around [`substitute`](@ref).
"""
function ocompose(f::WiringDiagram, gs::Vector{<:WiringDiagram})
  @assert length(gs) == nboxes(f)
  substitute(f, box_ids(f), gs)
end
function ocompose(f::WiringDiagram, i::Int, g::WiringDiagram)
  @assert 1 <= i <= nboxes(f)
  substitute(f, box_ids(f)[i], g)
end

# Substitution
##############

""" Substitute wiring diagrams for boxes.

Performs one or more substitutions. When performing multiple substitutions, the
substitutions are simultaneous.

This operation implements the operadic composition of wiring diagrams, see also
[`ocompose`](@ref).
"""
function substitute(d::WiringDiagram; kw...)
  substitute(d, filter(v -> box(d,v) isa WiringDiagram, box_ids(d)); kw...)
end
function substitute(d::WiringDiagram, v::Int; kw...)
  substitute(d, v, box(d,v)::WiringDiagram; kw...)
end
function substitute(d::WiringDiagram, vs::AbstractVector{Int}; kw...)
  substitute(d, vs, WiringDiagram[ box(d,v) for v in vs ]; kw...)
end
function substitute(d::WiringDiagram, v::Int, sub::WiringDiagram; kw...)
  substitute(d, [v], [sub]; kw...)
end

function substitute(d::WD, vs::AbstractVector{Int}, subs::Vector{<:WiringDiagram};
                    merge_wire_values=default_merge_wire_values) where WD <: WiringDiagram
  # In outline, the algorithm is:
  #
  # 1. Create an empty wiring diagram.
  # 2. Add *all* boxes of original diagram and the diagrams to be substituted
  #    (in the appropriate order).
  # 3. Add *all* wires of original diagram and the diagrams to be substituted.
  # 4. Remove the boxes that were substituted (hence also removing extraneous
  #    wires from step 3).
  #
  # This may seem convoluted, but it is the simplest way I know to handle the
  # problem of *instantaneous wires*. Some authors ban instantaneous wires, but
  # we need them to represent identities, braidings, etc.
  @assert length(vs) == length(subs)
  result = WD(d.value, input_ports(d), output_ports(d))

  # Add boxes by interleaving, in the correct order, the non-substituted boxes
  # of the original diagram and the internal boxes of the substituted diagrams.
  # At the very end, add the substituted boxes too.
  vmap = Dict(input_id(d) => input_id(result), output_id(d) => output_id(result))
  sub_maps = Dict(zip(vs, ((sub, Dict{Int,Int}()) for sub in subs)))
  for v in box_ids(d)
    if haskey(sub_maps, v)
      sub, sub_map = sub_maps[v]
      for u in box_ids(sub)
        sub_map[u] = add_box!(result, box(sub, u))
      end
    else
      vmap[v] = add_box!(result, box(d, v))
    end
  end
  for v in vs
    vmap[v] = add_box!(result, box(d, v))
  end

  # Add the wires of the original diagram, then add the internal wires of the
  # substituted diagrams.
  for wire in wires(d)
    add_wire!(result, Wire(wire.value,
      set_box(wire.source, vmap[wire.source.box]),
      set_box(wire.target, vmap[wire.target.box])))
  end
  for v in vs
    substitute_wires!(result, vmap[v], sub_maps[v]...;
      merge_wire_values=merge_wire_values)
  end

  # Finally, remove the substituted boxes. Because they were added last, this
  # will not change the IDs of the other boxes.
  rem_boxes!(result, (vmap[v] for v in vs))
  result
end

""" Substitute wires of sub-diagram into containing wiring diagram.
"""
function substitute_wires!(d::WiringDiagram, v::Int,
                           sub::WiringDiagram, sub_map::Dict{Int,Int};
                           merge_wire_values=default_merge_wire_values)
  for wire in wires(sub)
    src = get(sub_map, wire.source.box, 0)
    tgt = get(sub_map, wire.target.box, 0)
    # Special case: wire from input port to output port.
    if wire.source.box == input_id(sub) && wire.target.box == output_id(sub)
      for in_wire in in_wires(d, v, wire.source.port)
        for out_wire in out_wires(d, v, wire.target.port)
          add_wire!(d, Wire(
            merge_wire_values(in_wire.value, wire.value, out_wire.value),
            in_wire.source, out_wire.target))
        end
      end
    # Special case: wire from input port to internal box.
    elseif wire.source.box == input_id(sub)
      for in_wire in in_wires(d, v, wire.source.port)
        add_wire!(d, Wire(
          merge_wire_values(in_wire.value, wire.value, nothing),
          in_wire.source, set_box(wire.target, tgt)))
      end
    # Special case: wire from internal box to output port.
    elseif wire.target.box == output_id(sub)
      for out_wire in out_wires(d, v, wire.target.port)
        add_wire!(d, Wire(
          merge_wire_values(nothing, wire.value, out_wire.value),
          set_box(wire.source, src), out_wire.target))
      end
    # Default case: wire between two internal boxes.
    else
      add_wire!(d, Wire(
        merge_wire_values(nothing, wire.value, nothing),
        set_box(wire.source, src), set_box(wire.target, tgt)))
    end
  end
end

default_merge_wire_values(::Any, middle::Any, ::Any) = middle

# Encapsulation
###############

""" Encapsulate multiple boxes within a single sub-diagram.

This operation is a (one-sided) inverse to subsitution, see
[`substitute`](@ref).
"""
function encapsulate(d::WiringDiagram, vs::Vector{Int}; value=nothing, kw...)
  encapsulate(d, [vs]; values=[value], kw...)
end

function encapsulate(d::WD, vss::Vector{Vector{Int}}; discard_boxes::Bool=false,
                     make_box=Box, values=nothing) where WD <: WiringDiagram
  if isempty(vss); return d end
  if any(isempty(vs) for vs in vss)
    error("Cannot encapsulate an empty set of boxes")
  end
  if !allunique(reduce(vcat, vss))
    error("Cannot encapsulate overlapping sets of boxes")
  end
  if isnothing(values)
    values = repeat([nothing], length(vss))
  end
  result = WD(d.value, input_ports(d), output_ports(d))

  # Add boxes, both encapsulated and non-encapsulated, to new wiring diagram.
  encapsulated_representatives = Dict(
    minimum(vs) => (vs, value) for (vs, value) in zip(vss, values))
  all_encapsulated = Set(v for vs in vss for v in vs)
  vmap = Dict(input_id(d) => input_id(result), output_id(d) => output_id(result))
  port_map = Dict{Port,Port}()
  for v in box_ids(d)
    if haskey(encapsulated_representatives, v)
      vs, value = encapsulated_representatives[v]
      sub, sub_map = encapsulated_subdiagram(d, vs;
        discard_boxes=discard_boxes, make_box=make_box, value=value)
      subv = add_box!(result, sub)
      merge!(port_map, Dict(port => Port(subv, data...)
                            for (port, data) in sub_map))
    elseif v ∉ all_encapsulated
      vmap[v] = add_box!(result, box(d, v))
    end
  end

  # Add wires to new wiring diagram.
  for wire in wires(d)
    src, tgt = wire.source, wire.target
    new_src = if haskey(vmap, src.box); set_box(src, vmap[src.box])
      elseif haskey(port_map, src); port_map[src]
      else; continue end
    new_tgt = if haskey(vmap, tgt.box); set_box(tgt, vmap[tgt.box])
      elseif haskey(port_map, tgt); port_map[tgt]
      else; continue end
    add_wire!(result, Wire(new_src, new_tgt))
  end
  result
end

""" Create an encapsulating box for a set of boxes in a wiring diagram.

To a first approximation, the union of input ports of the given boxes will
become the inputs ports of the encapsulating box and likewise for the output
ports. However, when copies or merges occur, as in a cartesian or cocartesian
category, a simplification procedure may reduce the number of ports on the
encapsulating box.

Specifically:

1. Each input port of an encapsulated box will have at most one incoming wire
from the encapsulating outer box, and each output port of an encapsulated box
will have at most one outgoing wire to the encapsulating outer box.

2. A set of ports connected to the same outside (non-encapsulated) ports will be
simplified into a single port of the encapsulating box.

See also `induced_subdiagram`.
"""
function encapsulated_subdiagram(d::WD, vs::Vector{Int}; discard_boxes::Bool=false,
                                 make_box=Box, value=nothing) where WD <: WiringDiagram
  # Add encapsulated box to new wiring diagram.
  inputs, outputs = [], []
  result = discard_boxes ? nothing : WD(value, inputs, outputs)
  vmap = if discard_boxes
    Dict(v => nothing for v in vs)
  else
    Dict(v => add_box!(result, box(d, v)) for v in vs)
  end

  # Process wires into, or out of, encapsulated boxes.
  port_map = Dict{Port,NamedTuple{(:kind,:port),Tuple{PortKind,Int}}}()
  inner_port_map = Dict{Tuple{Vector{Port},Any},Int}()
  for v in vs
    # Add input ports to encapsulating box and corresponding wire.
    for (port, value) in enumerate(input_ports(d, v))
      tgt = Port(v, InputPort, port)
      srcs = sort!([ wire.source for wire in in_wires(d, tgt)
                     if !haskey(vmap, wire.source.box) ])
      if isempty(srcs) continue end
      src = get!(inner_port_map, (srcs, value)) do
        p = discard_boxes ? length(push!(inputs, value)) : add_outer_in_port!(result, value)
        port_data = port_map[tgt] = (kind=InputPort, port=p)
        port_data.port
      end
      if discard_boxes; continue end
      add_wire!(result,
        Wire(Port(input_id(result), OutputPort, src), set_box(tgt, vmap[v])))
    end

    # Add output ports to encapsulating box and corresponding wire.
    for (port, value) in enumerate(output_ports(d, v))
      src = Port(v, OutputPort, port)
      tgts = sort([ wire.target for wire in out_wires(d, src)
                    if !haskey(vmap, wire.target.box) ])
      if isempty(tgts) continue end
      tgt = get!(inner_port_map, (tgts, value)) do
        p = discard_boxes ? length(push!(outputs, value)) : add_outer_out_port!(result, value)
        port_data = port_map[src] = (kind=OutputPort, port=p)
        port_data.port
      end
      if discard_boxes; continue end
      add_wire!(result,
        Wire(set_box(src, vmap[v]), Port(output_id(result), InputPort, tgt)))
    end

    # Add wires between two encapsulated boxes.
    if discard_boxes; continue end
    for wire in out_wires(d, v)
      src, tgt = wire.source, wire.target
      if haskey(vmap, src.box) && haskey(vmap, tgt.box) # Clause #1 always true.
        add_wire!(result,
          Wire(set_box(src, vmap[src.box]), set_box(tgt, vmap[tgt.box])))
      end
    end
  end

  if discard_boxes
    result = make_box(value, inputs, outputs)
  end
  (result, port_map)
end

end


using Test
using MathOptInterface
const MOI = MathOptInterface
const MOIB = MOI.Bridges

struct VariableDummyBridge <: MOIB.Variable.AbstractBridge
    id::Int
end
MOIB.Variable.unbridged_map(::VariableDummyBridge, ::MOI.VariableIndex) = nothing

map = MOIB.Variable.Map()
@testset "Empty" begin
    @test isempty(map)
    @test !MOIB.Variable.has_bridges(map)
    @test isempty(keys(map))
end

b1 = VariableDummyBridge(1)
set1 = MOI.EqualTo(0.0)
F1 = MOI.SingleVariable
S1 = typeof(set1)
v1, c1 = MOIB.Variable.add_key_for_bridge(map, () -> b1, set1)
cannot_unbridge_err = ErrorException(
    "Cannot unbridge function because some variables are bridged by variable" *
    " bridges that do not support reverse mapping, e.g., `ZerosBridge`."
)
@testset "Scalar set" begin
    @test v1.value == c1.value == -1
    @test MOIB.Variable.constraint(map, v1) == c1
    @test haskey(map, v1)
    @test map[v1] == b1
    @test MOIB.Variable.constrained_set(map, v1) == S1
    @test MOIB.Variable.number_with_set(map, S1) == 1
    @test MOIB.Variable.constraints_with_set(map, S1) == [c1]
    @test MOIB.Variable.function_for(map, c1) == MOI.SingleVariable(v1)
    @test_throws cannot_unbridge_err MOIB.Variable.unbridged_function(
        map, MOI.VariableIndex(1))

    @test MOIB.Variable.number_of_variables(map) == 1
    @test MOIB.Variable.list_of_constraint_types(map) == Set([(F1, S1)])
    @test length(map) == 1
    @test !isempty(map)
    @test MOIB.Variable.has_bridges(map)
    @test collect(keys(map)) == [v1]
    @test collect(values(map)) == [b1]
end

b2 = VariableDummyBridge(2)
set2 = MOI.Zeros(4)
F2 = MOI.VectorOfVariables
S2 = typeof(set2)
v2, c2 = MOIB.Variable.add_keys_for_bridge(map, () -> b2, set2)
@testset "Vector set" begin
    @test v2[1].value == c2.value == -2
    @test MOIB.Variable.has_keys(map, v2)
    @test !MOIB.Variable.has_keys(map, v2[4:-1:1])
    for i in 1:4
        @test MOIB.Variable.constraint(map, v2[i]) == c2
        @test haskey(map, v2[i])
        @test map[v2[i]] == b2
        @test MOIB.Variable.constrained_set(map, v2[i]) == S2
        @test MOIB.Variable.length_of_vector_of_variables(map, v2[i]) == 4
        @test MOIB.Variable.index_in_vector_of_variables(map, v2[i]) == MOIB.Variable.IndexInVector(i)
    end
    @test MOIB.Variable.number_with_set(map, S2) == 1
    @test MOIB.Variable.constraints_with_set(map, S2) == [c2]
    @test MOIB.Variable.function_for(map, c2) == MOI.VectorOfVariables(v2)
    @test_throws cannot_unbridge_err MOIB.Variable.unbridged_function(
        map, MOI.VariableIndex(1))

    @test MOIB.Variable.number_of_variables(map) == 5
    @test MOIB.Variable.list_of_constraint_types(map) == Set([(F1, S1), (F2, S2)])
    @test length(map) == 2
    @test sprint(MOIB.print_num_bridges, map) == "\nwith 2 variable bridges"
    @test !isempty(map)
    @test MOIB.Variable.has_bridges(map)
    @test collect(keys(map)) == [v1; v2]
    @test collect(values(map)) == [b1, b2]
end

b3 = VariableDummyBridge(3)
set3 = MOI.Zeros(0)
v3, c3 = MOIB.Variable.add_keys_for_bridge(map, () -> b3, set3)
@testset "Vector set of length 0" begin
    @test isempty(v3)
    @test c3.value == 0

    bridges = collect(values(map))
    @test sort([b.id for b in bridges]) == 1:2
    elements = sort(collect(map), by = el -> el.second.id)
    @test elements[1].first == v1
    @test elements[1].second == b1
    @test elements[2].first == v2[1]
    @test elements[2].second == b2
end

@testset "Delete" begin
    delete!(map, v1)
    @test MOIB.Variable.number_of_variables(map) == 4
    @test length(map) == 1
    @test sprint(MOIB.print_num_bridges, map) == "\nwith 1 variable bridge"
    @test !isempty(map)
    @test MOIB.Variable.has_bridges(map)
    elements = collect(map)
    @test elements[1].first == v2[1]
    @test elements[1].second == b2
    @test MOIB.Variable.number_with_set(map, S1) == 0
    @test MOIB.Variable.number_with_set(map, S2) == 1
    @test isempty(MOIB.Variable.constraints_with_set(map, S1))
    @test MOIB.Variable.constraints_with_set(map, S2) == [c2]
    @test MOIB.Variable.list_of_constraint_types(map) == Set([(F2, S2)])
    @test collect(keys(map)) == v2
    @test collect(values(map)) == [b2]
    @test !haskey(map, v1)
    @test MOIB.Variable.has_keys(map, v2)

    rev_v2 = reverse(v2)
    err = ArgumentError(
        "`$rev_v2` is not a valid key vector as returned by `add_keys_for_bridge`."
    )
    @test_throws err delete!(map, rev_v2)

    delete!(map, v2[3])
    left = [1, 2, 4]
    @test MOIB.Variable.number_of_variables(map) == 3
    @test length(map) == 1
    @test !isempty(map)
    @test MOIB.Variable.has_bridges(map)
    elements = collect(map)
    @test elements[1].first == v2[1]
    @test elements[1].second == b2
    @test MOIB.Variable.number_with_set(map, S1) == 0
    @test MOIB.Variable.number_with_set(map, S2) == 1
    @test isempty(MOIB.Variable.constraints_with_set(map, S1))
    @test MOIB.Variable.constraints_with_set(map, S2) == [c2]
    @test MOIB.Variable.list_of_constraint_types(map) == Set([(F2, S2)])
    @test collect(keys(map)) == v2[left]
    @test collect(values(map)) == [b2]
    @test !haskey(map, v1)
    @test !haskey(map, v2[3])
    @test !MOIB.Variable.has_keys(map, v2)
    @test MOIB.Variable.has_keys(map, v2[left])
    for (j, i) in enumerate(left)
        @test MOIB.Variable.constraint(map, v2[i]) == c2
        @test haskey(map, v2[i])
        @test map[v2[i]] == b2
        @test MOIB.Variable.constrained_set(map, v2[i]) == S2
        @test MOIB.Variable.length_of_vector_of_variables(map, v2[i]) == 3
        @test MOIB.Variable.index_in_vector_of_variables(map, v2[i]) == MOIB.Variable.IndexInVector(j)
    end
    @test MOIB.Variable.function_for(map, c2) == MOI.VectorOfVariables(v2[left])

    delete!(map, v2[1])
    left = [2, 4]
    @test MOIB.Variable.number_of_variables(map) == 2
    @test length(map) == 1
    @test !isempty(map)
    @test MOIB.Variable.has_bridges(map)
    elements = collect(map)
    @test elements[1].first == v2[1]
    @test elements[1].second == b2
    @test MOIB.Variable.number_with_set(map, S1) == 0
    @test MOIB.Variable.number_with_set(map, S2) == 1
    @test isempty(MOIB.Variable.constraints_with_set(map, S1))
    @test MOIB.Variable.constraints_with_set(map, S2) == [c2]
    @test MOIB.Variable.list_of_constraint_types(map) == Set([(F2, S2)])
    @test collect(keys(map)) == v2[left]
    @test collect(values(map)) == [b2]
    @test !haskey(map, v1)
    @test !haskey(map, v2[1])
    @test !haskey(map, v2[3])
    @test !MOIB.Variable.has_keys(map, v2)
    @test !MOIB.Variable.has_keys(map, v2[[2, 3, 4]])
    @test !MOIB.Variable.has_keys(map, v2[[1, 2, 4]])
    @test MOIB.Variable.has_keys(map, v2[left])
    for (j, i) in enumerate(left)
        @test MOIB.Variable.constraint(map, v2[i]) == c2
        @test haskey(map, v2[i])
        @test map[v2[i]] == b2
        @test MOIB.Variable.constrained_set(map, v2[i]) == S2
        @test MOIB.Variable.length_of_vector_of_variables(map, v2[i]) == 2
        @test MOIB.Variable.index_in_vector_of_variables(map, v2[i]) == MOIB.Variable.IndexInVector(j)
    end
    @test MOIB.Variable.function_for(map, c2) == MOI.VectorOfVariables(v2[left])

    delete!(map, v2[left])
    @test MOIB.Variable.number_of_variables(map) == 0
    @test length(map) == 0
    @test isempty(map)
    @test MOIB.Variable.has_bridges(map)
    @test isempty(collect(map))
    @test MOIB.Variable.number_with_set(map, S1) == 0
    @test MOIB.Variable.number_with_set(map, S2) == 0
    @test isempty(MOIB.Variable.constraints_with_set(map, S1))
    @test isempty(MOIB.Variable.constraints_with_set(map, S2))
    @test isempty(MOIB.Variable.list_of_constraint_types(map))
    @test isempty(keys(map))
    @test isempty(values(map))
    @test !haskey(map, v1)
    @test !MOIB.Variable.has_keys(map, v2)
    @test !MOIB.Variable.has_keys(map, v2[left])
end

@testset "EmptyMap" begin
    map = MOIB.Variable.EmptyMap()
    empty!(map)
    @test isempty(map)
    @test length(map) == 0
    @test isempty(keys(map))
    @test isempty(values(map))
    @test !MOIB.Variable.has_bridges(map)
    @test iszero(MOIB.Variable.number_of_variables(map))
    @test iszero(MOIB.Variable.number_with_set(map, S1))
    @test iszero(MOIB.Variable.number_with_set(map, S2))
    @test isempty(MOIB.Variable.constraints_with_set(map, S1))
    @test isempty(MOIB.Variable.constraints_with_set(map, S2))
    @test sprint(MOIB.print_num_bridges, map) == ""
end


export evaluate, random_evaluate, fitness_evaluate

"""
    evaluate(e::AbstractEvolution)

Default evaluate function. Sets each element of the fitness array of each
individual to -Inf.
"""
function evaluate(e::AbstractEvolution)
    for i in eachindex(e.population)
        for d in 1:e.config.d_fitness
            e.population[i].fitness[d] = -Inf
        end
    end
end

"sets the fitness value of each individual to rand()"
function random_evaluate(e::AbstractEvolution)
    for i in eachindex(e.population)
        for d in 1:e.config.d_fitness
            e.population[i].fitness[d] = rand()
        end
    end
end

function null_evaluate(i::Individual)
    i.fitness[:] .= -Inf
end

"""
    function fitness_evaluate(e::AbstractEvolution; fitness::Function=null_evaluate)

sets the fitness of each individual to the Array of values returned by fitness
"""
function fitness_evaluate(e::AbstractEvolution, fitness::Function=null_evaluate)
    for i in eachindex(e.population)
        e.population[i].fitness[:] = fitness(e.population[i])[:]
    end
end

# TODO: dependencies, test distributed
# function distributed_evaluate!(e::AbstractEvolution; fitness::Function=null_evaluate)
#     fits = SharedArrays.SharedArray{Float64}(e.config["d_fitness"],
#                                              length(e.population))
#     @sync @distributed for i in eachindex(e.population)
#         fits[:, i] = fitness(e.population[i])
#     end
#     for i in eachindex(e.population)
#         e.population[i].fitness .= fits[:, i]
#     end
# end


#Function for quickly determining the energy of a state
#25/07/2017

function envalfunc(atom,nn,ll,jj)
  ZZ, spin, eneigval, params = GetAtomParams(atom,nn,ll,jj)
  return eneigval
end


using LinearAlgebra

import Libdl
const depsfile = joinpath(@__DIR__, "deps.jl")

settings = joinpath(first(DEPOT_PATH), "prefs", "FFTW")
mkpath(dirname(settings))
if haskey(ENV, "JULIA_FFTW_PROVIDER")
    provider = ENV["JULIA_FFTW_PROVIDER"]
    open(f -> println(f, provider), settings, "w")
elseif isfile(settings)
    provider = readchomp(settings)
else
    provider = "FFTW"
    open(f -> println(f, provider), settings, "w")
end

if provider == "MKL"
    const mkllib = Sys.iswindows() ? "mkl_rt" : "libmkl_rt"
    # If BLAS was compiled with MKL and the user wants MKL-based FFTs, we'll oblige.
    open(depsfile, "w") do io
        println(io, """
            using IntelOpenMP_jll, MKL_jll
            check_deps() = nothing
            const libfftw3 = MKL_jll.libmkl_rt_path
            const libfftw3f = libfftw3
        """)
    end
elseif provider == "FFTW"
    open(depsfile, "w") do io
        println(io, """
            using FFTW_jll
            check_deps() = nothing
        """)
    end
else
    error("Unrecognized JULIA_FFTW_PROVIDER \"$provider\".\n",
          "To fix this, set ENV[\"JULIA_FFTW_PROVIDER\"] to \"FFTW\" or \"MKL\"\n",
          "and rerun Pkg.build(\"FFTW\").")
end


using GridWorlds
using Test
using Random

ENVS = [EmptyGridWorld, FourRooms, GoToDoor]
ACTIONS = [TURN_LEFT, TURN_RIGHT, MOVE_FORWARD]

@testset "GridWorlds.jl" begin
    for env in ENVS
        @testset "$(env)" begin
            w = env()
            @test typeof(w.agent_pos) == CartesianIndex{2}
            @test typeof(w.agent.dir) <: LRUD
            @test size(w.world.world, 1) == length(w.world.objects)
            @test 1 ≤ w.agent_pos[1] ≤ size(w.world.world, 2)
            @test 1 ≤ w.agent_pos[2] ≤ size(w.world.world, 3)

            for _=1:1000
                w = w(rand(ACTIONS))
                @test 1 ≤ w.agent_pos[1] ≤ size(w.world.world, 2)
                @test 1 ≤ w.agent_pos[2] ≤ size(w.world.world, 3)
                @test w.world[WALL, w.agent_pos] == false
                view = get_agent_view(w)
                @test typeof(view) <: BitArray{3}
                @test size(view,1) == length(w.world.objects)
                @test size(view,2) == 7
                @test size(view,3) == 7
            end
        end
    end
end


"""
`Rotate180 <: ImageTransformation`

Description
============

Rotates the image 180 degrees. This is a special case rotation
because it can be performed very efficiently by simply rearranging
the existing pixels. Furthermore, the output images is guaranteed
to have the same dimensions as the input image.

If created using the parameter `chance`, the operation will be
lifted into a `Either`. See the documentation of `Either` for more
information.

Usage
======

    Rotate180()

    Rotate180(chance)

Arguments
==========

- **`chance`** : The probability of the operation to be applied to
the image if the operation is used in `transform`. Must be within
the set [0, 1].

Methods
========

- **`transform`** : Applies the transformation to the given Image
or set of images

Author(s)
==========

- Christof Stocker (Github: https://github.com/Evizero)

Examples
========

    using Images
    using Augmentor

    # Create a test image with 4 pixel.
    img = grayim(UInt8[200 150; 50 1])

    # Apply the image operation to the image
    img_new = transform(Rotate180(), img)

    # Check manually if the result is as expected
    @assert img_new == grayim(UInt8[1 50; 150 200])

see also
=========

`ImageTransformation`, `Either`, `transform`
"""
immutable Rotate180 <: ImageTransformation
end

Rotate180(chance) = ProbableOperation(Rotate180(); chance = chance)

Base.showcompact(io::IO, op::Rotate180) = print(io, "Rotate 180 degrees.")
multiplier(::Rotate180) = 1

function transform{T<:AbstractImage}(op::Rotate180, img::T)
    result = copyproperties(img, rotate_expand(img, 1π))::T
    _log_operation!(result, op)
end


@kwargs function emul!(
    w::GBVector,
    u::GBVector,
    v::GBVector;
    op::MonoidSemiringOrBinary = BinaryOps.TIMES
)
    size(w) == size(u) == size(v) || throw(DimensionMismatch())
    op = getoperator(op, eltype(w))
    if op isa libgb.GrB_Semiring
        libgb.GrB_Vector_eWiseMult_Semiring(w, mask, accum, op, u, v, desc)
        return w
    elseif op isa libgb.GrB_Monoid
        libgb.GrB_Vector_eWiseMult_Monoid(w, mask, accum, op, u, v, desc)
        return w
    elseif op isa libgb.GrB_BinaryOp
        libgb.GrB_Vector_eWiseMult_BinaryOp(w, mask, accum, op, u, v, desc)
        return w
    else
        error("Unreachable")
    end
end

@kwargs function emul(
    u::GBVector,
    v::GBVector;
    op::MonoidSemiringOrBinary = BinaryOps.TIMES
)
    if op isa OpUnion
        t = ztype(op)
    else
        t = optype(eltype(u), eltype(v))
    end
    w = GBVector{t}(size(u))
    return emul!(w, u, v; op, mask , accum, desc)
end

@kwargs function emul!(
    C::GBMatrix,
    A::GBMatOrTrans,
    B::GBMatOrTrans;
    op::MonoidSemiringOrBinary = BinaryOps.TIMES
)
    size(C) == size(A) == size(B) || throw(DimensionMismatch())
    A, desc, B = _handletranspose(A, desc, B)
    op = getoperator(op, eltype(C))
    if op isa libgb.GrB_Semiring
        libgb.GrB_Matrix_eWiseMult_Semiring(C, mask, accum, op, A, B, desc)
        return C
    elseif op isa libgb.GrB_Monoid
        libgb.GrB_Matrix_eWiseMult_Monoid(C, mask, accum, op, A, B, desc)
        return C
    elseif op isa libgb.GrB_BinaryOp
        libgb.GrB_Matrix_eWiseMult_BinaryOp(C, mask, accum, op, A, B, desc)
        return C
    end
end

@kwargs function emul(
    A::GBMatOrTrans,
    B::GBMatOrTrans;
    op::MonoidSemiringOrBinary = BinaryOps.TIMES
)
    if op isa OpUnion
        t = ztype(op)
    else
        t = optype(eltype(A), eltype(B))
    end
    C = GBMatrix{t}(size(A))
    return emul!(C, A, B; op, mask, accum, desc)
end

@kwargs function eadd!(
    w::GBVector,
    u::GBVector,
    v::GBVector;
    op::MonoidSemiringOrBinary = BinaryOps.PLUS
)
    size(w) == size(u) == size(v) || throw(DimensionMismatch())
    op = getoperator(op, eltype(w))
    if op isa libgb.GrB_Semiring
        libgb.GrB_Vector_eWiseAdd_Semiring(w, mask, accum, op, u, v, desc)
        return w
    elseif op isa libgb.GrB_Monoid
        libgb.GrB_Vector_eWiseAdd_Monoid(w, mask, accum, op, u, v, desc)
        return w
    elseif op isa libgb.GrB_BinaryOp
        libgb.GrB_Vector_eWiseAdd_BinaryOp(w, mask, accum, op, u, v, desc)
        return w
    else
        error("Unreachable")
    end
end

@kwargs function eadd(
    u::GBVector,
    v::GBVector;
    op::MonoidSemiringOrBinary = BinaryOps.PLUS
)
    if op isa OpUnion
        t = ztype(op)
    else
        t = optype(eltype(u), eltype(v))
    end
    w = GBVector{t}(size(u))
    return eadd!(w, u, v; op, mask, accum, desc)
end

@kwargs function eadd!(
    C::GBMatrix,
    A::GBMatOrTrans,
    B::GBMatOrTrans;
    op::MonoidSemiringOrBinary = BinaryOps.PLUS
)
    size(C) == size(A) == size(B) || throw(DimensionMismatch())
    A, desc, B = _handletranspose(A, desc, B)
    op = getoperator(op, eltype(C))
    if op isa libgb.GrB_Semiring
        libgb.GrB_Matrix_eWiseAdd_Semiring(C, mask, accum, op, A, B, desc)
        return C
    elseif op isa libgb.GrB_Monoid
        libgb.GrB_Matrix_eWiseAdd_Monoid(C, mask, accum, op, A, B, desc)
        return C
    elseif op isa libgb.GrB_BinaryOp
        libgb.GrB_Matrix_eWiseAdd_BinaryOp(C, mask, accum, op, A, B, desc)
        return C
    else
        error("Unreachable")
    end
end

@kwargs function eadd(
    A::GBMatOrTrans,
    B::GBMatOrTrans;
    op::MonoidSemiringOrBinary = BinaryOps.PLUS
)
    if op isa OpUnion
        t = ztype(op)
    else
        t = optype(eltype(A), eltype(B))
    end
    C = GBMatrix{t}(size(A))
    return eadd!(C, A, B; op, mask, accum, desc)
end

@kwargs Base.broadcasted(::typeof(+), u::GBVector, v::GBVector) = eadd(u, v)
@kwargs Base.broadcasted(::typeof(*), u::GBVector, v::GBVector) = emul(u, v)

@kwargs Base.broadcasted(::typeof(+), A::GBMatOrTrans, B::GBMatOrTrans) = eadd(A, B)
@kwargs Base.broadcasted(::typeof(*), A::GBMatOrTrans, B::GBMatOrTrans) = emul(A, B)


#useful macro for conditionally running things in parallel
macro mpar(cond, expr)
  quote
    if $(esc(cond))
        :($(Threads.@threads($expr)))
    else
        :($($expr))
    end
  end
end

const spinlock = Threads.SpinLock()

using Dates, DataFrames

function hypersort!(df::AbstractDataFrame, sorts::Vector{Symbol}, hyperidx::Vector{Int},
  threaded::Bool=false)

  #assume the other levels are sorted
  if length(sorts) > 1
    hypersort!(df, sorts[2:end], hsidx, threaded)
  end

  target::Symbol = sorts[1]
  T::Type = eltype(df[!,target])
  gdf::Vector{SubDataFrame} = collect(groupby(df, sorts[1]))
  targetv::Vector{Float64} = (sdf::SubDataFrame->sdf[1,target]).(gdf)
  Ntarget::Int = length(targetv)

  #map the values to the index array
  cumgroups::Vector{Int} = (sdf::SubDataFrame->size(sdf,1)).(gdf)
  cumsum!(cumgroups) #this is the upperbound on the index rows
  shyperidxs::Vector{SubArray{Int}} = (i->
    view(hyperidx, (i==1 ? 1 : cumgroups[i-1]):cumgroups[i])).(1:Ntarget)
  rowmapping::Dict{T, SubArray{Int}} = Dict{T, SubArray{Int}}(zip(targetv, hsidxs))

end
function hypersort(df::AbstractDataFrame, sorts::Vector{Symbol}; threaded::Bool=false)
  hyperidx::Vector{Int} = collect(1:size(df,1))
  hypersort!(df, sorts, hyperidx, threaded)
end

#------------------------------------------
#=function sortdf3!(df::DataFrame, sorts::Vector{Symbol}; threaded::Bool = false)

  firstsort::Symbol = sorts[1]
  sort!(df, firstsort)
  (length(sorts) == 1) && (return df)

  finalsorts::Vector{Symbol} = sorts[2:end]

  dfout::DataFrame = DataFrame(view(df, falses(size(df, 1)),:))
  sdfs = groupby(df, firstsort)
  dfouts::Vector{DataFrame} = (i->deepcopy(dfout)).(1:length(sdfs))
  @mpar threaded for i ∈ 1:length(sdfs)
    sdf::SubDataFrame = sdfs[i]
    dfouts[i] = sort(sdf, finalsorts)
  end

  (d->append!(dfout,d)).(dfouts)

  return dfout
end

using DataFrames, Dates, Finometrics
function testsorts(N::Int = 10_000_000)
  local df::DataFrame
  local dfs1::DataFrame
  local dfs2::DataFrame
  local dfs3::DataFrame

  df = DataFrame(ones(N,100))
  df.rid = rand(1:25000,N)
  df.dt = rand(Date(1980,1,1):Day(1):Date(2018,12,31),N)
  df.rid2 = rand(1:1000,N)

  dfs1 = deepcopy(df)
  print("\ntime sort1: ")
  @time sort!(dfs1, [:rid, :dt, :rid2])

  Base.GC.gc()

  dfs2 = deepcopy(df)
  print("time sort2: ")
  @time begin
    sort!(dfs2, :rid)
    dfs2s = groupby(dfs2, :rid)
    Threads.@threads for i ∈ 1:length(dfs2s)
      sdfs2::SubDataFrame = dfs2s[i]
      sdfs2 .= sort(sdfs2, [:dt, :rid2])
    end
  end

  dfs3 = deepcopy(df)
  print("time sort3: ")
  @time dfs3 = sortdf3!(dfs3, [:rid, :dt, :rid2], threaded=true)

  dfs4 = deepcopy(df)
  print("time sort4: ")
  @time dfs4 .= sort(dfs4, [:rid, :dt, :rid2])

  @assert sum(dfs1.rid .== dfs2.rid)==N
  @assert sum(dfs1.dt .== dfs2.dt)==N
  @assert sum(dfs1.rid2 .== dfs2.rid2)==N

  @assert sum(dfs1.rid .== dfs3.rid)==N
  @assert sum(dfs1.dt .== dfs3.dt)==N
  @assert sum(dfs1.rid2 .== dfs3.rid2)==N

  @assert sum(dfs1.rid .== dfs4.rid)==N
  @assert sum(dfs1.dt .== dfs4.dt)==N
  @assert sum(dfs1.rid2 .== dfs4.rid2)==N

  return nothing

end

testsorts(100_000)=#

using StatsModels, DataFrames, GLM
implicit_intercept(::Type{<:Any}) = true

#=function mwe(N=20)
  df = DataFrame(x = rand(N), z = (i->Symbol(:s, i)).(rand(1:3, N)))
  LHS::Nothing, RHS = nothing, :(x+z)

  #f = term(0) ~ StatsModels.terms!(StatsModels.parse!(:(x+y)))
  f = @eval(@formula(x ~ $RHS))
  #f = @eval(@formula(0 ~ $RHS))
  f = apply_schema(f, schema(f,df), StatisticalModel)
  m = modelcols(f.rhs, df)
  #mf::ModelFrame = ModelFrame(f, df)
  #m = ModelMatrix(mf).m
  display(m)
end

mwe()=#

function mwe(N=10)
  df = DataFrame(x = rand(N),
    zm = Vector{Union{Missing, Int}}(rand(1:4,N)),
    z = Vector{Int}(rand(1:4,N)))

  println("W/out missings")
  f = @eval(@formula(x ~ z))
  f = apply_schema(f, schema(f,df), StatisticalModel)
  m = modelcols(f.rhs, df)
  display(m)

  println("With missings")
  f = @eval(@formula(x ~ zm))
  f = apply_schema(f, schema(f,df), StatisticalModel)
  m = modelcols(f.rhs, df)
  display(m)
end

mwe()


#=
Copyright (c) 2018-2022 Chris Coey, Lea Kapelevich, and contributors

This Julia package Hypatia.jl is released under the MIT license; see LICENSE
file in the root directory or at https://github.com/chriscoey/Hypatia.jl
=#

insts = OrderedDict()
insts["minimal"] = [
    ((3, 4, true, true, true, true),),
    ((3, 4, false, true, true, true),),
    ((3, 4, true, false, true, true),),
    ((3, 4, false, false, true, true),),
    ((3, 4, true, true, false, true),),
    ((3, 4, false, false, false, true),),
    ((3, 4, true, true, false, false),),
    ((3, 4, false, false, false, false),),
    ]
insts["fast"] = [
    ((12, 24, true, true, true, true),),
    ((12, 24, false, true, true, true),),
    ((12, 24, true, false, true, true),),
    ((12, 24, false, false, true, true),),
    ((12, 24, true, true, false, true),),
    ((12, 24, false, false, false, true),),
    ((12, 24, true, true, false, false),),
    ((12, 24, false, false, false, false),),
    ]
insts["various"] = [
    ((5, 100, true, true, true, true),),
    ((5, 100, true, true, false, true),),
    ((10, 200, true, true, true, true),),
    ((10, 200, true, true, false, true),),
    ]
return (MatrixCompletionNative, insts)


function crop!(df, tStart::DateTime, tStop::DateTime)
  flt1 = df[:date] .>= tStart
  flt2 = df[:date] .<= tStop
  flt = flt1 & flt2
  df = df[flt, :]
end

function crop!(df, tStart::ASCIIString, tStop::ASCIIString)
  crop!(df, DateTime(tStart), DateTime(tStop))
end

function crop(df, tStart::DateTime, tStop::DateTime)
  flt1 = df[:date] .>= tStart
  flt2 = df[:date] .<= tStop
  flt = flt1 & flt2
  return df[flt, :]
end

function crop(df, tStart::ASCIIString, tStop::ASCIIString)
  crop(df, DateTime(tStart), DateTime(tStop))
end

function clean!(df)
  flt = ones(Bool, size(df, 1))
  for var in [:peakArea, :peakCenter, :peakWidth]
    flt &= get_outliers(df, var=var)
  end
  df = df[flt, :]
end

function clean(df)
  flt = ones(Bool, size(df, 1))
  for var in [:peakArea, :peakCenter, :peakWidth]
    flt &= get_outliers(df, var=var)
  end
  try
    flt_offNadir = df[:offNadirAngle_deg] .<= 25.0
    flt &= flt_offNadir
  end
  return df[flt, :]
end

function clean_h5(path)
  fileNames = readdir(path)
  for fname in fileNames
    fid = h5open(joinpath(path, fname),"r+")
    for grp in fid
      for dataset in grp
        if ismatch(r"M06.2", name(dataset))
          println(name(dataset))
          o_delete(grp, name(dataset))
        end
      end
    end
    close(fid)
  end
end

function doCO2Correction(tCO, nCO, tCO2, nCO2)
  dtMax = DateTime(2000,1,1,0,10) - DateTime(2000,1,1,0,0)
  nCOCorrected = Float64[]
  tCOCorrected = DateTime[]
  wasCorrected = zeros(Int64, length(nCO))
  for i=1:length(nCO)
    for j=1:length(nCO2)
      if ((tCO2[j] > tCO[i]) & ((tCO2[j] - tCO[i]) < dtMax))
        newValue = nCO[i] - 0.099 * nCO2[j]
        if newValue > 0
          push!(nCOCorrected, newValue)
          push!(tCOCorrected, tCO[i])
          wasCorrected[i] = 1
          break
        end
      end
    end
  end
  return tCOCorrected, nCOCorrected, wasCorrected
end

function moving_avg(y, window_size)
  myAvg = copy(y)
  for i=window_size+1:length(y)-window_size
    myAvg[i] = median(y[i-window_size:i+window_size])
  end
  return myAvg
end

function pds2h5(path, homeDir="/home/abieler/rosetta/data/dfms/h5/")
  fileNames = readdir(path)

  for fileName in sort(fileNames)
    # remove the .TAB of fileName
    bName = fileName[1:end-4]
    yrStr, tStr, modeStr = matchall(r"(\d+)", bName)
    if (modeStr[end] == '2') & (modeStr != "0572") & (modeStr[1:2] != "06")
      h5Str = joinpath(homeDir, yrStr[1:6] * ".h5")
      y, gainStep, t, m0, p_pt = parseDataFileBothRows(joinpath(path, fileName))
      mstr = "m" * string(round(Int,m0))
      dataSetName = mstr * "/" * bName
      try
        h5write(h5Str, joinpath(dataSetName, "gainStep"), gainStep)
        h5write(h5Str, joinpath(dataSetName, "commandedMass"), m0)
        h5write(h5Str, joinpath(dataSetName, "p_pt"), p_pt)
        h5write(h5Str, joinpath(dataSetName, "y"), y)
        h5write(h5Str, joinpath(dataSetName, "tStart"), string(t))
      catch
      end
    end
  end
end

function remove_outliers(df; N=50, var=:peakArea)
  flt = get_outliers(df, N=N, var=var)
  return df[flt,:]
end

function remove_outliers!(df; N=50, var=:peakArea)
  flt = get_outliers(df, N=N, var=var)
  df = df[flt,:]
end

function get_outliers(df; N=50, var=:peakArea)
  y = convert(Array, df[var])
  nDataPoints = length(y)
  flt = ones(Bool, nDataPoints)
  for i = N+1:nDataPoints-N
    data = append!(y[i-N:i-1], y[i+1:i+N])
    if abs(median(data) - y[i]) > (3*std(data))
      flt[i] = false
    end
  end
  return flt
end


mutable struct image_t <: LCMType
    utime::Int64
    width::Int32
    height::Int32
    row_stride::Int32
    pixelformat::Int32
    size::Int32
    data::Vector{UInt8}
    nmetadata::Int32
    metadata::Vector{image_metadata_t}
end

@lcmtypesetup(image_t,
    data => (size,),
    metadata => (nmetadata,)
)


# __precompile__()
module EchoJulia

using Reexport

export transform

@reexport using SimradRaw
@reexport using SimradEK60
@reexport using EchogramPyPlot
#@reexport using EchogramPlots
@reexport using EchogramImages
@reexport using EchoviewEvr
@reexport using EchoviewEcs
@reexport using SimradEK60TestData

function between(p, starttime, endtime)

    if starttime == nothing
        starttime = p.filetime
    end

    if endtime == nothing
        endtime = p.filetime
    end

    starttime <= p.filetime <= endtime

end

function transform(datagrams::Vector{SimradRaw.Datagram}; calibration=nothing, starttime=nothing, endtime=nothing)
    ps = collect(pings(datagrams))
    transform(ps, calibration=calibration, starttime=starttime, endtime=endtime)
end

function transform(ps::Vector{SimradEK60.EK60Ping}; calibration=nothing, starttime=nothing, endtime=nothing)

    transducers= []
    if calibration != nothing
        transducers = calibration
    end

    ps = [p for p in ps if between(p, starttime, endtime)]

    frequencies = unique([p.frequency for p in ps])

    dict = Dict()

    index = 1
    for f in frequencies
        fr = trunc(Int,f / 1000)
        psf = [p for p in ps if p.frequency == f]

        gain = nothing
        equivalentbeamangle = nothing
        soundvelocity =  nothing
        absorptioncoefficient = nothing
        transmitpower = nothing
        pulselength = nothing
        sacorrection = nothing

        if length(transducers) >= index
            transducer = transducers[index]
            gain = get(transducer, "Ek60TransducerGain", nothing)
            equivalentbeamangle = get(transducer,"TwoWayBeamAngle", nothing)
            soundvelocity=  get(transducer,"SoundSpeed", nothing)
            absorptioncoefficient= get(transducer,"AbsorptionCoefficient", nothing)
            transmitpower= get(transducer,"TransmittedPower", nothing)
            pulselength= get(transducer,"TransmittedPulseLength", nothing)
            sacorrection= get(transducer,"EK60SaCorrection", nothing)
            if pulselength != nothing
                pulselength = pulselength / 1000
            end
        end

        dict["Sv$fr"] = SimradEK60.Sv(psf,
                                      gain=gain,
                                      equivalentbeamangle=equivalentbeamangle,
                                      soundvelocity=soundvelocity,
                                      absorptioncoefficient=absorptioncoefficient,
                                      transmitpower=transmitpower,
                                      pulselength=pulselength,
                                      sacorrection=sacorrection);

        dict["ntheta$fr"] = alongshipangle(psf)
        dict["nphi$fr"] = athwartshipangle(psf)
        dict["r$fr"] = R(psf, soundvelocity=soundvelocity)
        dict["t$fr"] = SimradEK60.filetime(psf) # timestamps

        if gain != nothing
            dict["gain$fr"] = gain
        else
            dict["gain$fr"] = [x.gain for x in psf]
        end
        if equivalentbeamangle != nothing
            dict["equivalentbeamangle$fr"]= equivalentbeamangle
        else
            dict["equivalentbeamangle$fr"]= [x.equivalentbeamangle for x in psf]
        end
        if soundvelocity != nothing
            dict["soundvelocity$fr"]= soundvelocity
        else
            dict["soundvelocity$fr"]=  [x.soundvelocity for x in psf]
        end
        if absorptioncoefficient != nothing
            dict["absorptioncoefficient$fr"] = absorptioncoefficient
        else
            dict["absorptioncoefficient$fr"] = [x.absorptioncoefficient for x in psf]
        end
        if transmitpower != nothing
            dict["transmitpower$fr"] = transmitpower
        else
            dict["transmitpower$fr"] = [x.transmitpower for x in psf]
        end
        if pulselength != nothing
            dict["pulselength$fr"] = pulselength
        else
            dict["pulselength$fr"] = [x.pulselength for x in psf]
        end
        if sacorrection != nothing
            dict["sacorrection$fr"] = sacorrection
        else
            dict["sacorrection$fr"] =[x.sacorrection for x in psf]
        end

        dict["sampleinterval$fr"] =[x.sampleinterval for x in psf]

        index +=1
    end

    dict["DESCR"] = "Converted by EchoJulia."

    return dict
end

end # module


module RanDAGen

"""
Configuration Options

    This package provides the following configuration options for customizing the generated
    DAG's.

Local
- `fatness::Float64`: Defaults to 0.5
- `ccr::Int64`: Defaults to 0
- `data_range::UnitRange{Int64}`: Defaults to 2048:11264
- `alpha_range::UnitRange{Float64}`: Defaults to 0.0:0.2
- `regular::Float64`: Defaults to 0.9 | Must be between 0 and 1
- `jump::Int64`: Defaults to 0
- `density::Float64`: Defaults to 0.5
- `max_task::Int64`: Defaults to 100
- `output_stream::IOStream`: Defaults to stdout
"""
using Distributions

include("statistics.jl")
export Statistics, print_stats

greet() = print("Hello World!")

end # module


#Period types
value{P<:Period}(x::P) = x.value

# The default constructors for Periods work well in almost all cases
# P(x) = new((convert(Int64,x))
# The following definitions are for Period-specific safety
for p in (:Year,:Month,:Week,:Day,:Hour,:Minute,:Second,:Millisecond)
    # This ensures that we can't convert between Periods
    @eval $p(x::Period) = throw(ArgumentError("Can't convert $(typeof(x)) to $($p)"))
    # Unless the Periods are the same type
    @eval $p(x::$p) = x
    # Convenience method for show()
    @eval _units(x::$p) = " " * lowercase(string($p)) * (abs(value(x)) == 1 ? "" : "s")
    # periodisless
    @eval periodisless(x::$p,y::$p) = value(x) < value(y)
    # String parsing (mainly for IO code)
    @eval $p(x::String) = $p(parseint(Int64,x))
    # Period accessors
    @eval $p(x::TimeType) = $p($(symbol(lowercase(string(p))))(x))
end
# Now we're safe to define Period-Number conversions
# Anything an Int64 can convert to, a Period can convert to
Base.convert{T<:Number}(::Type{T},x::Period) = convert(T,value(x))
Base.convert{T<:Period}(::Type{T},x::Real) = T(int64(x))

#Print/show/traits
Base.string{P<:Period}(x::P) = string(value(x),_units(x))
Base.show(io::IO,x::Period) = print(io,string(x))
Base.zero{P<:Period}(::Union(Type{P},P)) = P(0)
Base.one{P<:Period}(::Union(Type{P},P)) = P(1)
Base.typemin{P<:Period}(::Type{P}) = P(typemin(Int64))
Base.typemax{P<:Period}(::Type{P}) = P(typemax(Int64))

# Default values (as used by TimeTypes)
default{T<:DatePeriod}(p::Union(T,Type{T})) = one(p)
default{T<:TimePeriod}(p::Union(T,Type{T})) = zero(p)

(-){P<:Period}(x::P) = P(-value(x))
Base.isless{P<:Period}(x::P,y::P) = isless(value(x),value(y))
Base.isless{R<:Real}(x::Period,y::R) = isless(int64(y),value(x))
Base.isless{R<:Real}(y::R,x::Period) = isless(int64(y),value(x))
=={P<:Period}(x::P,y::P) = ===(value(x),value(y))
=={R<:Real}(x::Period,y::R) = ==(int64(y),value(x))
=={R<:Real}(y::R,x::Period) = ==(int64(y),value(x))

Base.isless(x::Period,y::Period) = throw(ArgumentError("Can't compare $(typeof(x)) and $(typeof(y))"))
==(x::Period,y::Period) = throw(ArgumentError("Can't compare $(typeof(x)) and $(typeof(y))"))

#Period Arithmetic:
import Base.div, Base.mod, Base.gcd, Base.lcm
let vec_ops = [:.+,:.-,:.*,:.%,:div]
    for op in [:+,:-,:*,:%,:mod,:gcd,:lcm,vec_ops]
        @eval begin
        #Period-Period
        ($op){P<:Period}(x::P,y::P) = P(($op)(value(x),value(y)))
        #Period-Real
        ($op){P<:Period}(x::P,y::Real) = P(($op)(value(x),convert(Int64,y)))
        ($op){P<:Period}(x::Real,y::P) = P(($op)(convert(Int64,x),value(y)))
        end
        #Vectorized
        if op in vec_ops
            @eval begin
            ($op){P<:Period}(x::AbstractArray{P}, y::P) = reshape([$op(i,y) for i in x], size(x))
            ($op){P<:Period}(y::P, x::AbstractArray{P}) = reshape([$op(i,y) for i in x], size(x))
            end
        end
    end
end

# intfuncs
Base.gcdx{T<:Period}(a::T,b::T) = ((g,x,y)=gcdx(value(a),value(b)); return T(g),x,y)
Base.abs{T<:Period}(a::T) = T(abs(value(a)))

periodisless(::Period,::Year)        = true
periodisless(::Period,::Month)       = true
periodisless(::Year,::Month)         = false
periodisless(::Period,::Week)        = true
periodisless(::Year,::Week)          = false
periodisless(::Month,::Week)         = false
periodisless(::Period,::Day)         = true
periodisless(::Year,::Day)           = false
periodisless(::Month,::Day)          = false
periodisless(::Week,::Day)           = false
periodisless(::Period,::Hour)        = false
periodisless(::Minute,::Hour)        = true
periodisless(::Second,::Hour)        = true
periodisless(::Millisecond,::Hour)   = true
periodisless(::Period,::Minute)      = false
periodisless(::Second,::Minute)      = true
periodisless(::Millisecond,::Minute) = true
periodisless(::Period,::Second)      = false
periodisless(::Millisecond,::Second) = true
periodisless(::Period,::Millisecond) = false

# Stores multiple periods in greatest to least order by type, not values
type CompoundPeriod
    periods::Array{Period,1}
end
function Base.string(x::CompoundPeriod)
    s = ""
    for p in x.periods
        s *= ", " * string(p)
    end
    return s[3:end]
end
Base.show(io::IO,x::CompoundPeriod) = print(io,string(x))
# E.g. Year(1) + Day(1)
(+)(x::Period,y::Period) = CompoundPeriod(sort!(Period[x,y],rev=true,lt=periodisless))
(+)(x::CompoundPeriod,y::Period) = (sort!(push!(x.periods,y) ,rev=true,lt=periodisless); return x)
# E.g. Year(1) - Month(1)
(-)(x::Period,y::Period) = CompoundPeriod(sort!(Period[x,-y],rev=true,lt=periodisless))
(-)(x::CompoundPeriod,y::Period) = (sort!(push!(x.periods,-y),rev=true,lt=periodisless); return x)

# Capture TimeType+-Period methods
(+)(a::TimeType,b::Period,c::Period) = (+)(a,b+c)
(-)(a::TimeType,b::Period,c::Period) = (-)(a,b-c)
(+)(a::TimeType,b::Period,c::Period,d::Period...) = (+)((+)(a,b+c),d...)
(-)(a::TimeType,b::Period,c::Period,d::Period...) = (-)((-)(a,b-c),d...)

function (+)(x::TimeType,y::CompoundPeriod)
    for p in y.periods
        x += p
    end
    return x
end
(+)(x::CompoundPeriod,y::TimeType) = y + x

# Convert fixed value Periods to # of milliseconds
typealias FixedPeriod Union(Week,Day,Hour,Minute,Second,Millisecond)

toms(c::Millisecond) = value(c)
toms(c::Second)      = 1000*value(c)
toms(c::Minute)      = 60000*value(c)
toms(c::Hour)        = 3600000*value(c)
toms(c::Day)         = 86400000*value(c)
toms(c::Week)        = 604800000*value(c)
toms(c::Month)       = 86400000.0*30.436875*value(c)
toms(c::Year)        = 86400000.0*365.2425*value(c)

Millisecond{T<:FixedPeriod}(c::T) = Millisecond(toms(c))

days(c::Millisecond) = div(value(c),86400000)
days(c::Second)      = div(value(c),86400)
days(c::Minute)      = div(value(c),1440)
days(c::Hour)        = div(value(c),24)
days(c::Day)         = value(c)
days(c::Week)        = 7*value(c)
days(c::Year)        = 365.2425*value(c)
days(c::Month)       = 30.436875*value(c)

Day{T<:FixedPeriod}(c::T) = Day(days(c))


using MetaGraphs
using LightGraphs
using GraphMLReader
using GraphIO
using TimerOutputs

abstract type MyAbstractCity end

"""
Structure City
Instantiate a structure contening the information about the city (POI, Working Place, etc.)

name_file::String : name of the graph file (in graphml format)
n_node::Int32 : number of nodes of the graph
id_poi::Array{Int32,1} : id of all POI
id_aliment::Array{Int32,1} : id of all restaurants
id_ecole::Array{Int32,1} : id of all school
id_work::Array{Int32,1} : id of all working place
area_work::Array{Float64,1} : area of all working place (expressed in m2)
proba_resto::Array{Float64,2} : probability to go to the restaurant for each working place
"""

export City

struct City <: MyAbstractCity

    name_file::String
    n_node::Int32
    id_poi::Array{Int32,1}
    id_aliment::Array{Int32,1}
    id_ecole::Array{Int32,1}
    id_work::Array{Int32,1}
    area_work::Array{Float64,1}
    proba_resto::Array{Float64,2}

    function City()

        # Read the graph file (from GraphMLReader)
        name_file = ParamFile.GRAPH_FILE ;
        graph = loadgraphml("Data/$(lpad(name_file,1)).graphml") ;
        # Compute the number of nodes (nv: LightGraphs)
        n_node = nv(graph) ;
        # Find the id of POI, food and school
        id_poi, id_aliment, id_ecole = poi(graph,n_node) ;
        # find the id of Working Place and the area expressed in m2
        id_work, area_work = working_place(graph,n_node) ;
        # Compute the probability to go to the restaurant for each working place
        if(Journee.CHOIX_ALEATOIRE_RESTO)
            proba_resto = ones(Float64,(length(id_work),length(id_aliment)))/length(id_aliment) ;
        else
            proba_resto = proba_restaurant() ;
        end

        return new(
        name_file,
        n_node,
        id_poi,
        id_aliment,
        id_ecole,
        id_work,
        area_work,
        proba_resto
        )

    end

end

"""
POI(graph, n_node)

    Function used to find the id node of all POI
    Input:
    - graph: undirected Int64 metagraph with Float64 weights
    - n_node ({Int32}): total number of nodes
    Output:
    - id_poi (Array{Int32,.}): Vector containing the id of all POI

"""
function poi(graph::MetaGraph,n_node::Int)

    # Initialisation of the vector
    id_poi = Array{Int32}([]) ;

    # Find all nodes of POI type
    @inbounds for i=1:n_node
        if(get_prop(graph,i,:type)=="POI")
            append!(id_poi,i) ;
        end
    end

    id_aliment = Array{Int32}([]) ;

    @inbounds for i=1:length(id_poi)
        type_poi = get_prop(graph,id_poi[i],:typeNameICI) ;
        if((type_poi == "loisir.restauration") || (type_poi == "commerce.alimentation"))
            append!(id_aliment,i) ;
        end

    end

    id_aliment = id_poi[id_aliment] ;

    id_ecole = Array{Int32}([]) ;

    @inbounds for i=1:length(id_poi)
        type_poi = get_prop(graph,id_poi[i],:typeNameICI) ;
        if((type_poi == "education.primaire") || (type_poi == "education.secondaire") || (type_poi == "education.superieure") || (type_poi == "education.particuliere") )
        #if((type_poi == "education.primaire") || (type_poi == "education.secondaire"))
            append!(id_ecole,i) ;
        end
    end

    id_ecole = id_poi[id_ecole] ;

    return(id_poi,id_aliment, id_ecole)

end

"""
WorkingPlace(graph, n_node)

    Function used to find the id node of all working place
    Input:
    - graph: undirected Int64 metagraph with Float64 weights
    - n_node ({Int32}): total number of nodes
    Output:
    - id_work (Array{Int32,.}): Vector containing the id of all working place
    - area_work (Array{Int32,.}): Vector containing the area of each working place

"""
function working_place(graph::MetaGraph,n_node::Int)

    id_work = Array{Int32}([]) ;

    @inbounds for i=1:n_node
        if(get_prop(graph,i,:type)=="WorkingPlace")
            append!(id_work,i) ;
        end
    end

    n_work = length(id_work) ;
    area_work = rand(Uniform(50,1500),n_work) ;

    return(id_work,area_work)

end

"""
Function proba_restaurant()
Fonction permettant de définir une probabilité liée à la distance entre le restaurant et l'entreprise

"""
function proba_restaurant()

    distance = readdlm("Data/Distance_resto.txt") ;
    nb_resto = size(distance,2) ;
    nb_travail = size(distance,1) ;
    proba_resto = zeros(Float64,(nb_travail,nb_resto)) ;

    @inbounds for i=1:nb_travail
        Ind = findall(distance[i,:] .!= 0) ;
        nb = sum(1/distance[i,Ind]) ;
        proba_resto[i,Ind] = (1/distance[i,Ind]/nb) ;
    end

    return(proba_resto) ;

end


"""
Here we define the interface that all contexts implement and provide a simple implementation

Constructor takes compute graph and any implementation specific parameters and initialises initial
tensor and parameter storage

Each context implements the following functions to access tensor and parameter information:
gettensor(ctx, sym): get the tensor for the given symbol
settensor!(ctx, value, sym): set the tensor data for the given symbol
deletetensor!(ctx, sym): delete the tensor after it is no longer required
Base.getindex(ctx, sym): get the parameter value corresponding to given symbol
Base.setindex!(ctx, value, sym): set the parameter value corresponding to given symbol
Base.haskey(ctx, sym): Check if the parameter key exists
Base.zeros(ctx, size): create array of zeros with appropritate type for context
Base.zero(ctx): create scalar with same type used in context
Base.eltype(ctx): return the element type of numeric datastructures
set_open_bonds!(ctx, bitstring::String): Set output parameters according to provided bitstring
set_slice_vals!(ctx, slice_values::Vector{Int}): Set output parameters according to provided slice values
(c::ctx)(): Execute the compute graph in the provided context. Returns final tensor
compute_amplitude!(ctx, bitstring; num_slices=nothing): Contracts the network to compute the given bitstring. Includes reduction over slices

Also provides an implementation of each command which uses above functions to get required
tensors and parameters. For example contraction command this function is defined as

(c::ContractionCommand)(ctx): implenents the contraction described by the given contraction index with the context

Each context will also implement the following functions to be implemented by distributed contexts
ctxmap(f, ctx, items): Applies function f to items
ctxreduce(f, ctx, items): Performs reduction using the function f on items
ctxgather(ctx, items): Gathers all items
"""

using QXContexts.ComputeGraphs
using OMEinsum
using DataStructures
using CUDA

export gettensor, settensor!, deletetensor!, set_open_bonds!, set_slice_vals!
export AbstractContext, QXContext, compute_amplitude!
export ctxmap, ctxgather, ctxreduce

abstract type AbstractContext end

##################################################################################
# Provide implementation of each command
##################################################################################
"""
    (c::ContractCommand)(ctx::AbstractContext)

Execute a contraction command in the given context
"""
function (c::ContractCommand)(ctx::AbstractContext)
    output_idxs = Tuple(c.output_idxs)
    left_idxs = Tuple(c.left_idxs)
    right_idxs = Tuple(c.right_idxs)

    @debug "ncon DSL command: $(c.output_name)[$(output_idxs)] = $(c.left_name)[$(c.left_idxs)] * $(c.right_name)[$(c.right_idxs)]"
    @debug "ncon shapes: left_size=$(size(gettensor(ctx, c.left_name))), right_size=$(size(gettensor(ctx, c.right_name)))"
    @nvtx_range "NCON $(c.output_name)" begin
        settensor!(ctx, EinCode((left_idxs, right_idxs), output_idxs)(gettensor(ctx, c.left_name), gettensor(ctx, c.right_name)), c.output_name)
    end
    deletetensor!(ctx, c.left_name)
    deletetensor!(ctx, c.right_name)
    nothing
end

"""
    (c::LoadCommand)(ctx::AbstractContext)

Generic implementation of load command
"""
function (c::LoadCommand)(ctx::AbstractContext)
    @nvtx_range "Load $(c.label)" begin
        settensor!(ctx, gettensor(ctx, c.label), c.name)
    end
end

"""
    (c::SaveCommand)(ctx::AbstractContext)

Generic implementation of save command
"""
function (c::SaveCommand)(ctx::AbstractContext)
    @nvtx_range "Load $(c.label)" begin
        settensor!(ctx, gettensor(ctx, c.label), c.name)
    end
end

"""
    (c::ReshapeCommand)(ctx::AbstractContext)

Implementation of reshape command
"""
function (c::ReshapeCommand)(ctx::AbstractContext)
    tensor_dims = size(gettensor(ctx, c.input))
    new_dims = [prod([tensor_dims[y] for y in x]) for x in c.dims]
    @nvtx_range "Reshape $(c.output)" begin
        settensor!(ctx, reshape(gettensor(ctx, c.input), new_dims...), c.output)
    end
    @debug "Reshape DSL command: name=$(c.input)($(tensor_dims)) -> $(c.output)($(new_dims))"
    nothing
end

"""
    (c::ViewCommand)(ctx::AbstractContext)

Execute the given view command using provided context
"""
function (c::ViewCommand)(ctx::AbstractContext)
    bond_val = haskey(ctx, c.slice_sym) ? ctx[c.slice_sym] : nothing
    @nvtx_range "View $(c.output_sym)" begin
        if bond_val !== nothing
            dims = size(gettensor(ctx, c.input_sym))
            view_index_list = [i == c.bond_index ? UnitRange(bond_val, bond_val) : UnitRange(1, dims[i]) for i in 1:length(dims)]
            new_tensor = @view gettensor(ctx, c.input_sym)[view_index_list...]
            settensor!(ctx, new_tensor, c.output_sym)
            @debug "view DSL command: $(c.output_sym) = $(c.input_sym)[$(view_index_list)]"
        else
            settensor!(ctx, gettensor(ctx, c.input_sym), c.output_sym)
            @debug "view DSL command: $(c.output_sym) = $(c.input_sym)"
        end
    end
    nothing
end

"""
    (c::OutputCommand)(ctx::AbstractContext)

Execute the given output command using provided context
"""
function (c::OutputCommand)(ctx::AbstractContext)
    @nvtx_range "Output $(c.idx)" begin
        sym = Symbol("o$(c.idx)")
        @assert haskey(ctx, sym) "Output $sym not set in context"
        out_val = ctx[sym]
        settensor!(ctx, gettensor(ctx, Symbol("output_$out_val")), c.name)
        @debug "output DSL command: $(c.name) = o$(c.idx)"
    end
end

##########################################################################################
# Provide concreate implementation of a context
#########################################################################################

"""Data structure for context implementation"""
struct QXContext{T} <: AbstractContext
    params::Dict{Symbol, Int}
    tensors::Dict{Symbol, T}
    cg::ComputeGraph
    slice_dims::OrderedDict{Symbol, Int}
    output_dims::OrderedDict{Symbol, Int}
end

"""
    QXContext{T}(cg::ComputeGraph) where T

Constuctor which initialises instance from a compute graph
"""
function QXContext{T}(cg::ComputeGraph) where T
    tensors = Dict{Symbol, T}()
    # load tensors from compute graph into this dictionary
    for (k, v) in pairs(cg.tensors)
        tensors[k] = convert(T, v)
    end
    slice_dims = convert(OrderedDict, params(cg, ViewCommand))
    output_dims = convert(OrderedDict, params(cg, OutputCommand))
    dims = collect(values(output_dims))
    if length(dims) > 0
        @assert all(dims .== dims[1]) "Multiple output dimensions not supported"
        for d in 1:dims[1]
            t = zeros(eltype(T), dims[1])
            t[d] = 1.
            tensors[Symbol("output_$(d-1)")] = convert(T, t)
        end
    end
    sort!(slice_dims)
    sort!(output_dims)
    QXContext{T}(Dict{Symbol, Int}(), tensors, cg, slice_dims, output_dims)
end

QXContext(cg::ComputeGraph) = QXContext{Array{ComplexF32}}(cg)

"""
    gettensor(ctx::QXContext, sym)

Function to retrieve tensors by key
"""
function gettensor(ctx::QXContext, sym)
    @assert haskey(ctx.tensors, sym) "Tensor $sym does not exist in this context"
    ctx.tensors[sym]
end

"""
    settensor!(ctx::QXContext, sym)

Function to set tensors by key
"""
function settensor!(ctx::QXContext, value, sym)
    ctx.tensors[sym] = value
end

"""
    deletetensor!(ctx::QXContext, sym)
Function to delte tensors by key
"""
function deletetensor!(ctx::QXContext, sym)
    delete!(ctx.tensors, sym)
end

"""Implement has key to check if parameter by this name present"""
Base.haskey(ctx::QXContext, sym) = haskey(ctx.params, sym)

"""
    Base.getindex(ctx::QXContext, sym)::Int

Implement getindex for retrieving parameter values by key
"""
function Base.getindex(ctx::QXContext, sym)::Int
    @assert haskey(ctx, sym) "Parameter $sym does not exist in this context"
    ctx.params[sym]
end

"""
    Base.setindex!(ctx::QXContext, sym)::Int

Implement setindex! for setting parameter values
"""
function Base.setindex!(ctx::QXContext, value, sym)
    ctx.params[sym] = value
end

Base.zeros(::QXContext{T}, size) where T = convert(T, zeros(eltype(T), size))
Base.zero(::QXContext{T}) where T = zero(eltype(T))
Base.eltype(::QXContext{T}) where T = eltype(T)

"""
    set_open_bonds!(ctx::QXContext, bitstring::String)

Given a bitstring, set the open bonds to values so contracting the network will
calculate the amplitude of this bitstring
"""
function set_open_bonds!(ctx::QXContext, bitstring::String="")
    if bitstring == "" bitstring = "0"^length(ctx.output_dims) end
    @assert length(bitstring) == length(ctx.output_dims) "Bitstring length must match nubmer of outputs"
    for (i, key) in enumerate(keys(ctx.output_dims)) ctx[key] = parse(Int, bitstring[i]) end
end

"""
    set_slice_vals!(ctx::QXContext, slice_values::Vector{Int})

For each bond that is being sliced set the dimension to slice on.
"""
function set_slice_vals!(ctx::QXContext, slice_values::Vector{Int})
    # set slice values and remove any already set
    for (i, key) in enumerate(keys(ctx.slice_dims))
        if i > length(slice_values)
            delete!(ctx.params, key)
        else
            ctx[key] = slice_values[i]
        end
    end
end

"""
    (ctx::QXContext)()

Funciton to execute compute graph when struct is called. Returns final tensor
"""
function (ctx::QXContext)()
    for n in get_commands(ctx.cg) n(ctx) end
    gettensor(ctx, output(ctx.cg))
end

"""
    compute_amplitude!(ctx::QXContext, bitstring::String; max_slices=nothing)

Calculate a single amplitude with the given context and bitstring. Involves a sum over
contributions from each slice. Can optionally set the number of bonds. By default all slices
are used.
"""
function compute_amplitude!(ctx::QXContext, bitstring::String; max_slices=nothing)
    set_open_bonds!(ctx, bitstring)
    amplitude = nothing
    for p in SliceIterator(ctx.cg, max_slices=max_slices)
        set_slice_vals!(ctx, p)
        if amplitude === nothing
            amplitude = ctx()
        else
            amplitude += ctx()
        end
    end
    amplitude = convert(Array, amplitude) # if a gpu array convert back to gpu
    if ndims(amplitude) == 0
        amplitude = amplitude[]
    end
    amplitude
end

"""Map over items as placeholder for more complicated contexts"""
ctxmap(f, ctx::QXContext, items) = map(f, items)

"""Simple gather as placeholder for distributed contexts"""
ctxgather(ctx::QXContext, items) = items

"""Simple gather as placeholder for distributed contexts"""
ctxreduce(f, ctx::QXContext, items) = reduce(f, items)


export Algorithm

abstract type Algorithm end

abstract type Detect <: Algorithm end
abstract type Align <:Algorithm end
abstract type Cluster <:Algorithm end
abstract type Feature <:Algorithm end
abstract type Reduction <:Algorithm end
abstract type Threshold <:Algorithm end

abstract type Sorting end

#Data Structure to store range of a spike and cluster ID
struct Spike
    inds::UnitRange{Int64}
    id::UInt8
end

Spike()=Spike(0:0,0)

struct s_sizes
    win::Int16
    s_end::Int16
    win2::Int16
end

s_sizes(win)=s_sizes(win,win+div(win,2),div(win,2))

function output_buffer(channels::Int64,par=false)

    nums=zeros(UInt16,channels)

    if par==false
        buf=Spike[Spike() for i=1:100,j=1:channels]
    else
        buf=convert(SharedArray{Spike,2},Spike[Spike() for i=1:100,j=1:channels])
        nums=convert(SharedArray{UInt16,1},nums)
    end

    (buf,nums)
end

global sorting_num = 1

function MakeSorting()
end

function gen_sorting(D::Detect,C::Cluster,A::Align,F::Feature,R::Reduction,T::Threshold,in_type=Int16)

    global sorting_num

    f_type=feature_type(F,in_type)

    @eval begin
        mutable struct $(Symbol("Sorting_$sorting_num")) <: Sorting
            d::($(typeof(D)))
            c::($(typeof(C)))
            a::($(typeof(A)))
            f::($(typeof(F)))
            r::($(typeof(R)))
            t::($(typeof(T)))
            id::Int16
            sigend::Array{$(in_type),1}
            index::Int16
            p_temp::Array{$(in_type),1}
            features::Array{$(f_type),1}
            fullfeature::Array{$(f_type),1}
            dims::Array{Int16,1}
            thres::Float64
            #waveform::Array{$(in_type),1}
            cent::Int16
	    s::s_sizes
        end

        function MakeSorting(D::($(typeof(D))),C::($(typeof(C))),A::($(typeof(A))),F::($(typeof(F))),R::($(typeof(R))),T::($(typeof(T))),window::Int64,in_type=Int16)

            #determine size of alignment output
            wavelength=mysize(A,window)

            #determine feature size
            fulllength=mysize(F,wavelength)

            f_type=feature_type(F,in_type)

            if typeof(R)==ReductionNone
                reducedims=fulllength
            else
                R=typeof(R)(fulllength,R.mydims)
                reducedims=R.mydims
            end
            F=typeof(F)(wavelength,reducedims)
            C=typeof(C)(reducedims)
            p_temp=zeros(in_type,window*2)
            $(Symbol("Sorting_$sorting_num"))(D,C,A,F,R,T,
                    1,zeros(in_type,window+div(window,2)),0,
                    p_temp,zeros(f_type,reducedims),zeros(f_type,fulllength),
                    collect(1:reducedims),1.0,1,s_sizes(window))
        end
    end

    sorting_num+=1
    nothing
end

function create_multi(d::Detect,c::Cluster,a::Align,f::Feature,r::Reduction,t::Threshold,num::Int64,window=50,in_type=Int16)

    if method_exists(MakeSorting,(typeof(d),typeof(c),typeof(a),typeof(f),typeof(r),typeof(t),Int64,DataType))
    else
        gen_sorting(d,c,a,f,r,t,in_type)
    end

    if VERSION > v"0.7-"
        st=Array{typeof(Base.invokelatest(MakeSorting,d,c,a,f,r,t,window,in_type))}(undef, num)
    else
        st=Array{typeof(Base.invokelatest(MakeSorting,d,c,a,f,r,t,window,in_type))}(num)
    end

    for i=1:num
        st[i]=Base.invokelatest(MakeSorting,d,c,a,f,r,t,window,in_type)
        st[i].id=i
    end

    st
end

function create_multi(d::Detect,c::Cluster,a::Align,f::Feature,r::Reduction,t::Threshold,num::Int64,cores::UnitRange{Int64},window=50,in_type=Int16)

    st=create_multi(d,c,a,f,r,t,num,window,in_type)
    st=distribute(st,procs=collect(cores))
end


## functions for writing the midi codes as byte sequences

brev(v::UInt8) = v
brev(v::UInt16) = UInt8[(v>>8)%UInt8,(v)%UInt8]
brev(v::UInt32) = UInt8[(v>>24)%UInt8,(v>>16)%UInt8,(v>>8)%UInt8,(v)%UInt8]
brev(v::UInt64) = UInt8[(v>>56)%UInt8,(v>>48)%UInt8,(v>>40)%UInt8,(v>>32)%UInt8,(v>>24)%UInt8,(v>>16)%UInt8,(v>>8)%UInt8,(v)%UInt8]

msb14(v::UInt16) = (v>>7)%UInt8&0x7f
lsb14(v::UInt16) = (v)%UInt8&0x7f

function write14(out::IOStream, v::Integer)
	@assert 0 <= v < 2^14
	write(out,msb14(v%UInt16)) ;  write(out,lsb14(v%UInt16))
end

function varb(v::Integer)
	@assert 0 <= v < 2^28
	if v<2^7
		return UInt8[(v)%UInt8]
	elseif v<2^14
		return UInt8[(v>>7)%UInt8|0x80,(v)%UInt8&0x7f]
	elseif v<2^21
		return UInt8[(v>>14)%UInt8|0x80,(v>>7)%UInt8|0x80,(v)%UInt8&0x7f]
	else
		return UInt8[(v>>21)%UInt8|0x80,(v>>14)%UInt8|0x80,(v>>7)%UInt8|0x80,(v)%UInt8&0x7f]
	end
end

function writeFileHeaderT0(out::IOStream, tpq::Int)
	@assert 0 < tpq < 2^15
	write(out,ascii("MThd"))
 	write(out,UInt8[0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x01])
	write(out,brev(tpq%UInt16))
end

function writeTrackHeader(out::IOStream, sz::Int)
	@assert 0 <= sz < 2^31
	write(out,ascii("MTrk"))
	write(out,brev(sz%UInt32))
end

function bytesTimeSig(num::Int, den::Int, clocks::Int, sub::Int)
	@assert 0 < num <= 32 ;  @assert 0 <= den <= 5
	@assert 0 < clocks < 128 ;  @assert 0 < sub <= 128
	UInt8[0xff,0x58,0x04,num%UInt8,den%UInt8,clocks%UInt8,sub%UInt8]
end

bytesDt(v::Int)  = varb(v)

function bytesTempo(bpm::Int)
	@assert 0 < bpm
	micspq = (div(10^6*60,bpm))%UInt32
	@assert 0 < micspq < 2^24
	vcat(UInt8[0xff,0x51,0x03],brev(micspq%UInt32)[2:4])
end

function bytesProgSelect(chan::Int, prog::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= prog < 128
	UInt8[0xC0|chan%UInt8,prog%UInt8]
end

function bytesNoteOn(chan::Int, pch::Int, vel::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= pch < 128 ;  @assert 0 <= vel < 128
	UInt8[0x90|chan%UInt8,pch%UInt8,vel%UInt8]
end

function bytesNoteOff(chan::Int, pch::Int, vel::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= pch < 128 ;  @assert 0 <= vel < 128
	UInt8[0x80|chan%UInt8,pch%UInt8,vel%UInt8]
end

function bytesPitchWheel(chan::Int, val::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= val < 2^14
	UInt8[0xe0|chan%UInt8,lsb14(val%UInt16),msb14(val%UInt16)]
end

function bytesChanAfter(chan::Int, val::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= val < 128
	UInt8[0xd0|chan%UInt8,val%UInt8]
end

function bytesPolyAfter(chan::Int, pitch::Int, val::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= pitch < 128 ;  @assert 0 <= val < 128
	UInt8[0xa0|chan%UInt8,pitch%UInt8,val%UInt8]
end

function bytesControl7(chan::Int, num::Int, val::Int)
	@assert 0 <= chan < 16 ;  @assert 64 <= num < 102 || 120 <= num < 128 ;  @assert 0 <= val < 128
	UInt8[0xb0|chan%UInt8,num%UInt8,val%UInt8]
end

function bytesControlMsb14(chan::Int, num::Int, val::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= num < 32 ;  @assert 0 <= val < 2^14
	UInt8[0xb0|chan%UInt8,num%UInt8,msb14(val%UInt16)]
end

function bytesControlLsb14(chan::Int, num::Int, val::Int)
	@assert 0 <= chan < 16 ;  @assert 0 <= num < 32 ;  @assert 0 <= val < 2^14
	UInt8[0xb0|chan%UInt8,(num+32)%UInt8,lsb14(val%UInt16)]
end

function bytesTrackEnd()
	UInt8[0xff,0x2f,0x00]
end

function bytesTrackMarker(mes::ASCIIString)
	len = length(mes) ;  @assert 0<len<128
	bs = UInt8[0xff,0x06,len%UInt8]
	for x in mes
		push!(bs,x%UInt8)
	end
	bs
end


using SnpArrays
using Knockoffs
using DataFrames
using CSV

res = 0
for chr in 1:22
    chr == 10 && continue
    plinkfile = "/scratch/users/bbchu/ukb_SHAPEIT/knockoffs/ukb_gen_chr$(chr)_ibd1_res$(res)"
    xdata = SnpData(plinkfile)
    isknockoff = endswith.(xdata.snp_info[!, :snpid], ".k")
    original, knockoff = Int[], Int[]
    for i in 1:size(xdata)[2]
        isknockoff[i] ? push!(knockoff, i) : push!(original, i)
    end

    @time decorrelate_knockoffs(plinkfile, original, knockoff;
        outfile = "ukb_gen_chr$(chr)_ibd1_res$(res)_decorrelated",
        outdir = pwd(),
        r2_threshold = 0.95)
end


using Raylib

const screenWidth = 800
const screenHeight = 450

function main()
    Raylib.InitWindow(
        screenWidth, screenHeight,
        "raylib [core] example - keyboard input"
    )

    ballPosition = Raylib.rayvector(screenWidth/2, screenHeight/2)

    Raylib.SetTargetFPS(60)
    while !Raylib.WindowShouldClose()    # Detect window close button or ESC key
        # update
        if Raylib.IsKeyDown(Int(Raylib.KEY_RIGHT))
            ballPosition = Raylib.rayvector(ballPosition[1]+2, ballPosition[2])
        elseif Raylib.IsKeyDown(Int(Raylib.KEY_LEFT))
            ballPosition = Raylib.rayvector(ballPosition[1]-2, ballPosition[2])
        elseif Raylib.IsKeyDown(Int(Raylib.KEY_UP))
            ballPosition = Raylib.rayvector(ballPosition[1], ballPosition[2]-2)
        elseif Raylib.IsKeyDown(Int(Raylib.KEY_DOWN))
            ballPosition = Raylib.rayvector(ballPosition[1], ballPosition[2]+2)
        end

        # draw
        Raylib.BeginDrawing()
        Raylib.ClearBackground(Raylib.RAYWHITE)
        Raylib.DrawText("move the ball with arrow keys", 10, 10, 20, Raylib.DARKGRAY)
        Raylib.DrawCircleV(ballPosition, 50, Raylib.MAROON)
        Raylib.EndDrawing()
    end

    Raylib.CloseWindow()
end