import ClimateMachine.Mesh.Grids: DiscontinuousSpectralElementGrid

abstract type AbstractDiscretizedDomain end

"""
    DiscretizedDomain
"""
struct DiscretizedDomain{𝒜, ℬ, 𝒞} <: AbstractDiscretizedDomain
    domain::𝒜
    resolution::ℬ
    numerical::𝒞
end

function DiscretizedDomain(
    domain::ProductDomain;
    elements = nothing,
    polynomial_order = nothing,
    overintegration_order = nothing,
    FT = Float64,
    mpicomm = MPI.COMM_WORLD,
    array = ClimateMachine.array_type(),
    topology = StackedBrickTopology,
    brick_builder = uniform_brick_builder,
)

    grid = DiscontinuousSpectralElementGrid(
        domain,
        elements = elements,
        polynomialorder = polynomial_order .+ overintegration_order,
        FT = FT,
        mpicomm = mpicomm,
        array = array,
        topology = topology,
        brick_builder = brick_builder,
    )
    return DiscretizedDomain(domain, (; elements, polynomial_order, overintegration_order), grid)
end

function DiscretizedDomain(
    domain::SphericalShell;
    elements = nothing,
    polynomial_order = nothing,
    overintegration_order = nothing,
    FT = Float64,
    mpicomm = MPI.COMM_WORLD,
    array = ClimateMachine.array_type(),
)
    new_polynomial_order = convention(polynomial_order, Val(2))
    new_polynomial_order = new_polynomial_order .+ convention(overintegration_order, Val(2))
    horizontal, vertical = new_polynomial_order

    grid = DiscontinuousSpectralElementGrid(
        domain,
        elements = elements,
        polynomialorder = (; vertical, horizontal),
        FT = FT,
        mpicomm = mpicomm,
        array = array,
    )
    return DiscretizedDomain(domain, (; elements, polynomial_order, overintegration_order), grid)
end

function DiscretizedDomain(
    domain::SingleStack;
    elements = nothing,
    polynomial_order = nothing,
    overintegration_order = nothing,
    FT = Float64,
    mpicomm = MPI.COMM_WORLD,
    array = ClimateMachine.array_type(),
)
    new_polynomial_order = convention(polynomial_order, Val(2))
    new_polynomial_order = new_polynomial_order .+ convention(overintegration_order, Val(2))
    horizontal, vertical = new_polynomial_order

    grid = DiscontinuousSpectralElementGrid(
        domain,
        elements = elements,
        polynomialorder = (; vertical, horizontal),
        FT = FT,
        mpicomm = mpicomm,
        array = array,
    )
    return DiscretizedDomain(domain, (; elements, polynomial_order, overintegration_order), grid)
end

"""
function DiscontinuousSpectralElementGrid(domain::ProductDomain; elements = nothing, polynomialorder = nothing)
# Description
Computes a DiscontinuousSpectralElementGrid as specified by a product domain
# Arguments
-`domain`: A product domain object
# Keyword Arguments
-`elements`: A tuple of integers ordered by (Nx, Ny, Nz) for number of elements
-`polynomialorder`: A tupe of integers ordered by (npx, npy, npz) for polynomial order
-`FT`: floattype, assumed Float64 unless otherwise specified
-`topology`: default = StackedBrickTopology
-`mpicomm`: default = MPI.COMM_WORLD
-`array`: default = ClimateMachine.array_type()
-`brickbuilder`: default = uniform_brick_builder,
  brickrange=uniform_brick_builder(domain, elements)
# Return
A DiscontinuousSpectralElementGrid object
"""
function DiscontinuousSpectralElementGrid(
    domain::ProductDomain;
    elements,
    polynomialorder,
    FT = Float64,
    mpicomm = MPI.COMM_WORLD,
    array = ClimateMachine.array_type(),
    topology = StackedBrickTopology,
    brick_builder = uniform_brick_builder,
)

    if elements === nothing
        error_message = "Please specify the number of elements as a tuple whose size is commensurate with the domain,"
        error_message *= " e.g., a 3 dimensional domain would need a specification like elements = (10,10,10)."
        error_message *= " or elements = (vertical = 8, horizontal = 5)"

        @error(error_message)
        return nothing
    end

    if polynomialorder === nothing
        error_message = "Please specify the polynomial order as a tuple whose size is commensurate with the domain,"
        error_message *= "e.g., a 3 dimensional domain would need a specification like polynomialorder = (3,3,3)."
        error_message *= " or polynomialorder = (vertical = 8, horizontal = 5)"

        @error(error_message)
        return nothing
    end

    dimension = ndims(domain)

    if (dimension < 2) || (dimension > 3)
        error_message = "SpectralElementGrid only works with dimensions 2 or 3. "
        error_message *= "The current dimension is " * string(ndims(domain))

        println("The domain is ", domain)
        @error(error_message)
        return nothing
    end

    elements = convention(elements, Val(dimension))
    if ndims(domain) != length(elements)
        @error("Incorrectly specified elements for the dimension of the domain")
        return nothing
    end

    polynomialorder = convention(polynomialorder, Val(dimension))
    if ndims(domain) != length(polynomialorder)
        @error("Incorrectly specified polynomialorders for the dimension of the domain")
        return nothing
    end

    brickrange = brick_builder(domain, elements, FT = FT)

    if dimension == 2
        boundary = ((1, 2), (3, 4))
    else
        boundary = ((1, 2), (3, 4), (5, 6))
    end

    periodicity = domain.periodicity
    connectivity = dimension == 2 ? :face : :full

    topl = topology(mpicomm, brickrange; periodicity = periodicity, boundary = boundary, connectivity = connectivity)

    grid =
        DiscontinuousSpectralElementGrid(topl, FloatType = FT, DeviceArray = array, polynomialorder = polynomialorder)

    return grid
end

function DiscontinuousSpectralElementGrid(
    domain::SphericalShell;
    elements,
    polynomialorder,
    mpicomm = MPI.COMM_WORLD,
    boundary = (1, 2),
    FT = Float64,
    array = Array,
)
    Rrange = grid1d(domain.radius, domain.radius + domain.height, nelem = elements.vertical)

    topl = StackedCubedSphereTopology(mpicomm, elements.horizontal, Rrange; boundary = boundary)

    grid = DiscontinuousSpectralElementGrid(
        topl,
        FloatType = FT,
        DeviceArray = array,
        polynomialorder = (polynomialorder.horizontal, polynomialorder.vertical),
        meshwarp = equiangular_cubed_sphere_warp,
    )
    return grid
end

"""
function DiscontinuousSpectralElementGrid(
    domain::SingleStack;
"""
function DiscontinuousSpectralElementGrid(
    domain::SingleStack;
    elements,
    polynomialorder,
    mpicomm = MPI.COMM_WORLD,
    boundary = ((0, 0), (0, 0), (5, 6)),
    periodicity = (true, true, false),
    meshwarp = (x...) -> identity(x),
    FT = Float64,
    array = Array,
)
    xmin, xmax = zero(FT), domain.xmax
    ymin, ymax = zero(FT), domain.ymax

    zmin, zmax = (domain.zmin, domain.zmax)

    brickrange = (
        grid1d(xmin, xmax, nelem = elements.horizontal),
        grid1d(ymin, ymax, nelem = elements.horizontal),
        grid1d(zmin, zmax, nelem = elements.vertical),
    )

    topl = StackedBrickTopology(mpicomm, brickrange, periodicity = periodicity, boundary = boundary)

    grid = DiscontinuousSpectralElementGrid(
        topl,
        FloatType = FT,
        DeviceArray = array,
        polynomialorder = (polynomialorder.horizontal, polynomialorder.vertical),
        meshwarp = meshwarp,
    )
    return grid
end

"""
    Brick builder
"""
function uniform_brick_builder(domain::ProductDomain, elements; FT = Float64)
    dimension = ndims(domain)

    tuple_ranges = []
    for i in 1:dimension
        push!(tuple_ranges, range(FT(domain[i].min); length = elements[i] + 1, stop = FT(domain[i].max)))
    end

    brickrange = Tuple(tuple_ranges)
    return brickrange
end

"""
    Conventions for polynomial order and overintegration order
"""
function convention(a::NamedTuple{(:vertical, :horizontal), T}, ::Val{3}) where {T}
    return (a.horizontal, a.horizontal, a.vertical)
end

function convention(a::Number, ::Val{3})
    return (a, a, a)
end

function convention(a::NamedTuple{(:vertical, :horizontal), T}, ::Val{2}) where {T}
    return (a.horizontal, a.vertical)
end

function convention(a::Number, ::Val{2})
    return (a, a)
end

function convention(a::Tuple, b)
    return a
end


#### thermodynamics

export new_thermo_state, recover_thermo_state

"""
    new_thermo_state(atmos::AtmosModel, state::Vars, aux::Vars)

Create a new thermodynamic state, based on the `state`, and _not_
the `aux` state.

!!! note
    This method calls the iterative saturation adjustment
    procedure for EquilMoist models.
"""
new_thermo_state(atmos::AtmosModel, state::Vars, aux::Vars) =
    new_thermo_state(atmos, atmos.moisture, state, aux)

function new_thermo_state(
    atmos::AtmosModel,
    moist::DryModel,
    state::Vars,
    aux::Vars,
)
    e_int = internal_energy(atmos, state, aux)
    return PhaseDry(atmos.param_set, e_int, state.ρ)
end

function new_thermo_state(
    atmos::AtmosModel,
    moist::EquilMoist,
    state::Vars,
    aux::Vars,
)
    e_int = internal_energy(atmos, state, aux)
    return PhaseEquil(
        atmos.param_set,
        e_int,
        state.ρ,
        state.moisture.ρq_tot / state.ρ,
        moist.maxiter,
        moist.tolerance,
    )
end

function new_thermo_state(
    atmos::AtmosModel,
    moist::NonEquilMoist,
    state::Vars,
    aux::Vars,
)
    e_int = internal_energy(atmos, state, aux)
    q = PhasePartition(
        state.moisture.ρq_tot / state.ρ,
        state.moisture.ρq_liq / state.ρ,
        state.moisture.ρq_ice / state.ρ,
    )

    return PhaseNonEquil{eltype(state), typeof(atmos.param_set)}(
        atmos.param_set,
        e_int,
        state.ρ,
        q,
    )
end

"""
    recover_thermo_state(atmos::AtmosModel, state::Vars, aux::Vars)

An atmospheric thermodynamic state.

!!! warn
    While recover_thermo_state is an ideal long-term solution,
    right now we are directly calling new_thermo_state to avoid
    inconsistent aux states in kernels where the aux states are
    out of sync with the boundary state.

# TODO:
    - Allow a safe way to call
    `recover_thermo_state(state, moist::EquilMoist, ...)`
"""
recover_thermo_state(atmos::AtmosModel, state::Vars, aux::Vars) =
    new_thermo_state(atmos, atmos.moisture, state, aux)

function recover_thermo_state(
    atmos::AtmosModel,
    moist::DryModel,
    state::Vars,
    aux::Vars,
)
    e_int = internal_energy(atmos, state, aux)
    return PhaseDry(atmos.param_set, e_int, state.ρ)
end

function recover_thermo_state(
    atmos::AtmosModel,
    moist::EquilMoist,
    state::Vars,
    aux::Vars,
)
    e_int = internal_energy(atmos, state, aux)
    return PhaseEquil{eltype(state), typeof(atmos.param_set)}(
        atmos.param_set,
        e_int,
        state.ρ,
        state.moisture.ρq_tot / state.ρ,
        aux.moisture.temperature,
    )
end

function recover_thermo_state(
    atmos::AtmosModel,
    moist::NonEquilMoist,
    state::Vars,
    aux::Vars,
)
    e_int = internal_energy(atmos, state, aux)
    q = PhasePartition(
        state.moisture.ρq_tot / state.ρ,
        state.moisture.ρq_liq / state.ρ,
        state.moisture.ρq_ice / state.ρ,
    )

    return PhaseNonEquil{eltype(state), typeof(atmos.param_set)}(
        atmos.param_set,
        e_int,
        state.ρ,
        q,
    )
end


# __precompile__()

module MotionPlanning

using Reexport
@reexport using StaticArrays
@reexport using Distances
# import PyPlot; const plt = PyPlot
using Distributions; import Base.eltype; eltype{T}(::Dirichlet{T}) = T    # TODO: submit Distributions PR
using SCS
using Iterators
using NearestNeighbors
using DataStructures

### Includes
include("primitivetypes.jl")
include("utilities/utils.jl")
include("collisioncheckers.jl")
include("goals.jl")
include("nearneighbors.jl")
include("statespaces.jl")
include("problems.jl")
include("sampling.jl")
include("planners.jl")
include("postprocessors.jl")
# include("plotting.jl")

# TODO: test suite!
# TODO: T<:AbstractFloat - either ensure type stability (at least for states), or just go with Float64 everything
# TODO: change up lazy vcat [.,.] syntax in anticipation of v0.4; also consider push!/append!
# TODO: half-baked controls caching - do it or kill it
# TODO: "prototype" (.h-ish) files to specify required methods; julia type system makes OOP a pain
# TODO: FMT re-solving is wonky with different sample sizes
# TODO: Think about parameterizing MPProblem on the StateSpace type (for ease of postprocessing)
# TODO: Requires.jl for conditional dependency loading

end # module


# PdePumiInterface3DG.jl: implements AbstractMesh for a 3D Discontinuous
#                        Galerkin mesh

export PumiMeshDG3

# Element = an entire element (verts + edges + interior face)
# Type = a vertex or edge or interior face
#

# the vert, edge, face Nptrs are numberings that map a MeshEntity to its
# index the the arrays verts, edges, faces, except that they are zero based indices

# internally, all operations and loops are done by looping over nodes
# in the Pumi order (ie. vertex nodes counter clockwise, edge nodes
# counterclockwise, the face nodes counterclockwise starting in with the
# node nearest the first vertex).
# The external interface remaps the nodes into an order specified
# by the nodemaps nodemapSbptoPumi and nodemapPumitoSbp.
# The external interface is compreised of the arrays:
# dofs, coords, dxidx, jac, sparsity_bnds
# Care must be taken with the visualization files to map back to the Pumi
# order

# For DG meshes, two different FieldShapes are used: the coordinate FieldShape,
# which coveres vertices (and might cover mid edge nodes in the future for
# curved meshes), and the solution Fieldshape, which describes the nodes
# Pumi *only* stores the coordinates of nodes in the coordinate FieldShape,
# so the interface between what Pumi knows about elements and what Julia
# knows about element is the follow:
#  Pumi keeps track of vertices (and possibly mid edge nodes)
#  SBP calculates the location of all nodes from the vertex coordinates
#  SBP calculates the mapping jacobian from the node coordinates
#
# In order to store data on the mesh in the solution FieldShape, Pumi
# must know at least the number and classification of the nodes.
# In practice, we define a Pumi node ordering as follows:
#  vertex nodes, from the bottom left, going counter-clockwise
#  edge nodes: edge 1, edge 2, edge 3
#  face nodes: starting at the node closest to vertex 1, proceeding
#              counter-clockwise, outermost to innermost
# This requirement is stronger than what is actually necessary, but
# having an explicitly defined node ordering convention and a general
# mechanism to convert from Pumi to SBP ordering is a better interface
#
@doc """
### PumiInterface.PumiMeshDG3

  This is an implementation of AbstractMesh for a 2 dimensional equation.
  The constructor for this type extracts all the needed information from Pumi,
  so the solver never needs access to Pumi.

  Fields:
    m_ptr: a Ptr{Void} to the Pumi mesh
    mshape_ptr: a Ptr{Void} do the FieldShape of the solution field (not the
                coordinate field)
    coordshape_ptr: a pointer to the FieldShape of the coordinate field

    f_ptr:  a Ptr{Void} to the Pumi apf::Field holding the solution
    shape_type: an enum describing what shape functions to use
    min_node_dist: the minimum distance between nodes on a reference element
    vert_NPtr: a pointer to the apf::Numbering object that numbers the
               vertices (0-based numbering)
    edge_Nptr: a pointer to the Pumi apf::Numbering that numbers the mesh
               edges (0-based numbering)
    el_Nptr:  a pointer to the Pumi apf::Numbering that numbers the mesh
              elements (0-based numbering)
    coloring_Nptr: a pointer to the Pumi Numbering containing the element
                   coloring

    numVert: number of vertices in the mesh
    numEdge: number of edges in the mesh
    numEl: number of elements in the mesh
    order: degree of the shape functions
    numDof: total number of degrees of freedom
    numNodes: number of nodes in the mesh
    numDofPerNode: number of degrees of freedom on each node
    numBoundaryFaces: number of edges on the boundary of the domain
    numInterfaces: number of internal edges (edges not on boundary)
    numNodesPerElements: number of nodes on an element
    numNodesPerType: array of length 3 that tells how many nodes are on a mesh
                     vertex, edge, and element
    numNodesPerFace: number of nodes on each face of the element (edges in 2D)
    numEntitiesPertype: [numVert, numEdges, numEl]
    numTypesPerElement: array of length 3 telling how many vertices, edges
                         and faces in each element

    typeOffsetsPerElement: array of length 3 telling the starting index of the
                           vertices, edges, and faces in an array where
                           all quantities for the vertices are stored, then
                           the edges, then the faces

    typeOffsetsPerElement_: Int32 version of the above
    nodemapSBPtoPumi: array of UInt8s that maps the SBP ordering of nodes to
                      the Pumi ordering of nodes
    nodemapPumiToSBP: array of UInt8s that maps the Pumi node ordering to the
                      SBP one.

    coloringDistance: the distance k of the distance-k graph coloring used to
                      color the elements (graph vertices are elements and graph
                      edges exist where elements share an edge)

    dofs: a 3D array holding the degree of freedom numbers for the elements,
          both locally owned and the ghost elements.  The first mesh.numEl
          elements are the locally owned ones, the rest are for the ghost
          element.  Their offsets are specified by mesh.shared_element_offsets.
          The global dof number is the number stored in this array +
          dof_offset  (even for the non-local elements)
"""->
mutable struct PumiMeshDG3{T1, Tface <: AbstractFace{Float64}} <: PumiMesh3DG{T1}   # 2d pumi mesh, triangle only
  m_ptr::Ptr{Void}  # pointer to mesh
  mshape_ptr::Ptr{Void} # pointer to the FieldShape of the node field
  coordshape_ptr::Ptr{Void}  # pointer to FieldShape of the coordinate
                             # field
  f_ptr::Ptr{Void} # pointer to apf::field for storing solution
  fnew_ptr::Ptr{Void}  # pointer to apf::Field used for visualization
  mnew_ptr::Ptr{Void}  # pointer to mesh used for visualization
  fnewshape_ptr::Ptr{Void} # FieldShape of fnew_ptr
  mexact_ptr::Ptr{Void}  # pointer to subtriangulated mesh used for
                         # exact visualization) - only here for compatability
                         # with 2D
  fexact_ptr::Ptr{Void}  # pointer to field on mexact_ptr
  fexactshape_ptr::Ptr{Void}  # apf::FieldShape of fexact_ptr

  shr_ptr::Ptr{Void} # pointer to apf::Sharing object
  normalshr_ptr::Ptr{Void}  # NormalSharing object
  shape_type::Int  #  type of shape functions
  min_node_dist::Float64  # minimum distance between nodes
  min_el_size::Float64 # size of the smallest element (units of length)
  volume::T1  # volume of mesh
  f::IOStream
  vert_Nptr::Ptr{Void}  # numbering of vertices (zero based)
  edge_Nptr::Ptr{Void}  # numbering of edges (zero based)
  face_Nptr::Ptr{Void}  # number of faces (zero based)
  el_Nptr::Ptr{Void}  # numbering of elements (faces)  (zero based)
  coloring_Nptr::Ptr{Void}  # coloring of mesh for sparse jacobian calculation
  entity_Nptrs::Array{Ptr{Void}, 1}  # [vert_Nptr, edge_Nptr, el_Nptr], 0-based
  numVert::Int  # number of vertices in the mesh
  numEdge::Int # number of edges in the mesh
  numFace::Int # number of faces (=edge) in the mesh
  numEl::Int  # number of elements (faces)
  numGlobalEl::Int  # number of local elements + number of non-local elements
  numSharedEl::Int  # number of non-local elements
  order::Int # order of shape functions
  numDof::Int # number of degrees of freedom
  numNodes::Int  # number of nodes
  numDofPerNode::Int  # number of dofs per node
  numBoundaryFaces::Int # number of edges on the exterior boundary
  numInterfaces::Int # number of internal interfaces (including periodic)
  numPeriodicInterfaces::Int  # number of periodic interfaces
  numNodesPerElement::Int  # number of nodes per element
  numFacesPerElement::Int  # number of faces per element
  numNodesPerType::Array{Int, 1}  # number of nodes classified on each vertex, edge, face
  numNodesPerFace::Int  # number of face nodes
  numEntitiesPerType::Array{Int, 1} # [numVert, numEdge, numEl]
  numTypePerElement::Array{Int, 1}  # number of verts, edges, faces per element
  typeOffsetsPerElement::Array{Int, 1} # the starting index of the vert, edge, and face nodes in an element
  typeOffsetsPerElement_::Array{Int32, 1}  # Int32 version of above
  nodemapSbpToPumi::Array{UInt8, 1}  # maps nodes of SBP to Pumi order
  nodemapPumiToSbp::Array{UInt8, 1}  # maps nodes of Pumi to SBP order

  # minimal bookkeeping info for coordinate field
  coord_order::Int
  coord_numNodesPerElement::Int
  coord_numNodesPerType::Array{Int, 1}
  coord_typeOffsetsPerElement::Array{Int, 1}
  coord_numNodes::Int  # total number of coordinate nodes (on this part)
  coord_numNodesPerFace::Int
  coord_xi::Array{Float64, 2}  # xi coordinates of nodes on reference element
                                # in the Pumi order
                                # dim x coords_numNodesPerElement
  coord_facexi::Array{Float64, 2}  # like coord_xi, but for the face of an
                                     # element
  coord_nodenums_Nptr::Ptr{Void}  # numbering for nodes of coordinate field,

  geoNums::GeometricDofs

  # constants needed by Pumi
  el_type::Int  # apf::Type for the elements of the mesh
  face_type::Int # apf::Type for the faces of the mesh

  # parallel bookkeeping info
  comm::MPI.Comm  # MPI Communicator
  myrank::Int  # MPI rank, zero based
  commsize::Int # MPI comm size
  peer_parts::Array{Int, 1}  # array of part numbers that share entities with
                             # the current part
  npeers::Int  # length of the above array
  peer_face_counts::Array{Int, 1}  # number of edges on each part
  send_waited::Array{Bool, 1}  # indicates whether the send_reqs have been
                               # waited on already
  send_reqs::Array{MPI.Request, 1}  # Request objects for sends
  send_stats::Array{MPI.Status, 1} # Status objects for sends
  recv_waited::Array{Bool, 1}  # similar to send_waited
  recv_reqs::Array{MPI.Request, 1}  # Request objects for sends
  recv_stats::Array{MPI.Status, 1}  # status objects for sends

  ref_verts::Array{Float64, 2}  # 2 x 3 array giving the coordinates
                                # of the vertices of the reference
                                # element in parametric space
  dim::Int  # dimension of mesh (2 or 3D)
  isDG::Bool  # is this a DG mesh (always true)
  isInterpolated::Bool # does the solution need to be interpolated to
                       # the boundaries
  coloringDistance::Int  # distance between elements of the same color, measured in number of edges
  numColors::Int  # number of colors
  maxColors::Int  # maximum number of colors on any processor
  numBC::Int  # number of boundary conditions

  # hold pointers to mesh entities
  verts::Array{Ptr{Void},1}  # holds pointers to mesh vertices
  edges::Array{Ptr{Void},1}  # pointers to mesh edges
  faces::Array{Ptr{Void},1}  # alias for edges
  elements::Array{Ptr{Void},1}  # pointers to faces

  # used for high order elements to determine the orientations of edges and
  # faces (and vertices, too, for consistency)
  # stored in Pumi node ordering, not SBP
  elementNodeOffsets::Array{UInt8, 2}
  # truth values if the entity is oriented consistently with the element
  # Array of length 3, whose elements are numTypePerelement x numEl
  typeNodeFlags::Array{BitArray{2}, 1}
  triangulation::Array{Int32, 2}  # sub triangulation of an element

  nodestatus_Nptr::Ptr{Void}  # node status pointer, status >= 2 -> give it a
                              # node number
  nodenums_Nptr::Ptr{Void}    # node numbering itself
  dofnums_Nptr::Ptr{Void}  # pointer to Numbering of dofs (result of reordering)
#  boundary_nums::Array{Int, 2}  # array of [element number, edgenumber] for each edge on the boundary

  bndry_funcs::Array{BCType, 1}  # array of boundary functors (Abstract type)
  bndry_funcs_revm::Array{BCType_revm, 1}  # reverse mode functors
  bndry_funcs_revq::Array{BCType_revq, 1}
  bndry_normals::Array{T1, 3}  # array of normals to each face on the boundary
#  bndry_facenums::Array{Array{Int, 1}, 1}  # hold array of faces corresponding to each boundary condition
  bndry_offsets::Array{Int, 1}  # location in bndryfaces where new type of BC starts
                                # and one past the end of the last BC type
				# array has length numBC + 1
  bndry_geo_nums::Array{Array{Int, 1}, 1}  # array of arrays, where the
                                           # outer array is of length numBC
                                           # and the inner arrays contains
                                           # the geometric face numbers of this
                                           # BC

  bndryfaces::Array{Boundary, 1}  # store data on external boundary of mesh
  interfaces::Array{Interface, 1}  # store data on internal edges
  vert_coords::Array{T1, 3}  # dim x numVertPerElement x numEl array of
                              # coordinates of the vertices of each element
  vert_coords_bar::Array{T1, 3}  # adjoint part
  coords::Array{T1, 3}  # store coordinates of all nodes
  coords_bndry::Array{T1, 3}  # store coordinates of nodes on boundary,
                              # 3 x numFaceNodes x numBoundaryFaces

  coords_bndry_bar::Array{T1, 3} # adjoint part
  coords_interface::Array{T1, 3} # store coordinates of nodes on interfaces
                                 # 3 x numFaceNodes x numInterfaces
  coords_sharedface::Array{Array{T1, 3}, 1}  # coordinates of shared interface nodes
  dxidx::Array{T1, 4}  # store scaled mapping jacobian
  dxidx_bar::Array{T1, 4}
  dxidx_face::Array{T1, 4} # store scaled mapping jacobian at face nodes
                           # 2 x 2 x numfacenodes x numInterfaces
  dxidx_face_bar::Array{T1, 4}
  dxidx_sharedface::Array{Array{T1, 4}, 1}  # array of arrays for dxidx
  dxidx_sharedface_bar::Array{Array{T1, 4}, 1}
                                            # on shared edges
  dxidx_bndry::Array{T1, 4} # store scaled mapping jacobian at boundary nodes,
                            # similar to dxidx_face
  dxidx_bndry_bar::Array{T1, 4}
  jac::Array{T1,2}  # store mapping jacobian output
  jac_bar::Array{T1, 2}
  jac_face::Array{T1,2}  # store jacobian determanent at face nodes
                         # numfacenodes x numInterfaces
  jac_face_bar::Array{T1, 2}
  jac_sharedface::Array{Array{T1, 2}, 1}  # array of arrays for shared
                                          # edge jacobian determinent
  jac_sharedface_bar::Array{Array{T1, 2}, 1}

  jac_bndry::Array{T1, 2} # store jacobian determinant at boundry nodes
                          # similar to jac_bndry
  jac_bndry_bar::Array{T1, 2}

  nrm_bndry::Array{T1, 3}  # dim x numfacenodes x numBoundaryFaces array holding
                           # normal vector in x-y to each face node  on the boundary
  nrm_bndry_bar::Array{T1, 3}  # similar to nrm_bndry

  nrm_face::Array{T1, 3}  # dim x numfacenodes x numInterfaces array holding
                          # normal vector to each face node on elementL of an
                          # interface
  nrm_face_bar::Array{T1, 3}

  nrm_sharedface::Array{Array{T1, 3}, 1}  # array of arrays.  Outer array is of
                          # lench npeers.  Inner arrays are of size dim x
                          # numfacenodes x number of faces shared with current
                          # peer
  nrm_sharedface_bar::Array{Array{T1, 3}, 1}

  dof_offset::Int  # local to global offset for dofs
  dofs::Array{Int, 3}  # store dof numbers of solution array to speed assembly
  element_vertnums::Array{Int32, 2}  # map from elements to their vertex numbers
                                  # numVertPerElement x numEl

  sparsity_bnds::Array{Int32, 2}  # store max, min dofs for each dof
  sparsity_nodebnds::Array{Int32, 2}  # store min, max nodes for each node

  sparsity_counts::Array{Int32, 2}  # store number of local, remote dofs
                                    # for each dof
  sparsity_counts_node::Array{Int32, 2}  # store number of local, remote dofs
                                         # for each node
  color_masks::Array{BitArray{1}, 1}  # array of bitarray masks used to control element perturbations when forming jacobian, number of arrays = number of colors
  neighbor_colors::Array{UInt8, 2}  # 4 by numEl array, holds colors of edge-neighbor elements + own color
  neighbor_nums::Array{Int32, 2}  # 4 by numEl array, holds element numbers of neighbors + own number, in same order as neighbor_colors
  pertNeighborEls::Array{Int32, 2}  # numEl by numcolors array, for each color,
                                    # stores the element number of the element
				    # whose perturbation is affecting the
				    # current element
  pertNeighborEls_edge::Array{Int32, 2}  # numEl by 3 (number of edges per element)
                                         # stores the element number of the
					 # neighboring element that shares
					 #  edge, where edge is the local edge
					 # index (ie. edge 1,2, or 3)
  color_cnt::Array{Int32, 1}  # number of elements in each color

  interp_op::Array{Float64, 2}  # 3 x numNodesPerEl matrix that interpolates
                                # solution values to the vertices
  interp_op2::Array{Float64, 2}  # numNodesPerEl x coord_numNodesPerElement
                                 # array that interpolates from the
                                 # coordinate field to the solution field

  ###### parallel data #####

  # array of arrays of Boundary objects, one array for each peer part
  # the Boundary describes the local part of a shared interface
  bndries_local::Array{Array{Boundary, 1}, 1}
  # array of arrays of Boundary objects from the *remote* process describing its
  # part of a shared interface, on array for each peer part
  # ie. the element number is the element number of the remote part.  Use with
  # caution.
  bndries_remote::Array{Array{Boundary, 1}, 1} # hold a Boundary object from

  # array of arrays of Interface objects giving a complete description of the
  # interfaces that span part boundaries.  elementL is always the local element
  # and elementR is always the element that lives on the remote process.
  # elementR has been given an element number > numEl
  shared_interfaces::Array{Array{Interface, 1}, 1}

  # array of length npeers + 1, holding the first element number for
  # each peer part, with the last element being 1 more than the highest element
  # number for the last peer
  shared_element_offsets::Array{Int, 1}
  local_element_counts::Array{Int, 1}  # number of element on the local
                                       # side of each peer boundary
  remote_element_counts::Array{Int, 1}

  # element numbers of the elements on each peer boundary
  local_element_lists::Array{Array{Int32, 1}, 1}

  # color masks for the non-local elements, contains npeers x numColor
  # BitArrays, where each BitArray has length = the number of ghost elements
  # on the current peer boundary
  shared_element_colormasks::Array{Array{BitArray{1}, 1}, 1}
  remote_metrics::Array{RemoteMetrics{T1}, 1}  # metric information for
                                                 # remote elements
  remote_metrics_bar::Array{RemoteMetrics{T1}, 1}  # metric information for

  #TODO: remove this once SBP interface is clarified
  sbpface::Tface  # SBP object needed to do interpolation
  topo::ElementTopology{3}  # SBP topology
  topo_pumi::ElementTopology{3}  # Pumi topology

  vert_sharing::VertSharing
  coordscatter::ScatterData{T1}  # abstract type, used by coords3DTo1D

  # temporarily allow nested meshes for the staggered grid work
  mesh2::AbstractMesh
  sbp2::AbstractSBP

  # interpolation operators
  # these are usually stored in the solution mesh but not in the flux mesh
  I_S2F::Matrix{Float64}  # interpolation operator from solution to flux grid,
                          # numNodesPerElement_f x numNodesPerElement_s
  I_S2FT::Matrix{Float64}  # transpose of above
  I_F2S::Matrix{Float64}  # interpolation operator from flux grid to solution
                          # grid, numNodesPerElement_s x numNodesPerElement_f
  I_F2ST::Matrix{Float64} # transpose of above

  fields::AttachedData
  numberings::AttachedData

  """
    This inner constructor loads a Pumi mesh from files and sets a few fields
    that must be consistent with how the mesh was loaded.

    **Inputs**

     * dmg_name: name of Pumi geometry file
     * smb_name: name of Pumi mesh files (without file number)

   **Keywords**

    * shape_type: integer identifying the field shape of the coordinate field,
                  default -1 (keep existing field)
    * order: order of coordinate field, unused if coordinate field is not
             changed, default 1
    * comm: MPI Communicator the mesh will be defind on, must have the same
            number of processes as the mesh

  """
  function PumiMeshDG3{T1, Tface}(dmg_name::AbstractString, smb_name::AbstractString;
                       order::Integer=1, shape_type::Integer=-1,
                       comm=MPI.COMM_WORLD) where {T1, Tface <: AbstractFace{Float64}}
    if !MPI.Initialized()
      MPI.Init()
    end

    mesh = new{T1, Tface}()
    mesh.isDG = true
    mesh.dim = 3

    mesh.comm = comm
    topo2 = ElementTopology{2}(apf.tri_edge_verts.')
    mesh.topo_pumi = ElementTopology{3}(apf.tet_tri_verts.',
                                        apf.tet_edge_verts.', topo2=topo2)
    mesh.myrank = MPI.Comm_rank(mesh.comm)
    mesh.commsize = MPI.Comm_size(mesh.comm)
    myrank = mesh.myrank
    mesh.fields = AttachedData()
    mesh.numberings = AttachedData()

    if myrank == 0
      println("\nConstructing PumiMeshDG3 Object")
      println("  smb_name = ", smb_name)
      println("  dmg_name = ", dmg_name)
    end

    mesh.m_ptr, dim = apf.loadMesh(dmg_name, smb_name, order, shape_type=shape_type)

    apf.pushMeshRef(mesh.m_ptr)
    recordAllFields(mesh)
    recordAllNumberings(mesh)
    if dim != mesh.dim
      throw(ErrorException("loaded mesh is not 3 dimensional"))
    end

    return mesh
  end  # end inner constructor

  """
    This inner constructor returns an uninitailized mesh object.
  """
  function PumiMeshDG3{T1, Tface}() where {T1, Tface}
    return new{T1, Tface}()
  end

end  # end PumiMeshDG3 type declaration

"""
  This outer constructor populates the essential fields of the mesh
  that are populated when a mesh is loaded from a file, but instead
  of loading a mesh from a file, it copies the fields from the old mesh.

  The mesh.m_ptr field is not populated because this function could be used
  for creating submeshes.
"""
function PumiMeshDG3(old_mesh::PumiMeshDG3{T},
                     sbpface::Tface=old_mesh.sbpface) where {T, Tface}

  mesh = PumiMeshDG3{T, Tface}()  # get uninitailized object

  # set essential fields from old_mesh
  mesh.isDG = true
  mesh.dim = 3
  mesh.comm = old_mesh.comm
  topo2 = ElementTopology{2}(apf.tri_edge_verts.')
  mesh.topo_pumi = ElementTopology{3}(apf.tet_tri_verts.',
                                      apf.tet_edge_verts.', topo2=topo2)

  mesh.sbpface = sbpface
  mesh.myrank = old_mesh.myrank
  mesh.commsize = old_mesh.commsize
  mesh.fields = AttachedData()
  mesh.numberings = AttachedData()

  return mesh
end

"""
  This outer constructor loads a mesh from a file, according to the
  options specified in the options dictionary.  The following keys are used:

  * smb_name
  * dmg_name

  **Inputs**

   * T: the DataType of the data fields of the mesh
   * sbp: SBP operator
   * topo: ElementTopology{3} describing the SBP operator reference element
           topology

  **Keyword**

   * dofpernode: number of degrees of freedom at each node in the mesh,
                 default 1
   * shape_type: the integer identifying the solution field shape, must match
                 the SBP operator
   * comm: the MPI Communicator the mesh is to be loaded on, must have the
           same number of processes as the mesh
"""
function PumiMeshDG3(::Type{T}, sbp::AbstractSBP, opts,
                     sbpface::Tface,
                     topo::ElementTopology{3};
                     dofpernode=1, shape_type=2, comm=MPI.COMM_WORLD) where {T, Tface}

  set_defaults(opts)

  # distinguish between coordinate field and solution field
  coord_shape_type = -1  # keep coordinate field already present
  coord_order = 1

  smb_name = opts["smb_name"]
  dmg_name = opts["dmg_name"]

  mesh = PumiMeshDG3{T, Tface}(dmg_name, smb_name, order=coord_order,
                               shape_type=coord_shape_type)

  mesh.sbpface = sbpface
  finishMeshInit(mesh, sbp, opts, topo, dofpernode=dofpernode, shape_type=shape_type)

  return mesh
end

"""
  This constructor creates a new mesh object after mesh adaptation has
  run.

  **Inputs**

   * old_mesh: old mesh object, mesh.m_ptr points to the adapted mesh (
               which means that mesh.m_ptr and the mesh object are inconsistent
               while this constructor runs
   * sbp: SBP operator
   * opts: options dictionary

  **Outputs**

   * mesh: new mesh object
"""
function PumiMeshDG3(old_mesh::PumiMeshDG3{T}, sbp, opts, sbpface::Tface=old_mesh.sbpface) where {T, Tface}

  mesh = PumiMeshDG3(old_mesh, sbpface)
  mesh.m_ptr = old_mesh.m_ptr
  apf.pushMeshRef(mesh.m_ptr)
  attachOrigFields(mesh, old_mesh.fields.orig)
  attachOrigNumberings(mesh, old_mesh.numberings.orig)

  finishMeshInit(mesh, sbp, opts, old_mesh.topo, dofpernode=old_mesh.numDofPerNode,
                 shape_type=old_mesh.shape_type)

  return mesh
end

"""
  Finish mesh initialization.  Most constructors use this to do the bulk of the
  initialization.  The options dictionary keys "order" and "coloring_distance"
  are used.  The following fields of the mesh must already be populated:

   * isDG
   * dim
   * comm
   * topo_pumi
   * myrank
   * commsize
   * m_ptr

  **Inputs**

   * mesh: partially initailized mesh object
   * sbp: SBP operator
   * opts: options dictionary
   * topo: the ElementTopology{3} describing the SBP reference element

  **Keyword**

   * dofpernode: number of degrees of freedom on each node, default 1
   * shape_type: integer identifying the solution field shape, must match the
                 SBP operator

"""
function finishMeshInit(mesh::PumiMeshDG3{T1}, sbp::AbstractSBP, opts,
                        topo::ElementTopology{3};
                        dofpernode=1, shape_type=2) where T1
#function finishMeshInit(dmg_name::AbstractString, smb_name::AbstractString, order, sbp::AbstractSBP, opts, sbpface, topo::ElementTopology{3}; dofpernode=1, shape_type=2, coloring_distance=2, comm=MPI.COMM_WORLD)
  # construct pumi mesh by loading the files named
  # dmg_name = name of .dmg (geometry) file to load (use .null to load no file)
  # smb_name = name of .smb (mesh) file to load
  # order = order of shape functions
  # opts: dictionary of options
  # dofpernode = number of dof per node, default = 1
  # shape_type = type of shape functions, 0 = lagrange, 1 = SBP, 2 = SBP DG1
  #              3 = DG2
  # coloring_distance : distance between elements of the same color, where distance is the minimum number of edges that connect the elements, default = 2

  # unpack options keys
  field_shape_type = shape_type
  order = opts["order"]
  coloring_distance = opts["coloring_distance"]
  myrank = mesh.myrank
  sbpface = mesh.sbpface

  mesh.numDofPerNode = dofpernode
  mesh.order = order
  mesh.shape_type = shape_type
  mesh.coloringDistance = coloring_distance
#  mesh.interp_op = interp_op
  mesh.ref_verts = [0.0 1 0; 0 0 1]  # ???
  mesh.numNodesPerFace = sbpface.numnodes
  mesh.topo = topo
  checkTopologyConsistency(mesh.topo, mesh.topo_pumi)

  myrank = mesh.myrank
  mesh.f = open("meshlog_$myrank.dat", "w")

  # force mesh to be interpolated
#  if sbp.numfacenodes == 0
  mesh.isInterpolated = true
#  else
#    mesh.isInterpolated = false
#    # leave mesh.sbpface undefined - bad practice
#  end

  mesh.coordshape_ptr, num_Entities, n_arr = apf.initMesh(mesh.m_ptr)
  for n in n_arr
    attachUserNumbering(mesh, n)
  end

#  num_Entities, mesh.m_ptr, mesh.coordshape_ptr, dim, n_arr = apf.init2(dmg_name, smb_name, mesh_order, shape_type=coord_shape_type)

  # create the solution field
  mesh.mshape_ptr = apf.getFieldShape(field_shape_type, order, mesh.dim)
  # use coordinate fieldshape for solution, because it will be interpolated
  # create new field only if it doesn't already exist (re-apf.init after mesh
  # adapt)
  mesh.f_ptr = apf.createPackedField(mesh, "solution_field", dofpernode, mesh.coordshape_ptr)

  mesh.fnew_ptr = mesh.f_ptr
  mesh.mnew_ptr = mesh.m_ptr
  mesh.fnewshape_ptr = mesh.coordshape_ptr
  mesh.mexact_ptr = C_NULL
  mesh.fexact_ptr = C_NULL
  mesh.fexactshape_ptr = C_NULL
  mesh.min_node_dist = minNodeDist(sbp, mesh.isDG)
  mesh.shr_ptr = apf.getSharing(mesh.m_ptr)
  mesh.normalshr_ptr = apf.getNormalSharing(mesh.m_ptr)

  # count the number of all the different mesh attributes
  mesh.numVert = convert(Int, num_Entities[1])
  mesh.numEdge =convert(Int,  num_Entities[2])
  mesh.numFace = convert(Int, num_Entities[3])
  mesh.numEl = convert(Int, num_Entities[4])
  mesh.numEntitiesPerType = [mesh.numVert, mesh.numEdge, mesh.numFace, mesh.numEl]
  mesh.numTypePerElement = [4, 6, 4, 1]
  mesh.numFacesPerElement = mesh.numTypePerElement[end-1]
  mesh.el_type = apf.TET
  mesh.face_type = apf.TRIANGLE

  num_nodes_v = apf.countNodesOn(mesh.mshape_ptr, 0)  # number of nodes on a vertex
  num_nodes_e = apf.countNodesOn(mesh.mshape_ptr, 1) # on edge
  num_nodes_f = apf.countNodesOn(mesh.mshape_ptr, 2) # on face
  num_nodes_r = apf.countNodesOn(mesh.mshape_ptr, apf.TET)  # on region
  num_nodes_entity = [num_nodes_v, num_nodes_e, num_nodes_f, num_nodes_r]
  # count numbers of different things per other thing
  # use for bookkeeping
  mesh.numNodesPerType = num_nodes_entity
  mesh.typeOffsetsPerElement = zeros(Int, 5)
  pos = 1
  mesh.typeOffsetsPerElement[1] = pos
  for i=2:5
    pos += mesh.numTypePerElement[i-1]*mesh.numNodesPerType[i-1]
    mesh.typeOffsetsPerElement[i] = pos
  end
  mesh.numNodesPerType, mesh.typeOffsetsPerElement = getNodeInfo(mesh.mshape_ptr, mesh.dim, mesh.numTypePerElement)
  mesh.typeOffsetsPerElement_ = [Int32(i) for i in mesh.typeOffsetsPerElement]

  # get coordinate field info
  mesh.coord_numNodesPerType, mesh.coord_typeOffsetsPerElement = getNodeInfo(mesh.coordshape_ptr, mesh.dim, mesh.numTypePerElement)
  mesh.coord_numNodesPerElement = mesh.coord_typeOffsetsPerElement[end] - 1
  mesh.coord_order = apf.getOrder(mesh.coordshape_ptr)
  mesh.coord_xi = getXiCoords(mesh.coord_order, mesh.dim)
  mesh.coord_facexi = getXiCoords(mesh.coord_order, mesh.dim-1)
  mesh.coord_numNodesPerFace = size(mesh.coord_facexi, 2)

  mesh. numNodesPerElement = mesh.typeOffsetsPerElement[end] - 1
  numnodes = mesh.numNodesPerElement*mesh.numEl
  mesh.numNodes = numnodes      # we assume there are no non-free nodes/dofs
  mesh.numDof = numnodes*dofpernode

  # build interpolation operator
  ref_coords = baryToXY(mesh.coord_xi, sbp.vtx)
  mesh.interp_op = SummationByParts.buildinterpolation(sbp, ref_coords)
  mesh.interp_op2 = SummationByParts.buildinterpolation(ref_coords, calcnodes(sbp), mesh.coord_order)

  # get nodemaps
  mesh.nodemapSbpToPumi, mesh.nodemapPumiToSbp = getNodeMaps(order, shape_type, mesh.numNodesPerElement, mesh.dim, mesh.isDG)

  # get pointers to mesh entity numberings
  mesh.vert_Nptr = n_arr[1] #getVertNumbering()
  mesh.edge_Nptr = n_arr[2] #getEdgeNumbering()
  mesh.face_Nptr = n_arr[3] #getFaceNumbering()
  mesh.el_Nptr = n_arr[4] #getElNumbering()
  mesh.entity_Nptrs = [mesh.vert_Nptr, mesh.edge_Nptr, mesh.face_Nptr, mesh.el_Nptr]

  # create the coloring_Nptr
  el_mshape = apf.getConstantShapePtr(mesh.dim)
  mesh.coloring_Nptr = apf.createNumberingJ(mesh, "coloring", 1, el_mshape)

  # create node status numbering (node, not dof)
  mesh.nodestatus_Nptr = apf.createNumberingJ(mesh, "dof status",
                                              1, mesh.mshape_ptr)
  # create node numbering
  mesh.nodenums_Nptr = apf.createNumberingJ(mesh, "reordered node numbers",
                                            1, mesh.mshape_ptr)

  # create dof numbering
  mesh.dofnums_Nptr = apf.createNumberingJ(mesh, "reordered dof numbers",
                                           dofpernode, mesh.mshape_ptr)

  # coordinate node numbering
  mesh.coord_nodenums_Nptr = apf.createNumberingJ(mesh, "coord node numbers",
                                                  mesh.dim, mesh.coordshape_ptr)

  xiNums_Nptr = apf.createNumberingJ(mesh, "geometric dof numbers",
                                     mesh.dim, mesh.coordshape_ptr)

  # get entity pointers
#  println("about to get entity pointers")
  mesh.verts, mesh.edges, mesh.faces, mesh.elements = getEntityPointers(mesh)

  checkConnectivity(mesh)

  # populate node status numbering
  populateNodeStatus(mesh)

  # do node reordering
  coord_numnodes = 0
  for i=1:(mesh.dim+1)
    coord_numnodes += mesh.coord_numNodesPerType[i]*mesh.numEntitiesPerType[i]
  end
  mesh.coord_numNodes = coord_numnodes

  if opts["reordering_algorithm"] == "adjacency"
    start_coords = opts["reordering_start_coords"]
    numberNodesWindy(mesh, start_coords)
    # tell the algorithm there is only 1 dof per node because we only
    # want to label nodes
#    apf.reorder(mesh.m_ptr, mesh.numNodes, 1,
#            mesh.nodestatus_Nptr, mesh.nodenums_Nptr, mesh.el_Nptr,
#	    start_coords[1], start_coords[2])
    apf.reorder(mesh.m_ptr, mesh.dim*mesh.coord_numNodes, mesh.dim,
            C_NULL, mesh.coord_nodenums_Nptr, C_NULL,
	    start_coords)

    numXiDof = apf.reorderXi(mesh.m_ptr, xiNums_Nptr, start_coords)

  elseif opts["reordering_algorithm"] == "default"
#    println("about to number nodes")
    numberNodesElement(mesh)

    # coordinate node numbering
    start_coords = zeros(3)
    apf.getPoint(mesh.m_ptr, mesh.verts[1], 0, start_coords)
    apf.reorder(mesh.m_ptr, mesh.dim*mesh.coord_numNodes, mesh.dim,
            C_NULL, mesh.coord_nodenums_Nptr, C_NULL,
            start_coords)

    numXiDof = apf.reorderXi(mesh.m_ptr, xiNums_Nptr, start_coords)
  else
    throw(ErrorException("invalid dof reordering algorithm requested"))
  end

  mesh.geoNums = GeometricDofs(mesh.m_ptr, mesh.coord_nodenums_Nptr,
                        xiNums_Nptr, mesh.dim*mesh.coord_numNodes, numXiDof)

#  println("finished numbering nodes")

#  println("about to number dofs")
  # do dof numbering
  populateDofNumbers(mesh)
#  println("finished numbering dofs")

  mesh.element_vertnums = getElementVertMap(mesh)
#  println("finished getting entity pointers")

  boundary_nums = getAllFaceData(mesh, opts)

  # use partially constructed mesh object to populate arrays
#  println("about to get entity orientations")
  mesh.elementNodeOffsets, mesh.typeNodeFlags = getEntityOrientations(mesh)
#  println("finished getting entity orientations")

  # start parallel initializiation
  colordata = getParallelInfo(mesh)
  mesh.vert_sharing = getVertexParallelInfo(mesh)
  mesh.coordscatter = initSendToOwner(mesh, mesh.coordshape_ptr, (mesh.dim,))

#  println("about to get degree of freedom numbers")
  getDofNumbers(mesh)  # store dof numbers
#  println("finished getting degree of freedom numbers")

  MPI.Barrier(mesh.comm)
  if coloring_distance == 2
    numc = colorMesh2(mesh, colordata)
    mesh.numColors = numc
    mesh.maxColors = MPI.Allreduce(numc, MPI.MAX, mesh.comm)
    mesh.color_masks = Array{BitArray{1}}(numc)  # one array for every color
    mesh.neighbor_colors = zeros(UInt8, mesh.numFacesPerElement+1, mesh.numEl)
    mesh.neighbor_nums = zeros(Int32, mesh.numFacesPerElement+1, mesh.numEl)
    cnt, mesh.shared_element_colormasks = getColors1(mesh, colordata, mesh.color_masks,
                                mesh.neighbor_colors, mesh.neighbor_nums; verify=opts["verify_coloring"] )
    mesh.pertNeighborEls = getPertNeighbors1(mesh)

  elseif coloring_distance == 0  # do a distance-0 coloring
    numc = colorMesh0(mesh)
    @assert numc == 1
    mesh.numColors = numc
    mesh.maxColors = numc
    mesh.color_masks = Array{BitArray{1}}(numc)
    mesh.neighbor_colors = zeros(UInt8, 0, 0)  # unneeded array for distance-0
    mesh.neighbor_nums = zeros(Int32, 0, 0)  # unneeded for distance-0
    getColors0(mesh, mesh.color_masks)
    mesh.pertNeighborEls = getPertNeighbors0(mesh)

  else
    throw(ErrorException("unsupported coloring distance requested"))
  end

  # get sparsity information
  # this takes into account the coloring distance
  #TODO: make getting sparsity bounds faster
  if opts["run_type"] != 1  # no a rk4 run
    mesh.sparsity_bnds = zeros(Int32, 0, 0)
#    mesh.sparsity_bnds = zeros(Int32, 2, mesh.numDof)
#    @time getSparsityBounds(mesh, mesh.sparsity_bnds)
#    mesh.sparsity_nodebnds = zeros(Int32, 2, mesh.numNodes)
#    @time getSparsityBounds(mesh, mesh.sparsity_nodebnds, getdofs=false)
#    println("finished getting sparsity bounds")

    mesh.sparsity_counts = zeros(Int32, 2, mesh.numDof)
    mesh.sparsity_counts_node = zeros(Int32, 2, mesh.numNodes)
    getSparsityCounts(mesh, mesh.sparsity_counts)
    getSparsityCounts(mesh, mesh.sparsity_counts_node, getdofs=false)

  end

  # TODO: make this a separate option from use_edge_res, make decision
  #       in read_input.jl
  if opts["use_edge_res"]  # if using edge based residual
    mesh.pertNeighborEls_edge = getPertEdgeNeighbors(mesh)
  end

  getAllCoordinatesAndMetrics(mesh, sbp, opts)

#  if mesh.dim == 2
#    @time createSubtriangulatedMesh(mesh)
#    println("finished creating sub mesh\n")
#  end

  checkFinalMesh(mesh)

  # print number of mesh entities
  printStats(mesh)

  # write data if requested
  myrank = mesh.myrank
  if opts["write_edge_vertnums"]
    rmfile("edge_vertnums_$myrank.dat")
    f = open("edge_vertnums_$myrank.dat", "a+")
    apf.printEdgeVertNumbers(mesh.edge_Nptr, mesh.vert_Nptr, fstream=f)
    close(f)
  end

  if opts["write_face_vertnums"]
    rmfile("face_vertnums_$myrank.dat")
    f = open("face_vertnums_$myrank.dat", "a+")
    apf.printFaceVertNumbers(mesh.face_Nptr, mesh.vert_Nptr, fstream=f)
    close(f)
  end

  if opts["write_element_vertnums"]
    rmfile("el_vertnums_$myrank.dat")
    f = open("el_vertnums_$myrank.dat", "a+")
    apf.printElementVertNumbers(mesh.el_Nptr, mesh.vert_Nptr, fstream=f)
    close(f)
  end

  if opts["write_boundarynums"]
    writedlm("boundary_nums_$myrank.dat", boundary_nums)
  end

  if opts["write_dxidx"]
    writedlm("dxidx_$myrank.dat", mesh.dxidx)
  end
#=
  if opts["write_jac2"]
    rmfile("jac_$myrank.dat")
    writedlm("jac_$myrank.dat", mesh.jac)
  end
=#

  if opts["write_coords"]
    rmfile("coords_$myrank.dat")
    writedlm("coords_$myrank.dat", mesh.coords)
#    printcoords("coords.dat", mesh.coords)
  end

  if opts["write_sparsity"]
    rmfile("sparsity_bnds_$myrank.dat")
    writedlm("sparsity_bnds_$myrank.dat", mesh.pertNeighborEls)
  end
#=
  if opts["write_sparsity_nodebnds"]
    println("writing sparsiy node bounds")
    rmfile("sparsity_nodebnds_$myrank.dat")
    writedlm("sparsity_nodebnds_$myrank.dat", mesh.sparsity_nodebnds)
  end
=#
  if opts["write_offsets"]
    rmfile("entity_offsets_$myrank.dat")
    writedlm("entity_offsets_$myrank.dat", mesh.elementNodeOffsets)
  end

  if opts["write_dofs"]
    rmfile("dofs_$myrank.dat")
    writedlm("dofs_$myrank.dat", mesh.dofs)
  end

  if opts["write_counts"]
    writeCounts(mesh)
  end

  if opts["write_interfaces"]
    f = open("interface_$myrank.dat", "a+")
    ifaces = mesh.interfaces
    for i=1:length(ifaces)
      face_i = ifaces[i]
      println(f, (face_i.elementL), " ", Int(face_i.elementR), " ", Int(face_i.faceL), " ", Int(face_i.faceR), " ", Int(face_i.orient))
    end
    close(f)
  end

  if opts["write_boundaries"]
    f = open("boundary_$myrank.dat", "a+")
    bndries = mesh.bndryfaces
    for i=1:length(mesh.bndryfaces)
      bndry_i = mesh.bndryfaces[i]
      println(f, Int(bndry_i.element), " ", Int(bndry_i.face))
    end
    close(f)
  end

  if opts["write_sharedboundaries"]
    for i=1:mesh.npeers
      fname = string("sharedboundaries_", i, "_", myrank, ".dat")
      f = open(fname, "a+")
      bndries_i = mesh.bndries_local[i]
      for j=1:length(bndries_i)
        bndry_j = bndries_i[j]
        println(f, Int(bndry_j.element), " ", Int(bndry_j.face))
      end
      close(f)
    end
  end

  writeVisFiles(mesh, "mesh_complete")

  myrank = mesh.myrank
  f = open("load_balance_$myrank.dat", "a+")
  println(f, mesh.numVert)
  println(f, mesh.numEdge)
  println(f, mesh.numEl)
  println(f, mesh.numInterfaces)
  println(f, mesh.numBoundaryFaces)
  println(f, sum(mesh.peer_face_counts))
  close(f)

  registerFinalizer(mesh)

#  close(mesh.f)
  return mesh
  # could use incomplete initilization to avoid copying arrays
#  return PumiMeshDG2(m_ptr, mshape_ptr, f_ptr, vert_Nptr, edge_Nptr, el_Nptr, numVert, numEdge, numEl, order, numdof, numnodes, dofpernode, bnd_edges_cnt, verts, edges, elements, dofnums_Nptr, bnd_edges_small)
end

function PumiMeshDG3Preconditioning(mesh_old::PumiMeshDG2, sbp::AbstractSBP, opts;
                                  coloring_distance=0)
# construct pumi mesh for preconditioner residual evaluations
# this operates by copying an existing mesh object (hence not loading a new
#  pumi mesh), and updating a few of its fields
# once there big picture of preconditioning is more clear, there should be
# options to control these steps.

  println("Creating Pumi mesh for preconditioning")
  mesh = deepcopy(mesh_old)  # create new PumiMesh object, sharing all
                        # mutable fields (like arrays)

  mesh.coloringDistance = coloring_distance
  # create the coloring_Nptr
  el_mshape = apf.getConstantShapePtr(2)
  mesh.coloring_Nptr = apf.createNumberingJ(mesh, "preconditioning coloring",
                                            1, el_mshape)

  if coloring_distance == 2
    numc = colorMesh2(mesh)
    mesh.numColors = numc

    mesh.color_masks = Array{BitArray{1}}(numc)  # one array for every color
    mesh.neighbor_colors = zeros(UInt8, 4, mesh.numEl)
    mesh.neighbor_nums = zeros(Int32, 4, mesh.numEl)
    getColors1(mesh, colordata, mesh.color_masks, mesh.neighbor_colors, mesh.neighbor_nums; verify=opts["verify_coloring"] )
    mesh.pertNeighborEls = getPertNeighbors1(mesh)

  elseif coloring_distance == 0  # do a distance-0 coloring
    numc = colorMesh0(mesh)
    @assert numc == 1
    mesh.numColors = numc
    mesh.color_masks = Array{BitArray{1}}(numc)
    mesh.neighbor_colors = zeros(UInt8, 0, 0)  # unneeded array for distance-0
    mesh.neighbor_nums = zeros(Int32, 0, 0)  # unneeded for distance-0
    getColors0(mesh, mesh.color_masks)
    mesh.pertNeighborEls = getPertNeighbors0(mesh)

  else
    throw(ErrorException("unsupported coloring distance requested"))
  end

  # get sparsity information
  # this takes into account the coloring distanc
  mesh.sparsity_bnds = zeros(Int32, 2, mesh.numDof)
  getSparsityBounds(mesh, mesh.sparsity_bnds)
  mesh.sparsity_nodebnds = zeros(Int32, 2, mesh.numNodes)
  getSparsityBounds(mesh, mesh.sparsity_nodebnds, getdofs=false)

  mesh.sparsity_counts = zeros(Int32, 2, mesh.numDof)
  mesh.sparsity_counts_node = zeros(Int32, 2, mesh.numNodes)
  getSparsityCounts(mesh, mesh.sparsity_counts)
  getSparsityCounts(mesh, mesh.sparsity_counts_node, getdofs=false)

  if opts["write_counts"]
    writeCounts(mesh, fname="countsp.txt")
  end

  return mesh

end


using HDF5
import ProgressMeter: Progress
using JLD2, FileIO

mutable struct NER
    worddict::Dict
    chardict::Dict
    tagset
    model
end

function NER()
    words = h5read(wordembeds_file, "key")
    worddict = Dict(words[i] => i for i=1:length(words))
    chardict = Dict("UNKNOWN" => 1)
    for word in words
        for c in Vector{Char}(word)
            get!(chardict, string(c), length(chardict)+1)
            get!(chardict, string(uppercase(c)), length(chardict)+1)
        end
    end
    NER(worddict, chardict, BIOES(), nothing)
end

function encode_word(ner::NER, words::Vector{String})
    worddict = ner.worddict
    unkword = worddict["UNKNOWN"]
    ids = map(words) do w
        w = lowercase(w)
        # w = replace(word, r"[0-9]", '0')
        get(worddict, w, unkword)
    end
    Var(ids)
end

function encode_char(ner::NER, words::Vector{String})
    chardict = ner.chardict
    unkchar = chardict["UNKNOWN"]
    batchdims = Int[]
    ids = Int[]
    for w in words
        # w = replace(word, r"[0-9]", '0')
        chars = Vector{Char}(w)
        push!(batchdims, length(chars))
        for c in chars
            push!(ids, get(chardict,string(c),unkchar))
        end
    end
    Var(ids, batchdims)
end

function readdata!(ner::NER, path::String)
    data = NTuple{3,Var}[]
    words, tags = String[], String[]
    lines = open(readlines, path)
    for i = 1:length(lines)
        line = lines[i]
        if isempty(line) || i == length(lines)
            isempty(words) && continue
            w = encode_word(ner, words)
            c = encode_char(ner, words)
            t = Var(encode(ner.tagset,tags))
            push!(data, (w,c,t))
            empty!(words)
            empty!(tags)
        else
            items = split(line, "\t")
            push!(words, String(items[1]))
            push!(tags, String(items[2]))
        end
    end
    data
end

function train(ner::NER, traindata::Vector, testdata::Vector)
    info("#Training examples:\t$(length(traindata))")
    info("#Testing examples:\t$(length(testdata))")
    info("#Words:\t$(length(ner.worddict))")
    info("#Chars:\t$(length(ner.chardict))")
    info("#Tags:\t$(length(ner.tagset.tag2id))")
    testdata = batch(testdata, 100)

    wordembeds = embeddings(h5read(wordembeds_file,"value"))
    charembeds = embeddings(Float32, length(ner.chardict), 20, init_w=Normal(0,0.05))
    ner.model = setup_model(wordembeds, charembeds, length(ner.tagset.tag2id))
    batchsize = 10
    opt = SGD()
    for epoch = 1:50
        println("epoch:\t$epoch")
        opt.rate = 0.0005 * batchsize / sqrt(batchsize) / (1 + 0.05*(epoch-1))
        println("learning rate: $(opt.rate)")

        shuffle!(traindata)
        batches = batch(traindata, batchsize)
        prog = Progress(length(batches))
        loss = 0.0
        for i in 1:length(batches)
            w, c, t = batches[i]
            h = ner.model(w, c)
            y = softmax_crossentropy(t, h)
            loss += sum(y.data)
            params = gradient!(y)
            foreach(opt, params)
            ProgressMeter.next!(prog)
        end
        loss /= length(batches)
        println("Loss:\t$loss")

        # test
        println("Testing...")
        pred = Int[]
        gold = Int[]
        for (w,c,t) in testdata
            y = ner.model(w, c)
            append!(pred, vec(argmax(y.data,1)))
            append!(gold, t.data)
        end
        length(pred) == length(gold) || throw("Length mismatch: $(length(pred)), $(length(gold))")

        ranges_p = decode(ner.tagset, pred)
        ranges_g = decode(ner.tagset, gold)
        fscore(ranges_g, ranges_p)
        println()
    end
end

function fscore{T}(golds::Vector{T}, preds::Vector{T})
    set = intersect(Set(golds), Set(preds))
    count = length(set)
    prec = round(count/length(preds), 5)
    recall = round(count/length(golds), 5)
    fval = round(2*recall*prec/(recall+prec), 5)
    println("Prec:\t$prec")
    println("Recall:\t$recall")
    println("Fscore:\t$fval")
end


mutable struct RandomOCClassifier <: OCClassifier
    data
    outlier_bias
    RandomOCClassifier(data) = new(data, 0.1)
    RandomOCClassifier(data, pools::Vector) = RandomOCClassifier(data)
    RandomOCClassifier(data, outlier_bias::Float64) = new(data, outlier_bias)
    RandomOCClassifier(data, outlier_bias::Float64, pools::Vector) = RandomOCClassifier(data, outlier_bias)
end

set_adjust_K!(model::RandomOCClassifier, adjust_K::Bool) = nothing

get_model_params(model::RandomOCClassifier) = (:Ϟ => 0.42, :param2 => 5)

function fit!(model::RandomOCClassifier, solver)
    debug(LOGGER, "[FIT] $(typeof(model)) always returns :Optimal.")
    return :Optimal
end

function predict(model::RandomOCClassifier, target::Array{T,2}) where T <: Real
    rand(size(target, 2)) * 2 .- model.outlier_bias * 2
end

initialize!(model::RandomOCClassifier, strategy::InitializationStrategy) = nothing

get_kernel(model::RandomOCClassifier) = nothing

invalidate_solution!(model::RandomOCClassifier) = nothing

set_pools!(model::RandomOCClassifier, pools::Vector{Symbol}) = nothing
set_pools!(model::RandomOCClassifier, pools::Dict{Symbol, Vector{Int}}) = nothing

function set_data!(model::RandomOCClassifier, data::Array{T, 2}) where T <: Real
    model.data = data
    return nothing
end


"""
    diff_optimizer(optimizer_constructor)::Optimizer

Creates a `DiffOpt.Optimizer`, which is an MOI layer with an internal optimizer
and other utility methods. Results (primal, dual and slack values) are obtained
by querying the internal optimizer instantiated using the
`optimizer_constructor`. These values are required for find jacobians with respect to problem data.

One define a differentiable model by using any solver of choice. Example:

```julia
julia> import DiffOpt, HiGHS

julia> model = DiffOpt.diff_optimizer(HiGHS.Optimizer)
julia> x = model.add_variable(model)
julia> model.add_constraint(model, ...)
```
"""
function diff_optimizer(optimizer_constructor)::Optimizer
    optimizer = MOI.instantiate(optimizer_constructor, with_bridge_type=Float64)
    # When we do `MOI.copy_to(diff, optimizer)` we need to efficiently `MOI.get`
    # the model information from `optimizer`. However, 1) `optimizer` may not
    # implement some getters or it may be inefficient and 2) the getters may be
    # unimplemented or inefficient through some bridges.
    # For this reason we add a cache layer, the same cache JuMP adds.
    caching_opt = MOIU.CachingOptimizer(
        MOIU.UniversalFallback(MOIU.Model{Float64}()),
        optimizer,
    )
    return Optimizer(caching_opt)
end

mutable struct Optimizer{OT <: MOI.ModelLike} <: MOI.AbstractOptimizer
    optimizer::OT

    model_constructors::Vector{Any}
    model_constructor::Any

    diff::Any

    index_map::Union{Nothing,MOI.Utilities.IndexMap}

    # sensitivity input cache using MOI like sparse format
    input_cache::InputCache

    function Optimizer(optimizer::OT) where {OT <: MOI.ModelLike}
        output = new{OT}(
            optimizer,
            Any[],
            nothing,
            nothing,
            nothing,
            InputCache(),
        )
        add_all_model_constructors(output)
        return output
    end
end

"""
   add_model_constructor(optimizer::Optimizer, model_constructor)

Add the constructor of [`AbstractModel`](@ref) for `optimizer` to choose
from when trying to differentiate.
"""
function add_model_constructor(optimizer::Optimizer, model_constructor)
    push!(optimizer.model_constructors, model_constructor)
    return
end

function MOI.add_variable(model::Optimizer)
    model.diff = nothing
    vi = MOI.add_variable(model.optimizer)
    return vi
end

function MOI.add_variables(model::Optimizer, N::Int)
    model.diff = nothing
    return VI[MOI.add_variable(model) for i in 1:N]
end

function MOI.add_constraint(model::Optimizer, f::MOI.AbstractFunction, s::MOI.AbstractSet)
    model.diff = nothing
    return MOI.add_constraint(model.optimizer, f, s)
end

function MOI.add_constraints(model::Optimizer, f::AbstractVector{F}, s::AbstractVector{S}) where {F<:MOI.AbstractFunction, S<: MOI.AbstractSet}
    model.diff = nothing
    return CI{F, S}[MOI.add_constraint(model, f[i], s[i]) for i in eachindex(f)]
end

function MOI.set(model::Optimizer, attr::MOI.ObjectiveFunction{F}, f::F) where {F<:MOI.AbstractFunction}
    model.diff = nothing
    MOI.set(model.optimizer, attr, f)
end

MOI.supports(model::Optimizer, attr::MOI.ObjectiveSense) = MOI.supports(model.optimizer, attr)

function MOI.set(model::Optimizer, attr::MOI.ObjectiveSense, sense::MOI.OptimizationSense)
    model.diff = nothing
    return MOI.set(model.optimizer, attr, sense)
end

function MOI.get(model::Optimizer, attr::MOI.AbstractModelAttribute)
    return MOI.get(model.optimizer, attr)
end

function MOI.get(model::Optimizer, attr::MOI.ListOfConstraintIndices)
    return MOI.get(model.optimizer, attr)
end

function MOI.get(model::Optimizer, attr::MOI.ConstraintSet, ci::CI)
    return MOI.get(model.optimizer, attr, ci)
end

function MOI.set(
    model::Optimizer, attr::MOI.ConstraintSet, ci::CI{F, S}, s::S
) where {F,S}
    model.diff = nothing
    return MOI.set(model.optimizer, attr, ci, s)
end

function MOI.get(
    model::Optimizer, attr::MOI.ConstraintFunction, ci::CI{F, S}
) where {F,S}
    return MOI.get(model.optimizer, attr, ci)
end

function MOI.set(
    model::Optimizer, attr::MOI.ConstraintFunction, ci::CI{F, S}, f::F
) where {F,S}
    model.diff = nothing
    return MOI.set(model.optimizer, attr, ci, f)
end

# `MOI.supports` methods

function MOI.supports(model::Optimizer, attr::MOI.AbstractModelAttribute)
    return MOI.supports(model.optimizer, attr)
end

function MOI.supports(model::Optimizer, attr::MOI.ObjectiveFunction)
    return MOI.supports(model.optimizer, attr)
end

function MOI.supports_constraint(
    model::Optimizer, ::Type{F}, ::Type{S},
) where {F<:MOI.AbstractFunction,S<:MOI.AbstractSet}
    MOI.supports_constraint(model.optimizer, F, S)
end
function MOI.supports(
    model::Optimizer, attr::MOI.ConstraintName, ::Type{CI{F, S}}
) where {F,S}
    return MOI.supports(model.optimizer, attr, CI{F,S})
end

MOI.get(model::Optimizer, attr::MOI.SolveTimeSec) = MOI.get(model.optimizer, attr)

function MOI.empty!(model::Optimizer)
    MOI.empty!(model.optimizer)
    model.diff = nothing
    model.index_map = nothing
    empty!(model.input_cache)
    return
end

function MOI.is_empty(model::Optimizer)
    return MOI.is_empty(model.optimizer) &&
           model.diff === nothing
end

function MOI.supports_incremental_interface(model::Optimizer)
    if !MOI.supports_incremental_interface(model.optimizer)
        error(
            "DiffOpt requires a solver that " *
            "`MOI.supports_incremental_interface`, which is not the case for " *
            "$(model.optimizer)")
    end
    return true
end

function MOI.copy_to(model::Optimizer, src::MOI.ModelLike)
    model.diff = nothing
    return MOIU.default_copy_to(model.optimizer, src)
end

function MOI.get(model::Optimizer, ::MOI.TerminationStatus)
    return MOI.get(model.optimizer, MOI.TerminationStatus())
end

function MOI.set(model::Optimizer, ::MOI.VariablePrimalStart,
                 vi::VI, value::Float64)
    model.diff = nothing
    MOI.set(model.optimizer, MOI.VariablePrimalStart(), vi, value)
end

function MOI.supports(model::Optimizer, attr::MOI.AbstractVariableAttribute,
                      ::Type{MOI.VariableIndex})
    return MOI.supports(model.optimizer, attr, MOI.VariableIndex)
end

function MOI.set(model::Optimizer, attr::MOI.AbstractVariableAttribute, v::MOI.VariableIndex, value)
    MOI.set(model.optimizer, attr, v, value)
end

function MOI.get(model::Optimizer, attr::MOI.AbstractVariableAttribute, vi::MOI.VariableIndex)
    return MOI.get(model.optimizer, attr, vi)
end

function MOI.delete(model::Optimizer, ci::CI{F,S}) where {F,S}
    model.diff = nothing
    MOI.delete(model.optimizer, ci)
end

function MOI.get(model::Optimizer, attr::MOI.ConstraintPrimal, ci::CI{F,S}) where {F,S}
    return MOI.get(model.optimizer, attr, ci)
end

function MOI.is_valid(model::Optimizer, v::VI)
    return MOI.is_valid(model.optimizer, v::VI)
end

MOI.is_valid(model::Optimizer, con::CI) = MOI.is_valid(model.optimizer, con)

function MOI.get(model::Optimizer, attr::MOI.ConstraintDual, ci::CI{F,S}) where {F,S}
    return MOI.get(model.optimizer, attr, ci)
end

function MOI.get(model::Optimizer, ::MOI.ConstraintBasisStatus, ci::CI{F,S}) where {F,S}
    return MOI.get(model.optimizer, MOI.ConstraintBasisStatus(), ci)
end

# helper methods to check if a constraint contains a Variable
function _constraint_contains(model::Optimizer, v::VI, ci::CI{MOI.VariableIndex})
    return v == MOI.get(model, MOI.ConstraintFunction(), ci)
end

function _constraint_contains(model::Optimizer, v::VI, ci::CI{MOI.ScalarAffineFunction{Float64}})
    func = MOI.get(model, MOI.ConstraintFunction(), ci)
    return any(term -> v == term.variable, func.terms)
end

function _constraint_contains(model::Optimizer, v::VI, ci::CI{MOI.VectorOfVariables})
    func = MOI.get(model, MOI.ConstraintFunction(), ci)
    return v in func.variables
end

function _constraint_contains(model::Optimizer, v::VI, ci::CI{MOI.VectorAffineFunction{Float64}})
    func = MOI.get(model, MOI.ConstraintFunction(), ci)
    return any(term -> v == term.scalar_term.variable, func.terms)
end

function MOI.delete(model::Optimizer, v::VI)
    model.diff = nothing
    MOI.delete(model.optimizer, v)
end

# for array deletion
function MOI.delete(model::Optimizer, indices::Vector{VI})
    model.diff = nothing
    for i in indices
        MOI.delete(model, i)
    end
end

function MOI.modify(
    model::Optimizer,
    ::MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}},
    chg::MOI.AbstractFunctionModification
)
    model.diff = nothing
    MOI.modify(
        model.optimizer,
        MOI.ObjectiveFunction{MOI.ScalarAffineFunction{Float64}}(),
        chg
    )
end

function MOI.modify(model::Optimizer, ci::CI, chg::MOI.AbstractFunctionModification)
    model.diff = nothing
    MOI.modify(model.optimizer, ci, chg)
end

function MOI.get(model::Optimizer, ::Type{MOI.VariableIndex}, name::String)
    return MOI.get(model.optimizer, MOI.VariableIndex, name)
end

function MOI.set(model::Optimizer, ::MOI.ConstraintName, con::CI, name::String)
    MOI.set(model.optimizer, MOI.ConstraintName(), con, name)
end

function MOI.get(model::Optimizer, ::Type{MOI.ConstraintIndex}, name::String)
    return MOI.get(model.optimizer, MOI.ConstraintIndex, name)
end

function MOI.get(model::Optimizer, ::MOI.ConstraintName, con::CI)
    return MOI.get(model.optimizer, MOI.ConstraintName(), con)
end

function MOI.get(model::Optimizer, ::MOI.ConstraintName, ::Type{CI{F, S}}) where {F,S}
    return MOI.get(model.optimizer, MOI.ConstraintName(), CI{F,S})
end

function MOI.set(model::Optimizer, ::MOI.Name, name::String)
    MOI.set(model.optimizer, MOI.Name(), name)
end

function MOI.get(model::Optimizer, ::Type{CI{F, S}}, name::String) where {F,S}
    return MOI.get(model.optimizer, CI{F,S}, name)
end

function MOI.supports(model::Optimizer, attr::MOI.TimeLimitSec)
    MOI.supports(model.optimizer, attr)
end
function MOI.set(model::Optimizer, ::MOI.TimeLimitSec, value::Union{Real, Nothing})
    MOI.set(model.optimizer, MOI.TimeLimitSec(), value)
end
function MOI.get(model::Optimizer, ::MOI.TimeLimitSec)
    return MOI.get(model.optimizer, MOI.TimeLimitSec())
end

MOI.supports(model::Optimizer, ::MOI.Silent) = MOI.supports(model.optimizer, MOI.Silent())

function MOI.set(model::Optimizer, ::MOI.Silent, value)
    MOI.set(model.optimizer, MOI.Silent(), value)
end

function MOI.get(model::Optimizer, ::MOI.Silent)
    return MOI.get(model.optimizer, MOI.Silent())
end

function MOI.get(model::Optimizer, attr::A) where {A <: Union{MOI.SolverName, MOI.SolverVersion}}
    return MOI.get(model.optimizer, attr)
end

function MOI.optimize!(model::Optimizer)
    model.diff = nothing
    MOI.optimize!(model.optimizer)
    return
end

"""
    ModelConstructor <: MOI.AbstractOptimizerAttribute

Determines which subtype of [`DiffOpt.AbstractModel`](@ref) to use for
differentiation. When set to `nothing`, the first one out of
`model.model_constructors` that support the problem is used.
"""
struct ModelConstructor <: MOI.AbstractOptimizerAttribute end

MOI.supports(::Optimizer, ::ModelConstructor) = true
MOI.get(model::Optimizer, ::ModelConstructor) = model.model_constructor
function MOI.set(model::Optimizer, ::ModelConstructor, model_constructor)
    model.model_constructor = model_constructor
end

function reverse_differentiate!(model::Optimizer)
    st = MOI.get(model.optimizer, MOI.TerminationStatus())
    if !in(st,  (MOI.LOCALLY_SOLVED, MOI.OPTIMAL))
        error("Trying to compute the reverse differentiation on a model with termination status $(st)")
    end
    diff = _diff(model)
    for (vi, value) in model.input_cache.dx
        MOI.set(diff, ReverseVariablePrimal(), model.index_map[vi], value)
    end
    return reverse_differentiate!(diff)
end

function _copy_forward_in_constraint(diff, index_map, con_map, constraints)
    for (index, value) in constraints
        MOI.set(diff, ForwardConstraintFunction(), con_map[index], MOI.Utilities.map_indices(index_map, value))
    end
end

function forward_differentiate!(model::Optimizer)
    st = MOI.get(model.optimizer, MOI.TerminationStatus())
    if !in(st,  (MOI.LOCALLY_SOLVED, MOI.OPTIMAL))
        error("Trying to compute the forward differentiation on a model with termination status $(st)")
    end
    diff = _diff(model)
    if model.input_cache.objective !== nothing
        MOI.set(diff, ForwardObjectiveFunction(), MOI.Utilities.map_indices(model.index_map, model.input_cache.objective))
    end
    for (F, S) in keys(model.input_cache.scalar_constraints.dict)
        _copy_forward_in_constraint(diff, model.index_map, model.index_map.con_map[F, S], model.input_cache.scalar_constraints[F, S])
    end
    for (F, S) in keys(model.input_cache.vector_constraints.dict)
        _copy_forward_in_constraint(diff, model.index_map, model.index_map.con_map[F, S], model.input_cache.vector_constraints[F, S])
    end
    forward_differentiate!(diff)
end

function _copy_constraint_start(dest, src, index_map::MOIU.DoubleDicts.IndexDoubleDictInner{F, S}, dest_attr, src_attr) where {F, S}
    for ci in MOI.get(src, MOI.ListOfConstraintIndices{F, S}())
        value = MOI.get(src, src_attr, ci)
        MOI.set(dest, dest_attr, index_map[ci], value)
    end
end

function _instantiate_with_bridges(model_constructor)
    model = MOI.Bridges.LazyBridgeOptimizer(
        MOI.instantiate(model_constructor)
    )
    # We don't add any variable bridge here because:
    # 1) If `ZerosBridge` is used, `MOI.Bridges.unbridged_function` does not work.
    #    This is in fact expected: since `ZerosBridge` drops the variable, we dont
    #    compute the derivative of the value of this variable as a function of its fixed value.
    #    This could be easily determined as the same as the derivative of the value but
    #    since the variable was also dropped from other constraints, we would ignore its impact on the other constraints.
    # 2) For affine variable bridges, `bridged_function` and `unbridged_function` don't treat the function as a derivative hence they will add constants
    MOI.Bridges.Constraint.add_all_bridges(model, Float64)
    MOI.Bridges.Objective.add_all_bridges(model, Float64)
    return model
end

function _diff(model::Optimizer)
    if model.diff === nothing
        _check_termination_status(model)
        model_constructor = MOI.get(model, ModelConstructor())
        if isnothing(model_constructor)
            model.diff = nothing
            for constructor in model.model_constructors
                model.diff = _instantiate_with_bridges(constructor)
                try
                    model.index_map = MOI.copy_to(model.diff, model.optimizer)
                catch err
                    if err isa MOI.UnsupportedConstraint || err isa MOI.UnsupportedAttribute
                        model.diff = nothing
                    else
                        rethrow(err)
                    end
                end
                if !isnothing(model.diff)
                    break
                end
            end
            if isnothing(model.diff)
                error("No differentiation model supports the problem. If you believe it should be supported, say by `DiffOpt.QuadraticProgram.Model`, use `MOI.set(model, DiffOpt.ModelConstructor, DiffOpt.QuadraticProgram.Model)` and try again to see an error indicating why it is not supported.")
            end
        else
            model.diff = _instantiate_with_bridges(model_constructor)
            model.index_map = MOI.copy_to(model.diff, model.optimizer)
        end
        vis_src = MOI.get(model.optimizer, MOI.ListOfVariableIndices())
        MOI.set(model.diff, MOI.VariablePrimalStart(), getindex.(Ref(model.index_map), vis_src), MOI.get(model.optimizer, MOI.VariablePrimal(), vis_src))
        for (F, S) in MOI.get(model.optimizer, MOI.ListOfConstraintTypesPresent())
            _copy_constraint_start(model.diff, model.optimizer, model.index_map.con_map[F, S], MOI.ConstraintPrimalStart(), MOI.ConstraintPrimal())
            _copy_constraint_start(model.diff, model.optimizer, model.index_map.con_map[F, S], MOI.ConstraintDualStart(), MOI.ConstraintDual())
        end
    end
    return model.diff
end

function _check_termination_status(model::Optimizer)
    if !in(
        MOI.get(model, MOI.TerminationStatus()), (MOI.LOCALLY_SOLVED, MOI.OPTIMAL)
        )
        error("problem status: ", MOI.get(model.optimizer, MOI.TerminationStatus()))
    end
end

# DiffOpt attributes redirected to `diff`

function _checked_diff(model::Optimizer, ::MOI.AnyAttribute, call)
    if model.diff === nothing
        error("Cannot get attribute `attr`. First call `DiffOpt.$call`.")
    end
    return model.diff
end

function MOI.get(model::Optimizer, attr::ReverseObjectiveFunction)
    return IndexMappedFunction(
        MOI.get(_checked_diff(model, attr, :reverse), attr),
        model.index_map,
    )
end
MOI.supports(::Optimizer, ::ForwardObjectiveFunction) = true
function MOI.get(model::Optimizer, ::ForwardObjectiveFunction)
    return model.input_cache.objective
end
function MOI.set(model::Optimizer, ::ForwardObjectiveFunction, objective)
    model.input_cache.objective = objective
    return
end

function MOI.get(model::Optimizer, attr::ForwardVariablePrimal, vi::MOI.VariableIndex)
    return MOI.get(_checked_diff(model, attr, :forward), attr, model.index_map[vi])
end
MOI.supports(::Optimizer, ::ReverseVariablePrimal, ::Type{VI}) = true
function MOI.get(model::Optimizer, ::ReverseVariablePrimal, vi::VI)
    return get(model.input_cache.dx, vi, 0.0)
end
function MOI.set(model::Optimizer, ::ReverseVariablePrimal, vi::VI, val)
    model.input_cache.dx[vi] = val
    return
end

function MOI.get(model::Optimizer, attr::ReverseConstraintFunction, ci::MOI.ConstraintIndex)
    return IndexMappedFunction(
        MOI.get(_checked_diff(model, attr, :reverse), attr, model.index_map[ci]),
        model.index_map,
    )
end
function MOI.get(model::Optimizer,
    ::ForwardConstraintFunction, ci::CI{MOI.ScalarAffineFunction{T},S}
) where {T,S}
    return get(model.input_cache.scalar_constraints, ci, zero(MOI.ScalarAffineFunction{T}))
end
function MOI.supports(
    ::Optimizer,
    ::ForwardConstraintFunction,
    ::Type{CI{F,S}}
) where {F<:Union{MOI.ScalarAffineFunction,MOI.VectorAffineFunction},S}
    return true
end
function MOI.get(model::Optimizer,
    ::ForwardConstraintFunction, ci::CI{MOI.VectorAffineFunction{T},S}
) where {T,S}
    func = get(model.input_cache.vector_constraints, ci, nothing)
    if func === nothing
        set = MOI.get(model, MOI.ConstraintSet(), ci)
        dim = MOI.dimension(set)
        return MOI.Utilities.zero_with_output_dimension(MOI.VectorAffineFunction{T}, dim)
    else
        return func
    end
end
function MOI.set(model::Optimizer,
    ::ForwardConstraintFunction,
    ci::CI{MOI.ScalarAffineFunction{T},S},
    func::MOI.ScalarAffineFunction{T},
) where {T,S}
    model.input_cache.scalar_constraints[ci] = func
    return
end
function MOI.set(model::Optimizer,
    ::ForwardConstraintFunction,
    ci::CI{MOI.VectorAffineFunction{T},S},
    func::MOI.VectorAffineFunction{T},
) where {T,S}
    model.input_cache.vector_constraints[ci] = func
    return
end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

module Meshes

using Tables
using StaticArrays
using SparseArrays
using CircularArrays
using SimpleTraits
using RecipesBase
using LinearAlgebra
using Random

using IterTools: ivec
using StatsBase: Weights
using SpecialFunctions: gamma
using Distances: PreMetric, Euclidean, Mahalanobis, evaluate
using ReferenceFrameRotations: angle_to_dcm
using NearestNeighbors: KDTree, BallTree, knn, inrange

# import categorical arrays as a temporary solution for plot recipes
using CategoricalArrays: CategoricalValue, levelcode

import Tables
import Random
import Base: values, ==, +, -, *
import StatsBase: sample
import Distances: evaluate
import NearestNeighbors: MinkowskiMetric

# Queryverse compatibility
import TableTraits
import IteratorInterfaceExtensions
const IIE = IteratorInterfaceExtensions

# IO utils
include("ioutils.jl")

# numerical tolerances
include("tolerances.jl")

# basic types
include("vectors.jl")
include("points.jl")
include("angles.jl")

# type traits
include("traits.jl")

# geometries
include("geometries.jl")

# collections
include("collections.jl")

# meshes
include("connectivities.jl")
include("topologies.jl")
include("toporelations.jl")
include("mesh.jl")

# mesh data
include("meshdata.jl")

# miscellaneous
include("conventions.jl")
include("paths.jl")
include("distances.jl")
include("neighborhoods.jl")
include("neighborsearch.jl")
include("supportfun.jl")
include("laplacian.jl")

# views and partitions
include("views.jl")
include("partitions.jl")

# algorithms
include("viewing.jl")
include("sampling.jl")
include("partitioning.jl")
include("intersections.jl")
include("discretization.jl")
include("simplification.jl")
include("refinement.jl")
include("smoothing.jl")
include("boundingboxes.jl")
include("hulls.jl")

# utilities
include("utils.jl")

# plot recipes
include("plotrecipes/domain.jl")
include("plotrecipes/data.jl")
include("plotrecipes/points.jl")
include("plotrecipes/geometries.jl")
include("plotrecipes/collections.jl")
include("plotrecipes/cartesiangrids.jl")
include("plotrecipes/partitions.jl")

export
  # points
  Point, Point1, Point2, Point3,
  Point1f, Point2f, Point3f,
  embeddim, paramdim,
  coordtype, coordinates,
  center, centroid,
  ⪯, ≺, ⪰, ≻,

  # vectors
  Vec, Vec1, Vec2, Vec3, Vec1f, Vec2f, Vec3f,

  # angles
  ∠,

  # domain traits
  Domain,
  embeddim, paramdim, coordtype,
  element, nelements,

  # data traits
  Data, Variable,
  domain, constructor, asarray,
  variables, name, mactype,

  # optional traits
  IsGrid, isgrid,

  # domain/data alias
  DomainOrData,

  # geometries
  Geometry,
  embeddim, paramdim, coordtype,
  measure, area, volume, boundary,
  center, centroid, isconvex, issimplex,

  # primitives
  Primitive,
  Line, Ray, Plane, BezierCurve,
  Box, Ball, Sphere, Cylinder,
  ncontrols, degree, Horner, DeCasteljau,
  radius, height, sides,
  measure, diagonal,

  # polytopes
  Polytope, Polygon, Polyhedron,
  Segment, Ngon, Triangle, Quadrangle,
  Pentagon, Hexagon, Heptagon,
  Octagon, Nonagon, Decagon,
  Chain, PolyArea,
  Tetrahedron, Pyramid, Hexahedron,
  vertices, nvertices,
  windingnumber, chains, segments,
  isclosed, issimple, hasholes,
  angles, innerangles, close!, open!,
  orientation, bridge, normal,

  # orientation algorithms
  WindingOrientation,
  TriangleOrientation,

  # point or geometry alias
  PointOrGeometry,

  # multi-types
  Multi,

  # collections
  Collection,
  PointSet,
  GeometrySet,

  # rotation conventions
  RotationConvention,
  GSLIB, Leapfrog, Datamine,
  TaitBryanExtr, TaitBryanIntr,
  EulerExtr, EulerIntr,
  axesseq, orientation, angleunits,
  mainaxis, isextrinsic, rotmat,

  # paths
  Path,
  LinearPath, RandomPath,
  ShiftedPath, SourcePath,
  traverse,

  # neighborhoods
  Neighborhood, MetricBall,
  NormBall, Ellipsoid,
  metric, radius,

  # neighbordhood search
  NeighborSearchMethod,
  BoundedNeighborSearchMethod,
  BallSearch,
  KNearestSearch,
  KBallSearch,
  BoundedSearch,
  search!, search,

  # miscellaneous
  supportfun,
  laplacematrix,

  # connectivities
  Connectivity,
  paramdim, indices,
  connect, materialize,
  issimplex,

  # topologies
  Topology,
  vertices, nvertices,
  faces, elements, facets,
  element, nelements,
  facet, nfacets,
  FullTopology,
  GridTopology,
  HalfEdgeTopology, HalfEdge,
  half4elem, half4vert,
  half4edge, half4pair,
  edge4pair,

  # topological relations
  TopologicalRelation,
  Boundary, Coboundary, Adjacency,

  # meshes
  Mesh,
  CartesianGrid, SimpleMesh,
  vertices, nvertices, topology,
  faces, elements, facets,
  element, nelements,
  facet, nfacets,
  topoconvert,
  spacing,

  # mesh data
  MeshData,
  meshdata,

  # views
  DomainView, DataView,

  # partitions
  Partition,
  indices, metadata,

  # viewing
  indices, slice,

  # sampling
  SamplingMethod,
  DiscreteSamplingMethod,
  ContinuousSamplingMethod,
  UniformSampling,
  WeightedSampling,
  BallSampling,
  RegularSampling,
  HomogeneousSampling,
  MinDistanceSampling,
  sample,

  # partitioning
  PartitionMethod,
  PredicatePartitionMethod,
  SPredicatePartitionMethod,
  RandomPartition,
  FractionPartition,
  BlockPartition,
  BisectPointPartition,
  BisectFractionPartition,
  BallPartition,
  PlanePartition,
  DirectionPartition,
  PredicatePartition,
  SpatialPredicatePartition,
  ProductPartition,
  HierarchicalPartition,
  partition, split, →,

  # intersections
  Intersection,
  NoIntersection,
  CrossingLines,
  OverlappingLines,
  CrossingSegments,
  MidTouchingSegments,
  CornerTouchingSegments,
  OverlappingSegments,
  OverlappingBoxes,
  FaceTouchingBoxes,
  CornerTouchingBoxes,
  intersecttype,
  hasintersect,

  # discretization
  DiscretizationMethod,
  FIST, Dehn1899,
  discretize,
  triangulate,

  # simplification
  SimplificationMethod,
  DouglasPeucker,
  simplify,
  decimate,

  # refinement
  RefinementMethod,
  QuadRefinement,
  CatmullClark,
  refine,

  # smoothing
  SmoothingMethod,
  TaubinSmoothing,
  smooth,

  # bounding boxes
  boundingbox,

  # hulls
  HullMethod,
  GrahamScan,
  hull,

  # utililities
  signarea,
  sideof

end # module


export GenomicPos,
    GenomicAxis

# ---- Genomic axis --------------------------------------------------------

"""
    GenomicAxis(;params...)

Create a genomic axis for the track plots.

# Parameters

- `:axis_line_width`
- `:axis_color`
- `:axis_font`
- `:axis_tick_length`

- `:width` width of track
- `:height` height of track

- `:margin_top`
- `:margin_bottom`
- `:margin_left`
- `:margin_right`

"""
struct GenomicAxis <: AbstractRefFrame
    # AbstractRefFrame
    transform::AbstractTransform
    children::Vector{AbstractRefFrame}
    params::Dict

    function GenomicAxis(;params...)
        new(Translation(0., 0.), AbstractRefFrame[], Dict(params))
    end
end

GenomicPos = GenomicAxis

function draw(cc::Cairo.CairoContext, tc::Vector{T}, gp::GenomicPos, xmin, xmax) where T <: AbstractRefFrame

    width = getparam(tc, :width)
    height = getparam(tc, :height, 1cm)

    units = Rect(xmin, 0, xmax - xmin, 1)
    bbox = Rect(getparam(tc, :margin_left),
                getparam(tc, :margin_top),
                width - getparam(tc, :margin_left) - getparam(tc, :margin_right),
                1)

    datactx = RefFrame(transform = Scaling(bbox, units))

    xticks = _ticks_default_linear([xmin, xmax])

    devx, devy = user2dev(vcat(tc, datactx), xticks, repmat([0], length(xticks)))

    axis_line_width = getparam(tc, :axis_line_width)
    Cairo.set_source_rgba(cc, rgba(getparam(tc, :axis_color))...)
    Cairo.set_font_face(cc, getparam(tc, :axis_font))
    Cairo.set_line_width(cc, axis_line_width)

    Cairo.move_to(cc, devx[1] - axis_line_width/2, devy[1])
    Cairo.line_to(cc, devx[end] + axis_line_width/2, devy[end])

    for i in 1:length(devx)
        Cairo.move_to(cc, devx[i], devy[i] - getparam(tc, :axis_tick_length))
        Cairo.line_to(cc, devx[i], devy[i])
        Cairo.text(cc, devx[i], devy[i] - 2*getparam(tc, :axis_tick_length), _format_ticklabel(xticks[i]),
                   halign="center", valign="bottom")
    end

    Cairo.stroke(cc)

    user2dev(gp, Rect(0.0, 0.0, width, height))
end


#################### Binary Metropolised Gibbs Sampler ####################

#################### Types and Constructors ####################

typealias BMGForm Union{Int, Vector{Vector{Int}}}

type BMGTune{F<:BMGForm} <: SamplerTune
  logf::Nullable{Function}
  k::F

  BMGTune() = new()

  BMGTune(x::Vector, k::F) = new(Nullable{Function}(), k)

  BMGTune(x::Vector, k::F, logf::Function) = new(Nullable{Function}(logf), k)
end

typealias BMGIntVariate SamplerVariate{BMGTune{Int}}
typealias BMGVecVariate SamplerVariate{BMGTune{Vector{Vector{Int}}}}

BMGVariate{F<:BMGForm}(x::Vector, logf::Function; k::F=1) =
  SamplerVariate{BMGTune{F}}(x, k, logf)

function validate(v::BMGIntVariate)
  n = length(v)
  v.tune.k <= n || throw(ArgumentError("k exceeds variate length $n"))
  validatebinary(v)
end

function validate(v::BMGVecVariate)
  n = length(v)
  mapreduce(maximum, max, v.tune.k) <= n ||
    throw(ArgumentError("indices in k exceed variate length $n"))
  validatebinary(v)
end

#################### Sampler Constructor ####################

function BMG{F<:BMGForm}(params::ElementOrVector{Symbol}; k::F=1)
  samplerfx = function(model::Model, block::Integer)
    block = SamplingBlock(model, block)
    v = SamplerVariate(block, k)
    sample!(v, x -> logpdf!(block, x))
    relist(block, v)
  end
  Sampler(params, samplerfx, BMGTune{F}())
end

#################### Sampling Functions ####################

sample!{F<:BMGForm}(v::SamplerVariate{BMGTune{F}}) = sample!(v, v.tune.logf)

function sample!{F<:BMGForm}(v::SamplerVariate{BMGTune{F}}, logf::Function)
  n = length(v)
  probs = Vector{Float64}(n)
  idx = randind(v)

  pbernoulli! = function(x, probs)
    for i in idx
      x_i = x[i]

      x[i] = 0.0
      logf0 = logf(x)
      x[i] = 1.0
      logf1 = logf(x)

      p = invlogit(logf1 - logf0)
      probs[i] = 0.0 < p < 1.0 ? p : 0.5

      x[i] = x_i
    end
    probs
  end

  x = v[:]
  pbernoulli!(x, probs)
  theta = map(p -> rand() < p, probs[idx])

  if n == 1
    v[idx] = theta
  else
    y = v[:]
    y[idx] = theta
    qy = mapreduce(i -> y[i] == 1.0 ? log(probs[i]) : log(1.0 - probs[i]), +,
                   idx)

    pbernoulli!(y, probs)
    qx = mapreduce(i -> x[i] == 1.0 ? log(probs[i]) : log(1.0 - probs[i]), +,
                   idx)

    if rand() < exp((logf(y) - qy) - (logf(x) - qx))
      v[idx] = theta
    end
  end

  v
end

randind(v::BMGIntVariate) = sample(1:length(v), v.tune.k, replace=false)
randind(v::BMGVecVariate) = sample(v.tune.k)


"""
Internal structure that remembers that `quantity` was requested by `request`.
"""
struct ResourceRequest{Q<:Real}
    quantity::Q
    request::Function
end

"""
Resource type for holding numeric values (like amount of liquid).
It stores current `quantity` of matter and its allowed `lo` and `hi` amounts.
Servers can get matter from the resource with optional maximum number of
requests pending for fulfillment.

Fields:
* `quantity`       current quantity in resource
* `lo`            minimum quantity of resource
* `hi`            maximum quantity of resource
* `fifo_requests` if `true` `requests` is FIFO, otherwise LIFO
* `max_requests`  maximum `requests` size
* `requests`      vector of request and requested quantity

Functions in `requests` must accept one argument `Scheduler`, so they should
know the amount they requested. When resource arrives to a queue there is a try
to immediately dispatch it to pending requests. When new request arrives there
is a try to immediately fulfill it.

Initially an empty `SimResource` with no requests is constructed.
Initial `quantity`, `lo` and `hi` may be provided. By default `SimResource` is
empty, and has minimum quantity of zero and unbounded maximum.
"""
mutable struct SimResource{Q<:Real} <: AbstractReservoir
    quantity::Q
    lo::Q
    hi::Q
    fifo_requests::Bool
    max_requests::Int
    requests::Vector{ResourceRequest{Q}}

    function SimResource{Q}(;quantity::Q=zero(Q), lo::Q=zero(Q), hi::Q=typemax(Q),
                      fifo_requests::Bool=true,
                      max_requests::Int=typemax(Int)) where Q
        lo <= quantity <= hi || error("wrong quantity/lo/hi combination")
        max_requests > 0 || error("max_requests must be positive")
        new(quantity, lo, hi, fifo_requests, max_requests,
            Vector{ResourceRequest{Q}}())
    end
end

function dispatch!(s::Scheduler, r::SimResource)
    # while is needed as large provide! can fulfill many requests
    while (!isempty(r.requests) &&
           r.lo <= r.quantity - r.requests[end].quantity <= r.hi)
        req = pop!(r.requests)
        r.quantity -= req.quantity
        register!(s, req.request)
    end
end

function request!(s::Scheduler, r::SimResource{Q}, quantity::Q,
                  request::Function) where Q
    rr = ResourceRequest{Q}(quantity, request)
    length(r.requests) < r.max_requests || return false, rr
    qend = r.fifo_requests ? pushfirst! : push!
    qend(r.requests, rr)
    dispatch!(s, r)
    return true, rr
end

function waive!(r::SimResource{Q}, res_request::ResourceRequest{Q}) where Q
    idx = findfirst(isequal(res_request), r.requests)
    isa(idx, Nothing) && return false
    deleteat!(r.requests, idx)
    return true
end

function provide!(s::Scheduler, r::SimResource{Q}, quantity::Q) where Q
    startq = r.quantity
    # we do not want to oveflow the container, but allow large provisions
    r.quantity = clamp(startq+quantity, r.lo, r.hi)
    added = r.quantity - startq
    dispatch!(s, r)
    return added # return how much was added
end


using Random: shuffle
using Distributed: @everywhere, @distributed
@everywhere include("gru_dynamic_struct.jl")

create_model_definitions(layer) =
begin
    eval(Meta.parse("@everywhere " * "struct Model\n" * *(["l$i::Layer\n" for i in 1:length(layers) +1]...) * "end"))
    eval(Meta.parse("@everywhere " * "(model::Model)(io) =\n" * "begin\n" * *(["io = model.l$i(io)\n" for i in 1:length(layers) +1]...) * "end"))
end

soften = arr -> (begin
    new_arr = []
    append!(new_arr, softmax(arr[1:Int(length(arr)/4)]))
    # soft = softmax(arr[1:Int(length(arr)/4)])
    # for i in 1:Int(length(arr)/4)
    #     push!(new_arr, soft[i])
    # end
    append!(new_arr, sigm.(arr[Int(length(arr)/4):Int(length(arr)*3/4)]))
    # for i in Int(length(arr)/4):Int(length(arr)*3/4)
    #     push!(new_arr, sigm(arr[i]))
    # end
new_arr
end)

prop(model, x) =
begin
    for mfield in fieldnames(Model)
        layer = getfield(model, mfield)
        layer.state = zeros(1, length(layer.bs))
    end
soften([model(t) for t in x][end])
end

cross_entropy(out, label) =
begin
    - sum(label .* log.(out))
end

train!(model, datas, lr) =
begin

    result = @distributed vcat for (x,y) in shuffle(datas)
        d = @diff cross_entropy(prop(model, x), y)
        grads = []
        for mfield in fieldnames(Model)
            layer = getfield(model, mfield)
            for lfield in fieldnames(Layer)
                push!(grads, grad(d, getfield(layer, lfield)))
            end
        end
        grads, value(d)
    end

    loss = 0.0
    for (g,l) in result
        loss += l
        i = 0
        for mfield in fieldnames(Model)
            layer = getfield(model, mfield)
            for lfield in fieldnames(Layer)
                i +=1
                setfield!(layer, lfield, Param(getfield(layer, lfield) - g[i] .* lr))
            end
        end
    end
    @show loss
end