# export readuwevt, uwpf, uwpf!
#
"""
    writesac(W::SeisEvent[, v=0])

Write all data in SeisEvent structure `W` to auto-generated SAC files. Event
header information is written from W.hdr; W.source is not used as there is no
standard header position for event source information.
"""
function writesac(S::SeisEvent; v::Integer=KW.v)
  tdata = Array{Float32,1}(undef, 0)
  reset_sacbuf()

  evid  = codeunits(S.hdr.id == "" ? "-12345  " : S.hdr.id)
  evid  = evid[1:min(length(evid),16)]
  BUF.sac_cv[9:length(evid)+8] .= evid

  # Values from event header
  S.hdr.loc.lat == 0.0 || setindex!(BUF.sac_fv, Float32(S.hdr.loc.lat), 40)
  S.hdr.loc.lon == 0.0 || setindex!(BUF.sac_fv, Float32(S.hdr.loc.lon), 41)
  S.hdr.loc.dep == 0.0 || setindex!(BUF.sac_fv, Float32(S.hdr.loc.dep), 42)
  S.hdr.mag.val == -5.0f0 || setindex!(BUF.sac_fv, Float32(S.hdr.mag.val), 44)
  BUF.sac_cv[9:length(evid)+8] .= evid
  t_evt = d2u(S.hdr.ot)

  # Ints, always
  BUF.sac_iv[7] = Int32(6)

  for i = 1:S.data.n
    BUF.sac_fv[8] = t_evt - S.data.t[i][1,2]*μs
    write_sac_channel(S.data, i, "", v)
  end
  return nothing
end


export iauTdbtt
"""
Time scale transformation:  Barycentric Dynamical Time, TDB, to
Terrestrial Time, TT.

This function is part of the International Astronomical Union's
SOFA (Standards of Fundamental Astronomy) software collection.

Status:  canonical.

Given:
   tdb1,tdb2  double    TDB as a 2-part Julian Date
   dtr        double    TDB-TT in seconds

Returned:
   tt1,tt2    double    TT as a 2-part Julian Date

Returned (function value):
               int       status:  0 = OK

Notes:

   1. tdb1+tdb2 is Julian Date, apportioned in any convenient way
   between the two arguments, for example where tdb1 is the Julian
   Day Number and tdb2 is the fraction of a day.  The returned
   tt1,tt2 follow suit.

   2. The argument dtr represents the quasi-periodic component of the
   GR transformation between TT and TCB.  It is dependent upon the
   adopted solar-system ephemeris, and can be obtained by numerical
   integration, by interrogating a precomputed time ephemeris or by
   evaluating a model such as that implemented in the SOFA function
   iauDtdb.   The quantity is dominated by an annual term of 1.7 ms
   amplitude.

   3. TDB is essentially the same as Teph, the time argument for the
   JPL solar system ephemerides.

References:

   McCarthy, D. D., Petit, G. (eds.), IERS Conventions (2003),
   IERS Technical Note No. 32, BKG (2004)

   IAU 2006 Resolution 3

This revision:  2013 June 18

SOFA release 2018-01-30

Copyright (C) 2018 IAU SOFA Board.  See notes at end.
"""

# int iauTdbtt(double tdb1, double tdb2, double dtr,
#              double *tt1, double *tt2 )

function iauTdbtt(tdb1::Real, tdb2::Real, dtr::Real)
   ref_tt1 = Ref{Float64}(0.0)
   ref_tt2 = Ref{Float64}(0.0)

   status = ccall((:iauTdbtt, libsofa_c), Cint,
                   (Cdouble, Cdouble, Cdouble, Ref{Cdouble}, Ref{Cdouble}),
                   convert(Float64, tdb1), convert(Float64, tdb2), convert(Float64, dtr),
                   ref_tt1, ref_tt2)

   return status, ref_tt1[], ref_tt2[]
end


# Create a "generated" function to add together two ModelStates. This results in
# a function for each type that is as fast as a custom + function.
@generated function Base.:+(x::T, y::T) where {T <: ModelState}
    expr = ( :(x.$f + y.$f) for f in fieldnames(T) ) # Make a tuple of expressions to add each field
    :( T($(expr...)) ) # Splat them into an expression for a constructor for a T type
end

# Create a generated function to multiple a ModelState by a scalar.
@generated function Base.:*(x::S, y::T) where {S <: Real, T <: ModelState}
    expr = ( :(x * y.$f) for f in fieldnames(T) ) # Make a tuple of expressions to add each field
    :( T($(expr...)) ) # Splat them into an expression for a constructor for a T type
end

# We use this to ignore nothing states when adding them together.
Base.:+(a::Nothing, b)          = b
Base.:+(a, b::Nothing)          = a
Base.:+(a::Nothing, b::Nothing) = a
Base.:*(a, b::Nothing)          = b

# There are built-in types for UT date-times, but they don't work the way that I
# want. This isn't mutable. Just make a new one when something needs to be
# different.
struct UtcDateTime
    Y::Int
    M::Int
    D::Int
    h::Int
    m::Int
    s::Float64
    # UtcDateTime(v::Vector{Number}) UtcDateTime(v[1], v[2], v[3], v[4], v[5], v[6]);
end

# Create a function to convert a vector to a UtcDateType.
Base.convert(::Type{UtcDateTime}, d::Vector{T}) where {T <: Real} = UtcDateTime(d...);


# file for function used for node/dof numbering

# this can be generalized to 3D trivially
function populateDofNumbers(mesh::PumiMesh)
# populate the dofnums_Nptr with values calculated from
# nodenums_Nptr

#  resetAllIts2(mesh.m_ptr)
  # mesh iterator increment, retreval functions
  # TODO: checks if using mutable arrays is causing dynamic dispatch
#  iterators_inc = [incrementVertIt, incrementEdgeIt, incrementFaceIt, incrementElIt]
#  iterators_get = [getVert, getEdge, getFace, getEl]
  num_entities = mesh.numEntitiesPerType
  num_nodes_entity = mesh.numNodesPerType

  for etype = 1:(mesh.dim+1) # loop over entity types
    if (num_nodes_entity[etype] != 0)  # if no nodes on this type of entity, skip
      for entity_ptr in apf.MeshIterator(mesh.m_ptr, etype-1)  # loop over all entities of this type
	for node = 1:num_nodes_entity[etype]
          nodenum = apf.getNumberJ(mesh.nodenums_Nptr, entity_ptr, node-1, 0)
	  if nodenum != 0
	    for i=1:mesh.numDofPerNode
	      dofnum_i = (nodenum -1)*mesh.numDofPerNode + i
  	      apf.numberJ(mesh.dofnums_Nptr, entity_ptr, node-1, i-1, dofnum_i)
	    end  # end loop over dofsPerNode
	  end   # end if nodenum != 0
	end  # end loop over node

#      iterators_inc[etype]()
      end  # end loop over entitiesa
    end  # end if
  end  # end loop over entity types

#  resetAllIts2(mesh.m_ptr)

#  apf.writeVtkFiles("dofs_numbered", mesh.m_ptr)
  return nothing
end

# this can be generalized trivially
function populateNodeStatus(mesh::PumiMesh)
# populate the nodestatus_Nptr with values
# currently we set all nodes to status 3 (free)

#  resetAllIts2(mesh.m_ptr)
  # mesh iterator increment, retreval functions
  # TODO: checks if using mutable arrays is causing dynamic dispatch
#  iterators_inc = [incrementVertIt, incrementEdgeIt, incrementFaceIt, incrementElIt]
#  iterators_get = [getVert, getEdge, getFace, getEl]
  num_entities = mesh.numEntitiesPerType
  num_nodes_entity = mesh.numNodesPerType

#  mesh.numNodesPerType = num_nodes_entity

  for etype = 1:(mesh.dim + 1) # loop over entity types
    if (num_nodes_entity[etype] != 0)  # if no nodes on this type of entity, skip
      for entity_ptr in apf.MeshIterator(mesh.m_ptr, etype-1)  # loop over all entities of this type
	for node = 1:num_nodes_entity[etype]
	  apf.numberJ(mesh.nodestatus_Nptr, entity_ptr, node-1, 0, 3)
	end  # end loop over node

#      iterators_inc[etype]()
      end  # end loop over entitiesa
    end  # end if
  end  # end loop over entity types

#  resetAllIts2(mesh.m_ptr)
  return nothing
end

# this can be generalized to 3D trivially
function getDofNumbers(mesh::PumiMeshDG)
# populate array of dof numbers, in same shape as solution array u (or q)

#println("in getDofNumbers")
#println("numNodesPerElement = ", mesh.numNodesPerElement)

  mesh.dofs = Array{Int}(mesh.numDofPerNode, mesh.numNodesPerElement, mesh.shared_element_offsets[end] - 1)

  for i=1:mesh.numEl
    #TODO: use the non-allocating version
    dofnums = getGlobalNodeNumbers(mesh, i)

    for j=1:mesh.numNodesPerElement
      for k=1:mesh.numDofPerNode  # loop over dofs on the node
        mesh.dofs[k, j, i] = dofnums[k,j]
      end
    end
  end
  # get dof number of non-local elements
  # post receives first
  send_reqs = mesh.send_reqs
  recv_reqs = mesh.recv_reqs
  for i=1:mesh.npeers
    # get the pointer of the start location
    start_el = mesh.shared_element_offsets[i]
    numel = mesh.shared_element_offsets[i+1] - start_el
    ndata = mesh.numDofPerNode*mesh.numNodesPerElement*numel
    lin_idx = sub2ind(size(mesh.dofs), 1, 1, start_el)
    ptr_start_el = pointer(mesh.dofs, lin_idx)
    peer_i = mesh.peer_parts[i]
    recv_reqs[i] = MPI.Irecv!(ptr_start_el, ndata, peer_i, 1, mesh.comm)
  end

  # now send data
  dof_sendbuf = Array{Array{Int, 3}}(mesh.npeers)
  for i=1:mesh.npeers
    elnums_i = mesh.local_element_lists[i]
    numel = length(elnums_i)
    dof_sendbuf[i] = Array{Int}(mesh.numDofPerNode, mesh.numNodesPerElement, numel)
    sendbuf_i = dof_sendbuf[i]
    for j=1:length(elnums_i)
      elnum_j = elnums_i[j]
      for k=1:mesh.numNodesPerElement
        for p=1:mesh.numDofPerNode
          sendbuf_i[p, k, j] = mesh.dofs[p, k, elnum_j]
        end
      end
    end  # end loop over current list of elements

    # now do the send
    send_reqs[i] = MPI.Isend(sendbuf_i, mesh.peer_parts[i], 1, mesh.comm)
  end

  # figure out the local to global offset of the dof numbers
  # compute number of local dofs
  ndof = 0
  for i=1:(mesh.dim + 1)
    ndof += mesh.numNodesPerType[i]*mesh.numEntitiesPerType[i]*mesh.numDofPerNode
  end
  println(mesh.f, "ndof = ", ndof)
  # get all processes dof offsets
  dof_offsets = MPI.Allgather(ndof, mesh.comm)
  println(mesh.f, "dof_offsets = ", dof_offsets)
  # figure out own dof_offset
  dof_offset = 0
  for i=1:mesh.myrank  # sum all dofs up to but *not* including self
    dof_offset += dof_offsets[i]
  end
  mesh.dof_offset = dof_offset

  # figure out peer dof_offsets
  peer_dof_offsets = Array{Int}(mesh.npeers)
  for i=1:mesh.npeers
    dof_offset_i = 0
    for j=1:mesh.peer_parts[i]
      dof_offset_i += dof_offsets[j]
    end
    peer_dof_offsets[i] = dof_offset_i
  end

  println(mesh.f, "peer_parts = ", mesh.peer_parts)
  println(mesh.f, "peer_dof_offsets = ", peer_dof_offsets)
  # adjust other elements dof such that dof + dof_offset of this process = global
  # dof number
  # store local dof number + (offset of owner - offset of host) in dofs array
  dofs = mesh.dofs
  for i=1:mesh.npeers
    idx, stat =  MPI.Waitany!(recv_reqs)
    # do subtraction first to avoid overflow
    offset = peer_dof_offsets[idx] - dof_offset
    start_el = mesh.shared_element_offsets[idx]
    end_el = mesh.shared_element_offsets[idx+1] - 1
    for j=start_el:end_el
      for k=1:mesh.numNodesPerElement
        for p=1:mesh.numDofPerNode
          dofs[p, k, j] += offset
        end
      end
    end

  end  # end loop over peers

  # wait for the sends to finish before returning
  for i=1:mesh.npeers
    MPI.Wait!(send_reqs[i])
  end

  #println("mesh.dof = ", mesh.dofs)

  return nothing

end

"""
  Node numbering algorithm that loops over elements and numbers their downward
  adjacencies, generating a decent sparsity pattern.  This algorithm is
  primarily intended for CG meshes, although I think it works for DG meshes too.
  This probably only works for 2D CG meshes.

  **Inputs**

   * mesh: a PumiMesh
   * number_dofs: Bool, if true, number the dofs (mesh.dofnums_Npts), otherwise
                  number the nodes (mesh.nodenums_Nptr)
"""
function numberNodes(mesh::PumiMesh, number_dofs=false)
# assign node numbers to entire mesh, or dof numbers if number_dofs=true,
# using the correct Numbering pointer
# assumes mesh elements have already been reordered
# this works for high order elements

  @assert mesh.dim == 2

  # initally number all dofs as numDof+1 to 2*numDof
  # this allows quick check to see if somthing is labelled or not
  numberNodesElement(mesh, number_dofs=number_dofs, start_at_one=false)

  # calculate number of nodes, dofs
  numnodes = mesh.numNodes

  num_entities = mesh.numEntitiesPerType
  num_nodes_entity = mesh.numNodesPerType

#  mesh.numNodesPerType = num_nodes_entity

  if number_dofs
    numbering_ptr = mesh.dofnums_Nptr
    curr_dof = mesh.numDof + 1
    dofpernode = mesh.numDofPerNode
    numDof = mesh.numDofPerNode*numnodes
  else  # do node numbering
    numbering_ptr = mesh.nodenums_Nptr
    curr_dof = mesh.numNodes + 1
    dofpernode = 1
    numDof = numnodes
  end

  verts_i = Array{Ptr{Void}}(12)
  edges_i = Array{Ptr{Void}}(12)
#  resetAllIts2(mesh.m_ptr)
  el_i_ptr = Ptr{Void}(0)  # hold current element
# TODO: move all if statements out one for loop (check only first dof on each node)
  curr_dof = 1
  for el_i_ptr in apf.MeshIterator(mesh.m_ptr, mesh.dim)
    # get vertices, edges for this element
    numVert = apf.getDownward(mesh.m_ptr, el_i_ptr, 0, verts_i)
#    println("verts_i = ", verts_i)
#    println("mesh.verts = ", mesh.verts)
    numEdge = apf.getDownward(mesh.m_ptr, el_i_ptr, 1, edges_i)
    for j=1:3  # loop over vertices, edges  #TODO: why is this 1:3?
      vert_j = verts_i[j]
      edge_j = edges_i[j]
      for k=1:num_nodes_entity[1] # loop over vertex nodes
        for p=1:dofpernode
          dofnum_k = apf.getNumberJ(numbering_ptr, vert_j, k-1, p-1)
          if dofnum_k > numDof  # still has initial number
            # give it new (final) number
            apf.numberJ(numbering_ptr, vert_j, 0, k-1, curr_dof)
            curr_dof += 1
          end
        end
      end  # end loop over vertex dofs

      # loop over nodes on edge
      for k=1:num_nodes_entity[2]  # loop over nodes
	for p=1:dofpernode  # loop over dofs
	  dofnum_p = apf.getNumberJ(numbering_ptr, edge_j, k-1, p-1)
	  if dofnum_p > numDof  # still has initial number
	    # give it new (final) number
	    apf.numberJ(numbering_ptr, edge_j, k-1, p-1, curr_dof)
	    curr_dof += 1
	  end
	end
      end
    end  # end loop over vertices, edges
    # label face nodes
    for k=1:num_nodes_entity[3]  # loop over nodes on face
        # visit the nodes in the SBP ordering, so consider k to be the SBP
        # node number
        # this only works when there are no nodes on edges or vertices, because
        # we would have to do additional translation to take them into account

      for p=1:dofpernode  # loop over dofs
	dofnum_p = apf.getNumberJ(numbering_ptr, el_i_ptr, k-1, p-1)
	if dofnum_p > numDof
	  apf.numberJ(numbering_ptr, el_i_ptr, k-1, p-1, curr_dof)
	  curr_dof += 1
	end
      end
    end  # end loop over face nodes
  end  # end loop over elements

  if (curr_dof -1) != numDof
    error("Warning: number of dofs assigned is not equal to the expected number" *
          "number of dofs assigned = $(curr_dof-1) ,expected number =  $numDof")
  end

  return nothing

end

"""
  This function numbers the mesh entities using a simple iteration over all
  entities of each dimension.  This generates a bad sparsity pattern for CG
  meshes.  For DG meshes the sparsity pattern is ok if Pumi reordered the
  elements in its database, otherwise the sparsity pattern is bad.

  **Inputs**

   * mesh: a PumiMesh

  **Keyword Arguments**

   * number_dofs: if true, number the dofs (mesh.dofnums_Nptr), otherwise
                  number the nodes (mesh.nodenums_Nptr), default false
   * start_at_one: if true, start numbering dofs at 1, otherwise start at
                   number of dofs/nodes + 1.  This is useful as an initial
                   numbering for other algorithms, because any dof/node
                   with a number greater than the number of dofs/nodes has not
                   been relabeled yet.
"""
function numberNodesElement(mesh::PumiMesh; number_dofs=false, start_at_one=true)
# number all nodes (or dofs) by iterating over all entities, from lowest
# dimension to highest, and numbering the nodes in order

  # calculate number of nodes, dofs
  numnodes = mesh.numNodes

  num_entities = mesh.numEntitiesPerType
  num_nodes_entity = mesh.numNodesPerType

  if number_dofs
    numbering_ptr = mesh.dofnums_Nptr
    curr_dof = mesh.numDof + 1
    dofpernode = mesh.numDofPerNode
    numDof = mesh.numDofPerNode*numnodes
  else  # do node numbering
    numbering_ptr = mesh.nodenums_Nptr
    curr_dof = mesh.numNodes + 1
    dofpernode = 1
    numDof = numnodes
  end

  if (numDof > 2^30 || numDof < 0)
    error("Warning: too many dofs, renumbering will fail")
  end

#  println("num_entities = ", num_entities)
#  println("num_nodes_entity = ", num_nodes_entity)

  if start_at_one
    curr_dof = 1
  else
    curr_dof = numDof+1
  end

  if mesh.isDG
    nodemap =  mesh.nodemapSbpToPumi
  else  # fake nodemap: always 1:1, this works because we apf.iterate over the
        # entities of each dimension, so every entity is visited exactly once
        # Such an algorith generates a horrible sparsity pattern, but this
        # is the default algorithm, so whatever
    nodemap = collect(eltype(mesh.nodemapSbpToPumi), 1:mesh.numNodesPerElement)
  end

  for etype = 1:(mesh.dim + 1) # loop over entity types
    if (num_nodes_entity[etype] != 0)  # if no nodes on this type of entity, skip
      for entity_ptr in apf.MeshIterator(mesh.m_ptr, etype - 1)  # loop over all entities of this type
	for node = 1:num_nodes_entity[etype]
          pumi_node = nodemap[node]  # this appears to only work for elements
                                     # with nodes on the interior, otherwise
                                     # there would need to be different node
                                     # maps for entities of different dimension
	  for dof = 1:dofpernode
	    apf.numberJ(numbering_ptr, entity_ptr, pumi_node-1, dof-1, curr_dof)
	    curr_dof += 1
	  end  # end loop over dof
	end  # end loop over node
      end  # end loop over entitiesa
    end  # end if
  end  # end loop over entity types

  return nothing
end

function getDofNumbers(mesh::PumiMesh2)
# populate array of dof numbers, in same shape as solution array u (or q)

mesh.dofs = Array{Int32}(mesh.numDofPerNode, mesh.numNodesPerElement, mesh.numEl)

for i=1:mesh.numEl
  dofnums = getGlobalNodeNumbers(mesh, i)

  for j=1:mesh.numNodesPerElement
    for k=1:mesh.numDofPerNode  # loop over dofs on the node
      mesh.dofs[k, j, i] = dofnums[k,j]
    end
  end
end

#println("mesh.dof = ", mesh.dofs)

return nothing

end

"""
  This function numbers the dof/nodes of the mesh using and adjacency-type
  algorithm, starting from the mesh element nearest the specified coordinates.
  The dofs/nodes are numbered in an increasing order.  This only works for
  DG meshes.  This function overwrites the mesh.el_Nptr numbering with
  the new element numbering, in addition to numbering the dofs/nodes.

  Note that this function updates the mesh.verts, mesh.edges, mesh.faces,
  mesh.elements arrays so they match the numbering (ie. mesh.element[1] has
  the number 0 in mesh.el_Nptr, because the numbering is zero based).

  The numbering of dofs/nodes is 1-based, and the numbering of elements is
  zero-based.

  **Inputs**

   * mesh: a DG mesh
   * start_coords: vector of length mesh.dim specifying starting location
                   of the numbering.  The element with the closests linear
                   centroid (based on vertex locations only) is selected as
                   the starting element.
  * number_dofs: if true, numbers the dofs (mesh.dofnums_Nptr), otherwise
                 numbers the nodes (mesh.nodenums_Nptr).
"""
function numberNodesWindy(mesh::PumiMeshDG, start_coords, number_dofs=false)
# number nodes and elements starting from a particular location

  # calculate number of nodes, dofs
  numDof = mesh.numDof
  if (numDof > 2^30 || numDof < 0)
    println("Warning: too many dofs, renumbering will fail")
  end

  if number_dofs
    numbering_ptr = mesh.dofnums_Nptr
    dofpernode = mesh.numDofPerNode
    numDof = mesh.numDofPerNode*numnodes
  else  # do node numbering
    numbering_ptr = mesh.nodenums_Nptr
    dofpernode = 1
    numDof = mesh.numNodes
  end
  dim = mesh.dim
  el_Nptr = mesh.el_Nptr
  adj_els = Array{Ptr{Void}}(mesh.numFacesPerElement)
  nodemap = mesh.nodemapSbpToPumi
  numDofPerElement = dofpernode*mesh.numNodesPerElement

  # initially number all components in range (numEl+1):(2*numEl)
  curr_elnum = mesh.numEl+1
  for el_i in apf.MeshIterator(mesh.m_ptr, dim)
    apf.numberJ(el_Nptr, el_i, 0, 0, curr_elnum)
    curr_elnum += 1
#    incrementIt(dim)
  end
  @assert (curr_elnum - 1) == 2*mesh.numEl

  # do the final numbering
  start_el = getStartEl(mesh, start_coords)
  que = FIFOQueue{Ptr{Void}}(size_hint = div(mesh.numEl, 2))
  push!(que, start_el)

  curr_elnum = mesh.numEl - 1
  curr_dof = numDof
  # the dreaded while loop
  while (!isempty(que))
#    print("\n")
    curr_el = pop!(que)

    # only unlabelled entities are added to the que, and they are added
    # exactly once, so no need to check here
    apf.numberJ(el_Nptr, curr_el, 0, 0, curr_elnum)
    curr_elnum -= 1

    # now label dofs
    # We have to label the dofs on each element in ascending order, so that
    # eqn.res = eqn.res_vec
    _curr_dof = curr_dof - numDofPerElement + 1
    for i=1:mesh.numNodesPerElement
      pumi_node = nodemap[i]
      for j=1:dofpernode
        apf.numberJ(numbering_ptr, curr_el, pumi_node-1, j-1, _curr_dof)
        _curr_dof += 1
      end
    end
    curr_dof -= numDofPerElement

    # add face adjacent neighbor elements to the que
    numadj = apf.countBridgeAdjacent(mesh.m_ptr, curr_el, dim-1, dim)
    apf.getBridgeAdjacent(adj_els)

    for i=1:numadj
      el_i = adj_els[i]
      elnum = apf.getNumberJ(el_Nptr, el_i, 0, 0)

       # if not numbered and not in que
       # when adding to que, scale elnum so it is in range (2*numEl+1):3*numEl
       # as long as there are at least 3 nodes per element, node numbers will
       # overflow before this
      if (elnum > mesh.numEl) && (elnum <= 2*mesh.numEl)
        apf.numberJ(el_Nptr, el_i, 0, 0, elnum + mesh.numEl)
        push!(que, el_i)
      end
    end  # end loop over adjacent elements

    # this can only happen if either 1) this algorithm is broken, or
    # 2) there are disjoint sets of elements on this process
    # Assume it is 2 and find a new starting element
    if curr_elnum != -1 && isempty(que)
      new_startel = getStartEl(mesh, start_coords, el_Nptr, mesh.numEl - 1)
      @assert new_startel != C_NULL
      push!(que, new_startel)
    end

  end  # end while loop

  #-----------------------------------------
  # Error checking and cleanup

  if curr_dof != 0
    println("Warning: number of dofs assigned is not equal to the expected number")
    println("number of dofs remaining = ", curr_dof)
    throw(ErrorException("dof numbering failed"))
  end

  if curr_elnum != -1
    throw(ErrorException("element numbering failed"))
  end
#=
  # the element number should have been zero based (oops), so decrement
  # the element numbers
  for i=1:mesh.numEl
    el_i = mesh.elements[i]
    idx = apf.getNumberJ(mesh.el_Nptr, el_i, 0, 0)
    apf.numberJ(mesh.el_Nptr, el_i, 0, 0, idx - 1)
  end
=#
  # need to update the element pointer array because element numbering has
  # changed
  #TODO: don't reallocate the arrays
  mesh.verts, mesh.edges, mesh.faces, mesh.elements = getEntityPointers(mesh)
  return nothing
end

"""
  Finds the element with the (linear) centroid closest to the specified
  coordinates.  Can optionally skip certain elements that are already numbered.

  **Inputs**

   * mesh
   * start_coords: vector of length `mesh.dim` containing the point that
                   the element will be closest to
   * el_Nptr: an apf::Numbering* that numbers the mesh elements (optional)
   * sentinel: the sentinal value for element numbers.  Element numbers less
               than or equal to this value will be skipped.  This argument
               is required if `el_Nptr` is provided. This value should be
               zero-based

"""
function getStartEl(mesh::PumiMeshDG, start_coords, el_Nptr::Ptr{Void}=C_NULL, sentinel::Integer=-1)
  # find the element with centroid closest to start_coords

  numVertsPerElement = mesh.numTypePerElement[1]
  coords = zeros(3, numVertsPerElement)
  centroid = zeros(3)
  dim = mesh.dim  # hoist the lookup

#  resetIt(dim)

  min_norm = typemin(Float64)
  min_el = C_NULL

  for el_i  in apf.MeshIterator(mesh.m_ptr, dim)

    # skip already numbered elements
    if el_Nptr != C_NULL
      elnum = apf.getNumberJ(el_Nptr, el_i, 0, 0)
      if elnum <= sentinel
        continue
      end
    end

    getElementCoords(mesh, el_i, coords)

    # compute centroid
    fill!(centroid, 0.0)
    for j=1:dim
      for k=1:numVertsPerElement
        centroid[j] += coords[j, k]
      end
      centroid[j] = centroid[j]/numVertsPerElement
    end

    # compute norm of distance from start_coords
    nrm = 0.0
    for j=1:dim
      nrm += (centroid[j] - start_coords[j])*(centroid[j] - start_coords[j])
    end
    # skip the square root because it doesn't affect the comparison

    if nrm > min_norm
      min_norm = nrm
      min_el = el_i
    end

  end

  return min_el
end

"""
  Verifies the element numbering, specifically that every element number
  is assigned exactly once, and all numbers are in the range of 0 to
  mesh.numEl -1
"""
function checkElementNumbering(mesh, el_Nptr)

  # check all numbers are assigned, none more than once
  elnums = zeros(Int, mesh.numEl)

  for el in apf.MeshIterator(mesh.m_ptr, mesh.dim)
    elnum = apf.getNumberJ(el_Nptr, el, 0, 0) + 1
    @assert elnum >= 1
    @assert elnum <= mesh.numEl
    elnums[elnum] += 1
  end

  for i=1:mesh.numEl
    @assert elnums[i] == 1
  end

  return nothing
end

"""
  Verifies the dof numbering, specifically that every dof number is assigned
  exactly once and all numbers are in the range 1 to mesh.numDof
"""
function checkDofNumbering(mesh, dof_Nptr)

  dofnums = zeros(UInt8, mesh.numDof)

  for el in apf.MeshIterator(mesh.m_ptr, mesh.dim)
    for j=1:mesh.numNodesPerElement
      for k=1:mesh.numDofPerNode
        dofnum = apf.getNumberJ(dof_Nptr, el, j-1, k-1)
        @assert dofnum > 0
        @assert dofnum <= mesh.numDof
        dofnums[dofnum] += 1
      end
    end
  end

  for i=1:mesh.numDof
    @assert dofnums[i] == 1
  end

  return nothing
end


"""

$(TYPEDEF)

Simple structure to contain the number of samples of each type
of calculation to compute final results

$(TYPEDFIELDS)

"""
@with_kw struct Samples
    md::Float64
    random::Int
end


using CSV
using DataFrames

function loadData()
    startDate = "1/22/20"
    endDate = "3/30/20"
    roi = [(0,"Italy"), (0,"Spain"), ("Hubei","China"), (0,"Germany"), (0,"France"), (0,"Iran"), (0,"United Kingdom"), (0,"Switzerland"), (0,"Turkey"), (0,"Belgium"), (0,"Netherlands"), (0,"Austria"), (0,"Korea, South")] #(Province/State, Country/Region); Province/State=0 if data empty

    for i = 1:length(roi)
        buildDataFrame(roi[i],startDate,endDate)
    end

end

function buildDataFrame(roi,startDate,endDate)
    csvfile_confirmed = CSV.File("time_series_covid19_confirmed_global.csv")
    csvfile_deaths = CSV.File("time_series_covid19_deaths_global.csv")
    csvfile_recovered = CSV.File("time_series_covid19_recovered_global.csv")
    dfconfirmed = DataFrame(csvfile_confirmed)
    dfdeaths = DataFrame(csvfile_deaths)
    dfrecovered = DataFrame(csvfile_recovered)

    # region of interest
    if roi[1] == 0
        rowind_confirmed = .!(typeof.(dfconfirmed[:,Symbol("Province/State")]) .== String) .& (dfconfirmed[:,Symbol("Country/Region")] .== roi[2])
        rowind_deaths = .!(typeof.(dfdeaths[:,Symbol("Province/State")]) .== String) .& (dfdeaths[:,Symbol("Country/Region")] .== roi[2])
        rowind_recovered = .!(typeof.(dfrecovered[:,Symbol("Province/State")]) .== String) .& (dfrecovered[:,Symbol("Country/Region")] .== roi[2])
    else
        rowind_confirmed = (dfconfirmed[:,Symbol("Province/State")] .== roi[1]) .& (dfconfirmed[:,Symbol("Country/Region")] .== roi[2])
        rowind_deaths = (dfdeaths[:,Symbol("Province/State")] .== roi[1]) .& (dfdeaths[:,Symbol("Country/Region")] .== roi[2])
        rowind_recovered = (dfrecovered[:,Symbol("Province/State")] .== roi[1]) .& (dfrecovered[:,Symbol("Country/Region")] .== roi[2])
    end
    dfconfirmed_roi = dfconfirmed[rowind_confirmed,:];
    dfdeaths_roi = dfdeaths[rowind_deaths,:];
    dfrecovered_roi = dfrecovered[rowind_recovered,:];

    # time period of interest
    colind = collect(1:size(dfconfirmed_roi,2))
    startInd = colind[names(dfconfirmed_roi) .== Symbol(startDate)][1]
    endInd = colind[names(dfconfirmed_roi) .== Symbol(endDate)][1]

    # tailored to roi and toi
    nameInd = names(dfconfirmed_roi)[startInd:endInd]
    dfconfirmed_roi_toi = dfconfirmed_roi[startInd:endInd]
    dfdeaths_roi_toi = dfdeaths_roi[startInd:endInd]
    dfrecovered_roi_toi = dfrecovered_roi[startInd:endInd]

    # combine confirm, death, recover, active
    dflength = size(dfconfirmed_roi_toi,2)
    dfroi = DataFrame()
    dfroi.date = nameInd
    dfroi.cum_case = stack(dfconfirmed_roi_toi,1:dflength)[:,:value]
    dfroi.cum_death = stack(dfdeaths_roi_toi,1:dflength)[:,:value]
    dfroi.cum_recover = stack(dfrecovered_roi_toi,1:dflength)[:,:value]
    dfroi.active = dfroi.cum_case - dfroi.cum_death - dfroi.cum_recover

    CSV.write("covid_timeseries_" * roi[2] * ".csv",dfroi)

end


#MDDatasets DataHR defninitions
#-------------------------------------------------------------------------------

#==Main types
===============================================================================#

#Hyper-rectangle -representation of data:
#-------------------------------------------------------------------------------
mutable struct DataHR{T} <: DataMD
	sweeps::Vector{PSweep}
	elem::Array{T}

	function DataHR{T}(sweeps::Vector{PSweep}, elem::Array{T}) where {T}
		if !elemallowed(DataMD, eltype(elem))
			msg = "Can only create DataHR{T} for T ∈ {DataF1, DataFloat, DataInt, DataComplex}"
			throw(ArgumentError(msg))
		elseif size(DataHR, sweeps) != size(elem)
			throw(ArgumentError("Number of sweeps must match dimensionality of elem"))
		end
		return new(sweeps, elem)
	end
end

#Shorthand (because default (non-parameterized) constructor was overwritten):
DataHR(sweeps::Vector{PSweep}, a::Array{T}) where T = DataHR{T}(sweeps, a)

#Construct DataHR from Vector{PSweep}:
(::Type{DataHR{T}})(sweeps::Vector{PSweep}) where T = DataHR{T}(sweeps, Array{T}(undef, size(DataHR, sweeps)...))

#Construct DataHR{DataF1} from DataHR{Number}
#Collapse inner-most sweep (last dimension), by default:
#TODO: use convert(...) instead?
function (::Type{DataHR{DataF1}})(d::DataHR{T}) where T<:Number
	sweeps = d.sweeps[1:end-1]
	x = d.sweeps[end].v

	#Collapsed last sweep:
	if length(sweeps) < 1
		y = d.elem
		return DataF1(x, reshape(y, length(y)))
	end

	#Reduced DataHR structure:
	result = DataHR{DataF1}(sweeps) #Construct empty results
	_sub = collect(subscripts(result))
	for inds in _sub
		y = d.elem[inds...,:]
		result.elem[inds...] = DataF1(x, reshape(y, length(y)))
	end
	return result
end

#Relay function, so people can blindly convert to DataHR{DataF1} using any DataHR:
(::Type{DataHR{DataF1}})(d::DataHR{DataF1}) = d

#==Type promotions
===============================================================================#
Base.promote_rule(::Type{T1}, ::Type{T2}) where {T1<:DataHR, T2<:Number} = DataHR

#==Accessor functions
===============================================================================#
#Compute the size of a DataHR array from a Vector{PSweep}:
function Base.size(::Type{DataHR}, sweeps::Vector{PSweep})
	dims = Int[]
	for s in sweeps
		push!(dims, length(s.v))
	end
	if 0 == length(dims) #Without sweeps, you can still have a single subset
		push!(dims, 1)
	end
	return tuple(dims...)
end

#Returns the dimension corresponding to the given parameter name:
function dimension(::Type{DataHR}, sweeps::Vector{PSweep}, id::String)
	dim = findfirst((s)->(id==s.id), sweeps)
	ensure(dim!=nothing, ArgumentError("Sweep not found: $id."))
	return dim
end
dimension(d::DataHR, id::String) = dimension(DataHR, d.sweeps, id)

#Returns an element subscripts iterator for a DataHR corresponding to Vector{PSweep}.
subscripts(::Type{DataHR}, sweeps::Vector{PSweep}) =
	subscripts(size(DataHR, sweeps))
subscripts(d::DataHR) = subscripts(d.elem)

#Dimensionality of DataHR array:
Base.ndims(::Type{DataHR}, sweeps::Vector{PSweep}) = max(1, length(sweeps))
Base.ndims(d::DataHR) = ndims(DataHR, d.sweeps)

#Obtain sweep info
#-------------------------------------------------------------------------------
sweeps(d::DataHR) = d.sweeps
sweep(d::DataHR, dim::Int) = d.sweeps[dim].v
sweep(d::DataHR, dim::String) = d.sweeps[dim].v

#Returns parameter sweep coordinates corresponding to given subscript:
function coordinates(d::DataHR, subscr::Tuple=0)
	result = []
	if length(d.sweeps) > 0
		for i in 1:length(subscr)
			push!(result, sweep(d, i)[subscr[i]])
		end
	end
	return result
end

"""
    paramlist(d::Vector{PSweep})

Return a list of parameter in a list of PSweep.
"""
paramlist(sl::Vector{PSweep}) = String[s.id for s in sl]

"""
    paramlist(d::DataHR)

Return a list of parameter values being swept.
"""
paramlist(d::DataHR) = paramlist(d.sweeps)

#==Help with construction
===============================================================================#

#Implement "fill(DataHR, ...) do sweepval" syntax:
function Base.fill!(fn::Function, d::DataHR)
	for inds in subscripts(d)
		d.elem[inds...] = fn(coordinates(d, inds)...)
	end
	return d
end
Base.fill(fn::Function, ::Type{DataHR{T}}, swlist::Vector{PSweep}) where T =
	fill!(fn, DataHR{T}(swlist))
Base.fill(fn::Function, ::Type{DataHR{T}}, sweep::PSweep) where T = fill(fn, DataHR{T}, PSweep[sweep])
#Default: DataHR -> creates DataHR{DataF1}??? Is this a bad idea?
Base.fill(fn::Function, ::Type{DataHR}, swlist::Vector{PSweep}) = fill(fn, DataHR{DataF1}, swlist)

#Fill with a specified value:
Base.fill(elem_0::Number, ::Type{DataHR}, swlist::Vector{PSweep}) =
	DataHR(swlist, fill(elem_0, size(DataHR, swlist)))
Base.fill(elem_0::DataF1, ::Type{DataHR}, swlist::Vector{PSweep}) =
	DataHR(swlist, fill!(Array{DataF1}(undef, size(DataHR, swlist)), elem_0)) #Must be array of abstract DatF1[]
Base.fill(elem_0::DF1_Num, ::Type{DataHR}, sweep::PSweep) = fill(elem_0, DataHR, PSweep[sweep])

#zeros & ones:
Base.zeros(::Type{DataHR}, swlist::Vector{PSweep}) = fill(0.0, DataHR, swlist)
Base.zeros(::Type{DataHR}, sweep::PSweep) = zeros(DataHR, PSweep[sweep])
Base.ones(::Type{DataHR}, swlist::Vector{PSweep}) = fill(1.0, DataHR, swlist)
Base.ones(::Type{DataHR}, sweep::PSweep) = ones(DataHR, PSweep[sweep])

#==Data generation
===============================================================================#
#Generate a DataHR object containing the value of a given swept parameter:
function parameter(::Type{DataHR}, sweeps::Vector{PSweep}, sweepno::Int)
	sw = sweeps[sweepno].v #Sweep of interest
	T = eltype(sw)
	result = DataHR{T}(sweeps)
	for inds in subscripts(result)
		result.elem[inds...] = sw[inds[sweepno]]
	end
	return result
end
parameter(::Type{DataHR}, sweeps::Vector{PSweep}, id::String) =
	parameter(DataHR, sweeps, dimension(DataHR, sweeps, id))
parameter(d::DataHR, id::String) = parameter(DataHR, d.sweeps, id)

#==Dataset reductions
===============================================================================#

#Like getindex(A, inds...), but with DataHR:
#TODO: Support array "view"s instead of copying with "getindex" for efficiency??
function getsubarrayind(d::DataHR{T}, inds...) where T
	sweeps = PSweep[]
	idx = 1
	for rng in inds
		sw = d.sweeps[idx]

		#Only provide a sweep if user selects a range of more than one element:
		addsweep = Colon == typeof(rng) || length(rng)>1
		if addsweep
			push!(sweeps, PSweep(sw.id, sw.v[rng]))
		end
		idx +=1
	end
	return DataHR{T}(sweeps, reshape(getindex(d.elem, inds...), size(DataHR, sweeps)))
end

#getindex(DataHR, inds...), using key/value pairs:
function getsubarraykw(d::DataHR{T}; kwargs...) where T
	sweeps = PSweep[]
	indlist = Vector{Int}[]
	for sweep in d.sweeps
		keepsweep = true
		arg = getkwarg(kwargs, Symbol(sweep.id))
		if arg != nothing
			inds = indices(sweep, arg)
			push!(indlist, inds)
			if length(inds) > 1
				keepsweep = false
				push!(sweeps, PSweep(sweep.id, sweep.v[inds...]))
			end
		else #Keep sweep untouched:
			push!(indlist, 1:length(sweep.v))
			push!(sweeps, sweep)
		end
	end
	return DataHR{T}(sweeps, reshape(getindex(d.elem, indlist...), size(DataHR, sweeps)))
end

#TODO: Support array "view"s instead of copying with "getindex" for efficiency??
#NOTE: Used to be called "sub", which was replaced by "view".
function getsubarray(d::DataHR{T}, args...; kwargs...) where T
	if length(kwargs) > 0
		return getsubarraykw(d, args...; kwargs...)
	else
		return getsubarrayind(d, args...)
	end
end

#==User-friendly show functions
===============================================================================#
#Also changes string():
Base.print(io::IO, ::Type{DataHR{DataF1}}) = print(io, "DataHR{DataF1}")
Base.print(io::IO, ::Type{DataHR{DataFloat}}) = print(io, "DataHR{DataFloat}")
Base.print(io::IO, ::Type{DataHR{DataInt}}) = print(io, "DataHR{DataInt}")
Base.print(io::IO, ::Type{DataHR{DataComplex}}) = print(io, "DataHR{DataComplex}")

function Base.show(io::IO, ds::DataHR)
	szstr = string(size(ds.elem))
	typestr = string(typeof(ds))
	print(io, "$typestr$szstr[\n")
	for inds in subscripts(ds)
		if isassigned(ds.elem, inds...)
			subset = ds.elem[inds...]
			print(io, " $inds: "); show(io, subset); println(io)
		else
			println(io, " $inds: UNDEFINED")
		end
	end
	print(io, "]\n")
end

function Base.show(io::IO, ds::DataHR{T}) where T<:Number
	szstr = string(size(ds.elem))
	typestr = string(typeof(ds))
	print(io, "$typestr$szstr:\n")
	print(io, ds.elem)
end

#Last line


# This file contains functions that are related to the AMR capabilities of the DG solver

# Refine elements in the DG solver based on a list of cell_ids that should be refined
function refine!(dg::Dg1D{Eqn, MeshType, NVARS, POLYDEG}, mesh::TreeMesh,
                 cells_to_refine::AbstractArray{Int}) where {Eqn, MeshType, NVARS, POLYDEG}
  # Return early if there is nothing to do
  if isempty(cells_to_refine)
    return
  end

  # Determine for each existing element whether it needs to be refined
  needs_refinement = falses(nelements(dg.elements))
  tree = mesh.tree
  # The "Ref(...)" is such that we can vectorize the search but not the array that is searched
  elements_to_refine = searchsortedfirst.(Ref(dg.elements.cell_ids[1:nelements(dg.elements)]),
                                          cells_to_refine)
  needs_refinement[elements_to_refine] .= true

  # Retain current solution data
  old_n_elements = nelements(dg.elements)
  old_u = dg.elements.u

  # Get new list of leaf cells
  leaf_cell_ids = leaf_cells(tree)

  # Initialize new elements container
  elements = init_elements(leaf_cell_ids, mesh, Val(NVARS), Val(POLYDEG))
  n_elements = nelements(elements)

  # Loop over all elements in old container and either copy them or refine them
  element_id = 1
  for old_element_id in 1:old_n_elements
    if needs_refinement[old_element_id]
      # Refine element and store solution directly in new data structure
      refine_element!(elements.u, element_id, old_u, old_element_id,
                      dg.amr_refine_right, dg.amr_refine_left, dg)
      element_id += 2^ndims(dg)
    else
      # Copy old element data to new element container
      @views elements.u[:, :, element_id] .= old_u[:, :, old_element_id]
      element_id += 1
    end
  end

  # Initialize new interfaces container
  interfaces = init_interfaces(leaf_cell_ids, mesh, Val(NVARS), Val(POLYDEG), elements)
  n_interfaces = ninterfaces(interfaces)

  # Initialize boundaries
  boundaries, n_boundaries_per_direction = init_boundaries(leaf_cell_ids, mesh, Val(NVARS), Val(POLYDEG), elements)
  n_boundaries = nboundaries(boundaries)

  # Sanity check
  if isperiodic(mesh.tree)
    @assert n_interfaces == 1*n_elements ("For 1D and periodic domains, n_surf must be the same as 1*n_elem")
  end

  # Update DG instance with new data
  dg.elements = elements
  dg.n_elements = n_elements
  dg.n_elements_global = n_elements
  dg.interfaces = interfaces
  dg.n_interfaces = n_interfaces
  dg.boundaries = boundaries
  dg.n_boundaries = n_boundaries
  dg.n_boundaries_per_direction = n_boundaries_per_direction
end

# Refine solution data u for an element, using L2 projection (interpolation)
function refine_element!(u, element_id, old_u, old_element_id,
                        refine_right, refine_left, dg::Dg1D)
  # Store new element ids
  left_id  = element_id
  right_id = element_id + 1

  # Interpolate to left element
  for i in 1:nnodes(dg)
    acc = zero(get_node_vars(u, dg, i, element_id))
    for k in 1:nnodes(dg)
      acc += get_node_vars(old_u, dg, k, old_element_id) * refine_left[i, k]
    end
    set_node_vars!(u, acc, dg, i, left_id)
  end

  # Interpolate to right element
  for i in 1:nnodes(dg)
    acc = zero(get_node_vars(u, dg, i, element_id))
    for k in 1:nnodes(dg)
      acc += get_node_vars(old_u, dg, k, old_element_id)  * refine_right[i, k]
    end
    set_node_vars!(u, acc, dg, i, right_id)
  end

end

# Coarsen elements in the DG solver based on a list of cell_ids that should be removed
function coarsen!(dg::Dg1D{Eqn, MeshType, NVARS, POLYDEG}, mesh::TreeMesh,
                  child_cells_to_coarsen::AbstractArray{Int}) where {Eqn, MeshType, NVARS, POLYDEG}
  # Return early if there is nothing to do
  if isempty(child_cells_to_coarsen)
    return
  end

  # Determine for each old element whether it needs to be removed
  to_be_removed = falses(nelements(dg.elements))
  # The "Ref(...)" is such that we can vectorize the search but not the array that is searched
  elements_to_remove = searchsortedfirst.(Ref(dg.elements.cell_ids[1:nelements(dg.elements)]),
                                          child_cells_to_coarsen)
  to_be_removed[elements_to_remove] .= true

  # Retain current solution data
  old_n_elements = nelements(dg.elements)
  old_u = dg.elements.u

  # Get new list of leaf cells
  leaf_cell_ids = leaf_cells(mesh.tree)

  # Initialize new elements container
  elements = init_elements(leaf_cell_ids, mesh, Val(NVARS), Val(POLYDEG))
  n_elements = nelements(elements)

  # Loop over all elements in old container and either copy them or coarsen them
  skip = 0
  element_id = 1
  for old_element_id in 1:old_n_elements
    # If skip is non-zero, we just coarsened 2^ndims elements and need to omit the following elements
    if skip > 0
      skip -= 1
      continue
    end

    if to_be_removed[old_element_id]
      # If an element is to be removed, sanity check if the following elements
      # are also marked - otherwise there would be an error in the way the
      # cells/elements are sorted
      @assert all(to_be_removed[old_element_id:(old_element_id+2^ndims(dg)-1)]) "bad cell/element order"

      # Coarsen elements and store solution directly in new data structure
      coarsen_elements!(elements.u, element_id, old_u, old_element_id,
                        dg.amr_coarsen_right, dg.amr_coarsen_left, dg)
      element_id += 1
      skip = 2^ndims(dg) - 1
    else
      # Copy old element data to new element container
      @views elements.u[:, :, element_id] .= old_u[:, :, old_element_id]
      element_id += 1
    end
  end

  # Initialize new interfaces container
  interfaces = init_interfaces(leaf_cell_ids, mesh, Val(NVARS), Val(POLYDEG), elements)
  n_interfaces = ninterfaces(interfaces)

  # Initialize boundaries
  boundaries, n_boundaries_per_direction = init_boundaries(leaf_cell_ids, mesh, Val(NVARS), Val(POLYDEG), elements)
  n_boundaries = nboundaries(boundaries)

  # Sanity check
  if isperiodic(mesh.tree)
    @assert n_interfaces == 1*n_elements ("For 1D and periodic domains, n_surf must be the same as 1*n_elem")
  end

  # Update DG instance with new data
  dg.elements = elements
  dg.n_elements = n_elements
  dg.n_elements_global = n_elements
  dg.interfaces = interfaces
  dg.n_interfaces = n_interfaces
  dg.boundaries = boundaries
  dg.n_boundaries = n_boundaries
  dg.n_boundaries_per_direction = n_boundaries_per_direction
end

# Coarsen solution data u for four elements, using L2 projection
function coarsen_elements!(u, element_id, old_u, old_element_id,
                            coarsen_right, coarsen_left, dg::Dg1D)
  # Store old element ids
  left_id  = old_element_id
  right_id = old_element_id + 1

  for i in 1:nnodes(dg)
    acc = zero(get_node_vars(u, dg, i, element_id))

    # Project from left element
    for k in 1:nnodes(dg)
      acc += get_node_vars(old_u, dg, k, left_id) * coarsen_left[i, k]
    end

    # Project from right element
    for k in 1:nnodes(dg)
      acc += get_node_vars(old_u, dg, k, right_id) * coarsen_right[i, k]
    end

    # Update value
    set_node_vars!(u, acc, dg, i, element_id)
  end
end

# Calculate an AMR indicator value for each element/leaf cell
#
# The indicator value λ ∈ [-1,1] is ≈ -1 for cells that should be coarsened, ≈
# 0 for cells that should remain as-is, and ≈ 1 for cells that should be
# refined.
#
# Note: The implementation here implicitly assumes that we have an element for
# each leaf cell and that they are in the same order.
#
# FIXME: This is currently implemented for each test case - we need something
# appropriate that is both equation and test case independent
function calc_amr_indicator(dg::Dg1D, mesh::TreeMesh, time)
  lambda = zeros(dg.n_elements)

  if dg.amr_indicator === :gauss
    base_level = 4
    max_level = 6
    threshold_high = 0.6
    threshold_low = 0.1

    # Iterate over all elements
    for element_id in 1:dg.n_elements
      # Determine target level from peak value
      peak = maximum(dg.elements.u[:, :, element_id])
      if peak > threshold_high
        target_level = max_level
      elseif peak > threshold_low
        target_level = max_level - 1
      else
        target_level = base_level
      end

      # Compare target level with actual level to set indicator
      cell_id = dg.elements.cell_ids[element_id]
      actual_level = mesh.tree.levels[cell_id]
      if actual_level < target_level
        lambda[element_id] = 1.0
      elseif actual_level > target_level
        lambda[element_id] = -1.0
      else
        lambda[element_id] = 0.0
      end
    end
  elseif dg.amr_indicator === :blast_wave
    base_level = 4
    max_level = 6
    blending_factor_threshold = 0.01

    # (Re-)initialize element variable storage for blending factor
    if (!haskey(dg.element_variables, :amr_indicator_values) ||
        length(dg.element_variables[:amr_indicator_values]) != dg.n_elements)
      dg.element_variables[:amr_indicator_values] = Vector{Float64}(undef, dg.n_elements)
    end
    if (!haskey(dg.element_variables, :amr_indicator_values_tmp) ||
        length(dg.element_variables[:amr_indicator_values_tmp]) != dg.n_elements)
      dg.element_variables[:amr_indicator_values_tmp] = Vector{Float64}(undef, dg.n_elements)
    end

    alpha     = dg.element_variables[:amr_indicator_values]
    alpha_tmp = dg.element_variables[:amr_indicator_values_tmp]
    calc_blending_factors!(alpha, alpha_tmp, dg.elements.u, dg.amr_alpha_max, dg.amr_alpha_min, dg.amr_alpha_smooth,
                           density_pressure, dg.thread_cache, dg)

    # Iterate over all elements
    for element_id in 1:dg.n_elements
      if alpha[element_id] > blending_factor_threshold
        target_level = max_level
      else
        target_level = base_level
      end

      # Compare target level with actual level to set indicator
      cell_id = dg.elements.cell_ids[element_id]
      actual_level = mesh.tree.levels[cell_id]
      if actual_level < target_level
        lambda[element_id] = 1.0
      elseif actual_level > target_level
        lambda[element_id] = -1.0
      else
        lambda[element_id] = 0.0
      end
    end
  else
    error("unknown AMR indicator '$(dg.amr_indicator)'")
  end

  return lambda
end


module SymbolMaps
export SymbolMap, submap
export integral!, derivative!

import Base./
using OneVision

const symbol_path_cache = Dict{Tuple{Symbol, Symbol}, Symbol}()
function symbol_path(s1::Symbol, s2::Symbol)::Symbol
    @get!(symbol_path_cache, (s1,s2), Symbol(string(s1, "/", s2)))
end

"""
Append the symbol `s2` to symbol `s1`.
"""
@inline function /(s1::Symbol, s2::Symbol)
    symbol_path(s1, s2)
end

@kwdef struct SymbolMap
    base_path::Symbol = :/
    data::Dict{Symbol,Any} = Dict{Symbol,Any}()
end

Base.getindex(m::SymbolMap, path::Symbol) = m.data[m.base_path / path]
Base.setindex!(m::SymbolMap, v, path::Symbol) =
    m.data[m.base_path / path] = v
Base.get!(m::SymbolMap, path::Symbol, default) =
    get!(m.data, m.base_path / path, default)
Base.get(m::SymbolMap, path::Symbol, default) =
    get(m.data, m.base_path / path, default)

"""
Creat a restricted view of the orginal SymbolMap.
"""
function submap(m::SymbolMap, path::Symbol)::SymbolMap
    SymbolMap(m.base_path / path, m.data)
end

function integral!(m::SymbolMap, path::Symbol, t::T, e::E)::E where {T,E}
    r = get(m, path, nothing)
    if r === nothing
        (m[path] = (t, zero(E)))[2]
    else
        (t0, v0) = r::Tuple{T,E}
        @assert(t > t0, "`integral!` with the path `$path` shouldn't be called multiple
                times in a single time step. (t = $t)")
        Δt = t - t0
        (m[path] = (t, v0 + e * Δt))[2]
    end
end

function derivative!(m::SymbolMap, path::Symbol, t::T, e::E)::E where {T,E}
    r = get(m, path, nothing)
    if r === nothing
        (m[path] = (t, zero(E)))[2]
    else
        (t0, v0) = r::Tuple{T,E}
        @assert(t > t0, "`derivative!` with the path `$path` shouldn't be called multiple
                times in a single time step. (t = $t)")
        Δt = t - t0
        (m[path] = (t, v0 + e / Δt))[2]
    end
end

end # module SymbolMaps


module ImageBase

export
    # two-fold downsampling
    # originally from ImageTransformations.jl
    restrict,

    # basic image statistics, from Images.jl
    minimum_finite,
    maximum_finite,
    meanfinite,
    varfinite,
    sumfinite

# Introduced in ColorVectorSpace v0.9.3
# https://github.com/JuliaGraphics/ColorVectorSpace.jl/pull/172
using ImageCore.ColorVectorSpace.Future: abs2

using Reexport

using Base.Cartesian: @nloops
@reexport using ImageCore
using ImageCore.OffsetArrays

include("diff.jl")
include("restrict.jl")
include("utils.jl")
include("statistics.jl")
include("deprecated.jl")

if VERSION >= v"1.4.2" # work around https://github.com/JuliaLang/julia/issues/34121
    include("precompile.jl")
    _precompile_()
end

end


module TestJoin
    using Base.Test
    using DataFrames

    name = DataFrame(ID = [1, 2, 3], Name = ["John Doe", "Jane Doe", "Joe Blogs"])
    job = DataFrame(ID = [1, 2, 4], Job = ["Lawyer", "Doctor", "Farmer"])

    # Join on symbols or vectors of symbols
    join(name, job, on = :ID)
    join(name, job, on = [:ID])

    # Soon we won't allow natural joins
    #@test_throws join(name, job)

    # Test output of various join types
    outer = DataFrame(ID = [1, 2, 3, 4],
                      Name = @data(["John Doe", "Jane Doe", "Joe Blogs", NA]),
                      Job = @data(["Lawyer", "Doctor", NA, "Farmer"]))

    # (Tests use current column ordering but don't promote it)
    right = outer[!isna(outer[:Job]), [:Name, :ID, :Job]]
    left = outer[!isna(outer[:Name]), :]
    inner = left[!isna(left[:Job]), :]
    semi = inner[:, [:ID, :Name]]
    anti = left[isna(left[:Job]), [:ID, :Name]]

    @test isequal(join(name, job, on = :ID), inner)
    @test isequal(join(name, job, on = :ID, kind = :inner), inner)
    @test isequal(join(name, job, on = :ID, kind = :outer), outer)
    @test isequal(join(name, job, on = :ID, kind = :left), left)
    @test isequal(join(name, job, on = :ID, kind = :right), right)
    @test isequal(join(name, job, on = :ID, kind = :semi), semi)
    @test isequal(join(name, job, on = :ID, kind = :anti), anti)

    # Join on multiple keys
    df1 = DataFrame(A = 1, B = 2, C = 3)
    df2 = DataFrame(A = 1, B = 2, D = 4)

    join(df1, df2, on = [:A, :B])

    # Test output of cross joins
    df1 = DataFrame(A = 1:2, B = 'a':'b')
    df2 = DataFrame(A = 1:3, C = 3:5)

    cross = DataFrame(A = [1, 1, 1, 2, 2, 2],
                      B = ['a', 'a', 'a', 'b', 'b', 'b'],
                      C = [3, 4, 5, 3, 4, 5])

    @test join(df1, df2[[:C]], kind = :cross) == cross

    # Cross joins handle naming collisions
    @test size(join(df1, df1, kind = :cross)) == (4, 4)

    # Cross joins don't take keys
    @test_throws ArgumentError join(df1, df2, on = :A, kind = :cross)
end


module DiracNotation

using Requires
include("prettyprint.jl")

function __init__()
    @require QuantumOptics = "6e0679c1-51ea-5a7c-ac74-d61b76210b0c" include("qo.jl")
end

end # module


#=
   The mag type used by Arb (fredrikj.net/arb/mag.html)
   see also (https://github.com/Nemocas/Nemo.jl/blob/master/src/arb/ArbTypes.jl)
   see also (https://github.com/thofma/Hecke.jl/blob/master/src/Misc/mag.jl)
=#

#=
type MagFloat
    radius_exponentOf2 :: Int        # exponentOf2
    radius_significand :: UInt64     # significand
end
=#

# define hash so other things work
const hash_arbmag_lo = (UInt === UInt64) ? 0x29f934c433d9a758 : 0x2578e2ce
const hash_0_arbmag_lo = hash(zero(UInt), hash_arbmag_lo)
if UInt===UInt64
   hash(z::MagFloat, h::UInt) = hash( reinterpret(UInt64, z.radius_exponentOf2), z.radius_significand )
else
   hash(z::MagFloat, h::UInt) = hash( reinterpret(UInt32, z.radius_exponentOf2) % UInt64, z.radius_significand )
end

# initialize and zero a variable of type MagFloat
function zero(::Type{MagFloat})
    z = MagFloat()
    # ccall(@libarb(mag_set_ui), Void, (Ptr{MagFloat}, UInt64), &z, zero(UInt64))
    return z
end
#zero(x::MagFloat) = zero(MagFloat)
function zero(x::MagFloat)
    return zero(MagFloat)
end

function one(::Type{MagFloat})
    z = MagFloat()
    ccall(@libarb(mag_set_ui), Void, (Ptr{MagFloat}, UInt64), &z, one(UInt64))
    return z
end
function one(x::MagFloat)
    return one(MagFloat)
end

#=
# how to write
for (T, postfix) in ((:Int, :int), (:Float64, :fp))
    @eval  ccall(  ___ )
end
# so Julia sees
ccall( (:convert_from_int, "myClib"), Void, (Ref{ Int     }, ), my_int_va
ccall( (:convert_from_fp,  "myClib"), Void, (Ref{ Float64 }, ), my_fp_var  )
# ------- from Yichao Yu, to get
ccall( (:convert_from_int, "myClib"), Void, (Ref{ Int     }, ), my_int_var )
# ------- the loop content is written this way
ccall( ($(QuoteNode(Symbol("convert_from", postfix))), "myClib"),
Void, (Ref{$T}, ), $(Symbol("my_", postfix, "_var"))
=#

for (T,M) in ((:UInt, :ui), (:Int, :si), (:Float64, :d))
  @eval begin
    function convert(::Type{MagFloat}, x::($T))
        z = MagFloat()
        ccall( :($(QuoteNode(Symbol("mag_set_", $M))), "libarb"), Void, (Ptr{$T}, ), &z )
        #ccall( ($(QuoteNode(Symbol("mag_set_", M))), "libarb"), Void, (Ref{$T}, ), z )
        return z
    end
  end
end

MagFloat(radius_exponentOf2::Int, radius_significand::Int64) =
    MagFloat(radius_exponentOf2, radius_significand % UInt64)

MagFloat(radius_exponentOf2::Int, radius_significand::Int32) =
    MagFloat(radius_exponentOf2, UInt64(radius_significand % UInt32) )

MagFloat(radius_exponentOf2::Int, radius_significand::Float64) =
    MagFloat(radius_exponentOf2, convert(UInt64, abs(radius_significand)) )

MagFloat(radius_exponentOf2::Int, radius_significand::Float32) =
    MagFloat(radius_exponentOf2, convert(UInt64, abs(radius_significand)) )

# conversions

# convert to MagFloat

Error_MagIsNegative() = throw(ErrorException("Magnitudes must be nonnegative."))

#=
function convert(::Type{MagFloat}, x::Float64)
    signbit(x) && Error_MagIsNegative()
    z = MagFloat()
    ccall(@libarb(mag_set_d), Void, (Ptr{MagFloat}, Ptr{Float64}), &z, &x)
    return z
end
convert(::Type{MagFloat}, x::Float32) = convert(MagFloat, convert(Float64, x))
convert(::Type{MagFloat}, x::Float16) = convert(MagFloat, convert(Float64, x))
=#

#=
   lowerbound returns upper bound of value
   upperbound returns lower bound of value
=#

if Int == Int64
  function upperbound(::Type{MagFloat}, x::UInt64)
    z = MagFloat()
    ccall(@libarb(mag_set_ui), Void, (Ptr{MagFloat}, Ptr{UInt64}), &z, &x)
    return z
  end
  function lowerbound(::Type{MagFloat}, x::UInt64)
    z = MagFloat()
    ccall(@libarb(mag_set_ui_lower), Void, (Ptr{MagFloat}, Ptr{UInt64}), &z, &x)
    return z
  end
  for T in (:UInt128, :UInt32, :UInt16, :UInt8)
    @eval upperbound(::Type{MagFloat}, x::($T)) = upperbound(MagFloat, convert(UInt64, x))
    @eval lowerbound(::Type{MagFloat}, x::($T)) = lowerbound(MagFloat, convert(UInt64, x))
  end
  for T in (:Int128, :Int32, :Int16, :Int8)
    @eval convert(::Type{MagFloat}, x::($T))  = convert(MagFloat, convert(Int64, x))
  end
else
  function upperbound(::Type{MagFloat}, x::UInt32)
    z = MagFloat()
    ccall(@libarb(mag_set_ui), Void, (Ptr{MagFloat}, Ptr{UInt32}), &z, &x)
    return z
  end
  function lowerbound(::Type{MagFloat}, x::UInt32)
    z = MagFloat()
    ccall(@libarb(mag_set_ui_lower), Void, (Ptr{MagFloat}, Ptr{UInt32}), &z, &x)
    return z
  end
  for T in (:UInt128, :UInt64, :UInt16, :UInt8)
    @eval upperbound(::Type{MagFloat}, x::($T)) = upperbound(MagFloat, convert(UInt32, x))
    @eval lowerbound(::Type{MagFloat}, x::($T)) = lowerbound(MagFloat, convert(UInt32, x))
  end
  for T in (:Int128, :Int64, :Int16, :Int8)
    @eval convert(::Type{MagFloat}, x::($T))  = convert(MagFloat, convert(Int32, x))
  end
end

# convert from MagFloat

function convert(::Type{Float64}, x::MagFloat)
    z = ccall(@libarb(mag_get_d), Float64, (Ptr{MagFloat}, ), &x)
    return z
end
function convert(::Type{Float32}, x::MagFloat)
    z = convert(Float64, x)
    convert(Float32, z)
    return z
end

# function convert(::Type{UInt}, x::MagFloat)

# promotions

for T in (:UInt, :Int, :Float32, :Float64)
    @eval promote_rule(::Type{MagFloat}, ::Type{$T}) = MagFloat
end

# string, show
#
function string(x::MagFloat)
    fp = convert(Float64, x)
    return string(fp)
end

function stringcompact(x::MagFloat)
    fp = convert(Float32, convert(Float64, x))
    return string(fp)
end

function show(io::IO, x::MagFloat)
    s = string(x)
    print(io, s)
    return nothing
end

function showcompact(io::IO, x::MagFloat)
    s = stringcompact(x)
    print(io, s)
    return nothing
end


@testset "Hawkes Fit" begin

    testEvents = sort(rand(100)*100)

    testSamples = HawkesProcesses.fit(testEvents, 100, 3)
    inferTests && @inferred HawkesProcesses.fit(testEvents, 100, 3)
    @test isa(testSamples, Tuple)
    @test length(testSamples) == 3
    @test size(testSamples[1]) == (3,)
    @test size(testSamples[2]) == (3,)
    @test size(testSamples[3]) == (3,)

end


module JuliaCon2020RANSAC

import Remark
import RANSAC
using RANSAC: IterLow1
import RANSACVisualizer
using ColorSchemes

using Logging
using LinearAlgebra
using ExtractMacro
using UnionFind
using NearestNeighbors: KDTree, inrange
using StaticArrays: SVector

export  makeslides,
        openslides

export  SLIDE_FOLDER

const SLIDE_FOLDER = joinpath(dirname(@__DIR__), "slides")

makeslides() = Remark.slideshow(SLIDE_FOLDER, options=Dict("ratio"=>"16:9"),
    title = "Efficient RANSAC in efficient Julia", documenter=false)

openslides() = Remark.open(SLIDE_FOLDER)

export  FittedTranslational,
        ExtractedTranslational

include("translational/measures.jl")
include("translational/profit.jl")
include("translational/translational.jl")

end # module


function amelioration(sol::Solution, prob::Problem)::Solution
    chgt::Bool = true

    chgt21::Bool = true
	chgt12::Bool = true
    chgt11::Bool = true
    chgt01::Bool = true

	nbChgt::Int64 = 0

	nbChgt21::Int64 = 0
	nbChgt12::Int64 = 0
	nbChgt11::Int64 = 0
	nbChgt01::Int64 = 0

    #while chgt

		chgt = false
		nbChgt += 1

		nbChgt21 = 0
		nbChgt12 = 0
		nbChgt11 = 0
		nbChgt01 = 0

		@time while chgt21
			sol, chgt21 = exchange21(sol, prob)
			nbChgt21 += 1
		end
		# println("nbChgt21 -> ", nbChgt21)
        @time while chgt12
			sol, chgt12 = exchange12(sol, prob)
			nbChgt12 += 1
			if chgt12
				chgt = true
			end
		end
		# println("nbChgt12 -> ", nbChgt12)
		@time if !prob.equalWeights
			while chgt11
				sol, chgt11 = exchange11(sol, prob)
				nbChgt11 += 1
				if chgt11
					chgt = true
				end
			end
		end
		# println("nbChgt11 -> ", nbChgt11)
		@time if sol.nbVarLibre != 0
			while chgt01
				sol, chgt01 = exchange01(sol, prob)
				nbChgt01 += 1
				if chgt01
					chgt = true
				end
			end
		end
		# println("nbChgt01 -> ", nbChgt01)
	# end
	return sol
end


##################################################################################
# This file is part of ModelBaseEcon.jl
# BSD 3-Clause License
# Copyright (c) 2020, Bank of Canada
# All rights reserved.
##################################################################################

export SteadyStateEquation

"""
    struct SteadyStateEquation <: AbstractEquation

Data structure representing an individual steady state equation.
"""
struct SteadyStateEquation <: AbstractEquation
    type::Symbol
    vinds::Vector{Int64}
    vsyms::Vector{Symbol}
    expr::ExtExpr
    eval_resid::Function
    eval_RJ::Function
end

########################################################

"""
    struct SteadyStateVariable

Holds the steady state solution for one variable.
"""
struct SteadyStateVariable{DATA<:AbstractVector{Float64},MASK<:AbstractVector{Bool}}
    # The corresponding entry in m.allvars. Needed for its type
    name::ModelVariable
    # Its index in the m.allvars array
    index::Int
    # A view in the SteadyStateData.values location for this variable
    data::DATA
    # A view into the mask array
    mask::MASK
end

for sym in (:lin, :log, :neglog, :steady, :exog, :shock)
    issym = Symbol("is", sym)
    eval(quote
        @inline $(issym)(s::SteadyStateVariable) = $(issym)(s.name)
    end)
end

@inline transformation(v::SteadyStateVariable) = transformation(v.name)
@inline inverse_transformation(v::SteadyStateVariable) = inverse_transformation(v.name)
@inline transform(x, v::SteadyStateVariable) = transform(x, v.name)
@inline inverse_transform(x, v::SteadyStateVariable) = inverse_transform(x, v.name)

#############################################################################
# Access to .level and .slope

Base.getproperty(v::SteadyStateVariable, name::Symbol) = begin
    if name ∈ fieldnames(SteadyStateVariable)
        return getfield(v, name)
    end
    data = getfield(v, :data)
    # we store transformed data, must invert to give back to user
    return name == :level ? inverse_transform(data[1], v) :
           name == :slope ?
           (islog(v) || isneglog(v) ? exp(data[2]) : data[2]) :
           getfield(v, name)
end

Base.setproperty!(v::SteadyStateVariable, name::Symbol, val) = begin
    data = getfield(v, :data)
    # we store transformed data, must transform user input
    if name == :level
        data[1] = transform(val, v)
    elseif name == :slope
        data[2] = (islog(v) || isneglog(v) ? log(val) : val)
    else
        setfield!(v, name, val)  # this will error (immutable)
    end
end

# use [] to get a time series of values.
# the ref= value is the t at which it equals its level
function Base.getindex(v::SteadyStateVariable, t; ref = 1)
    if eltype(t) != eltype(ref)
        throw(ArgumentError("Must provide reference time of the same type as the time index"))
    end
    int_t = convert(Int, first(t) - ref):convert(Int, last(t) - ref)
    # we must inverse transform internal data before returning to user
    return inverse_transform(v.data[1] .+ v.data[2] .* int_t, v)
end

# pretty printing

"""
    alignment5(io::IO, v::SteadyStateVariable)

Return a 5-tuple with the number of characters for the name, and the alignment 2-tuples for level and slope.

"""
function alignment5(io::IO, v::SteadyStateVariable)
    name = sprint(print, string(v.name.name), context = io, sizehint = 0)
    lvl_a = Base.alignment(io, v.level)
    if isshock(v) || issteady(v)
        slp_a = (0, 0)
    else
        slp_a = Base.alignment(io, v.slope)
    end
    (length(name), lvl_a..., slp_a...)
end

function alignment5(io::IO, vars::AbstractVector{SteadyStateVariable})
    a = (0, 0, 0, 0, 0)
    for v in vars
        a = max.(a, alignment5(io, v))
    end
    return a
end

function show_aligned5(io::IO, v::SteadyStateVariable, a = alignment5(io, v);
    mask = trues(2), sep1 = " = ",
    sep2 = islog(v) || isneglog(v) ? " * " : " + ",
    sep3 = islog(v) || isneglog(v) ? "^t" : "*t")
    name = sprint(print, string(v.name.name), context = io, sizehint = 0)
    if mask[1]
        lvl_a = Base.alignment(io, v.level)
        lvl = sprint(show, v.level, context = io, sizehint = 0)
    else
        lvl_a = (0, 1)
        lvl = "?"
    end
    if mask[2]
        slp_a = Base.alignment(io, v.slope)
        slp = sprint(show, v.slope, context = io, sizehint = 0)
    else
        slp_a = (0, 1)
        slp = "?"
    end
    print(io, "  ", repeat(' ', a[1] - length(name)), name,
        sep1, repeat(' ', a[2] - lvl_a[1]), lvl, repeat(' ', a[3] - lvl_a[2]))
    if (!issteady(v) && !isshock(v)) && !(v.data[2] + 1.0 ≈ 1.0)
        print(io, sep2, repeat(' ', a[4] - slp_a[1]), slp, repeat(' ', a[5] - slp_a[2]), sep3)
    end
end

@inline Base.show(io::IO, v::SteadyStateVariable) = show_aligned5(io, v)

########################################################

export SteadyStateData

"""
    SteadyStateData

Data structure that holds information about the steady state solution of the Model.
"""
struct SteadyStateData
    "List of steady state variables."
    vars::Vector{SteadyStateVariable}
    "Steady state solution vector."
    values::Vector{Float64}
    "`mask[i] == true if and only if `values[i]` holds the steady state value."
    mask::BitArray{1}
    "Steady state equations derived from the dynamic system."
    equations::Vector{SteadyStateEquation}
    "Steady state equations explicitly added with @steadystate."
    constraints::Vector{SteadyStateEquation}
    # default constructor
    SteadyStateData() = new([], [], [], [], [])
end

Base.push!(ssd::SteadyStateData, vars...) =
    for v in vars
        push!(ssd, v)
    end
Base.push!(ssd::SteadyStateData, var::Symbol) = push!(ssd, convert(ModelSymbol, var))
function Base.push!(ssd::SteadyStateData, var::ModelSymbol)
    for v in getfield(ssd, :vars)
        if v.name == var
            return v
        end
    end
    push!(getfield(ssd, :values), isexog(var) || isshock(var) ? 0.0 : 0.1, 0.0)
    push!(getfield(ssd, :mask), isexog(var) || isshock(var), isexog(var) || isshock(var) || issteady(var))
    ind = length(getfield(ssd, :vars)) + 1
    v = SteadyStateVariable(var, ind, @view(getfield(ssd, :values)[2ind.+(-1:0)]), @view(getfield(ssd, :mask)[2ind.+(-1:0)]))
    push!(getfield(ssd, :vars), v)
    return v
end

export alleqns
"""
    alleqns(ssd::SteadyStateData)

Return a list of all steady state equations.

The list contains all equations derived from the dynamic system and all explicitly added steady state constraints.
"""
@inline alleqns(ssd::SteadyStateData) = vcat(ssd.equations, ssd.constraints)

export neqns
"""
    neqns(ssd::SteadyStateData)

Return the total number of equations in the steady state system, including the ones derived from the dynamic system and the
ones added explicitly as steady state constraints.
"""
@inline neqns(ssd::SteadyStateData) = length(ssd.equations) + length(ssd.constraints)

export geteqn
"""
    geteqn(i, ssd::SteadyStateData)

Return the i-th steady state equation. Index i is interpreted as in the output of `alleqns`.
Calling `geteqn(i, sdd)` has the same effect as `alleqn(ssd)[i]`, but it's more efficient.

### Example
```julia
# Iterate all equations like this:
for i = 1:neqns(ssd)
    eqn = geteqn(i, ssd)
    # do something awesome with `eqn` and `i`
end
```
"""
function geteqn(i::Integer, ssd::SteadyStateData)
    ci = i - length(ssd.equations)
    return ci > 0 ? ssd.constraints[ci] : ssd.equations[i]
end

Base.show(io::IO, ::MIME"text/plain", ssd::SteadyStateData) = show(io, ssd)
Base.show(io::IO, ssd::SteadyStateData) = begin
    if issssolved(ssd)
        println(io, "Steady state solved.")
    else
        println(io, "Steady state not solved.")
    end
    len = length(ssd.constraints)
    if len == 0
        println(io, "No additional constraints.")
    else
        println(io, len, " additional constraint", ifelse(len > 1, "s.", "."))
        for c in ssd.constraints
            println(io, "    ", c)
        end
    end
end

#########
# Implement access to steady state values using dot notation and index notation

function Base.propertynames(ssd::SteadyStateData, private::Bool = false)
    if private
        return ((v.name.name for v in ssd.vars)..., fieldnames(SteadyStateData)...,)
    else
        return ((v.name.name for v in ssd.vars)...,)
    end
end

function Base.getproperty(ssd::SteadyStateData, sym::Symbol)
    if sym ∈ fieldnames(SteadyStateData)
        return getfield(ssd, sym)
    else
        for v in ssd.vars
            if v.name == sym
                return v
            end
        end
        throw(ArgumentError("Unknown variable $sym."))
    end
end

Base.getindex(ssd::SteadyStateData, ind::Int) = ssd.vars[ind]
Base.getindex(ssd::SteadyStateData, sym::ModelSymbol) = getproperty(ssd, sym.name)
Base.getindex(ssd::SteadyStateData, sym::Symbol) = getproperty(ssd, sym)
Base.getindex(ssd::SteadyStateData, sym::AbstractString) = getproperty(ssd, Symbol(sym))

@inline ss_symbol(ssd::SteadyStateData, vi::Int) = Symbol("#", ssd.vars[(1+vi)÷2].name.name, "#", (vi % 2 == 1) ? :lvl : :slp, "#")

#########################
#

export printsstate

"""
    printsstate([io::IO,] ssd::SteadyStateData)

Display steady state solution.

Steady state solution is presented in a table, where the first column is
the name of the variable, the second and third columns are the corresponding
values of the level and the slope. If the value is not determined
(as per its `mask` value) then it is displayed as "*".
"""
function printsstate(io::IO, model::AbstractModel)
    io = IOContext(io, :compact => get(io, :compact, true))
    ssd = model.sstate
    println(io, "Steady State Solution:")
    a = max.(alignment5(io, ssd.vars), (0, 0, 3, 0, 3))
    for v in ssd.vars
        show_aligned5(io, v, a, mask = v.mask)
        println(io)
    end
end
printsstate(model::AbstractModel) = printsstate(Base.stdout, model)

###########################
# Make steady state equation from dynamic equation

# Idea:
#   in the steady state equation, we assume that the variable y_ss
#   follows a linear motion expressed as y_ss[t] = y_ss#lvl + t * y_ss#slp
#   where y_ss#lvl and y_ss#slp are two unknowns we solve for.
#
#   The dynamic equation has mentions of lags and leads. We replace those
#   with the above expression.
#
#   Since we have two parameters to determine, we need two steady state equations
#   from each dynamic equation. We get this by writing the dynamic equation at
#   two different values of `t` - 0 and another one we call `shift`.
#
#   Shift is a an option in the model object, which the user can set to any integer
#   other than 0. The default is 10.

"""
    SSEqnData

Internal structure used for evaluation of the residual of the steady state equation
derived from a dynamic equation.

"""
struct SSEqnData{M<:AbstractModel}
    "Whether or not to add model.shift"
    shift::Bool
    "Reference to the model object (needed for the current value of shift"
    model::Ref{M}
    "Information needed to compute the Jacobian matrix of the transformation between steady state and dynamic unknowns"
    JT::Vector
    "The dynamic equation instance"
    eqn::Equation
    SSEqnData(s, m::M, jt, e) where {M<:AbstractModel} = new{M}(s, Ref(m), jt, e)
    SSEqnData(s, m::Ref{M}, jt, e) where {M<:AbstractModel} = new{M}(s, m, jt, e)
end

function sseqn_resid_RJ(s::SSEqnData)
    buffer = Vector{Float64}(undef, length(s.JT))
    @inline lag(jt) = ifelse(s.shift, jt.tlag + s.model[].shift, jt.tlag)
    function to_dyn_pt(pt::AbstractVector{Float64})
        # This function applies the transformation from steady
        # state equation unknowns to dynamic equation unknowns
        fill!(buffer, 0.0)
        for (i, jt) in enumerate(s.JT)
            if length(jt.ssinds) == 1
                pti = pt[jt.ssinds[1]]
            else
                pti = pt[jt.ssinds[1]] + lag(jt) * pt[jt.ssinds[2]]
            end
            buffer[i] += pti
        end
        return buffer
    end
    function to_ssgrad(pt::AbstractVector{Float64}, jj::AbstractVector{Float64})
        # This function inverts the transformation. jj is the gradient of the
        # dynamic equation residual with respect to the dynamic equation unknowns.
        # Here we compute the Jacobian of the transformation and use it to compute
        ss = zeros(size(pt))
        for (i, jt) in enumerate(s.JT)
            if length(jt.ssinds) == 1
                pti = pt[jt.ssinds[1]]
                ss[jt.ssinds[1]] += jj[i]
            else
                local lag_jt = lag(jt)
                pti = pt[jt.ssinds[1]] + lag_jt * pt[jt.ssinds[2]]
                ss[jt.ssinds[1]] += jj[i]
                ss[jt.ssinds[2]] += jj[i] * lag_jt
            end
        end
        # NOTE regarding the above: The dynamic equation is F(x_t) = 0
        # Here we're solving F(u(l+t*s)) = 0
        # The derivative is dF/dl = F' * u' and dF/ds = F' * u' * t
        # F' is in jj[i]
        # u(x) = x, so u'(x) = 1
        return ss
    end
    function _resid(pt::AbstractVector{Float64})
        return s.eqn.eval_resid(to_dyn_pt(pt))
    end
    function _RJ(pt::AbstractVector{Float64})
        R, jj = s.eqn.eval_RJ(to_dyn_pt(pt))
        return R, to_ssgrad(pt, jj)
    end
    return _resid, _RJ
end

"""
    make_sseqn(model::AbstractModel, eqn::Equation; shift::Int64=0)

Create a steady state equation from the given dynamic equation for the given model.

Internal function, do not call directly.

"""
function make_sseqn(model::AbstractModel, eqn::Equation, shift::Bool)
    local allvars = model.allvars
    @inline tvalue(t) = ifelse(shift, t + model.shift, t)
    # ssind converts the dynamic index (t, v) into
    # the corresponding indexes of steady state unknowns.
    # Returned value is a list of length 0, 1, or 2.
    function ssind((ti, vi),)::Array{Int64,1}
        no_slope = isshock(allvars[vi]) || issteady(allvars[vi])
        # The level unknown has index 2*vi-1.
        # The slope unknown has index 2*vi. However:
        #  * :steady and :shock variables don't have slopes
        #  * :lin and :log variables the slope is in the equation
        #    only if the effective t-index is not 0.
        if no_slope || (!shift && ti == 0)
            return [2vi - 1]
        else
            return [2vi - 1, 2vi]
        end
    end
    local ss = model.sstate
    # The steady state indexes.
    vinds = unique(vcat(map(ssind, eqn.vinds)...))
    # The corresponding steady state symbols
    vsyms = Symbol[ss_symbol(ss, vi) for vi in vinds]
    # In the next loop we build the matrix JT which transforms
    # from the steady state values to the dynamic point values.
    JT = []
    for (i, (ti, vi)) in enumerate(eqn.vinds)
        val = (ssinds = indexin(ssind((ti, vi)), vinds), tlag = ti)
        push!(JT, val)
    end
    type = ifelse(shift == 0, :tzero, :tshift)
    let sseqndata = SSEqnData(shift, Ref(model), JT, eqn)
        return SteadyStateEquation(type, vinds, vsyms, eqn.expr, sseqn_resid_RJ(sseqndata)...)
    end
end

###########################
# Make steady state equation from user input

"""
    setss!(model::AbstractModel, expr::Expr; type::Symbol, modelmodule::Module)

Add a steady state equation to the model. Equations added by `setss!` are in
addition to the equations generated automatically from the dynamic system.

Internal function, do not call directly. Use [`@steadystate`](@ref) instead.

"""
function setss!(model::AbstractModel, expr::Expr; type::Symbol, modelmodule::Module = moduleof(model))

    if expr.head != :(=)
        error("Expected an equation, not $(expr.head)")
    end

    @assert type ∈ (:level, :slope) "Unknown steady state equation type $type. Expected either `level` or `slope`."

    local ss = sstate(model)

    local allvars = model.allvars

    ###############################################
    #     ssprocess(val)
    #
    # Process the given value to extract information about mentioned parameters and variables.
    # This function has the side effect of populating the vectors
    # `vinds`, `vsyms`, `val_params` and `source`
    #
    # Algorithm is recursive over the given expression. The bottom of the recursion is the
    # processing of a `Number`, a `Symbol`, (or a `LineNumberNode`).
    #
    # we will store indices
    local vinds = Int64[]
    local vsyms = Symbol[]
    # we will store parameters mentioned in `expr` here
    local val_params = Symbol[]
    local source = LineNumberNode[]
    # nothing to do with a number
    ssprocess(val::Number) = val
    # a symbol could be a variable (shock, auxvar), a parameter, or unknown.
    function ssprocess(val::Symbol)
        if val ∈ keys(model.parameters)
            # parameter - keep track that it's mentioned
            push!(val_params, val)
            return val
        end
        vind = indexin([val], allvars)[1]
        if vind !== nothing
            # it's a vriable of some sort: make a symbol and an index for the
            # corresponding steady state unknown
            vsym = Symbol("#", type, "#", val, "#")
            push!(vsyms, vsym)
            push!(vinds, type == :level ? 2vind - 1 : 2vind)
            if need_transform(allvars[vind])
                func = inverse_transformation(allvars[vind])
                return :($func($vsym))
            else
                return vsym
            end
        end
        # what to do with unknown symbols?
        error("unknown parameter $val")
    end
    # a sorce line information: store it and remove it from the expression
    ssprocess(val::LineNumberNode) = (push!(source, val); nothing)
    # process an expression recursively
    function ssprocess(val::Expr)
        if val.head == :(=)
            # we process the lhs and rhs seperately: there shouldn't be any equal signs
            error("unexpected equation.")
        end
        if val.head == :block
            # in a begin-end block, process each line and gather the results
            args = filter(x -> x !== nothing, map(ssprocess, val.args))
            if length(args) == 1
                # Only one thing left - no need for the begin-end anymore
                return args[1]
            else
                # reassemble the processed expressions back into a begin-end block
                return Expr(:block, args...)
            end
        elseif val.head == :call
            # in a function call, process each argument, but not the function name (args[1]) and reassemble the call
            args = filter(x -> x !== nothing, map(ssprocess, val.args[2:end]))
            return Expr(:call, val.args[1], args...)
        else
            # whatever this it, process each subexpression and reassemble it
            args = filter(x -> x !== nothing, map(ssprocess, val.args))
            return Expr(val.head, args...)
        end
    end
    # end of ssprocess() definition
    ###############################################
    #
    lhs, rhs = expr.args
    lhs = ssprocess(lhs)
    rhs = ssprocess(rhs)
    expr.args .= MacroTools.unblock.(expr.args)
    #
    nargs = length(vinds)
    # In case there's no source information, add a dummy one
    push!(source, LineNumberNode(0))
    # create the resid and RJ functions for the new equation
    # To do this, we use `makefuncs` from evaluation.jl
    resid, RJ = let mparams = parameters(model)
        # create a list of expressions that assign the values of model parameters to
        # variables of the same name
        param_assigments = Expr(:block)
        for p in unique(val_params)
            push!(param_assigments.args, :(local $(p) = $(mparams).$(p)))
        end
        residual = Expr(:block, source[1], :($(lhs) - $(rhs)))
        funcs_expr = makefuncs(residual, vsyms, param_assigments; mod = modelmodule)
        modelmodule.eval(funcs_expr)
    end
    # We have all the ingredients to create the instance of SteadyStateEquation
    for i = 1:2
        # remove blocks with line numbers from expr.args[i]
        a = expr.args[i]
        if Meta.isexpr(a, :block)
            args = filter(x -> !isa(x, LineNumberNode), a.args)
            if length(a.args) == 1
                expr.args[i] = args[1]
            end
        end
    end
    sscon = SteadyStateEquation(type, vinds, vsyms, expr, resid, RJ)
    if nargs == 1
        # The equation involves only one variable. See if there's already an equation
        # with just that variable and, if so, remove it.
        for (i, ssc) in enumerate(ss.constraints)
            if ssc.type == type && length(ssc.vinds) == 1 && ssc.vinds[1] == sscon.vinds[1]
                ss.constraints[i] = sscon
                return sscon
            end
        end
    end
    push!(ss.constraints, sscon)
    return sscon
end

export @steadystate

"""
    @steadystate model [type] equation

Add a steady state equation to the model.

The steady state system of the model is automatically derived from the dynamic
system. Use this macro to define additional equations for the steady state.
This is particularly useful in the case of a non-linear model that might have
multiple steady state, or the steady state might be difficult to solve for,
to help the steady state solver find the one you want to use.

  * `model` is the model instance you want to update
  * `type` (optional) is the type of constraint you want to add. This can be `level`
  or `slope`. If missing, the default is `level`
  * `equation` is the expression defining the steady state constraint. In the
  equation, use variables and shocks from the model, but without any t-references.

"""
macro steadystate(model, type::Symbol, equation::Expr)
    thismodule = @__MODULE__
    modelmodule = __module__
    return esc(:($(thismodule).setss!($(model), $(Meta.quot(equation)); type = $(QuoteNode(type)))))  # , modelmodule=$(modelmodule))))
end

macro steadystate(model, equation::Expr)
    thismodule = @__MODULE__
    return esc(:($(thismodule).setss!($(model), $(Meta.quot(equation)); type = :level))) # , modelmodule=$(modelmodule))))
end

"""
    initssdata!(m::AbstractModel)

Initialize the steady state data structure of the given model.

Do not call directly. This is an internal function, called during
[`@initialize`](@ref)

"""
function initssdata!(model::AbstractModel)
    ss = sstate(model)
    empty!(ss.vars)
    empty!(ss.values)
    empty!(ss.mask)
    for var in model.allvars
        push!(ss, var)
    end
    empty!(ss.equations)
    for eqn in alleqns(model)
        push!(ss.equations, make_sseqn(model, eqn, false))
    end
    if !model.flags.ssZeroSlope
        for eqn in alleqns(model)
            push!(ss.equations, make_sseqn(model, eqn, true))
        end
    end
    empty!(ss.constraints)
    return nothing
end

export issssolved
"""
    issssolved(sstate::SteadyStateData)

Return `true` if the steady state has been solved, or `false` otherwise.

"""
@inline issssolved(ss::SteadyStateData) = all(ss.mask)


abstract type Layer end

abstract type Loss <: Layer end

abstract type Activation <: Layer end

abstract type AbstractRNN{Params, Embed} <: Layer end

const LayerOrNothing = Union{Layer,Nothing}
const ActOrNothing   = Union{Activation,Nothing}
const DictOrNothing  = Union{Dict,Nothing}
const VVecOrNothing  = Union{Vector{Vector{<:Integer}},Nothing}


#unitary test GenericStatemod
x0 = ones(6)
state0 = GenericState(x0)

@test isnan(state0.start_time) #Default value of start_time is 0
x1 = [1.0]
update!(state0, x = x1) #Check the update of state0
@test state0.x == x1
@test isnan(state0.start_time) #start_time must be unchanged

update!(state0, tmps = 1.0)
@test state0.x == x1 #must be unchanged
@test state0.start_time == 1.0


# convert a Reaction to a ChemicalHyperEdge
function Base.convert(::Type{ChemicalHyperEdge}, r::Catalyst.Reaction)
    t_subs = eltype(r.substrates)
    t_prods = eltype(r.products)
    t = (isequal(t_subs, t_prods) ? t_subs : Simulacrum.get_lowest_subtype(t_subs, t_prods))
    ChemicalHyperEdge(SpeciesSet(Vector{t}(r.substrates), r.substoich), SpeciesSet(Vector{t}(r.products), r.prodstoich), r.rate)
end
function Base.convert(::Type{ChemicalHyperEdge}, rs::AbstractVector{T}) where {T<:Catalyst.Reaction}
    convert.(ChemicalHyperEdge, rs)
end

# now these functions may be defined via the conversion defined above
isunimolecular(r::Catalyst.Reaction) = isunimolecular(convert(ChemicalHyperEdge, r))
ismultimolecular(r::Catalyst.Reaction) = !(isunimolecular(r))

# convert a ChemicalHyperEdge to a Reaction
# note that rate(che)[1] is needed -- not ideal
function Base.convert(::Type{Catalyst.Reaction}, che::ChemicalHyperEdge)
    Reaction(rate(che)[1], src(che), tgt(che), src_stoich(che), tgt_stoich(che))
end
function Base.convert(::Type{Catalyst.Reaction}, ches::AbstractVector{T}) where {T<:ChemicalHyperEdge}
    convert.(Catalyst.Reaction, ches)
end

# convert a ReactionSystem to a ChemicalHyperGraph
function Base.convert(::Type{ChemicalHyperGraph}, rsys::Catalyst.ReactionSystem)
    ChemicalHyperGraph(convert(ChemicalHyperEdge, equations(rsys)))
end

# return reactions from a chemical hypergraph
catalyst_reactions(chx::ChemicalHyperGraph) = convert(Catalyst.Reaction, hyperedges(chx))

# convert a ChemicalHyperGraph to a ReactionSystem
function Base.convert(::Type{Catalyst.ReactionSystem}, chx::ChemicalHyperGraph; p = [], iv = Symbolics.variable(:t))
    Catalyst.ReactionSystem(catalyst_reactions(chx), iv, vertices(chx), p, name = :rsys)
end

# overload the ChemicalHyperGraph constructor to Reaction and ReactionSystem types
ChemicalHyperGraph(r::Catalyst.Reaction) = ChemicalHyperGraph(convert(ChemicalHyperEdge, r))
function ChemicalHyperGraph(rs::AbstractVector{T}) where {T<:Catalyst.Reaction}
    ChemicalHyperGraph(convert(ChemicalHyperEdge, rs))
end
ChemicalHyperGraph(rsys::Catalyst.ReactionSystem) = convert(ChemicalHyperGraph, rsys)


using Test

using CUDAdrv
include(joinpath(@__DIR__, "..", "test", "array.jl"))   # real applications: use CuArrays.jl

dev = CuDevice(0)
ctx = CuContext(dev)

md = CuModuleFile(joinpath(@__DIR__, "vadd.ptx"))
vadd = CuFunction(md, "kernel_vadd")

dims = (3,4)
a = round.(rand(Float32, dims) * 100)
b = round.(rand(Float32, dims) * 100)
c = similar(a)

d_a = CuTestArray(a)
d_b = CuTestArray(b)
d_c = CuTestArray(c)

len = prod(dims)
cudacall(vadd, Tuple{CuPtr{Cfloat},CuPtr{Cfloat},CuPtr{Cfloat}}, d_a, d_b, d_c; threads=len)

@test a+b ≈ Array(d_c)

CUDAdrv.unsafe_destroy!(ctx)


#=
    Vector valued bounds
        VectorOfVariables -in- Nonnegatives
        VectorOfVariables -in- Nonpositives
        VectorOfVariables -in- Zeros

    SOS
        VectorOfVariables -in- SOS1
        VectorOfVariables -in- SOS2
=#
constrdict(model::LinQuadOptimizer, ::VVCI{MOI.Nonnegatives}) = cmap(model).vv_nonnegatives
constrdict(model::LinQuadOptimizer, ::VVCI{MOI.Nonpositives}) = cmap(model).vv_nonpositives
constrdict(model::LinQuadOptimizer, ::VVCI{MOI.Zeros}) = cmap(model).vv_zeros

constrdict(model::LinQuadOptimizer, ::VVCI{SOS1}) = cmap(model).sos1
constrdict(model::LinQuadOptimizer, ::VVCI{SOS2}) = cmap(model).sos2

function set_variable_bound(model::LinQuadOptimizer, columns::Vector{Int}, bounds::Vector{Float64}, sense)
    @assert length(columns) == length(bounds)
    change_variable_bounds!(model, columns, bounds, sense)
end

function set_variable_bound(model::LinQuadOptimizer, vs::Vector{VarInd}, sets::Vector{LE})
    set_variable_bound(model, get_column.(model, vs), [s.upper for s=sets],
                       fill(backend_type(model, Val{:Upperbound}()), length(vs)))
end
function set_variable_bound(model::LinQuadOptimizer, vs::Vector{VarInd}, sets::Vector{GE})
    set_variable_bound(model, get_column.(model, vs), [s.lower for s=sets],
                       fill(backend_type(model, Val{:Lowerbound}()), length(vs)))
end
function set_variable_bound(model::LinQuadOptimizer, vs::Vector{VarInd}, sets::Vector{EQ})
    set_variable_bound(model, get_column.(model, vs), [s.value for s=sets],
                       fill(backend_type(model, Val{:Upperbound}()), length(vs)))
    set_variable_bound(model, get_column.(model, vs), [s.value for s=sets],
                       fill(backend_type(model, Val{:Lowerbound}()), length(vs)))
end
function set_variable_bound(model::LinQuadOptimizer, vs::Vector{VarInd}, sets::Vector{IV})
    set_variable_bound(model, get_column(model, v), [s.upper for s=sets],
                       fill(backend_type(model, Val{:Upperbound}()), length(vs)))
    set_variable_bound(model, get_column(model, v), [s.lower for s=sets],
                       fill(backend_type(model, Val{:Lowerbound}()), length(vs)))
end

function MOI.addconstraints!(model::LinQuadOptimizer, funcs::VecVar, sets::Vector{S}) where S <: LinSets
    # __assert_supported_constraint__(model, VecVar, S)
    # __check_for_conflicting__(model, variable, set,
    # S, MOI.Semicontinuous{Float64}, MOI.Semiinteger{Float64}, MOI.ZeroOne)
    num_vars = length(sets)
    set_variable_bound(model, funcs.variables, sets)
    # TODO we have to return some sort of constrdict here, need help!

    # model.last_constraint_reference += 1
    # index = VVCI{S}(model.last_constraint_reference)
    # dict = constrdict(model, index)
    # dict[index] = variable.variable
    # return index
end

function MOI.addconstraint!(model::LinQuadOptimizer, func::VecVar, set::S) where S <: VecLinSets
    __assert_supported_constraint__(model, VecVar, S)
    @assert length(func.variables) == MOI.dimension(set)
    rows = get_number_linear_constraints(model)
    num_vars = MOI.dimension(set)
    add_linear_constraints!(model,
        CSRMatrix{Float64}(collect(1:num_vars),
                           get_column.(Ref(model), func.variables),
                           ones(num_vars)),
        fill(backend_type(model, set), num_vars),
        zeros(num_vars)
    )
    model.last_constraint_reference += 1
    index = VVCI{S}(model.last_constraint_reference)
    dict = constrdict(model, index)
    dict[index] = collect((rows + 1):(rows + num_vars))
    append!(model.constraint_primal_solution, fill(NaN, num_vars))
    append!(model.constraint_dual_solution, fill(NaN, num_vars))
    append!(model.constraint_constant, fill(0.0, num_vars))
    return index
end

function MOI.delete!(model::LinQuadOptimizer, index::VVCI{S}) where S <: VecLinSets
    __assert_valid__(model, index)
    delete_constraint_name(model, index)
    dict = constrdict(model, index)
    # we delete rows from largest to smallest here so that we don't have
    # to worry about updating references in a greater numbered row, only to
    # modify it later.
    for row in sort(dict[index], rev=true)
        delete_linear_constraints!(model, row, row)
        deleteat!(model.constraint_primal_solution, row)
        deleteat!(model.constraint_dual_solution, row)
        deleteat!(model.constraint_constant, row)
        # shift all the other references
        shift_references_after_delete_affine!(model, row)
    end
    delete!(dict, index)
end

#=
    Get constraint set of vector variable bound
=#

MOI.canget(::LinQuadOptimizer, ::MOI.ConstraintSet, ::Type{VVCI{S}}) where S <: VecLinSets = true
function MOI.get(model::LinQuadOptimizer, ::MOI.ConstraintSet, index::VVCI{S}) where S <: VecLinSets
    S(length(model[index]))
end

#=
    Get constraint function of vector variable bound (linear ctr)
=#

MOI.canget(::LinQuadOptimizer, ::MOI.ConstraintFunction, ::Type{VVCI{S}}) where S <: VecLinSets = true
function MOI.get(model::LinQuadOptimizer, ::MOI.ConstraintFunction, index::VVCI{<: VecLinSets})
    rows = model[index]
    variables = VarInd[]
    sizehint!(variables, length(rows))
    for row in rows
        cols, coefs = get_linear_constraint(model, row)
        if length(cols) != 1
            error("Unexpected constraint")
        end
        push!(variables, model.variable_references[cols[1]])
    end
    return VecVar(variables)
end

#=
    SOS constraints
=#

function MOI.addconstraint!(model::LinQuadOptimizer, func::VecVar, set::S) where S <: Union{MOI.SOS1, MOI.SOS2}
    __assert_supported_constraint__(model, VecVar, S)
    make_problem_type_integer(model)
    add_sos_constraint!(model, get_column.(Ref(model), func.variables),
                        set.weights, backend_type(model, set))
    model.last_constraint_reference += 1
    index = VVCI{S}(model.last_constraint_reference)
    dict = constrdict(model, index)
    dict[index] = length(cmap(model).sos1) + length(cmap(model).sos2) + 1
    return index
end

function MOI.delete!(model::LinQuadOptimizer, index::VVCI{<:Union{SOS1, SOS2}})
    __assert_valid__(model, index)
    delete_constraint_name(model, index)
    dict = constrdict(model, index)
    sos_row = dict[index]
    delete_sos!(model, sos_row, sos_row)
    deleteref!(cmap(model).sos1, sos_row, index)
    deleteref!(cmap(model).sos2, sos_row, index)
    if !has_integer(model)
        make_problem_type_continuous(model)
    end
end

MOI.canget(::LinQuadOptimizer, ::MOI.ConstraintSet, ::Type{VVCI{S}}) where S <: Union{MOI.SOS1, MOI.SOS2} = true
function MOI.get(model::LinQuadOptimizer, ::MOI.ConstraintSet, index::VVCI{S}) where S <: Union{MOI.SOS1, MOI.SOS2}
    indices, weights, types = get_sos_constraint(model, model[index])
    set = S(weights)
    @assert types == backend_type(model, set)
    return set
end

MOI.canget(::LinQuadOptimizer, ::MOI.ConstraintFunction, ::Type{VVCI{S}}) where S <: Union{MOI.SOS1, MOI.SOS2} = true
function MOI.get(model::LinQuadOptimizer, ::MOI.ConstraintFunction, index::VVCI{<:Union{SOS1, SOS2}})
    indices, weights, types = get_sos_constraint(model, model[index])
    return VecVar(model.variable_references[indices])
end