#=
This file is auto-generated. Do not edit.
=#

#! format: off

"""
    mutable struct RECurrentControlB <: InnerControl
        Q_Flag::Int
        PQ_Flag::Int
        Vdip_lim::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
        T_rv::Float64
        dbd_pnts::Tuple{Float64, Float64}
        K_qv::Float64
        Iqinj_lim::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
        V_ref0::Float64
        K_vp::Float64
        K_vi::Float64
        T_iq::Float64
        I_max::Float64
        ext::Dict{String, Any}
        states::Vector{Symbol}
        n_states::Int
    end

Parameters of the Inner Control part of the REECB model in PSS/E

# Arguments
- `Q_Flag::Int`: Q Flag used for I_qinj, validation range: `(0, 1)`
- `PQ_Flag::Int`: PQ Flag used for the Current Limit Logic, validation range: `(0, 1)`
- `Vdip_lim::NamedTuple{(:min, :max), Tuple{Float64, Float64}}`: Limits for Voltage Dip Logic `(Vdip, Vup)`
- `T_rv::Float64`: Voltage Filter Time Constant, validation range: `(0, nothing)`
- `dbd_pnts::Tuple{Float64, Float64}`: Voltage error deadband thresholds `(dbd1, dbd2)`
- `K_qv::Float64`: Reactive current injection gain during over and undervoltage conditions, validation range: `(0, nothing)`
- `Iqinj_lim::NamedTuple{(:min, :max), Tuple{Float64, Float64}}`: Limits for Iqinj `(I_qh1, I_ql1)`
- `V_ref0::Float64`: User defined reference. If 0, PSID initializes to initial terminal voltage, validation range: `(0, nothing)`
- `K_vp::Float64`: Voltage regulator proportional gain (used when QFlag = 1), validation range: `(0, nothing)`
- `K_vi::Float64`: Voltage regulator integral gain (used when QFlag = 1), validation range: `(0, nothing)`
- `T_iq::Float64`: Time constant for low-pass filter for state q_V when QFlag = 0, validation range: `(0, nothing)`
- `I_max::Float64`: Maximum limit on total converter current, validation range: `(0, nothing)`
- `ext::Dict{String, Any}`
- `states::Vector{Symbol}`: The states of the RECurrentControlB depends on the Flags
- `n_states::Int`: The states of the RECurrentControlB depends on the Flags
"""
mutable struct RECurrentControlB <: InnerControl
    "Q Flag used for I_qinj"
    Q_Flag::Int
    "PQ Flag used for the Current Limit Logic"
    PQ_Flag::Int
    "Limits for Voltage Dip Logic `(Vdip, Vup)`"
    Vdip_lim::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
    "Voltage Filter Time Constant"
    T_rv::Float64
    "Voltage error deadband thresholds `(dbd1, dbd2)`"
    dbd_pnts::Tuple{Float64, Float64}
    "Reactive current injection gain during over and undervoltage conditions"
    K_qv::Float64
    "Limits for Iqinj `(I_qh1, I_ql1)`"
    Iqinj_lim::NamedTuple{(:min, :max), Tuple{Float64, Float64}}
    "User defined reference. If 0, PSID initializes to initial terminal voltage"
    V_ref0::Float64
    "Voltage regulator proportional gain (used when QFlag = 1)"
    K_vp::Float64
    "Voltage regulator integral gain (used when QFlag = 1)"
    K_vi::Float64
    "Time constant for low-pass filter for state q_V when QFlag = 0"
    T_iq::Float64
    "Maximum limit on total converter current"
    I_max::Float64
    ext::Dict{String, Any}
    "The states of the RECurrentControlB depends on the Flags"
    states::Vector{Symbol}
    "The states of the RECurrentControlB depends on the Flags"
    n_states::Int
end

function RECurrentControlB(Q_Flag, PQ_Flag, Vdip_lim, T_rv, dbd_pnts, K_qv, Iqinj_lim, V_ref0, K_vp, K_vi, T_iq, I_max, ext=Dict{String, Any}(), )
    RECurrentControlB(Q_Flag, PQ_Flag, Vdip_lim, T_rv, dbd_pnts, K_qv, Iqinj_lim, V_ref0, K_vp, K_vi, T_iq, I_max, ext, PowerSystems.get_REControlB_states(Q_Flag), 2, )
end

function RECurrentControlB(; Q_Flag, PQ_Flag, Vdip_lim, T_rv, dbd_pnts, K_qv, Iqinj_lim, V_ref0, K_vp, K_vi, T_iq, I_max, ext=Dict{String, Any}(), states=PowerSystems.get_REControlB_states(Q_Flag), n_states=2, )
    RECurrentControlB(Q_Flag, PQ_Flag, Vdip_lim, T_rv, dbd_pnts, K_qv, Iqinj_lim, V_ref0, K_vp, K_vi, T_iq, I_max, ext, states, n_states, )
end

# Constructor for demo purposes; non-functional.
function RECurrentControlB(::Nothing)
    RECurrentControlB(;
        Q_Flag=0,
        PQ_Flag=0,
        Vdip_lim=(min=0.0, max=0.0),
        T_rv=0,
        dbd_pnts=(0.0, 0.0),
        K_qv=0,
        Iqinj_lim=(min=0.0, max=0.0),
        V_ref0=0,
        K_vp=0,
        K_vi=0,
        T_iq=0,
        I_max=0,
        ext=Dict{String, Any}(),
    )
end

"""Get [`RECurrentControlB`](@ref) `Q_Flag`."""
get_Q_Flag(value::RECurrentControlB) = value.Q_Flag
"""Get [`RECurrentControlB`](@ref) `PQ_Flag`."""
get_PQ_Flag(value::RECurrentControlB) = value.PQ_Flag
"""Get [`RECurrentControlB`](@ref) `Vdip_lim`."""
get_Vdip_lim(value::RECurrentControlB) = value.Vdip_lim
"""Get [`RECurrentControlB`](@ref) `T_rv`."""
get_T_rv(value::RECurrentControlB) = value.T_rv
"""Get [`RECurrentControlB`](@ref) `dbd_pnts`."""
get_dbd_pnts(value::RECurrentControlB) = value.dbd_pnts
"""Get [`RECurrentControlB`](@ref) `K_qv`."""
get_K_qv(value::RECurrentControlB) = value.K_qv
"""Get [`RECurrentControlB`](@ref) `Iqinj_lim`."""
get_Iqinj_lim(value::RECurrentControlB) = value.Iqinj_lim
"""Get [`RECurrentControlB`](@ref) `V_ref0`."""
get_V_ref0(value::RECurrentControlB) = value.V_ref0
"""Get [`RECurrentControlB`](@ref) `K_vp`."""
get_K_vp(value::RECurrentControlB) = value.K_vp
"""Get [`RECurrentControlB`](@ref) `K_vi`."""
get_K_vi(value::RECurrentControlB) = value.K_vi
"""Get [`RECurrentControlB`](@ref) `T_iq`."""
get_T_iq(value::RECurrentControlB) = value.T_iq
"""Get [`RECurrentControlB`](@ref) `I_max`."""
get_I_max(value::RECurrentControlB) = value.I_max
"""Get [`RECurrentControlB`](@ref) `ext`."""
get_ext(value::RECurrentControlB) = value.ext
"""Get [`RECurrentControlB`](@ref) `states`."""
get_states(value::RECurrentControlB) = value.states
"""Get [`RECurrentControlB`](@ref) `n_states`."""
get_n_states(value::RECurrentControlB) = value.n_states

"""Set [`RECurrentControlB`](@ref) `Q_Flag`."""
set_Q_Flag!(value::RECurrentControlB, val) = value.Q_Flag = val
"""Set [`RECurrentControlB`](@ref) `PQ_Flag`."""
set_PQ_Flag!(value::RECurrentControlB, val) = value.PQ_Flag = val
"""Set [`RECurrentControlB`](@ref) `Vdip_lim`."""
set_Vdip_lim!(value::RECurrentControlB, val) = value.Vdip_lim = val
"""Set [`RECurrentControlB`](@ref) `T_rv`."""
set_T_rv!(value::RECurrentControlB, val) = value.T_rv = val
"""Set [`RECurrentControlB`](@ref) `dbd_pnts`."""
set_dbd_pnts!(value::RECurrentControlB, val) = value.dbd_pnts = val
"""Set [`RECurrentControlB`](@ref) `K_qv`."""
set_K_qv!(value::RECurrentControlB, val) = value.K_qv = val
"""Set [`RECurrentControlB`](@ref) `Iqinj_lim`."""
set_Iqinj_lim!(value::RECurrentControlB, val) = value.Iqinj_lim = val
"""Set [`RECurrentControlB`](@ref) `V_ref0`."""
set_V_ref0!(value::RECurrentControlB, val) = value.V_ref0 = val
"""Set [`RECurrentControlB`](@ref) `K_vp`."""
set_K_vp!(value::RECurrentControlB, val) = value.K_vp = val
"""Set [`RECurrentControlB`](@ref) `K_vi`."""
set_K_vi!(value::RECurrentControlB, val) = value.K_vi = val
"""Set [`RECurrentControlB`](@ref) `T_iq`."""
set_T_iq!(value::RECurrentControlB, val) = value.T_iq = val
"""Set [`RECurrentControlB`](@ref) `I_max`."""
set_I_max!(value::RECurrentControlB, val) = value.I_max = val
"""Set [`RECurrentControlB`](@ref) `ext`."""
set_ext!(value::RECurrentControlB, val) = value.ext = val


export PumiMesh3

mutable struct PumiMesh3{T1} <: PumiMesh3CG{T1}   # 3d pumi mesh, tetrahedron only
  m_ptr::Ptr{Void}  # pointer to mesh
  mshape_ptr::Ptr{Void} # pointer to mesh's FieldShape
  f_ptr::Ptr{Void} # pointer to apf::field for storing solution during mesh adaptation

  shape_type::Int  # type of shape functions

  vert_Nptr::Ptr{Void}  # numbering of vertices
  edge_Nptr::Ptr{Void}  # numbering of edges
  face_Nptr::Ptr{Void} # numbering of faces
  el_Nptr::Ptr{Void}  # numbering of elements (faces)

  numVert::Int  # number of vertices in the mesh
  numEdge::Int # number of edges in the mesh
  numFace::Int # number of faces in the mesh
  numEl::Int  # number of elements (regions)
  order::Int # order of shape functions
  numDof::Int # number of degrees of freedom
  numNodes::Int  # number of nodes
  numDofPerNode::Int  # number of dofs per node
  numBoundaryFaces::Int # number of faces on the exterior boundary
  numInterfaces::Int # number of internal interfaces
  numNodesPerElement::Int  # number of nodes per element
  numNodesPerType::Array{Int, 1}  # number of nodes classified on each vertex, edge, face, length 4
  numEntities::Array{Int, 1}  # number of entities of each type in the mesh, length 4
  numEntitiesPerElement::Array{Int, 1}  # number of vertices, edges, and regions in each element

  # parallel info
  comm::MPI.Comm  # MPI Communicator
  myrank::Int  # MPI rank, zero based
  commsize::Int # MPI comm size

  # hold pointers to mesh entities
  verts::Array{Ptr{Void},1}  # holds pointers to mesh vertices
  edges::Array{Ptr{Void},1}  # pointers to mesh edges
  faces::Array{Ptr{Void},1}  # pointers to mesh faces
  elements::Array{Ptr{Void},1}  # pointers to regions

  dofnums_Nptr::Ptr{Void}  # pointer to Numbering of dofs (result of reordering)
  boundary_nums::Array{Int, 2}  # array of [element number, edgenumber] for each edge on the boundary

  bndryfaces::Array{Boundary, 1}  # store data on external boundary of mesh
  interfaces::Array{Interface, 1}  # store data on internal edges

  coords::Array{T1, 3}  # store coordinates of all nodes
  dxidx::Array{T1, 4}  # store scaled mapping jacobian
  jac::Array{T1,2}  # store mapping jacobian output

  dofs::Array{Int, 3}  # store dof numbers of solution array to speed assembly

 function PumiMesh3{T1}(dmg_name::AbstractString, smb_name::AbstractString, order, sbp::AbstractSBP; dofpernode=1, shape_type=0)  where {T1}
  # construct pumi mesh by loading the files named
  # dmg_name = name of .dmg (geometry) file to load (use .null to load no file)
  # smb_name = name of .smb (mesh) file to load
  # order = order of shape functions
  # dofpernode = number of dof per node, default = 1
  # shape_type = type of shape functions to use, 0 = lagrange, 1 = SBP

  mesh = new{T1}()
  mesh.comm = MPI.COMM_WORLD
  mesh.commsize = MPI.Comm_size(MPI.COMM_WORLD)
  mesh.myrank = MPI.Comm_rank(MPI.COMM_WORLD)

  @assert mesh.commsize == 1  # not paralllezed yet

  mesh.numDofPerNode = dofpernode
  mesh.order = order
  mesh.numNodesPerElement = getNumNodes(order)
  mesh.shape_type = shape_type
  @time mesh.numEntities, mesh.m_ptr, mesh.mshape_ptr, n_arr = apf.init(dmg_name, smb_name, order, shape_type=shape_type)
  mesh.f_ptr = apf.findField(mesh.m_ptr, "solution_field")
  if mesh.f_ptr == C_NULL
    @time mesh.f_ptr = apf.createPackedField(mesh, "solution_field", dofpernode)
  end

  mesh.numVert = convert(Int, mesh.numEntities[1])
  mesh.numEdge = convert(Int,  mesh.numEntities[2])
  mesh.numFace = convert(Int, mesh.numEntities[3])
  mesh.numEl = convert(Int, mesh.numEntities[4])

  # get number of nodes on each type
  mesh.numNodesPerType = zeros(Int, 4)
  mesh.numNodesPerType[1] = apf.countNodesOn(mesh.mshape_ptr, 0)  # number of nodes on a vertex
  mesh.numNodesPerType[2]= apf.countNodesOn(mesh.mshape_ptr, 1) # on edge
  mesh.numNodesPerType[3] = apf.countNodesOn(mesh.mshape_ptr, 2) # on face
  mesh.numNodesPerType[4]= apf.countNodesOn(mesh.mshape_ptr, 4) # on element interior
  mesh.numEntitiesPerElement = [4, 6, 4, 1]

  mesh.numNodesPerElement = 0
  for i=1:4
    mesh.numNodesPerElement += mesh.numNodesPerType[i]*mesh.numEntitiesPerElement[i]
  end

  # get pointers to mesh entity numberings
  mesh.vert_Nptr = n_arr[1] #getVertNumbering()
  mesh.edge_Nptr = n_arr[2] #getEdgeNumbering()
  mesh.face_Nptr = n_arr[3] #getFaceNumbering()
  mesh.el_Nptr = n_arr[4] #getElNumbering()

   mesh.verts = Array{Ptr{Void}}(mesh.numVert)
  mesh.edges = Array{Ptr{Void}}(mesh.numEdge)
  mesh.faces = Array{Ptr{Void}}(mesh.numFace)
  mesh.elements = Array{Ptr{Void}}(mesh.numEl)
  mesh.dofnums_Nptr = apf.createNumberingJ(mesh, "reordered dof numbers",
                                           dofpernode, mesh.mshape_ptr)  # 1 dof per node

  # get pointers to all MeshEntities
  # also initilize the field to zero
  for (i, e) in enumerate(apf.MeshIterator(mesh.m_ptr, 0))
    mesh.verts[i] = e
  end

  for (i, e) in enumerate(apf.MeshIterator(mesh.m_ptr, 1))
    mesh.edges[i] = e
  end

  for (i, e) in eumerate(apf.MeshIterator(mesh.m_ptr, 2))
    mesh.faces[i] = e
  end

  for (i, e) in enumerate(apf.MeshIterator(mesh.m_ptr, 3))
    mesh.elements[i] = e
  end

  # use partially constructed mesh object to populate arrays
  checkConnectivity(mesh)

#  numberDofs(mesh)

  countBoundaryFaces(mesh)
  mesh.bndryfaces = Array{Boundary}(mesh.numBoundaryFaces)
  getBoundaryArray(mesh)

  numberDofs(mesh)
  getDofNumbers(mesh)

  countBoundaryFaces(mesh)
  mesh.bndryfaces = Array{Boundary}(mesh.numBoundaryFaces)
  getBoundaryArray(mesh)

  mesh.numInterfaces = mesh.numFace - mesh.numBoundaryFaces
  mesh.interfaces = Array{Interface}(mesh.numInterfaces)
  getInterfaceArray(mesh)

  getCoordinatesAndMetrics(mesh, sbp)
#

#=
  mesh.bndryfaces = Array{Boundary}(mesh.numBoundaryFaces)
  getBoundaryArray(mesh)

  mesh.numInterfaces = mesh.numEdge - mesh.numBoundaryFaces
  mesh.interfaces = Array{Interface}(mesh.numInterfaces)
  getInterfaceArray(mesh)

  getCoordinates(mesh, sbp)  # store coordinates of all nodes into array
  getDofNumbers(mesh)  # store dof numbers

  mesh.dxidx = Array{T1}(2, 2, sbp.numnodes, mesh.numEl)
  mesh.jac = Array{T1}(sbp.numnodes, mesh.numEl)
  mappingjacobian!(sbp, mesh.coords, mesh.dxidx, mesh.jac)
=#

  printStats(mesh)

  apf.writeVtkFiles("mesh_complete", mesh.m_ptr)
  return mesh
  end

end

function getNumNodes(order::Integer)
# get the number of nodes on an element
# assumes SBP elements

  nnodes = 0
  if order == 1
    nnodes = 4
  elseif order == 2
    nnodes = 11
  elseif order == 3
    nnodes = 24
  elseif order == 4
    nnodes = 45
  else
    println("Warning, unsupported order elements are requested")
  end

  return nnodes
end

function countBoundaryFaces(mesh::PumiMesh3)
  # count boundary face
  # store array of [element number, global edge number]
#  resetFaceIt()
  bnd_faces_cnt = 0
  bnd_faces = Array{Int}(mesh.numFace, 2)
  it = apf.MeshIterator(mesh.m_ptr, 2)
  for i=1:mesh.numFace
    face_i = apf.iterate(mesh.m_ptr, it)
    numRegion = apf.countAdjacent(mesh.m_ptr, face_i, 3)  # should be count upward

    if numRegion == 1  # if an exterior edge
      elements = apf.getAdjacent(numRegion)
      elnum = apf.getNumberJ(mesh.el_Nptr, elements[1], 0, 0) + 1

#      println("face number ", i, " is part of only one element")

      bnd_faces_cnt += 1
      bnd_faces[bnd_faces_cnt, 1] = elnum
      bnd_faces[bnd_faces_cnt, 2] = i
    end
  end
  apf.free(mesh.m_ptr, it)

  mesh.boundary_nums = bnd_faces[1:bnd_faces_cnt, :] # copy, bad but unavoidable
  mesh.numBoundaryFaces = bnd_faces_cnt

return nothing

end  # end function

function getBoundaryArray(mesh::PumiMesh3)
# get an array of type Boundary for SBP
# creating an an array of a user defined type seems like a waste of memory operations
# bnd_array is a vector of type Boundary with length equal to the number of edges on the boundary of the mesh

#  mesh.bndryfaces = Array{Boundary}(mesh.numBoundaryFaces)

  for i=1:mesh.numBoundaryFaces
    elnum = mesh.boundary_nums[i,1]
    facenum_global = mesh.boundary_nums[i,2]
    facenum_local = getBoundaryEntityLocalNum(mesh, facenum_global, 2)
    mesh.bndryfaces[i] = Boundary(elnum, facenum_local)
  end

  return nothing
end

function getBoundaryEntityLocalNum(mesh::PumiMesh3, entity_num::Integer, entity_dim::Integer)
# gets the local edge number of a specified edge that is on the boundary
# of the mesh
# entity_num is an the  num from the output of getBoundaryEdgeNums() (ie. the global entity number)
# entity_dim is the dimension of the entity (0 = vertex, 1 = edge, 2 = face)
# the local edge number is the edge number within the element (1st,s 2nd, 3rd...)
  if entity_dim == 1
    entity_i = mesh.edges[entity_num]
  elseif entity_dim == 2
    entity_i = mesh.faces[entity_num]
  else
    println("unsupported entity dimension requested for getBoundaryEntityLocalNum")
    println("requested entity dimension = ", entity_dim)
  end

  # get mesh face or region associated with edge
  apf.countAdjacent(mesh.m_ptr, entity_i, entity_dim+1)
  parent = apf.getAdjacent(1)[1]  # get the single face (not an array)

  local_num = apf.getEntityLocalNumber(mesh.m_ptr, entity_i, parent, entity_dim, entity_dim+1)

  return local_num

end

function numberDofs(mesh::PumiMesh3)
# assign dof numbers to entire mesh
# calculates numNodes, numDof
# assumes mesh elements have already been reordered
# this method minimizes the assembly/disassembly time
# because blocks of entries in res go into res_vec

  # calculate total number of nodes
  # Continuous Galerkin only
  numnodes = 0
  for i=1:4
    numnodes += mesh.numNodesPerType[i]*mesh.numEntities[i]
  end

  numDof = mesh.numDofPerNode*numnodes

  if (numDof > 2^30)
    println("Warning: too many dofs, renumbering will fail")
  end

  # save numbers to mesh
  mesh.numNodes = numnodes
  mesh.numDof = numDof

  # initally number all dofs as numDof+1 to 2*numDof
  # this allows quick check to see if somthing is labelled or not

#  resetAllIts2(mesh.m_ptr)
  # mesh iterator increment, retreval functions
#  iterators_inc = [incrementVertIt, incrementEdgeIt, incrementFaceIt, incrementElIt]
#  iterators_get = [getVert, getEdge, getFace, getEl]
  num_entities = mesh.numEntities
  num_nodes_entity = mesh.numNodesPerType

  #  curr_dof = 1
  curr_dof = numDof+1
  for etype = 1:length(num_nodes_entity) # loop over entity types
    if (num_nodes_entity[etype] != 0)  # if no nodes on this type of entity, skip
      it = apf.MeshIterator(mesh.m_ptr, etype - 1)
      for entity = 1:num_entities[etype]  # loop over all entities of this type
#	println("  entity number: ", entity)
        entity_ptr = apf.iterate(mesh.m_ptr, it)
#	entity_ptr = iterators_get[etype]()  # get entity

	for node = 1:num_nodes_entity[etype]
#	  println("    node : ", node)
	  for dof = 1:mesh.numDofPerNode
	    apf.numberJ(mesh.dofnums_Nptr, entity_ptr, node-1, dof-1, curr_dof)
#	    println("      entity ", entity_ptr, " labelled ", curr_dof)
	    curr_dof += 1
	  end  # end loop over dof
	end  # end loop over node

#      iterators_inc[etype]()
      end  # end loop over entitiesaa
      apf.free(mesh.m_ptr, it)
    end  # end if
  end  # end loop over entity types

  # final dof numbering
# move all if statements out one for loop (check only first dof on each node)
  curr_dof = 1
  for i=1:mesh.numEl
#    println("element number: ", i)

    el_i = mesh.elements[i]
    # get vertices, edges for this element
    (verts_i, numVert) = apf.getDownward(mesh.m_ptr, mesh.elements[i], 0)
#    println("verts_i = ", verts_i)
#    println("mesh.verts = ", mesh.verts)
    (edges_i, numEdge) = apf.getDownward(mesh.m_ptr, mesh.elements[i], 1)

    if mesh.numEntitiesPerElement[3] > 1  # don't call get downward if el_i is a face
      (face_i, numFace) = apf.getDownward(mesh.m_ptr, mesh.elements[i], 2)
      regions_i = [mesh.elements[i]]  # make this an array for consistency
      numRegions = 1
    else  # this is a 2d mesh
      face_i = [mesh.elements[i]]
      numFace = 1
      numRegion = 0
    end

    # loop over vertices
    for j=1:numVert  # loop over vertices, edges
#      println("  vertex number: ", j)
      vert_j = verts_i[j]
#      println("  vert_j = ", vert_j)
#      println("  edge_j = ", edge_j)
      for k=1:mesh.numDofPerNode # loop over vertex dofs
#	println("    dof number: ", k)
#	println("    vert_j = ", vert_j)
        dofnum_k = apf.getNumberJ(mesh.dofnums_Nptr, vert_j, 0, k-1)
	if dofnum_k > numDof  # still has initial number
	  # give it new (final) number
	  apf.numberJ(mesh.dofnums_Nptr, vert_j, 0, k-1, curr_dof)
#	  println("numbering a vertex ", curr_dof)
	  curr_dof += 1
	end
      end  # end loop over vertex dofs
    end  # end loop over vertices

    # now loop over edges
    for j=1:numEdge
#      println("edge number: ", j)
      edge_j = edges_i[j]
      # loop over nodes on edge
      for k=1:num_nodes_entity[2]  # loop over all nodes on the edge
	for p=1:mesh.numDofPerNode  # loop over dofs
	  dofnum_p = apf.getNumberJ(mesh.dofnums_Nptr, edge_j, k-1, p-1)
	  if dofnum_p > numDof  # still has initial number
	    # give it new (final) number
	    apf.numberJ(mesh.dofnums_Nptr, edge_j, k-1, p-1, curr_dof)
	    curr_dof += 1
	  end
	end
      end  # en loop over nodes on edge
    end  # end loop over edges

    # now loop over faces
    for j=1:numFace
#      println("face number: ", j)
      face_j = face_i[j]
      for k=1:num_nodes_entity[3]  # loop over nodes on faces
	for p=1:mesh.numDofPerNode  # loop over dofs
	  dofnum_p = apf.getNumberJ(mesh.dofnums_Nptr, face_j, k-1, p-1)
	  if dofnum_p > numDof
	    apf.numberJ(mesh.dofnums_Nptr, face_j, k-1, p-1, curr_dof)
	    curr_dof += 1
	  end
	end
      end  # end loop over face nodes
    end  # end loop over faces

    # now loop over region
    for j=1:numRegions
#      println("region number: ", j)
      region_j = regions_i[j]
      for k=1:num_nodes_entity[4]
	for p=1:mesh.numDofPerNode
	  dofnum_p = apf.getNumberJ(mesh.dofnums_Nptr, region_j, k-1, p-1)
	  if dofnum_p > numDof
	    apf.numberJ(mesh.dofnums_Nptr, region_j, k-1, p-1, curr_dof)
	    curr_dof += 1
	  end
	end
      end  # end loop over nodes on the region
    end  # end loop over region

  end  # end loop over elements

  if (curr_dof -1) != numDof
    println("Warning: number of dofs assigned is not equal to teh expected number")
    println("number of dofs assigned = ", curr_dof-1, " ,expected number = ", numDof)
  end

#  resetAllIts2(mesh.m_ptr)
  return nothing

end

function getDofNumbers(mesh::PumiMesh3)
# populate array of dof numbers, in same shape as solution array u (or q)

mesh.dofs = Array{Int}(mesh.numDofPerNode, mesh.numNodesPerElement, mesh.numEl)

for i=1:mesh.numEl
  dofnums = getGlobalNodeNumbers(mesh, i)

  for j=1:mesh.numNodesPerElement
    for k=1:mesh.numDofPerNode  # loop over dofs on the node
      mesh.dofs[k, j, i] = dofnums[k,j]
    end
  end
end

#println("mesh.dof = ", mesh.dofs)

return nothing

end

function getGlobalNodeNumbers(mesh::PumiMesh3, elnum::Integer)
# gets global node numbers of all dof on all nodes of the element
# output formap is array [numdofpernode, nnodes]  (each column contains dof numbers for a node)
el_i = mesh.elements[elnum]
#type_i = apf.getType(mesh.m_ptr, el_i)

# use apf::getElementNumbers to ensure everything is in the proper orientation

  nnodes = mesh.numNodesPerElement
  numdof = nnodes*mesh.numDofPerNode
  nums = zeros(Int32, numdof)

apf.getElementNumbers(mesh.dofnums_Nptr, el_i, numdof, nums::Array{Int32, 1})

# reshape into output format
# underlying memory is shared
dofnums_reshaped = reshape(nums, mesh.numDofPerNode, nnodes)

return dofnums_reshaped

end

function getCoordinates(mesh::PumiMesh3, sbp::AbstractSBP)
# populate the coords array of the mesh object

mesh.coords = Array{Float64}(3, sbp.numnodes, mesh.numEl)

println("entered getCoordinates")

coords_i = zeros(3, 4)
coords_it = zeros(4, 3)
for i=1:mesh.numEl  # loop over elements

  el_i = mesh.elements[i]
  (sizex, sizey) = size(coords_i)
  apf.getElCoords(mesh.m_ptr, el_i, coords_i, sizex, sizey)  # populate coords

#  println("coords_i = ", coords_i)

  coords_it[:,:] = coords_i.'
 # println("coords_it = ", coords_it)
  mesh.coords[:, :, i] = calcnodes(sbp, coords_it)
#  println("mesh.coords[:,:,i] = ", mesh.coords[:,:,i])
end

return nothing

end

function getInterfaceArray(mesh::PumiMesh3)
# get array of [elementL, elementR, faceL, faceR] for each internal face,
# where elementL and R are the elements that use the face, faceL R are the
# local number of the edge within the element
# interfaces is the array to be populated with the data, it must be
# number of internal edges by 1

  # only need internal boundaries (not external)
#  num_ext_edges = size(getBoundaryEdgeNums(mesh))[1]  # bad memory efficiency
#  num_int_edges = getNumEdges(mesh) - num_ext_edges

#  new_bndry = Boundary(2, 3)
#   println("new_bndry = ", new_bndry)

#  new_interface = Interface(1, 2, 3, 4)
#   println("new_interface = ", new_interface)

#   println("num_int_edges = ", num_int_edges)

#  interfaces = Array{typeof(new_interface}(num_int_edges)
  # get number of nodes affecting an edge
  num_face_nodes = apf.countAllNodes(mesh.mshape_ptr, 2)

  # store whether the node map has been calculated for each case
  # index = relative rotation of faces + 1
  nodemap_mask = zeros(Bool, 3)
  nodemaps = Array{Array{Uint8,1}}(3)

#  nodemap = collect(num_edge_nodes:(-1):1)

  pos = 1 # current position in interfaces
  for i=1:mesh.numFace
#     println("i = ", i)
#     println("pos = ", pos)
    # get number of elements using the edge
    adjacent_nums, num_adjacent = getAdjacentEntityNums(mesh, i, 2, 3)
#     println("num_adjacent = ", num_adjacent)
#     println("adjacent_nums = ", adjacent_nums)
    if num_adjacent > 1  # internal edge
#       println("this is an internal edge")
      element1 = adjacent_nums[1]
      element2 = adjacent_nums[2]

#      coords_1 = x[:, :, element1]
#      coords_2 = x[:, :, element2]

      coords_1 = zeros(3,4)
      coords_2 = zeros(3,4)
      apf.getElCoords(mesh.m_ptr, mesh.elements[element1], coords_1, 3, 4)
      apf.getElCoords(mesh.m_ptr, mesh.elements[element2], coords_2, 3, 4)

      # calculate centroid
      centroid1 = sum(coords_1, 2)
      centroid2 = sum(coords_2, 2)

      if abs(centroid1[1] - centroid2[2]) < 1e-10  # if big enough difference
        if centroid1[1] < centroid2[1]
    elementL = element1
    elementR = element2
        else
    elementL = element2
    elementR = element1
        end
      else  # use y coordinate to decide
        if centroid1[2] < centroid2[2]
    elementL = element1
    elementR = element2
        else
    elementL = element2
    elementR = element1
        end
      end

      faceL = apf.getEntityLocalNumber(mesh.m_ptr, mesh.faces[i], mesh.elements[elementL], 2, 3)
      faceR = apf.getEntityLocalNumber(mesh.m_ptr, mesh.faces[i], mesh.elements[elementR], 2, 3)
#      edgeL = getFaceLocalNum(mesh, i, elementL)
#      edgeR = getFaceLocalNum(mesh, i, elementR)

      # construct nodemap if needed
      rel_rotate = calcRelRotate(mesh, elementL, elementR, i)
      if !nodemap_mask[rel_rotate + 1]  # if nodemap not yet constructed
	nodemaps[rel_rotate + 1] = constructNodemap(mesh, rel_rotate)
	nodemap_mask[rel_rotate + 1] = true
      end

      mesh.interfaces[pos] = Interface(elementL, elementR, faceL, faceR, nodemaps[rel_rotate + 1])
#       println("updating pos")
      pos += 1

  #    print("\n")

    end  # end if internal edge

#     print("\n")
  end  # end loop over edges

  return nothing

end  # end function

function calcRelRotate(mesh::PumiMesh3, elementL::Integer, elementR::Integer, facenum::Integer)
# calculate the rotation of faceR relative to faceL, where faceL and faceR
# are shared between the two element
# this uses getAligmnet, which describes how to transform the face from
# its current orientation to the canonical orientation as part of its parent
# element
# the difference in the required rotations is the relative rotation
# elementL : global element index
# elementR : global element index
# faceL : local face number of shared face on elementL
# faceR : local face number of shared face on elementR
# face : global face number of the shared face

eL = mesh.elements[elementL]
eR = mesh.elements[elementR]
face = mesh.faces[facenum]

# get rotations to bring faces into canonical orientation
# for each element
# we can ignore flip because one of the faces will always be flipped
which, flipL, rotateL = apf.getAlignment(mesh.m_ptr, eL, face)
which, flipR, rotateR = apf.getAlignment(mesh.m_ptr, eR, face)

if flipL && flipR
  println("Warning both faces are flipped, nodemap may  be incorrect")
end

# sum rotations because they each rotate ccw in their parent
# element's orientation, so they rotate in opposite directions
rel_rotate = rotateR + rotateL
#println("rel_rotate before wrapping = ", rel_rotate)
rel_rotate = wrapNumber(rel_rotate, 0, 2)
#println("rel_rotate after wrapping = ", rel_rotate)

return rel_rotate

end  # end function

function constructNodemap(mesh::PumiMesh3, rel_rotate::Integer)
# calculate node map from elementL in canonical order to the canonical order
# on elementR
# both orders are canonical because the dofs array ensure everything
# is in canonical order for the element
# element must be a tetrahedron
# make nodemap UInt8?
# calculate total number of nodes on a face
num_nodes_face = 0

numEntitiesPerFace = [3, 3, 1]

for i=1:3
  num_nodes_face += numEntitiesPerFace[i]*mesh.numNodesPerType[i]
end

# get number of nodes classified on edges and face
nnodes_edge = mesh.numNodesPerType[2]
nnodes_face = mesh.numNodesPerType[3]

nodemap = zeros(Uint8, num_nodes_face)

for i=1:3  # loop over verts
  index_i = wrapNumber(i + rel_rotate, 1, 3)  # correct for relative rotation
#  println("after first wrap, index_i = ", index_i)
  index_i = 5 - index_i  # permuate because one triangle is flipped
#  println("pre wrapping, vertex index_i = ", index_i)
  index_i = wrapNumber(index_i, 1, 3)
#  println("post wrapping, vert index_i = ", index_i)
  nodemap[i] = index_i
end

for i=1:3  # loop over edges
  edgeR = 3 - i + 1 - rel_rotate

#  println("before wrap, edgeR = ", edgeR)
  edgeR = wrapNumber(edgeR, 1, 3)
#  println("after wrap, edgeR = ", edgeR)
  for j=1:nnodes_edge
    nodemap_index = 3 + nnodes_edge*(i-1) + j
    nodemap_val = 3 + nnodes_edge*(edgeR-1) + (nnodes_edge -j + 1)
#    println("nodemap_index = ", nodemap_index)
#    println("nodemap_val = ", nodemap_val)
    nodemap[nodemap_index] = nodemap_val
  end
end

#=
# threshold = location of wraparound
threshold = rel_rotate + 1
sum1 = 2 + (mesh.order - 2)*rel_rotate  # index sum below threshold
sum2 = 2 + (mesh.order - 2)*rel_rotate + nnodes_face  # index sum above threshold
=#

if ((rel_rotate % 2) == 0)
  threshold = rel_rotate + 1
  sum1 = nnodes_face - 1
  sum2 = 2*nnodes_face - 1
else  # rel_rotate is odd
  threshold = 0  # always above threshold
  sum1 = 0  # irrelevent
  sum2 = nnodes_face + 1
end

if mesh.order == 3
  threshold = 0
  sum1 = 2  # irrelevent
  sum2 = 2
end

if mesh.order == 4
  if rel_rotate == 0
    threshold = 1
    sum1 = 2
    sum2 = 5
  elseif rel_rotate == 1
    threshold == 0
    sum1 = 4  # irrelevent
    sum2 = 4
  else  # rel_rotate = 2
    threshold = 2
    sum1 = 3
    sum2 = 5
  end
end

nodemap_index = 3 + 3*nnodes_edge
for j=1:nnodes_face  # loop over interior face points
  if j <= threshold  # choose which sum to use
    sum = sum1
  else
    sum = sum2
  end
#  println("face loop j = ", j, " sum = ", sum)
  nodemap[nodemap_index + j] = nodemap_index + sum - j
end

return nodemap

end


using Mimi
using Test
using DataFrames
using VegaLite
using Electron

import Mimi:
    dataframe_or_scalar, _spec_for_item, menu_item_list, getdataframe, dim_names, time_labels

# Helper function returns true if VegaLite is verison 3 or above, and false otherwise
function _is_VegaLite_v3()
    return isdefined(VegaLite, :vlplot) ? true : false
end

include("../examples/compositecomp-model.jl") # constructs and runs model
# m's structure is as follows:
#
#          top
#        /    \
#       A       B
#     /  \     /  \
#    1    2   3    4

# 1.  dataframe helper functions
@test typeof(dataframe_or_scalar(m, :top, :fooA1)) == Float64 # same for fooA2, foo3, foo4
@test typeof(dataframe_or_scalar(m, :top, :var_3_1)) == DataFrame
@test typeof(dataframe_or_scalar(m, :top, :par_1_1)) == DataFrame

#2.  Specs and menu
items = [:fooA1, :fooA2, :foo3, :foo4, :var_3_1, :par_1_1]
for item in items
    static_spec = _spec_for_item(m, :top, item; interactive = false)
    interactive_spec = _spec_for_item(m, :top, item)
    if length(dim_names(m, :top, item)) == 0
        name =  string(:top, " : ", item, " = ", m[:top, item])
    else
        name = string(:top, " : ", item)
    end
    @test static_spec["name"] == interactive_spec["name"] == name
end

s = menu_item_list(m)
@test collect(keys(s)) == ["pars", "vars"]
@test length(collect(keys(s["pars"]))) == 5
@test length(collect(keys(s["vars"]))) == 1

#3.  explore(m::Model)
w = explore(m)
@test typeof(w) == Electron.Window
close(w)

#4.  Mim.plot(m::Model, comp_name::Symbol, datum_name::Symbol;
#       dim_name::Union{Nothing, Symbol} = nothing)
items = [:fooA1, :fooA2, :foo3, :foo4, :var_3_1, :par_1_1]
for item in items
    p_type = _is_VegaLite_v3() ? VegaLite.VLSpec : VegaLite.VLSpec{:plot}
    @test typeof(Mimi.plot(m, :top, item)) == p_type
end


# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/Materials.jl/blob/master/LICENSE

module Materials

abstract type AbstractMaterial end
abstract type AbstractMaterialState end

export AbstractMaterial, AbstractMaterialState
export integrate_material!, update_material!, reset_material!

"""
    :+(a::T, b::T) where T <: AbstractMaterialState

Fieldwise addition for material states.
"""
@generated function Base.:+(a::T, b::T) where T <: AbstractMaterialState
   expr = [:(a.$p + b.$p) for p in fieldnames(T)]
   return :(T($(expr...)))
end

"""
    integrate_material!(material::AbstractMaterial)

Integrate one timestep. The input `material.variables` represents the old
problem state.

Abstract method. Must be implemented for each material type. When integration is
done, the method **must** write the new state into `material.variables_new`.

**Do not** write into `material.variables`; actually committing the timestep
(i.e. accepting that one step of time evolution and applying it permanently)
is the job of `update_material!`.
"""
function integrate_material!(material::M) where M <: AbstractMaterial
    error("One needs to define how to integrate material $M!")
end

"""
    update_material!(material::AbstractMaterial)

Commit the result of `integrate_material!`.

In `material`, we add `ddrivers` into `drivers`, `dparameters` into
`parameters`, and replace `variables` by `variables_new`. Then we
automatically invoke `reset_material!`.
"""
function update_material!(material::AbstractMaterial)
    material.drivers += material.ddrivers
    material.parameters += material.dparameters
    material.variables = material.variables_new
    reset_material!(material)
    return nothing
end

"""
    reset_material!(material::AbstractMaterial)

In `material`, we zero out `ddrivers`, `dparameters` and `variables_new`. This
clears out the tentative state produced when a timestep has been computed, but
has not yet been committed.

Used internally by `update_material!`.
"""
function reset_material!(material::AbstractMaterial)
    material.ddrivers = typeof(material.ddrivers)()
    material.dparameters = typeof(material.dparameters)()
    material.variables_new = typeof(material.variables_new)()
    return nothing
end

include("utilities.jl")
using .Utilities
export Symm2, Symm4
export delta, II, IT, IS, IA, IV, ID, isotropic_elasticity_tensor, isotropic_compliance_tensor
export lame, delame, debang, find_root

include("perfectplastic.jl")
using .PerfectPlasticModule
export PerfectPlastic, PerfectPlasticDriverState, PerfectPlasticParameterState, PerfectPlasticVariableState

include("chaboche.jl")
using .ChabocheModule
export Chaboche, ChabocheDriverState, ChabocheParameterState, ChabocheVariableState

include("memory.jl")
using .MemoryModule
export Memory, MemoryDriverState, MemoryParameterState, MemoryVariableState

include("DSA.jl")
using .DSAModule
export DSA, DSADriverState, DSAParameterState, DSAVariableState

include("increments.jl")
using .Increments
export uniaxial_increment!, biaxial_increment!, stress_driven_uniaxial_increment!,
       general_increment!, stress_driven_general_increment!, find_dstrain!

end


#####
##### AlphaZero.jl
##### Jonathan Laurent, Carnegie Mellon University (2019-2020)
#####

module AlphaZero

# Submodules
export MCTS, MinMax, GameInterface, GI, Report, Network, Benchmark
# AlphaZero parameters
export Params, SelfPlayParams, LearningParams, ArenaParams
export MctsParams, MemAnalysisParams
export SamplesWeighingPolicy, CONSTANT_WEIGHT, LOG_WEIGHT, LINEAR_WEIGHT
export AbstractSchedule, ConstSchedule, PLSchedule, StepSchedule
# Players and games
export AbstractGame, AbstractPlayer
export think, select_move, reset_player!, player_temperature, apply_temperature
export play_game, interactive!
export MctsPlayer, RandomPlayer, EpsilonGreedyPlayer, NetworkPlayer, Human
export ColorPolicy, ALTERNATE_COLORS, BASELINE_WHITE, CONTENDER_WHITE
# Networks
export AbstractNetwork, OptimiserSpec, Nesterov, CyclicNesterov, Adam
export SimpleNet, SimpleNetHP, ResNet, ResNetHP
# Training environments
export Env, train!, get_experience
# User interface
export UserInterface
export Session, resume!, save, play_interactive_game
export Explorer, start_explorer

include("util.jl")
import .Util
using .Util: Option, @unimplemented, apply_temperature

include("game.jl")
using .GameInterface
const GI = GameInterface

include("mcts.jl")
import .MCTS

include("networks/network.jl")
using .Network

using Base: @kwdef
using DataStructures: CircularBuffer
using Distributions: Categorical, Dirichlet
using Statistics: mean

include("schedule.jl")
include("params.jl")
include("report.jl")
include("memory.jl")
include("learning.jl")
include("play.jl")
include("training.jl")

include("minmax.jl")
import .MinMax

include("benchmark.jl")
using .Benchmark

# We provide a library of predefined network architectures for convenience.
# Right now, it is included in the main AlphaZero.jl package. In the future,
# we may want to separate it so as to drop the Knet and Flux dependencies.

const USE_KNET_FOR_NETLIB = true # The Flux netlib is currently broken

if USE_KNET_FOR_NETLIB
  @eval begin
    include("networks/knet.jl")
    using .KNets
  end
else
  @eval begin
    include("networks/flux.jl")
    using .FluxNets
  end
end

# The default user interface is included here for convenience but it could be
# replaced or separated from the main AlphaZero.jl package (which would also
# enable dropping some dependencies such as Crayons or JSON3).

include("ui/ui.jl")
using .UserInterface

end


# Categorical distribution that works with arbitrary objects (Rather than consecutive ints)
struct ObjectCategorical
    cat::Categorical
    objs
    ObjectCategorical(objs::T) where {T <: AbstractArray} = new(Categorical(length(objs)), objs)
    ObjectCategorical(objs::T) where {T <: Tuple} = new(Categorical(length(objs)), [objs...])
end

Base.rand(d::ObjectCategorical, sz::Int...) = d.objs[rand(d.cat, sz...)]
function Distributions.logpdf(d::ObjectCategorical, x)
    if x isa AbstractArray
        x = mapslices(findfirst, d.objs .== reshape(x, 1,:), dims=1)
        return logpdf.(d.cat, x)
    else
        x = findfirst(d.objs .== x)
        return logpdf(d.cat, x)
    end

end

# Constant Layer
struct ConstantLayer{T}
    vec::T
end

Flux.@functor ConstantLayer
(m::ConstantLayer)(x::AbstractArray) = m.vec

## Useful functions
whiten(v) = whiten(v, mean(v), std(v))
whiten(v, μ, σ) = (v .- μ) ./ σ

to2D(W) = reshape(W, :, size(W, ndims(W))) # convert a multidimensional weight matrix to 2D

# Weighted mean aggregator
weighted_mean(weights) = (y) -> mean(y .* weights)

function LinearAlgebra.norm(grads::Flux.Zygote.Grads; p::Real = 2)
    v = []
    for θ in grads.params
        !isnothing(grads[θ]) && push!(v, norm(grads[θ] |> cpu, p))
    end
    norm(v, p)
end

## Early stopping
# Early stopping function that terminates training on validation error increase
function stop_on_validation_increase(π, 𝒫, 𝒟_val, loss; window=5)
    k = "validation_error"
    (infos) -> begin
        ve = loss(π, 𝒫, 𝒟_val) # Compute the validation error
        infos[end][k] = ve # store it
        N = length(infos)
        if length(infos) >= 2*window
            curr_window = mean([infos[i][k] for i=N-window+1:N])
            old_window = mean([infos[i][k] for i=N-2*window+1:N-window])
            return curr_window >= old_window # check if the error has gone up
        end
        false
    end
end

## Losses
function td_loss(;loss=Flux.mse, name=:Qavg, s_key=:s, a_key=:a, weight=nothing)
    (π, 𝒫, 𝒟, y; info=Dict()) -> begin
        Q = value(π, 𝒟[s_key], 𝒟[a_key])

        # Store useful information
        ignore() do
            info[name] = mean(Q)
        end

        loss(Q, y, agg = isnothing(weight) ? mean : weighted_mean(𝒟[weight]))
    end
end

function double_Q_loss(;name1=:Q1avg, name2=:Q2avg, kwargs...)
    l1 = td_loss(;name=name1, kwargs...)
    l2 = td_loss(;name=name2, kwargs...)

    (π, 𝒫, 𝒟, y; info=Dict()) -> begin
        .5f0*(l1(π.C.N1, 𝒫, 𝒟, y, info=info) + l2(π.C.N2, 𝒫, 𝒟, y, info=info))
    end
end

function multi_td_loss(;names, indices=1:length(names), kwargs...)
    ls = [td_loss(;name=name, kwargs...) for name in names]

    (π, 𝒫, 𝒟, ys; info=Dict()) -> begin
        mean([loss(net, 𝒫, 𝒟, y, info=info) for (loss, net, y) in zip(ls, π.networks[indices], ys)])
    end
end

td_error(π, 𝒟, y) = abs.(value(π, 𝒟[:s], 𝒟[:a])  .- y)

## Scheduling
struct LinearDecaySchedule{R<:Real} <: Function
    start::R
    stop::R
    steps::Int
end

function (schedule::LinearDecaySchedule)(i)
    rate = (schedule.start - schedule.stop) / schedule.steps
    val = schedule.start - i*rate
    val = max(schedule.stop, val)
end

function MultitaskDecaySchedule(steps::Int, task_ids; start = 1.0, stop = 0.1)
    schedule = LinearDecaySchedule(start, stop, steps)
    function val(i)
        taskindex = ceil(Int, i / steps)
        taskindex < 1 && return start
        taskindex > length(task_ids) && return stop
        taskid = task_ids[taskindex]
        used_steps = steps*sum(task_ids[1:taskindex-1] .== taskid)
        schedule(used_steps + mod1(i, steps))
    end
end

function logcompσ(x::Real)
    # Computes log(1 - sigmoid(x)) in a numerically stable way
    return logσ(x) - x
end


# tools for the regularization of the system.

# update regularization values in classic mode if there is a failure during factorization
function update_regu_trycatch!(regu, cnts, T, T0)
  T == Float32 && return 1
  T0 == Float128 && T == Float64 && return 1
  if cnts.c_pdd == 0 && cnts.c_catch == 0
    regu.δ *= T(1e2)
    regu.δ_min *= T(1e2)
    regu.ρ *= T(1e5)
    regu.ρ_min *= T(1e5)
  elseif cnts.c_pdd == 0 && cnts.c_catch != 0
    regu.δ *= T(1e1)
    regu.δ_min *= T(1e1)
    regu.ρ *= T(1e0)
    regu.ρ_min *= T(1e0)
  elseif cnts.c_pdd != 0 && cnts.c_catch == 0
    regu.δ *= T(1e5)
    regu.δ_min *= T(1e5)
    regu.ρ *= T(1e5)
    regu.ρ_min *= T(1e5)
  else
    regu.δ *= T(1e1)
    regu.δ_min *= T(1e1)
    regu.ρ *= T(1e1)
    regu.ρ_min *= T(1e1)
  end
  return 0
end

function update_regu!(regu)
  if regu.δ >= regu.δ_min
    regu.δ /= 10
  end
  if regu.ρ >= regu.ρ_min
    regu.ρ /= 10
  end
end

# update regularization, and corrects if the magnitude of the diagonal of the matrix is too high
function update_regu_diagK2!(regu, K_nzval, diagind_K, nvar, pdd, l_pdd, mean_pdd, cnts, T, T0)
  l_pdd[cnts.k % 6 + 1] = pdd
  mean_pdd = mean(l_pdd)

  if T == Float64 &&
     cnts.k > 10 &&
     mean_pdd != zero(T) &&
     std(l_pdd ./ mean_pdd) < T(1e-2) &&
     cnts.c_pdd < 5
    regu.δ_min /= 10
    regu.δ /= 10
    cnts.c_pdd += 1
  end
  if T == Float64 &&
     cnts.k > 10 &&
     cnts.c_catch <= 1 &&
     @views minimum(K_nzval[diagind_K[1:nvar]]) < -one(T) / regu.δ / T(1e-6)
    regu.δ /= 10
    regu.δ_min /= 10
    cnts.c_pdd += 1
  elseif T != T0 &&
         cnts.c_pdd < 2 &&
         @views minimum(K_nzval[diagind_K[1:nvar]]) < -one(T) / regu.δ / T(1e-5)
    return 1
  elseif T == Float128 &&
         cnts.k > 10 &&
         cnts.c_catch <= 1 &&
         @views minimum(K_nzval[diagind_K[1:nvar]]) < -one(T) / regu.δ / T(1e-15)
    regu.δ /= 10
    regu.δ_min /= 10
    cnts.c_pdd += 1
  end

  update_regu!(regu)

  return 0
end

function update_regu_diagK2_5!(regu, D, pdd, l_pdd, mean_pdd, cnts, T, T0)
  l_pdd[cnts.k % 6 + 1] = pdd
  mean_pdd = mean(l_pdd)

  if T == Float64 &&
     cnts.k > 10 &&
     mean_pdd != zero(T) &&
     std(l_pdd ./ mean_pdd) < T(1e-2) &&
     cnts.c_pdd < 5
    regu.δ_min /= 10
    regu.δ /= 10
    cnts.c_pdd += 1
  end
  if T == Float64 &&
     cnts.k > 10 &&
     cnts.c_catch <= 1 &&
     @views minimum(D) < -one(T) / regu.δ / T(1e-6)
    regu.δ /= 10
    regu.δ_min /= 10
    cnts.c_pdd += 1
  elseif T != T0 && cnts.c_pdd < 2 && @views minimum(D) < -one(T) / regu.δ / T(1e-5)
    return 1
  elseif T == Float128 &&
         cnts.k > 10 &&
         cnts.c_catch <= 1 &&
         @views minimum(D) < -one(T) / regu.δ / T(1e-15)
    regu.δ /= 10
    regu.δ_min /= 10
    cnts.c_pdd += 1
  end

  update_regu!(regu)

  return 0
end


################################################################################
# Visuals
################################################################################
function visualize(env::Environment, traj::Vector{Vector{T}}) where T
	@assert size(traj[1]) == size(env.state)
    storage = generate_storage(env.mechanism, [env.representation == :minimal ? minimal_to_maximal(env.mechanism, x) : x for x in traj])
    visualize(env.mechanism, storage,
        vis=env.vis)
end

################################################################################
# Miscellaneous
################################################################################
type2symbol(H) = Symbol(lowercase(String(H.name.name)))

function get_control_mask(n_inputs, indices)
    n_outputs = length(indices)
    m = zeros(n_outputs, n_inputs)
    for (i,ind) in enumerate(indices)
        m[i,ind] = 1.0
    end
    return m
end


#using Revise
module Endowment
using Revise

if  pwd() ∉ LOAD_PATH
  push!(LOAD_PATH,pwd())
end

if "$(pwd())\\data" ∉ LOAD_PATH
  push!(LOAD_PATH,"$(pwd())\\data")
end

#TODO (maybe): Refactor for metadata of processed data fields
#Revert CSV from master (test once this is done)
#Use this to print
#=
using Weave
codeName = "train"
weave(Pkg.dir("$(pwd())\\$(codeName).jl"),
  informat="script",
  out_path = "$(pwd())\\output\\$(codeName)_Appendix.html",
  doctype = "md2html")
=#

#############Dependencies####################
using DataFrames, Distributions, GZip, Dates,
   Measures, Formatting, HTTP, ODBC, Statistics,CodecZlib,
  StatsBase, Distributions, Finometrics, Distributed,
  Serialization, uCSV, CSV, #, StatsPlots,#, PlotlyJS#, UnicodePlots
   Gadfly, Cairo#, Fontconfig#FileIO, CSVFiles

Gadfly.push_theme(:default)
#plotlyjs()

##################Exports########################
export initializePar, ###Acquire NCCSData
cleanup,
downloadNCCSCSVs,
preProcessData,
validateSingleTypeCSV,
readSingleTypeCSV,
readMetaIndex,
formMetaData,
MetaTuple,
MetaDictionary,
NCCSFile,
NCCS_PATH,
downloadIRSPDFs,
processNCCS,
NCCSData,
constructPrimaryOutput,
reportandanalyze,
getfielddf

#############################Shared Constants###########
const TEX_PATH = "C:\\Users\\Clinton\\Dropbox\\Apps\\Overleaf\\Endowment Project"
const WORKING_PATH = pwd() * "\\working"
const DATA_PATH = pwd() * "\\data"
const NCCS_PATH = DATA_PATH * "\\NCCS"
const DOCUMENTATION_PATH = DATA_PATH * "\\documentation"
const IRS_PATH = DOCUMENTATION_PATH * "\\irs"
const FRED_PATH = DATA_PATH * "\\FRED"

const WRDS_PATH = DATA_PATH * "\\WRDS"
const WRDS_DATE_FORMAT = "yyyy-mm-dd"
const REFRESH_WRDS = false

const FISCAL_DATE_FORMAT = "yyyymm"
const IRS_SOI_PATH = DATA_PATH * "\\IRSSOI"

const FF_PATH = DATA_PATH * "\\FF"
const FF_NAME = "FF"
const FF_DATE_FORMAT = "yyyymm"

const OPEN990_PATH = DATA_PATH * "\\open990"
const OPEN990_FIELD_PATH = OPEN990_PATH * "\\fieldfiles"
const OPEN990_DATE_FORMAT = "yyyymm"
const OPEN990_FIELD_INDEX_NAME = "fieldfileindex"
const OPEN990_FIELD_DATA_NAME = "fieldfile"
#const OPEN990_SUBMITTED__ON_DATE_FORMAT = "m/d/yyyy"

const INDUSTRY_CROSS_YEAR = 2013
const COMPUSTAT_NAME = "$INDUSTRY_CROSS_YEAR-COMPUSTAT"
const SOI_NAME = "$INDUSTRY_CROSS_YEAR-SOI"
const INDUSTRY_COMPARISON_SOURCE = #=:wrds =# :soiwrds

#reqquire all asset levels above this mark (set to $10_000_000 for testing, 50,000 breadth)
#NOTE: the below is the main specification. DO NOT DELETE!
const REVENUE_LOWER_BOUND = 500_000.0
const WEALTH_LOWER_BOUND = 1_000_000.0

#NOTE: the below specification is mostly for the wealth distribution graphs
#const REVENUE_LOWER_BOUND = 100_000.0
#const WEALTH_LOWER_BOUND = 1.0

#NOTE: the below is for testing
#const REVENUE_LOWER_BOUND = 10_000_000.0
#const WEALTH_LOWER_BOUND = 10_000_000.0

const DATA_SUFFIX = ("W$(Int(round(WEALTH_LOWER_BOUND/1000)))" *
  "R$(Int(round(REVENUE_LOWER_BOUND/1000)))")

# NOTE: Below limit leads to a hard drop for records below this mark.
# Roughly deflation adjusted 1988(EOY-126.1)-> 2015 (EOY-236.525)
const PREFILTER_REVENUE_LOWER_BOUND = REVENUE_LOWER_BOUND * 0.5
const PREFILTER_WEALTH_LOWER_BOUND = WEALTH_LOWER_BOUND * 0.5

const IRS_INDEX_NAME = "IRS List"
const HEADER_NAME = "header.tex"
const FOOTER_NAME = "footer.tex"
const METADATA_NAME = "NCCSMetadata"
const META_INDEX_NAME = "NCCSMetaIndex"
const META_PATH = DOCUMENTATION_PATH * "\\raw meta"
const SUB_TABLES_PATH = META_PATH * "\\subtables"
const FILES_INDEX_NAME = "NCCSFiles"
const KEEPER_NAME = "Keeper List"
const SUMMARY_NAME = "summaryTable"
const PROCESSED_DATA_NAME = "endowmentData_$DATA_SUFFIX"
const CONSOLIDATED_NAME = "summaryTableConsolidated"
const PRIMARY_OUTPUT_NAME = "primaryOutput_$DATA_SUFFIX"
const VAR_TYPES = Dict(:Numeric=>Float64, :Character=>String)
const FILE_TYPES_NAME_TO_CODE = Dict("Public Charity" => :pc,
    "Private Foundation" => :pf, "Other 501c" => :co)
const FRED_INFLATION_FILE = "inflation"
const FRED_DATE_FORMAT = "m/d/yyyy"
const TABLE_PATH = TEX_PATH * "\\sub\\tab"

const GRAPH_PATH = TEX_PATH * "\\sub\\fig"
const GRAPH_EXT = "pdf"
const GRAPH_SUFFIX = "$(DATA_SUFFIX)"
const GRAPH_SIZE = (1000,750)

const TABLE_SUFFIX = "$(DATA_SUFFIX)"

const DECIMALS = 2
const DECIMALS_RETURN = 2

const MAX_SERIALIZATION_WORKERS = 6
const MAX_MAPPING_WORKERS = 6

const MBool = Union{Bool, Missing}
const MInt = Union{Int, Missing}
const MFloat64 = Union{Float64, Missing}
const MSymbol = Union{Symbol, Missing}
const MString = Union{String, Missing}

#const PlotContainer = T where T <: Plots.Plot #holds graphs
const PlotContainer = Union{Plot,Gadfly.Compose.Context,Vector{Plot}} #holds graphs

const REVENUE_CUTOFF = 1. #require at least one year with a higher value-
# set to a higher value than the lwoer bound for an effect

const ENDOWMENT_RETURNS_NAME = "endowmentReturns"
const LARGE_DIFFERENCE_CUTOFF = .2
const MAX_RETURN_CUTOFF = 1.
const MIN_RETURN_CUTOFF = -0.9
const MIN_POINTS_PER_CATEGORY = 50
const BASE_YEAR = 2015

#technically this limit is on gross receipts s.t. gross receipts > revenue
#currently warns if breached
const LOWER_EZ_REVENUE_LIMIT = 50_000.0

const PERSISTENCE_PERIODS = [1,2,3,4]
const PERSISTENCE_YEARS = [2015]
const PERSISTENCE_CUTOFFS = [0.3333, 0.6667, 1.0]
const LBENCHMARKS = [nothing, :bmsp500, :bmff3, :bmff5]
const TOO_SMALL_TO_LOG = 1e-3 #anything smaller than this is more likely to be a decimal error

const TAIL_THRESHOLD = 0.05
const TAIL_MIN_ENTRIES_PER_FILE = 50

const CROSSTAB_CUTOFFS = [0.5, 1.0]
const CROSSTAB_YEAR_RANGE = 2009:2015

const MISSING_STRINGS = ["NUL", "NULL", "NIL", "missing", "None"]
const VERIFY_FIELDS = [:name, :ein, :fisyr]
const OVERRIDE_STAR_STRINGS = ["\\ostar","\\dstar","\\tstar"]

const MIN_POINTS_PLUS_K_FOR_BETA = 1
const LITE_WINSORIZE_PROP = 0.005 #set to 0.05 for enhanced winsorization
const WINSORIZE_PROP = 0.025

include("EndowmentCode\\CodeDictionaries.jl")

#default compressor algorithms
#I_CSV_STREAMER(p::String) = GzipDecompressorStream(open(p))
#I_CSV_STREAMER(p::String) = gzopen(p)
I_CSV_STREAMER(p::String) = open(GzipDecompressorStream, p)
I_CSV_STREAMER(F::Function, p::String) = open(F, GzipDecompressorStream, p)
#I_CSV_STREAMER(F::Function, p::String) = gzopen(F, p, "r")
#I_CSV_STREAMER(p::String) = Stream(format"CSV", GzipDecompressorStream(open(p)))
#I_CSV_STREAMER(p::String) = IOBuffer(read(gzopen(p)))
I_JLS_STREAMER(p::String) = open(GzipDecompressorStream, p)
I_JLS_STREAMER(F::Function, p::String) = open(F, GzipDecompressorStream, p)
O_JLS_STREAMER(p::String) = GzipCompressorStream(open(p, "w"))
O_JLS_STREAMER(F::Function, p::String) = open(F, GzipCompressorStream, p, "w")

#csvopen(iStream::Any)::DataFrame = load(Stream(format"CSV", iStream)) |> DataFrame
#csvsave(oStream::Any, df::T where T<:AbstractDataFrame) = save(Stream(format"CSV", oStream),df)

include("EndowmentCode\\EndowmentPar.jl") #holds supporting functions for parallelization
include("EndowmentCode\\DownloadAndValidate\\MetaTuples.jl") #holds methods related to field-level metadata
include("EndowmentCode\\DownloadAndValidate\\MetaDictionaries.jl") #holds methods related to file level metadata
include("EndowmentCode\\DownloadAndValidate\\Metadata.jl") #holds all other metadata information
include("EndowmentCode\\DownloadAndValidate\\DownloadNCCSCSVs.jl") #holds methods related to downloading the data
include("EndowmentCode\\DownloadAndValidate\\DownloadIRSPDFs.jl") #holds methods related to downlaoding IRS files
include("EndowmentCode\\DownloadAndValidate\\NCCSFiles.jl") #holds methods related to the data files
include("EndowmentCode\\DownloadAndValidate\\NCCSSummary.jl") #Gets summary information on the files
include("EndowmentCode\\PreprocessAndMap\\MappingFunctions.jl")
include("EndowmentCode\\PreprocessAndMap\\Mapping.jl")
include("EndowmentCode\\ReportAndAnalyze\\WRDSData.jl") #contains scripts relating to acquisition of wrds data
include("EndowmentCode\\DownloadAndValidate\\EndowmentData.jl") #holds high-level scripts and other miscellaneous

include("EndowmentCode\\ReportAndAnalyze\\FinancialData.jl")
include("EndowmentCode\\ReportAndAnalyze\\Open990.jl")
include("EndowmentCode\\ReportAndAnalyze\\ComputeNCCS.jl")
include("EndowmentCode\\ReportAndAnalyze\\TopInstitutions.jl")
include("EndowmentCode\\ReportAndAnalyze\\AnalyzeBeta.jl")
include("EndowmentCode\\ReportAndAnalyze\\Persistence.jl")
include("EndowmentCode\\ReportAndAnalyze\\DescriptiveTables.jl")
include("EndowmentCode\\ReportAndAnalyze\\IndustryFigures.jl")
include("EndowmentCode\\ReportAndAnalyze\\DescriptiveFigures.jl")
include("EndowmentCode\\ReportAndAnalyze\\CrossTab.jl")
include("EndowmentCode\\ReportAndAnalyze\\SensitivitySpecifications.jl")
include("EndowmentCode\\ReportAndAnalyze\\SensitivityRegressions.jl")
include("EndowmentCode\\ReportAndAnalyze\\ExtractTails.jl")
include("EndowmentCode\\ReportAndAnalyze\\RegressionSpecificationsV2.jl")
include("EndowmentCode\\ReportAndAnalyze\\RegressionsV2.jl")
include("EndowmentCode\\ReportAndAnalyze\\ReportAndAnalyze.jl")

end


struct HydroDispatchReservoirBudgetUpperBound <: PSI.AbstractHydroReservoirFormulation end

function PSI.DeviceRangeConstraintSpec(
    ::Type{<:PSI.RangeConstraint},
    ::Type{PSI.ActivePowerVariable},
    ::Type{T},
    ::Type{<:HydroDispatchReservoirBudgetUpperBound},
    ::Type{<:PM.AbstractPowerModel},
    feedforward::Union{Nothing, PSI.AbstractAffectFeedForward},
    use_parameters::Bool,
    use_forecasts::Bool,
) where {T <: PSY.HydroGen}
    if !use_parameters && !use_forecasts
        return PSI.DeviceRangeConstraintSpec(;
            range_constraint_spec = PSI.RangeConstraintSpec(;
                constraint_name = PSI.make_constraint_name(
                    PSI.RangeConstraint,
                    PSI.ActivePowerVariable,
                    T,
                ),
                variable_name = PSI.make_variable_name(PSI.ActivePowerVariable, T),
                limits_func = x -> (min = 0.0, max = PSY.get_active_power(x)),
                constraint_func = PSI.device_range!,
                constraint_struct = PSI.DeviceRangeConstraintInfo,
            ),
        )
    end

    return PSI.DeviceRangeConstraintSpec(;
        timeseries_range_constraint_spec = PSI.TimeSeriesConstraintSpec(
            constraint_name = PSI.make_constraint_name(
                PSI.RangeConstraint,
                PSI.ActivePowerVariable,
                T,
            ),
            variable_name = PSI.make_variable_name(PSI.ActivePowerVariable, T),
            parameter_name = use_parameters ? PSI.ACTIVE_POWER : nothing,
            forecast_label = "max_hourly_hydro_budget",
            multiplier_func = x -> PSY.get_max_active_power(x),
            constraint_func = use_parameters ? PSI.device_timeseries_param_ub! :
                              PSI.device_timeseries_ub!,
        ),
    )
end


function create_response_file(pm::PredictionModel, test::Bool, target_dir::String, filename::String)
    response_file = joinpath(target_dir, filename)
    f = open(response_file, "w")
    @printf(f, "CellLine")
    drugs = collect(keys(pm.data.results))
    for drug in drugs
        @printf(f, "\t%s", drug.id)
    end
    @printf(f, "\n")

    for cl in collect(values(pm.data.cell_lines))
        # if these two are equal
        if test == cl.in_test_set
            @printf(f, "%s", cl.id)
            for drug in drugs
                if haskey(pm.data.results[drug].outcome_values, cl)
                    @printf(f, "\t%.9f", pm.data.results[drug].outcome_values[cl])
                else
                    @printf(f, "\tNA")
                end
            end
            @printf(f, "\n")
        end
    end
    close(f)
    response_file
end

function write_results(pm::PredictionModel, p::PredictionModelParameters,
        parent_dir::String, filename::String,
        predictions::Dict{Drug, Vector{Float64}}, ranks::Dict{Drug, Vector{Int64}})
    # write predictions and true values
    mkpath(joinpath(parent_dir, "prediction"))
    prediction_file = joinpath(parent_dir, "prediction", filename)
    f = open(prediction_file, "w")
    @printf(f, "DrugAnonID\tType")
    for d in collect(keys(pm.data.results))
        @printf(f, "\t%s (prediction)\t%s (measured)", d.id, d.id)
    end
    @printf(f, "\n")
    for (cl_id, cl) in enumerate(collect(values(pm.data.cell_lines)))
        @printf(f, "%s", cl.id)
        if cl.in_test_set @printf(f, "\tTest") else @printf(f, "\tTraining") end
        for drug in collect(keys(pm.data.results))
            # drug = pm.data.drugs["Drug$d_id"]
            target = 0.
            if haskey(pm.data.results, drug) && haskey(pm.data.results[drug].outcome_values, cl)
                target = pm.data.results[drug].outcome_values[cl]
            # elseif haskey(pm.data.test_results, drug) && haskey(pm.data.test_results[drug].outcome_values, cl)
            #     target = pm.data.test_results[drug].outcome_values[cl]
            end
            @printf(f, "\t%.5f\t%.5f", predictions[drug][cl_id], target)
        end
        @printf(f, "\n")
    end
    close(f)
    # write rankings
    mkpath(joinpath(parent_dir, "ranking"))
    ranking_file = joinpath(parent_dir, "ranking", filename)
    # ranks = Dict{Drug, Vector{Int64}}()
    # we predict neg log values, i.e. higher value means lower concentration
    # which means higher susceptibility of the cell line to the drug and thus higher rank (lower number)
    # hence, we sort in reverse order
    for d in collect(keys(predictions))
        ranks[d] = zeros(Int64, length(predictions[d]))
        ranks[d][sortperm(predictions[d], rev=true)] = collect(1:length(predictions[d]))
    end
    f = open(ranking_file, "w")
    @printf(f, "DrugAnonID")
    for drug in collect(keys(pm.data.results))
        @printf(f, ",%s", drug.id)
    end
    @printf(f, "\n")
    for (cl_id, cl) in enumerate(collect(values(pm.data.cell_lines)))
        @printf(f, "%s", cl.id)
        for drug in collect(keys(pm.data.results))
           @printf(f, ",%d", ranks[drug][cl_id])
       end
       @printf(f, "\n")
    end
    close(f)

    # write model
    mkpath(joinpath(parent_dir, "model"))
    model_file = joinpath(parent_dir, "model", filename)
    f = open(model_file, "w")
    # T, K, N
    @printf(f, "T\tK\n")
    @printf(f, "%d\t%d\n", p.T, p.K)
    @printf(f, "N\n")
    N = collect(values(p.N))
    [@printf(f, "%d\t", n) for n in N[1:end-1]]
    @printf(f, "%d\n", N[end])
    # gamma
    @printf(f, "ɣ\n")
    [@printf(f, "%.5f\t", expected_value(val)) for val in p.ɣ[1:end-1]]
    @printf(f, "%.5f\n", expected_value(p.ɣ[end]))
    # b
    @printf(f, "b\n")
    [@printf(f, "%.5f\t", expected_value(val)) for val in p.b[1:end-1]]
    @printf(f, "%.5f\n", expected_value(p.b[end]))
    # a
    @printf(f, "a\n")
    for a_t in p.a
        @printf(f, "%s\n", a_t.var_name)
        a_t_exp = expected_value(a_t)
        [@printf(f, "%.5f\t", val) for val in a_t_exp[1:end-1]]
        @printf(f, "%.5f\n", a_t_exp[end])
    end
    # lambda
    @printf(f, "λ\n")
    for (t, λ_t) in enumerate(p.λ)
        λ_t_exp = expected_value.(λ_t)
        @printf(f, "λ[%d]:\t", t)
        for val in λ_t_exp[1:end-1]
            @printf(f, "%.5f\t", val)
        end
        @printf(f, "%.5f\n", λ_t_exp[end])
    end
    # epsilon
    @printf(f, "ε\n")
    [@printf(f, "%.5f\t", expected_value(val)) for val in p.ε[1:end-1]]
    @printf(f, "%.5f\n", expected_value(p.ε[end]))
    # nu
    @printf(f, "ν\n")
    [@printf(f, "%.5f\t", expected_value(val)) for val in p.ν[1:end-1]]
    @printf(f, "%.5f\n", expected_value(p.ν[end]))
    # G
    @printf(f, "G\n")
    for t in 1:p.T, k in 1:p.K
        exp_g = expected_value(p.G[t,k])
        @printf(f, "%s:\t", p.G[t,k].var_name)
        [@printf(f, "%.5f\t", val) for val in exp_g[1:end-1]]
        @printf(f, "%.5f\n", exp_g[end])
    end
    # omega
    @printf(f, "⍵\n")
    [@printf(f, "%.5f\t", expected_value(val)) for val in p.⍵[1:end-1]]
    @printf(f, "%.5f\n", expected_value(p.⍵[end]))
    # e
    @printf(f, "e\n")
    [@printf(f, "%.5f\t", expected_value(val)) for val in p.e[1:end-1]]
    @printf(f, "%.5f\n", expected_value(p.e[end]))

    close(f)

end