"""
Abstract supertype for layerwise training specification.
May be specifications for a normal RBM layer (see `TrainLayer`) or
multiple combined specifications for a partitioned layer
(see `TrainPartitionedLayer`).
"""
abstract type AbstractTrainLayer end

mutable struct TrainLayer <: AbstractTrainLayer
   epochs::Int
   usedefaultepochs::Bool
   learningrate::Float64
   learningrates::Vector{Float64}
   usedefaultlearningrate::Bool
   sdlearningrate::Float64
   sdlearningrates::Vector{Float64}
   monitoring::Function
   rbmtype::DataType
   nhidden::Int
   nvisible::Int
   batchsize::Int
   usedefaultbatchsize::Bool
   pcd::Bool
   cdsteps::Int
   startrbm::AbstractRBM
   optimizer::AbstractOptimizer
   optimizers::Vector{AbstractOptimizer}
   fisherscoring::Int
   inversedispersion::Array{Float64,1}
   dispersionquantile::Float64
   estimatedispersion::String
   lambda::Float64
   zeroinflation::Bool
end

"""
Specify parameters for training one RBM-layer in a DBM.

# Optional keyword arguments:
* The optional keyword arguments `rbmtype`, `nhidden`, `epochs`,
  `learningrate`/`learningrates`, `sdlearningrate`/`sdlearningrates`,
  `batchsize`, `pcd`, `cdsteps`, `startrbm` and `optimizer`/`optimizers`
  are passed to `fitrbm`. For a detailed description, see there.
  If a negative value is specified for `learningrate` or `epochs`, this indicates
  that a corresponding default value should be used
  (parameter defined by call to `stackrbms`).
* `monitoring`: also like in `fitrbm`, but may take a `DataDict` as third argument
   (see function `stackrbms` and its argument `monitoringdata`).
* `nvisible`: Number of visible units in the RBM. Only relevant for partitioning.
   This parameter is derived as much as possible by `stackrbms`.
   For `MultimodalDBM`s with a partitioned first layer, it is necessary to specify
   the number of visible nodes for all but at most one partition in the input layer.
"""
function TrainLayer(;
      epochs::Int = -1,
      learningrate::Float64 = -Inf,
      learningrates::Vector{Float64} = Vector{Float64}(),
      sdlearningrate::Float64 = 0.0,
      sdlearningrates::Vector{Float64} = Vector{Float64}(),
      monitoring = nomonitoring,
      rbmtype::DataType = BernoulliRBM,
      nhidden::Int = -1,
      nvisible::Int = -1,
      batchsize::Int = -1,
      pcd::Bool = true,
      cdsteps::Int = 1,
      startrbm::AbstractRBM = NoRBM(),
      optimizer::AbstractOptimizer = NoOptimizer(),
      optimizers::Vector{<:AbstractOptimizer} = Vector{AbstractOptimizer}(),
      fisherscoring::Int = 0,
      inversedispersion::Array{Float64,1} = (ones(1500) .* 1.0),   #TODO: Adapt to proper number of dimensions
      dispersionquantile::Float64 = 0.5,
      estimatedispersion::String = "gene",
      lambda::Float64 = 10.0,
      zeroinflation::Bool = false)

   usedefaultepochs = (epochs < 0)
   usedefaultlearningrate = (learningrate < 0 && isempty(learningrates))
   usedefaultbatchsize = (batchsize <= 0)
   TrainLayer(
         (usedefaultepochs ? 10 : epochs),
         usedefaultepochs,
         (usedefaultlearningrate ? 0.005 : learningrate),
         learningrates,
         usedefaultlearningrate,
         sdlearningrate, sdlearningrates,
         monitoring, rbmtype, nhidden, nvisible,
         (usedefaultbatchsize ? 1 : batchsize),
         usedefaultbatchsize,
         pcd, cdsteps,
         startrbm,
         optimizer, optimizers, fisherscoring, inversedispersion, dispersionquantile, estimatedispersion, lambda, zeroinflation)
end

"""
Encapsulates a vector of `TrainLayer` objects for training a partitioned layer.
"""
struct TrainPartitionedLayer <: AbstractTrainLayer
   parts::Vector{TrainLayer}
end

const AbstractTrainLayers = Vector{<:AbstractTrainLayer}

"""
A dictionary containing names of data sets as keys and the data sets (matrices
with samples in rows) as values.
"""
const DataDict = Dict{String,Array{Float64,2}}

"""
Converts a vector to a vector of the most specific type that all
elements share as common supertype.
"""
function converttomostspecifictype(v::Vector)
   Vector{mostspecifictype(v)}(v)
end

"""
    mostspecifictype(v)
Returns the most specific supertype for all elements in the vector `v`.
"""
function mostspecifictype(v::Vector)
   mapreduce(typeof, typejoin, v)
end

"""
    propagateforward(rbm, datadict, factor)
Returns a new `DataDict` containing the same labels as the given `datadict` but
as mapped values it contains the hidden potential in the `rbm` of the original
datasets. The factor is applied for calculating the hidden potential and is 1.0
by default.
"""
function propagateforward(rbm::AbstractRBM, datadict::DataDict, factor::Float64 = 1.0)
   DataDict(p[1] => hiddenpotential(rbm, p[2], factor) for p = pairs(datadict))
end

function partitioneddata(datadict::DataDict, visrange::UnitRange{Int})
   DataDict(p[1] => p[2][:, visrange] for p = pairs(datadict))
end

function setdefaultsforunspecified!(trainlayer::TrainLayer,
         learningrate::Float64, epochs::Int, batchsize::Int,
         optimizer::AbstractOptimizer)

   if trainlayer.usedefaultlearningrate
      trainlayer.learningrate = learningrate
   end
   if trainlayer.usedefaultepochs
      trainlayer.epochs = epochs
   end
   if trainlayer.usedefaultbatchsize
      trainlayer.batchsize = batchsize
   end
   if trainlayer.optimizer === NoOptimizer()
      trainlayer.optimizer = optimizer
   end

   if isempty(trainlayer.learningrates)
      # learning rates for each epoch have neither been explicitly specified
      # --> set same value for each epoch
      trainlayer.learningrates =
            fill(trainlayer.learningrate, trainlayer.epochs)
   end
   if isempty(trainlayer.sdlearningrates)
      # same for sdlearningrates
      trainlayer.sdlearningrates =
            fill(trainlayer.sdlearningrate, trainlayer.epochs)
   end
   trainlayer
end

function setdefaultsforunspecified!(trainpartitionedlayer::TrainPartitionedLayer,
      learningrate::Float64, epochs::Int, batchsize::Int,
      optimizer::AbstractOptimizer)

   setdefaultsforunspecified!(trainpartitionedlayer.parts,
         learningrate, epochs, batchsize, optimizer)
end

function setdefaultsforunspecified!(trainlayers::Vector{T},
      learningrate::Float64, epochs::Int, batchsize::Int,
      optimizer::AbstractOptimizer
      ) where {T <: AbstractTrainLayer}

   for trainlayer in trainlayers
      setdefaultsforunspecified!(trainlayer,
            learningrate, epochs, batchsize, optimizer)
   end
   trainlayers
end

"""
    stackrbms(x; ...)
Performs greedy layerwise training for Deep Belief Networks or greedy layerwise
pretraining for Deep Boltzmann Machines and returns the trained model.

# Optional keyword arguments (ordered by importance):
* `predbm`: boolean indicating that the greedy layerwise training is
   pre-training for a DBM.
   If its value is false (default), a DBN is trained.
* `nhiddens`: vector containing the number of nodes of the i'th hidden layer in
   the i'th entry
* `epochs`: number of training epochs
* `learningrate`: learningrate, default 0.005
* `batchsize`: size of minibatches, defaults to 1
* `trainlayers`: a vector of `TrainLayer` objects. With this argument it is possible
   to specify the training parameters for each layer/RBM individually.
   If the number of training epochs and the learning rate are not specified
   explicitly for a layer, the values of `epochs` and `learningrate` are used.
   For more information see help of `TrainLayer`.
* `monitoringdata`: a data dictionary (see type `DataDict`)
   The data is propagated forward through the
   network to monitor higher levels.
   If a non-empty dictionary is given, the monitoring functions in the
   `trainlayers`-arguments must accept a `DataDict` as third argument.
* `samplehidden`: boolean indicating that consequent layers are to be trained
   with sampled values instead of the deterministic potential,
   which is the default.
"""
function stackrbms(x::Array{Float64,2};
      nhiddens::Vector{Int} = Vector{Int}(),
      epochs::Int = 10,
      predbm::Bool = false,
      samplehidden::Bool = false,
      learningrate::Float64 = 0.005,
      batchsize::Int = 1,
      optimizer::AbstractOptimizer = NoOptimizer(),
      trainlayers::AbstractTrainLayers = Vector{TrainLayer}(),
      monitoringdata::DataDict = DataDict())

   stackrbms_checkmonitoringdata(x, monitoringdata)

   trainlayers = stackrbms_preparetrainlayers(trainlayers, x, epochs,
         learningrate, nhiddens, batchsize, optimizer)

   nrbms = length(trainlayers)
   dbmn = Vector{AbstractRBM}(undef, nrbms)

   upfactor = downfactor = 1.0
   if predbm
      upfactor = 2.0
   end

   dbmn[1] = stackrbms_trainlayer(x, trainlayers[1];
         monitoringdata = monitoringdata,
         upfactor = upfactor, downfactor = downfactor)

   hiddenval = x
   for i = 2:nrbms
      hiddenval = hiddenpotential(dbmn[i-1], hiddenval, upfactor)

      if samplehidden
         bernoulli!(hiddenval)
      end

      if !isempty(monitoringdata)
         monitoringdata = propagateforward(dbmn[i-1], monitoringdata, upfactor)
      end

      if predbm
         upfactor = downfactor = 2.0
         if i == nrbms
            upfactor = 1.0
         end
      else
         upfactor = downfactor = 1.0
      end

      dbmn[i] = stackrbms_trainlayer(hiddenval, trainlayers[i];
            monitoringdata = monitoringdata,
            upfactor = upfactor, downfactor = downfactor)
   end

   dbmn = converttomostspecifictype(dbmn)
   dbmn
end

function stackrbms_checkmonitoringdata(x::Matrix{Float64}, monitoringdata::DataDict)
   if !isempty(monitoringdata)
      if any(map(m -> size(m, 2), values(monitoringdata)) .!= size(x, 2))
         error("Matrices in the dictionary `monitoringdata` must have the same number of columns as the input matrix.")
      end
   end
end

""" Prepares the layerwise training specifications for `stackrbms` """
function stackrbms_preparetrainlayers(
      trainlayers::AbstractTrainLayers,
      x::Matrix{Float64},
      epochs::Int,
      learningrate::Float64,
      nhiddens::Vector{Int},
      batchsize::Int,
      optimizer::AbstractOptimizer)

   if isempty(trainlayers)
      # construct default "trainlayers"
      if isempty(nhiddens)
         nhiddens = fill(size(x,2), 2) # default value for nhiddens
      end
      trainlayers = map(n -> TrainLayer(nhidden = n), nhiddens)
      setdefaultsforunspecified!(trainlayers,
            learningrate, epochs, batchsize, optimizer)
      return trainlayers
   end

   # We are here, Argument "trainlayers" has been specified
   # --> check for correct specification and derive/set some parameters

   # avoid modification of given object
   trainlayers = deepcopy(trainlayers)

   if !isempty(nhiddens)
      warn("Argument `nhiddens` not used.")
   end

   setdefaultsforunspecified!(trainlayers,
         learningrate, epochs, batchsize, optimizer)

   function derive_nvisibles!(layer::AbstractTrainLayer,
         prevlayer::AbstractTrainLayer)
      # do nothing
   end

   function derive_nvisibles!(layer::TrainPartitionedLayer,
         prevlayer::TrainPartitionedLayer)

      if length(layer.parts) == length(prevlayer.parts)
         for j in eachindex(layer.parts)
            if layer.parts[j].nvisible < 0
               layer.parts[j].nvisible = prevlayer.parts[j].nhidden
            end
         end
      end
      if any(map(t -> t.nvisible < 0, layer.parts))
         error("Parameter `nvisible` could not be derived.")
      end
   end

   # derive number of visible nodes for higher layers
   for i = 2:length(trainlayers)
      derive_nvisibles!(trainlayers[i], trainlayers[i-1])
   end

   # check number of visible nodes for first layer
   if isa(trainlayers[1], TrainPartitionedLayer)
      partswithoutnvisible = findall(t -> t.nvisible <= 0, trainlayers[1].parts)
      if length(partswithoutnvisible) > 1
         error("The number of visible nodes is unspecified for more " *
               "than one segment in the first layer. ")
      end

      nvariables = size(x, 2)
      nvisiblestotal = mapreduce(t -> max(t.nvisible, 0), +, trainlayers[1].parts)

      if length(partswithoutnvisible) == 1
         # if only one is unspecified, derive it
         partwithunspecifiednvisible = trainlayers[1].parts[partswithoutnvisible[1]]
         partwithunspecifiednvisible.nvisible = nvariables - nvisiblestotal
      elseif nvisiblestotal != nvariables
         error("Number of visible nodes for first layer " *
               "($nvisiblestotal) is not of same length as " *
               "number of columns in `x` ($(size(x,2))).")
      end
   end

   trainlayers
end

""" Trains a layer without partitioning for `stackrbms`. """
function stackrbms_trainlayer(x::Matrix{Float64},
      trainlayer::TrainLayer;
      monitoringdata::DataDict = DataDict(),
      upfactor::Float64 = 1.0, downfactor::Float64 = 1.0)

   if isempty(monitoringdata)
      monitoring = trainlayer.monitoring
   elseif trainlayer.monitoring != nomonitoring
      monitoring = (rbm, epoch) ->
            trainlayer.monitoring(rbm, epoch, monitoringdata)
   end

   fitrbm(x;
         upfactor = upfactor, downfactor = downfactor,
         nhidden = trainlayer.nhidden,
         epochs = trainlayer.epochs,
         rbmtype = trainlayer.rbmtype,
         learningrates = trainlayer.learningrates,
         batchsize = trainlayer.batchsize,
         cdsteps = trainlayer.cdsteps,
         pcd = trainlayer.pcd,
         sdlearningrate = trainlayer.sdlearningrate,
         sdlearningrates = trainlayer.sdlearningrates,
         monitoring = monitoring,
         startrbm = trainlayer.startrbm,
         optimizer = trainlayer.optimizer,
         optimizers = trainlayer.optimizers,
         fisherscoring = trainlayer.fisherscoring,
         inversedispersion = trainlayer.inversedispersion,
         dispersionquantile = trainlayer.dispersionquantile,
         estimatedispersion = trainlayer.estimatedispersion,
         zeroinflation = trainlayer.zeroinflation,
         lambda = trainlayer.lambda)
end

""" Trains a partitioned layer for `stackrbms`. """
function stackrbms_trainlayer(x::Matrix{Float64},
      trainpartitionedlayer::TrainPartitionedLayer;
      monitoringdata::DataDict = DataDict(),
      upfactor::Float64 = 1.0, downfactor::Float64 = 1.0)

   visranges = ranges(map(t -> t.nvisible, trainpartitionedlayer.parts))

   # prepare the arguments before the for-loop for each call
   # to avoid unnecessary copying
   trainingargs = map(
         i -> (
               x[:, visranges[i]],
               trainpartitionedlayer.parts[i],
               partitioneddata(monitoringdata, visranges[i])
         ),
         eachindex(trainpartitionedlayer.parts))

   rbms = @distributed (vcat) for arg in trainingargs
      stackrbms_trainlayer(arg[1], arg[2];
            monitoringdata = arg[3],
            upfactor = upfactor, downfactor = downfactor)
   end

   commontype = mostspecifictype(rbms)
   PartitionedRBM{commontype}(Vector{commontype}(rbms))
end


module LagrangianParticleTracking

export LagrangianParticles, update_particle_properties!

using Adapt
using KernelAbstractions
using StructArrays

using Oceananigans.Grids
using Oceananigans.Architectures: device
using Oceananigans.Fields: interpolate, datatuple, compute!, location

import Base: size, length, show

abstract type AbstractParticle end

struct Particle{T} <: AbstractParticle
    x :: T
    y :: T
    z :: T
end

struct LagrangianParticles{P, R, T, D, Π}
        properties :: P
       restitution :: R
    tracked_fields :: T
          dynamics :: D
        parameters :: Π
end

@inline no_dynamics(args...) = nothing

"""
    LagrangianParticles(; x, y, z, restitution=1.0, dynamics=no_dynamics, parameters=nothing)

Construct some `LagrangianParticles` that can be passed to a model. The particles will have initial locations
`x`, `y`, and `z`. The coefficient of restitution for particle-wall collisions is specified by `restitution`.

`dynamics` is a function of `(lagrangian_particles, model, Δt)` that is called prior to advecting particles.
`parameters` can be accessed inside the `dynamics` function.
"""
function LagrangianParticles(; x, y, z, restitution=1.0, dynamics=no_dynamics, parameters=nothing)
    size(x) == size(y) == size(z) ||
        throw(ArgumentError("x, y, z must all have the same size!"))

    (ndims(x) == 1 && ndims(y) == 1 && ndims(z) == 1) ||
        throw(ArgumentError("x, y, z must have dimension 1 but ndims=($(ndims(x)), $(ndims(y)), $(ndims(z)))"))

    particles = StructArray{Particle}((x, y, z))

    return LagrangianParticles(particles; restitution, dynamics, parameters)
end

"""
    LagrangianParticles(particles::StructArray; restitution=1.0, tracked_fields::NamedTuple=NamedTuple(), dynamics=no_dynamics)

Construct some `LagrangianParticles` that can be passed to a model. The `particles` should be a `StructArray`
and can contain custom fields. The coefficient of restitution for particle-wall collisions is specified by `restitution`.

A number of `tracked_fields` may be passed in as a `NamedTuple` of fields. Each particle will track the value of each
field. Each tracked field must have a corresponding particle property. So if `T` is a tracked field, then `T` must also
be a custom particle property.

`dynamics` is a function of `(lagrangian_particles, model, Δt)` that is called prior to advecting particles.
`parameters` can be accessed inside the `dynamics` function.
"""
function LagrangianParticles(particles::StructArray; restitution=1.0, tracked_fields::NamedTuple=NamedTuple(),
                             dynamics=no_dynamics, parameters=nothing)

    for (field_name, tracked_field) in pairs(tracked_fields)
        field_name in propertynames(particles) ||
            throw(ArgumentError("$field_name is a tracked field but $(eltype(particles)) has no $field_name field! " *
                                "You might have to define your own particle type."))
    end

    return LagrangianParticles(particles, restitution, tracked_fields, dynamics, parameters)
end

size(lagrangian_particles::LagrangianParticles) = size(lagrangian_particles.properties)
length(lagrangian_particles::LagrangianParticles) = length(lagrangian_particles.properties)

function Base.show(io::IO, lagrangian_particles::LagrangianParticles)
    particles = lagrangian_particles.properties
    properties = propertynames(particles)
    fields = lagrangian_particles.tracked_fields
    print(io, "$(length(particles)) Lagrangian particles with\n",
        "├── $(length(properties)) properties: $properties\n",
        "└── $(length(fields)) tracked fields: $(propertynames(fields))")
end

include("update_particle_properties.jl")

end # module


# Lifting

# # Specializations
# const unidistattrs = [:succprob, :failprob, :maximum, :minimum, :islowerbounded,
#                       :isupperbounded, :isbounded, :std, :median, :mode, :modes,
#                       :skewness, :kurtosis, :isplatykurtic, :ismesokurtic,
#                       :isleptokurtic, :entropy, :mean]

# for func in unidistattrs
#   expr =
#   quote
#     $func(x::RandVar, israndvar::Type{Val{false}}) = Djl.$func(distribution(x))
#     $func(x::RandVar, israndvar::Type{Val{true}}) = $(:l *ₛ func)(x)
#     $(:l *ₛ func)(x::RandVar) = lift($func)(x, Val{false})
#     $func(x::RandVar) = $func(x, Val{elemtype(x) <: RandVar})
#   end
#   eval(expr)
# end

# Lift all dist ops
for dop in distops_names
  expr = :($(dop *ₛ :ᵣ)(x::RandVar) = lift($dop)(x))
  eval(expr)
end


# The MIT License (MIT)
#
# Copyright (c) 2016 Alexander Samoilov
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Ld = 365.
NA = 0.42
Mask_L = 1 # um

x = y = flipdim((-256:255) * (Ld / (2.*NA)*31) / 512, 2)

tm = zeros(length(x))


using SHA

@testset "Payload" begin

    host = "127.0.0.1"
    port = 8080
    server_url = "http://" * host * ":" * string(port)

    num_comm_rounds = 1
    fraction_clients = 0.1f0
    num_total_clients = 10

    base_config = Fed.Config.BaseConfig(
        server_url,
        num_comm_rounds,
        fraction_clients,
        num_total_clients
    )

    weights = rand(Float32, 199210)
    chunksize = 256
    is_client = false

    function serde(config)
        Fed.initialize_stats(config, length(weights))
        data = Fed.Serde.serialize_payload(config.payload_serde, weights)
        return Fed.Serde.deserialize_payload(config.payload_serde, data, "http://127.0.0.1:9090")
    end

    @testset "Vanilla" begin
        config = Fed.Config.VanillaConfig(base_config)
        @test serde(config) == weights
    end

    @testset "Quantized" begin
        config = Fed.Config.QuantizedConfig{UInt8}(base_config)
        res = serde(config)
        @test sum(res - weights) / length(weights) < 0.1 # quantzation generates a small error
    end

    @testset "Quantized diff" begin
        config = Fed.Config.QDiffConfig{UInt8}(base_config, chunksize, is_client)
        res = serde(config)
        @test sum(res - weights) / length(weights) < 0.1 # quantzation generates a small error
    end

    @testset "Quantized static diff" begin
        config = Fed.Config.QDiffStaticConfig{UInt8}(base_config, chunksize, is_client)
        res = serde(config)
        @test sum(res - weights) / length(weights) < 0.1 # quantzation generates a small error
    end

    @testset "GD" begin
        config = Fed.Config.GDConfig{UInt8}(base_config, chunksize, sha1, 0x05, host, 9090, is_client)
        res = serde(config)
        @test sum(res - weights) / length(weights) < 0.1 # quantzation generates a small error
    end
end


module EPOCHInput

export parse_input

using Unitful
using PhysicalConstants.CODATA2018: c_0, ε_0, μ_0, m_e, e
using BangBang

include("parse.jl")
include("constant.jl")
include("control.jl")
include("no_globals.jl")
include("no_parsing.jl")
include("output.jl")

end


module GPUChecker

# package code goes here
function gpu_usages()
	rawtext = readall(pipeline(`nvidia-smi`, `grep |`, `grep N/A`))
	str_usages = split(rawtext, '\n')
	usages = []
	for usg in str_usages
		if usg != ""
			usage, capacity = map(x->parse(Int, match(r"(\d+)", x).captures[1]),split(strip(split(usg, '|')[3]), '/'))
			push!(usages, (usage, capacity))
		end
	end
	return usages
end

function first_min_used_gpu()
	usages = gpu_usages()
	return indmin(usages) - 1
end

export gpu_usages
export first_min_used_gpu

end # module


using IcanProj
using NetCDF
using DataFrames
using PyPlot
using Statistics

# Global settings

path = "/data04/jmg/fsm_simulations/netcdf/fsmres_open/results_32"

# Load latent heat fluxes

variable = "latmo"

file_fine = joinpath(path, "$(variable)_1km.nc")

file_coarse = joinpath(path, "$(variable)_50km.nc")

df_links = link_results(file_fine, file_coarse)

latmo_coarse, latmo_aggregated = unify_results(file_fine, file_coarse, df_links, variable)

# Load snow water equivalents

variable = "swe"

file_fine = joinpath(path, "$(variable)_1km.nc")

file_coarse = joinpath(path, "$(variable)_50km.nc")

df_links = link_results(file_fine, file_coarse)

swe_coarse, swe_aggregated = unify_results(file_fine, file_coarse, df_links, variable)


using CSV
using DataFrames
using Plots

function visualize()
  data = DataFrame(
    CSV.File("pulitzer-circulation-data.csv", normalizenames = true))

  sort!(data, :Pulitzer_Prize_Winners_and_Finalists_1990_2014, rev = true)
  println(data)
  sort!(data, :Pulitzer_Prize_Winners_and_Finalists_1990_2014)

  x = data.Newspaper
  y = data.Pulitzer_Prize_Winners_and_Finalists_1990_2014
  # y = getproperty(data,
  #   Symbol("Pulitzer Prize Winners and Finalists, 1990-2014"))

  p = plot(x, y, seriestype = :bar, direction = :h, size = (1000, 2000),
    legend = false)
  savefig(p, "plot.png")
end

visualize()


#=
    * Need to consider Duration
    * Activeted levels for career paths are not updated
=#
requiredTypes = [ "AcademicLevel", "Affiliation", "Assignment", "CareerPath", "CareerStatus", "Job", "Rank", "SubJob" ]

for reqType in requiredTypes
    if !isdefined( Symbol( uppercase( string( reqType[ 1 ] ) ) * reqType[ 2:end ] ) )
        include(  reqType * ".jl" )
    end  # if !isdefined( Symbol( ...
end

using LightGraphs

export ConstructCareerPaths
function ConstructCareerPaths(g::DiGraph, DefinedLevels::Array{Vector{AbstractLevel}})
    CareerPaths = Vector{CareerPath}()

		if size(DefinedLevels, 1) < 1
			warn("No level defined.")
			return CareerPaths
		end
		InitPaths = CreateInitPaths(g, DefinedLevels[1])

		for path in InitPaths
			push!(CareerPaths, path)
		end

        for i in 2:length(DefinedLevels)
            NextLevelPaths!(CareerPaths, DefinedLevels[i-1], DefinedLevels[i])
        end

        for i in 1:length(CareerPaths)
            for j in 1:length(DefinedLevels)
                ActivateLevel(CareerPaths[i], typeof(DefinedLevels[j][1]))
            end
        end
    return CareerPaths
end

function CreatePathFromGraph(g::DiGraph, n::Int)
	CarPa = Vector{Array{Int}}()
  if length(neighbors(g, n)) == 0
      push!(CarPa, [-1])
  else
      for neighbor in neighbors(g, n)
          tempPaths = CreatePathFromGraph(g, neighbor)
          for path in tempPaths
              push!(CarPa, [n path])
          end
      end
  end
  return CarPa
end

function CreateInitPaths(g::DiGraph, StLevel::Array{AbstractLevel})

	# Traversing the graph
	InitPaths = Vector{Array{Int}}()
	if is_cyclic(g)
			warn("The graph is cyclic.")
			return Array{CareerPaths}()
	else
			for node in neighbors(g, 1)
					tmpCarPat = CreatePathFromGraph(g, node)
					for path in tmpCarPat
							push!(InitPaths, path)
					end
			end
	end
	InitCareerPaths = Array{CareerPath}(length(InitPaths))
	for path in 1:length(InitPaths)
		InitCareerPaths[path] = CareerPath()
		for p in InitPaths[path][1:(end-1)]
				push!(InitCareerPaths[path].Path, CareerStatus(academicLevel = StLevel[p-1]))
		end
	end
	#println("CreateInitPaths ", InitCareerPaths)
	return InitCareerPaths
end

SetterGetter = Dict{Any,Any}(
                    AcademicLevel => (get_AcademicLevel, set_AcademicLevel),
                    Rank => (get_Rank, set_Rank),
                    Assignment => (get_Assignment, set_Assignment),
                    Affiliation => (get_Affiliation, set_Affiliation),
                    Job => (get_Job, set_Job),
                    SubJob => (get_SubJob, set_SubJob))

function NextLevelPaths!(CareerPaths::Vector{CareerPath}, CurrentLevel::Vector{AbstractLevel}, NextLevel::Vector{AbstractLevel})
    # Getter and Setter for the current and next level
    #currentGet, currentSet = SetterGetter[typeof(CurrentLevel[1])]
    #nextGet, nextSet = SetterGetter[typeof(NextLevel[1])]

    if typeof(NextLevel[1]) == Rank
        Limits = sum(getMinStay(FirstLevel[i]) for i in 1:length(FirstLevel)) + sum(getMaxStay(FirstLevel[i]) for i in 1:length(FirstLevel))
        if Limits == 0
            ConstructRankLevelPathsFreeStLvl(CareerPaths, CurrentLevel, NextLevel)
        else
            ConstructRankLevelPathsLimitedDuration(CareerPaths, CurrentLevel, NextLevel)
        end
    else
        warn("$(typeof(NextLevel[1])) is under Construction.")
    end
end

function ConstructRankLevelPathsFreeStLvl(CareerPaths::Vector{CareerPath}, CurrentLevel::Vector{AbstractLevel}, NextLevel::Vector{AbstractLevel})
    currentGet, currentSet = SetterGetter[typeof(CurrentLevel[1])]
    nextGet, nextSet = SetterGetter[typeof(NextLevel[1])]
    # This is the number of paths that we will exploit
    nbInitPaths = length(CareerPaths)
    for i in 1:nbInitPaths
        PathToExploit = shift!(CareerPaths) # This path is used to generate the possible career paths which will be appended to CareerPaths at the end ***---------
        GeneratedPaths = Vector{CareerPath}()

        #for the first node in the career path we generate possible sub paths
        workinglevel = currentGet(PathToExploit.Path[1])
        for k in 1:length(workinglevel.NextLevel)
            tmppath = CareerPath()
            for l in 1:k
                tmpstatus = CareerStatus( academicLevel = PathToExploit.Path[1].AcademicLevel,
                                          rank = PathToExploit.Path[1].Rank,
                                          assignment = PathToExploit.Path[1].Assignment,
                                          affiliation = PathToExploit.Path[1].Affiliation,
                                          job = PathToExploit.Path[1].Job,
                                          subjob = PathToExploit.Path[1].SubJob)

                nextSet(tmpstatus, NextLevel[findIndex(workinglevel.NextLevel[l], NextLevel)])
                push!(tmppath.Path, tmpstatus)
            end
            push!(GeneratedPaths, tmppath)
        end

        # for each of the remaining nodes in the career path we generate the possible sub paths
        for j in 2:length(PathToExploit.Path)
            workinglevel = currentGet(PathToExploit.Path[j])
            nbPaths = length(GeneratedPaths)
            for p in 1:nbPaths
                workingpath = shift!(GeneratedPaths)

                if nextGet(workingpath.Path[end]).RankCode <= NextLevel[findIndex(workinglevel.NextLevel[1], NextLevel)].RankCode
                    println("StartingIndex = 1")
                    StartingIndex = 1
                elseif nextGet(workingpath.Path[end]).RankCode > NextLevel[findIndex(workinglevel.NextLevel[end], NextLevel)].RankCode
                    continue
                else
                    for indx in 1:length(workinglevel.NextLevel)
                        if nextGet(workingpath.Path[end]).Name == workinglevel.NextLevel[indx]
                            StartingIndex = indx
                            break
                        end
                    end
                    println("StartingIndex = $StartingIndex")
                end
                for k in StartingIndex:length(workinglevel.NextLevel)
                    tmppath = CareerPath(workingpath)
                    for l in StartingIndex:k
                        tmpstatus = CareerStatus( academicLevel = PathToExploit.Path[j].AcademicLevel,
                                                  rank = PathToExploit.Path[j].Rank,
                                                  assignment = PathToExploit.Path[j].Assignment,
                                                  affiliation = PathToExploit.Path[j].Affiliation,
                                                  job = PathToExploit.Path[j].Job,
                                                  subjob = PathToExploit.Path[j].SubJob)

                        nextSet(tmpstatus, NextLevel[findIndex(workinglevel.NextLevel[l], NextLevel)])
                        push!(tmppath.Path, tmpstatus)
                    end
                    push!(GeneratedPaths, tmppath)
                end
            end
        end

        append!(CareerPaths, GeneratedPaths) # Appending generated sub career paths ***---------
    end
end

function ConstructRankLevelPathsLimitedDuration(CareerPaths::Vector{CareerPath}, CurrentLevel::Vector{AbstractLevel}, NextLevel::Vector{AbstractLevel})
    warn("$(typeof(NextLevel[1])) with first level restrictions is under Construction.")

end

function findIndex(name::String, Level::Vector{AbstractLevel})
    for i in 1:length(Level)
        (name == Level[i].Name) && return i
    end
    return -1
end


#-------------------------------------------------------------------
#* EMSO Model Library (EML) Copyright (C) 2004 - 2007 ALSOC.
#*
#* This LIBRARY is free software; you can distribute it and/or modify
#* it under the therms of the ALSOC FREE LICENSE as available at
#* http://www.enq.ufrgs.br/alsoc.
#*
#* EMSO Copyright (C) 2004 - 2007 ALSOC, original code
#* from http://www.rps.eng.br Copyright (C) 2002-2004.
#* All rights reserved.
#*
#* EMSO is distributed under the therms of the ALSOC LICENSE as
#* available at http://www.enq.ufrgs.br/alsoc.
#*--------------------------------------------------------------------
#* Author: Gerson Balbueno Bicca
#* $Id: PHE.mso 250 2007-04-27 16:32:02Z bicca $
#*------------------------------------------------------------------
type Main_PHE
	Main_PHE()=begin
		new(
			PHE_HeatTransfer(Dict{Symbol,Any}(
				:Brief=>"PHE Heat Transfer",
				:Symbol=>" "
			)),
			PHE_PressureDrop(Dict{Symbol,Any}(
				:Brief=>"PHE Pressure Drop",
				:Symbol=>" "
			)),
			Physical_Properties(Dict{Symbol,Any}(
				:Brief=>"PHE Properties",
				:Symbol=>" "
			)),
			[:HeatTransfer,:PressureDrop,:Properties,]
		)
	end
	HeatTransfer::PHE_HeatTransfer
	PressureDrop::PHE_PressureDrop
	Properties::Physical_Properties
	variables::Array{Symbol,1}
	attributes::Dict{Symbol,Any}
end
export Main_PHE
function atributes(in::Main_PHE,_::Dict{Symbol,Any})
	fields::Dict{Symbol,Any}=Dict{Symbol,Any}()
	fields[:Pallete]=false
	fields[:Brief]="to be documented"
	fields[:Info]="to be documented"
	drive!(fields,_)
	return fields
end
Main_PHE(_::Dict{Symbol,Any})=begin
	newModel=Main_PHE()
	newModel.attributes=atributes(newModel,_)
	newModel
end


#****************************************************************************
#                     Molecular Dynamics Potentials (MDP)
#                            CESMIX-MIT Project
#
#  Contributing authors: Ngoc-Cuong Nguyen (cuongng@mit.edu, exapde@gmail.com)
#****************************************************************************

function genbo2grad(foldername, filename, u, xij, qi, qj, ti, tj, ai, aj, rho, mu, eta, kappa, natomtype)

# if (u[1]==0)
#     m = 0;
# else
#     n,m = size(u);
#     if (n==0) || (n == 1)
#         m = 0;
#     elseif (n == natomtype+1)
#         if u[n,1] == 0
#             m = 0;
#         end
#     else
#         error("Number of functions for the two-body bond order potential must be equal to 1 plus the number of atom types");
#     end
# end

if length(u)>0
    if (u[1]==0)
        m = 0;
        n = 0;
    else
        n,m = size(u);
        # if (n==0) || (n == 1)
        #     m = 0;
        # elseif (n == natomtype+1)
        #     if u[end,1] == 0
        #         m = 0;
        #     end
        # else
        #     error("Number of functions for the two-body bond order potential must be equal to 1 plus the number of atom types");
        # end
    end
else
    n = 0;
    m = 0;
end

gen = 1;
ifile = 0;
stropu = "";
strcpu = "";
strgpu = "";
potnum = 0;
fn = "PaircDensityGradient";

for i = 1:m
    ui = u[:,i];
    if (ui[n] != 0)
        potnum = potnum + 1;
        stropu1, strcpu1, strgpu1 = genpairgrad(foldername, filename*string(potnum), ui[1:(n-1)], xij, qi, qj, ti, tj, ai, aj, mu, eta, kappa, gen, ifile);
        stropu1 = modifystr(stropu1, natomtype);
        strcpu1 = modifystr(strcpu1, natomtype);
        strgpu1 = modifystr(strgpu1, natomtype);
        strgpu1 = replace(strgpu1, "\t\ti += blockDim.x * gridDim.x;\n\t\t}" => "\t\t}\n\t\ti += blockDim.x * gridDim.x;");
        stropu2, strcpu2, strgpu2 = gendensitygrad(foldername, fn*string(potnum), ui[n], rho, mu, eta, kappa, gen, ifile);
        stropu = stropu * stropu1 * "\n" * stropu2 * "\n";
        strcpu = strcpu * strcpu1 * "\n" * strcpu2 * "\n";
        strgpu = strgpu * strgpu1 * "\n" * strgpu2 * "\n";
    end
end

if potnum==0
    ifile = 0;
    gen = 0;
    stropu, strcpu, strgpu = genpairgrad(foldername, filename, u, xij, qi, qj, ti, tj, ai, aj, mu, eta, kappa, gen, ifile);
    tmpopu, tmpcpu, tmpgpu = gendensitygrad(foldername, fn, u, mu, mu, eta, kappa, gen, ifile);
    stropu = stropu  * "\n" * tmpopu * "\n";
    strcpu = strcpu  * "\n" * tmpcpu * "\n";
    strgpu = strgpu  * "\n" * tmpgpu * "\n";
    cppfiles(foldername, filename, stropu, strcpu, strgpu, 1);
else
    sp0, sp1, sp2, sp3 = getpotstr(2);
    sp0 = replace(sp0, "int ng)" => "int ng, int potnum)");
    sp1 = replace(sp1, "int);" => "int, int);");
    sp2 = replace(sp2, "int);" => "int, int);");
    tmpopu, tmpcpu, tmpgpu = genpotnum(filename, potnum, sp0, sp1, sp2, sp3);
    tmpopu = modstr11(tmpopu);
    tmpcpu = modstr11(tmpcpu);
    tmpgpu = modstr11(tmpgpu);
    stropu = stropu * tmpopu * "\n";
    strcpu = strcpu * tmpcpu * "\n";
    strgpu = strgpu * tmpgpu * "\n";

    sp0, sp1, sp2, sp3 = getpotstr(0);
    sp0 = replace(sp0, "int ng)" => "int ng, int potnum)");
    sp1 = replace(sp1, "int);" => "int, int);");
    sp2 = replace(sp2, "int);" => "int, int);");
    tmpopu, tmpcpu, tmpgpu = genpotnum(fn, potnum, sp0, sp1, sp2, sp3);
    tmpopu = modstr12(tmpopu);
    tmpcpu = modstr12(tmpcpu);
    tmpgpu = modstr12(tmpgpu);
    stropu = stropu * tmpopu;
    strcpu = strcpu * tmpcpu;
    strgpu = strgpu * tmpgpu;

    cppfiles(foldername, filename, stropu, strcpu, strgpu, 0);
end

end

function modifystr(oldstr, natomtype)

newstr = oldstr;
newstr = replace(newstr, "u_xij[" => "\tu_xij[");
for i = 1:natomtype
    s = "u[" * string(i-1) * " * i*" * string(natomtype) * "]";
    if contains(newstr, s)
        news = "if (tj[i] == " * string(i) * ") \n\t\t\t" * "u[i]";
        if (i>1)
            news = "}\n \t\t" * news;
        end
        newstr = replace(newstr, s => news);
    end
end
newstr = replace(newstr, "\t}\n}" => "\t\t}\n\t}\n}");

return newstr

end

function modstr11(tmpgpu)

tmpgpu = replace(tmpgpu, "(T *u," => "(T *u => T *u_xij,");
tmpgpu = replace(tmpgpu, "(u," => "(u, u_xij,");
tmpgpu = replace(tmpgpu, "(double *" => "(double *, double *");
tmpgpu = replace(tmpgpu, "(float *" => "(float *, float *");

return tmpgpu

end

function modstr12(tmpgpu)

tmpgpu = replace(tmpgpu, "(T *u," => "(T *u, T *u_rho,");
tmpgpu = replace(tmpgpu, "(u," => "(u, u_rho,");
tmpgpu = replace(tmpgpu, "(double *" => "(double *, double *");
tmpgpu = replace(tmpgpu, "(float *" => "(float *, float *");

return tmpgpu

end


#=
Simulated Annealing placement.
=#

################################################################################
# Type to keep track of previous moves.
################################################################################
abstract type AbstractCookie end
mutable struct MoveCookie{T} <: AbstractCookie
    cost_of_move::Float64
    index_of_moved_node::Int64
    move_was_swap::Bool
    index_of_other_node::Int64
    old_location::T
end

function MoveCookie(sa_struct::SAStruct)
    return MoveCookie(0.0, 0, false, 0, zero(eltype(sa_struct)))
end

################################################################################
# Types that control various parameters of the placement.
################################################################################

##########
# SAWarm #
##########
"""
Abstract type controlling the warming cycle of Simulated Annealing placement.

API
---
* [`warm!`](@ref)

Implementations
---------------
* [`DefaultSAWarm`](@ref)
"""
abstract type SAWarm end

#-----#
# API #
#-----#
"""
    warm!(warmer :: SAWarm, state :: SAState)

Increase `state.temperature` according to `warmer`. If `warmer` decides that
the system is done warming up, it must set `state.warming = false`.

Method List
-----------
$(METHODLIST)
"""
function warm! end

#----------------#
# Implementation #
#----------------#

"""
Default imeplementation of [`SAWarm`](@ref).

On each invocation, will multiply the temperature of
the anneal by `multiplier`. When the acceptance ratio rises above `ratio`,
warming routine will end.

To prevent unbounded warming, the `ratio` field is multiplied by the `decay`
field on each invocation.
"""
mutable struct DefaultSAWarm <: SAWarm
    ratio::Float64
    multiplier::Float64
    decay::Float64
end

@inline function warm!(w::DefaultSAWarm, state::SAState)
    # Compute acceptance ratio from the state
    acceptance_ratio = state.recent_accepted_moves / state.recent_successful_moves

    # Update temperature
    state.temperature *= w.multiplier
    # Check if acceptance ratio has achieved the desired ratio
    if acceptance_ratio > w.ratio
        state.warming = false
    else
        # Decay acceptance ratio
        w.ratio *= w.decay
    end

    return nothing
end

##########
# SACool #
##########

"""
Cooling schedule for Simulated Annealing placement.

API
---
* [`cool!`](@ref)

Implementations
---------------
* [`DefaultSACool`](@ref)

"""
abstract type SACool end

#-----#
# API #
#-----#

"""
    cool!(cooler :: SACool, state :: SAState)

When called, may decrease `state.temperature`.

Method List
-----------
$(METHODLIST)
"""
function cool! end

#----------------#
# Implementation #
#----------------#

"""
Default cooling schedule. Each invocation, will multiply the temperature
of the anneal by the `alpha` parameter.
"""
struct DefaultSACool <: SACool
    alpha::Float64
end

@inline cool!(c::DefaultSACool, state::SAState) = (state.temperature *= c.alpha)

####################
# Distance Limiter #
####################

"""
Type that controls the length contraction mechanism of Simulated Annealing
placement.

API
---
* [`limit!`](@ref)

Implmementations
----------------
* [`DefaultSALimit`](@ref)

"""
abstract type SALimit end

#-----#
# API #
#-----#
"""
    limit!(limiter :: SALimit, state :: SAState)

Mutate `state.distance_limit` when called.
"""
function limit! end

#----------------#
# Implementation #
#----------------#

"""
Default distance limit updater. Will adjust the distance limit so approximate
`ratio` of moves are accepted. Will not set `state.limit` lower than `minimum`.
"""
struct DefaultSALimit <: SALimit
    ratio::Float64
    # Minimum allows distance limit.
    minimum::Int
end
DefaultSALimit(ratio::Float64) = DefaultSALimit(ratio, 1)

function limit!(l::DefaultSALimit, state::SAState)
    # Compute acceptance ratio from state
    acceptance_ratio = state.recent_accepted_moves / state.recent_successful_moves
    # Update distance limit
    temporary = (1 - l.ratio + acceptance_ratio)
    state.distance_limit = clamp(
        state.distance_limit * temporary, l.minimum, state.max_distance_limit
    )
    return nothing
end

##########
# SADone #
##########

"""
Control when to terminate Simulated Annealing placement.

API
---
* [`sa_done`](@ref)

Implementations
---------------
* [`DefaultSADone`]

"""
abstract type SADone end

#-----#
# API #
#-----#

"""
    sa_done(doner :: SADone, state :: SAState)

Return `true` to finish placement.
"""
function sa_done end

#---------------#
# Implmentation #
#---------------#

"""
Default end detection. Will return `true` when objective deviation is
less than `atol`.
"""
struct DefaultSADone <: SADone
    atol::Float64
end

@inline function sa_done(d::DefaultSADone, state::SAState)
    return !state.warming && (state.deviation < d.atol)
end

################################################################################
# Placement Routine!
################################################################################

function place!(
    sa_struct::SAStruct{A,U,D};
    # Number of moves before doing a parameter update.
    move_attempts = 20000,
    initial_temperature = 1.0,
    supplied_state = nothing,
    movegen::MoveGenerator = CachedMoveGenerator(sa_struct),
    # Parameters for high-level control
    warmer::SAWarm = DefaultSAWarm(0.96, 2.0, 0.97),
    cooler::SACool = DefaultSACool(0.999),
    doner::SADone = DefaultSADone(10.0^-5),
    limiter::SALimit = DefaultSALimit(0.44, 2),
    kwargs...,
) where {A,U,D}
    @info "Running Simulated Annealing Placement."
    # Set the random number generator for repeatable results.

    # Unpack SA
    num_nodes = length(sa_struct.nodes)
    num_channels = length(sa_struct.channels)

    # Get the largest address
    max_addresses = size(sa_struct.pathtable)
    largest_address = maximum(max_addresses)

    initial_move_limit = distancelimit(movegen, sa_struct)

    # Initialize structure to help during placement.
    cost = map_cost(sa_struct)
    cookie = MoveCookie(sa_struct)

    # Main Simulated Annealing Loop
    loop = true

    # Initialize the main state variable. State variable's timer begins when
    # the structure is created.
    if supplied_state === nothing
        state = SAState(initial_temperature, Float64(initial_move_limit), cost)
    else
        state = supplied_state
        reset!(state)
    end

    # Initialize move generator
    initialize!(movegen, sa_struct, state.distance_limit)

    # Print out the header for the statistic columns.
    show_stats(state, true)
    first_display = true
    while loop
        ##################
        # Pre-loop setup #
        ##################

        # Invert temperature to perform floating point multiplication rather
        # than division. Set local counters for this iteration.
        one_over_T = 1 / state.temperature
        accepted_moves = 0
        successful_moves = 0
        objective = state.objective
        sum_cost_difference = zero(typeof(cost))

        ##############
        # Inner Loop #
        ##############
        while successful_moves < move_attempts
            # Try to generate a move. If it failed, try again.
            #@inbounds success = generate_move!(sa_struct, movegen, cookie)
            success = generate_move!(sa_struct, movegen, cookie)
            if !success
                continue
            end

            successful_moves += 1
            # Get the cost of the move
            cost_of_move = cookie.cost_of_move
            if cost_of_move <= zero(typeof(cost_of_move)) ||
               rand() < exp(-cost_of_move * one_over_T)
                accepted_moves += 1

                if cost_of_move > 0
                    sum_cost_difference += cost_of_move
                end

                objective += cost_of_move
            else
                undo_move(sa_struct, cookie)
            end
        end

        ###########################
        # Post iteration routines #
        ###########################
        # Update cost for numerical stability reasons
        state.objective = map_cost(sa_struct)
        # Sanity Check

        if !isapprox(objective, state.objective)
            @warn """
            Objective mismatch.
            actual: $(state.objective).
            calculated: $objective.
            """
        end

        # Update some statistics in the state variable
        state.recent_move_attempts = move_attempts
        state.recent_successful_moves = successful_moves
        state.recent_accepted_moves = accepted_moves
        # Quick check to avoid NaN's showing up in the case of zero accepted
        # moves.
        if iszero(accepted_moves)
            state.recent_deviation = 0
        else
            state.recent_deviation = sum_cost_difference / accepted_moves
        end
        state.aux_cost = aux_cost(sa_struct)

        # Adjust temperature
        state.warming ? warm!(warmer, state) : cool!(cooler, state)
        # Adjust distance limit - only if we're not warming up.
        state.warming || limit!(limiter, state)

        # State updates - check if we potentially need to do some recomputation
        # for the move generator
        update_movegen = update!(state)

        if update_movegen
            update!(movegen, sa_struct, state.distance_limit_int)
        end

        # Exit Condition
        loop = !sa_done(doner, state)
    end
    # Show final statistics
    show_stats(state)

    return state
end

################################################################################
# Movement related functions
################################################################################
@propagate_inbounds function move_with_undo(
    sa_struct::SAStruct, cookie, index::Int64, new_location
)
    node = sa_struct.nodes[index]

    # Store the old information
    old_location = location(node)
    cookie.index_of_moved_node = index
    cookie.old_location = old_location
    # Check to see if there is a node already mapped to this location
    occupying_node_index = sa_struct.grid[new_location]
    if occupying_node_index == 0
        cookie.move_was_swap = false

        base_cost = node_cost(sa_struct, index)
        move(sa_struct, index, new_location)
        moved_cost = node_cost(sa_struct, index)

        cookie.cost_of_move = moved_cost - base_cost
    else
        occupying_node = sa_struct.nodes[occupying_node_index]

        # Check if the node at the new location can be moved to the old location
        isvalid(sa_struct.maptable, getclass(occupying_node), old_location) || return false
        cookie.move_was_swap = true
        # Save the index of the occupying node
        cookie.index_of_other_node = occupying_node_index
        # Compute the cost before the move
        base_cost = node_pair_cost(sa_struct, index, occupying_node_index)
        # Swap nodes
        swap(sa_struct, index, occupying_node_index)
        # Get cost after move and store
        moved_cost = node_pair_cost(sa_struct, index, occupying_node_index)
        cookie.cost_of_move = moved_cost - base_cost
    end
    return true
end

@propagate_inbounds function undo_move(sa_struct::SAStruct, cookie)
    # If the last move was a swap, need a swap in order to undo it.
    if cookie.move_was_swap
        swap(sa_struct, cookie.index_of_moved_node, cookie.index_of_other_node)
        # Otherwise, a simple move is just fine.
    else
        move(sa_struct, cookie.index_of_moved_node, cookie.old_location)
    end
    return nothing
end


using StaticArrays

using JuLIP: AbstractCalculator,
             Atoms

import Base: Dict,
             ==

import JuLIP: cutoff,
              energy,
              forces,
              site_energies,
              virial

export NBodyIP,
       fast,
       load_ip,
       save_ip,
       energy,
       forces,
       site_energies,
       virial

# ----------- some auxiliaries we will use throughout -------------

"""
`push_str!(ex::Vector{Expr}, s::String)` : parse the string and
append it to the expressions.
"""
push_str!(ex::Vector{Expr}, s::String) = push!(ex, Meta.parse(s))

"""
`append_str!(ex::Vector{Expr}, s::Vector{String})` : parse the strings and
append them to the expressions.
"""
append_str!(ex::Vector{Expr}, s::Vector{String}) = append!(ex, Meta.parse.(s))

"""
`bo2edges(N)` : bodyorder-to-edges
"""
bo2edges(N::Integer) = (N * (N-1)) ÷ 2
bo2edges(::Val{N}) where {N} = (N * (N-1)) ÷ 2

"""
`edges2bo(M)`: "edges-to-bodyorder", an internal function that translates
the number of edges in a simplex into the body-order
"""
edges2bo(M::Integer) = (M <= 0) ? 1 : round(Int, 0.5 + sqrt(0.25 + 2 * M))

"""
`bo2angles(N)` : bodyorder-to-angles in one corner of a simplex
"""
bo2angles(N::Integer) = ((N-2) * (N-1)) ÷ 2
bo2angles(::Val{N}) where {N} = ((N-2) * (N-1)) ÷ 2

"""
`angles2bo(M)`: "angles-to-bodyorder", an internal function that translates
the number of angles in one corner of a simplex into the body-order
"""
angles2bo(A::Integer) = (M <= 0) ? 2 : round(Int, 3/2 + sqrt((9/4) + (2-2*A)))

# ----------- some generic functions that we like to have available globally
#             to overload as needed

"""
some measure of degree - could be but need not be polymomial degree; this
is basis-specific
"""
function degree end

"""
some measure of degree for bond-angles
"""
function wtdegree end

"""
`basisname(b) -> String`: returns a short string identifying the kind of
basis function.
"""
function basisname end

"""
`combinebasis`:  if `basis::Vector{AbstractCalculator}` with identical
`hash(::BASIS,...)`, then `combinebasis(basis, coeffs)` should return a new
calculator that combines all basis function into a single IP.
"""
function combinebasis end

# prototypes for the invariants
function invariants end
function invariants_d end
function invariants_ed end

# prototypes for obtaining the descriptor
function descriptor end

function evaluate_I end
function evaluate_I_d end
function evaluate_I_ed end

bodyorder(V::NBodyFunction{N}) where {N} = N

==(V1::NBodyIP, V2::NBodyIP) = V1.components == V2.components
cutoff(V::NBodyIP) = maximum( cutoff.(V.components) )
energy(V::NBodyIP, at::Atoms) = sum( energy(Vn, at)  for Vn in V.components )
forces(V::NBodyIP, at::Atoms) = sum( forces(Vn, at)  for Vn in V.components )
virial(V::NBodyIP, at::Atoms) = sum( virial(Vn, at)  for Vn in V.components )

"""
turn a potentially slow representation of an IP into a fast one,
by switching to a different representation.
"""
fast(IP::NBodyIP) = NBodyIP( fast.(IP.components) )

# fall-back
fast(V::Any) = V

function unique_components(basis)
   bods = hash.(Ref(BASIS()), basis)
   un = Int[]
   I = Vector{Int}[]
   for i = 1:length(bods)
      found = false
      for j = 1:length(un)
         if bods[un[j]] == bods[i]
            push!(I[j], i)
            found = true
            break
         end
      end
      if !found
         push!(un, i)
         push!(I, Int[i])
      end
   end
   return I
end

# construct an NBodyIP from a basis
function NBodyIP(basis, coeffs)
   components = AbstractCalculator[]
   newbasis = AbstractCalculator[]
   newcoeffs = Float64[]
   # if the basis contains any NBodyIPs
   # they should be treated differently
   for (i, b) in enumerate(basis)
      if b isa NBodyIP
         append!(components, b.components)
      else
         push!(newbasis, b)
         push!(newcoeffs, coeffs[i])
      end
   end
   basis, coeffs = newbasis, newcoeffs

   I = unique_components(basis)
   for J in I
      push!(components, combinebasis([basis[j] for j in J], coeffs[J]))
   end
   return NBodyIP(components)
end

# functionality for pure NBodyFunctions for computing
#  * site_energies
#  * energy
#  * forces
#  * virial
include("eval_nbody.jl")

# IO of NBodyIPs
include("io.jl")

# space transforms and cutoffs
include("descsupp.jl")

# ======= experimental ============

import JuLIP.Potentials: evaluate, evaluate_d

evaluate(V::NBodyFunction{2, <: BondLengthDesc}, r::AbstractFloat) =
      evaluate(V, SVector(r))
evaluate_d(V::NBodyFunction{2, <: BondLengthDesc}, r::AbstractFloat) =
      evaluate_d(V, SVector(r))[1]

evaluate(V::NBodyFunction{2, <: ClusterBLDesc}, r::AbstractFloat) =
      evaluate(V, SVector(r))
evaluate_d(V::NBodyFunction{2, <: ClusterBLDesc}, r::AbstractFloat) =
      evaluate_d(V, SVector(r))[1]

evaluate(V::NBodyFunction{2, <: BondAngleDesc}, r::T) where {T <: AbstractFloat} =
      evaluate(V, (SVector(r), SVector{0,T}()))
evaluate_d(V::NBodyFunction{2, <: BondAngleDesc}, r::T) where {T <: AbstractFloat} =
      evaluate_d(V, (SVector(r), SVector{0,T}()))[1]

function evaluate(V::NBodyFunction{N}, r::SVector{M}) where {N, M}
   # this assumes that D is a BondLengthDesc
   D = descriptor(V)::Union{BondLengthDesc, ClusterBLDesc}
   return evaluate_I(V, invariants(D, r)) * fcut(D, r)
end

function evaluate_d(V::NBodyFunction{N}, r::SVector{M}) where {N, M}
   D = descriptor(V)::BondLengthDesc
   fc, fc_d = fcut_d(D, r)
   Vn, Vn_d = evaluate_I_ed(V, invariants_ed(D, r))
   return fc * Vn_d + fc_d * Vn
end

function evaluate(V::NBodyFunction{N}, rθ::Tuple{SVector{M1}, SVector{M2}}
                  ) where {N, M1, M2}
   # this assumes that D is a BondAngleDesc
   D = descriptor(V)::BondAngleDesc
   return evaluate_I(V, invariants(D, rθ)) * fcut(D, rθ)
end

function evaluate_d(V::NBodyFunction{N}, rθ::Tuple{SVector{M1}, SVector{M2}}
                  ) where {N, M1, M2}
   D = descriptor(V)::BondAngleDesc
   fc, fc_d = fcut_d(D, rθ)
   Vn, Vn_d = evaluate_I_ed(V, invariants_ed(D, rθ))
   return fc * Vn_d + fc_d * Vn
end

"""
Take a basis and split it into individual basis groups.
"""
function split_basis(basis; splitfun = b -> hash(Val{:basis}(), b))
   # get some basis hashs of the individual basis functions
   tps = splitfun.(basis)
   Iord = Vector{Int}[]
   Bord = Any[]
   for tp in unique(tps)
      # find which elements of basis have type `tp`
      I = findall( [tp == t  for t in tps] )
      push!(Iord, I)
      push!(Bord, [b for b in basis[I]])
   end
   return Bord, Iord
end


# functions for calculating node coordinates and metrics for curvilinear
# meshes

#------------------------------------------------------------------------------
# allocators for curvilinear meshes

"""
  Allocates mesh.vert_coords
"""
function allocateMeshCoordinateArray(mesh::PumiMeshDG{Tmsh}, sbp::AbstractSBP) where Tmsh

  num_coord_nodes = mesh.coord_numNodesPerElement

  if !isFieldDefined(mesh, :vert_coords)
    mesh.vert_coords = Array{Tmsh}(mesh.dim, num_coord_nodes, mesh.numEl)
    mesh.vert_coords_bar = Array{Tmsh}(mesh.dim, num_coord_nodes, mesh.numEl)
  else
    fill!(mesh.vert_coords, 0.0)
  end

  return nothing
end

"""
  Allocates the following fields of mesh:
    coords
    dxidx
    jac

  Also allocates the following fields with size zero (because they are not used):
   dxidx_face
   jac_face
   dxidx_bndry
   jac_bndry
   dxidx_sharedface
   jac_sharedface
"""
function allocateCurvilinearCoordinateAndMetricArrays(mesh::PumiMeshDG{Tmsh},                                                         sbp::AbstractSBP) where Tmsh

  dim = mesh.dim
  sbpface = mesh.sbpface

  if !isFieldDefined(mesh, :coords, :dxidx, :jac)
    mesh.coords = zeros(Tmsh, mesh.dim, mesh.numNodesPerElement, mesh.numEl)
    mesh.dxidx = zeros(Tmsh, mesh.dim, mesh.dim, mesh.numNodesPerElement, mesh.numEl)
    mesh.jac = zeros(Tmsh, mesh.numNodesPerElement, mesh.numEl)

    mesh.dxidx_bar = zeros(mesh.dxidx)
    mesh.jac_bar = zeros(mesh.jac)

    # these arrays are not used for curvilinear meshes

    # interior faces
    mesh.dxidx_face = zeros(Tmsh, 0, 0, 0, 0)
    mesh.jac_face = zeros(Tmsh, 0, 0)

    mesh.dxidx_face_bar = zeros(mesh.dxidx_face)
    mesh.jac_face_bar = zeros(mesh.jac_face)

    # boundary faces
    mesh.dxidx_bndry = zeros(Tmsh, 0, 0, 0, 0)
    mesh.jac_bndry = zeros(Tmsh, 0, 0)

    mesh.dxidx_bndry_bar = zeros(mesh.dxidx_bndry)
    mesh.jac_bndry_bar = zeros(mesh.jac_bndry)

    # parallel shared faces
    mesh.dxidx_sharedface = Array{Array{Tmsh, 4}}(0)
    mesh.jac_sharedface = Array{Array{Tmsh, 2}}(0)

    mesh.dxidx_sharedface_bar = Array{Array{Tmsh, 4}}(0)
    mesh.jac_sharedface_bar = Array{Array{Tmsh, 2}}(0)
  else
    fill!(mesh.coords, 0.0)
    fill!(mesh.dxidx, 0.0)
    fill!(mesh.jac, 0.0)
  end

  return nothing
end

#------------------------------------------------------------------------------
# curvilinear functions

"""
  This function gets the coordinates that define the mesh
  (not the solution node coordinates) and puts them the mesh.vert_coords field
  of the mesh object.  For each element, the ordering of the coordinates
  is verts, then edges, then faces, then regions.  This function uses
  smart allocators to allocate the array if needed.

  Input:
    mesh: a DG mesh
    sbp: an SBP operators
"""
function getMeshCoordinates(mesh::PumiMeshDG{Tmsh}, sbp::AbstractSBP) where Tmsh

  allocateMeshCoordinateArray(mesh, sbp)

  coords_tmp = zeros(Float64, mesh.dim, mesh.coord_numNodesPerElement)
  for i=1:mesh.numEl
    el_i = mesh.elements[i]
    coords_i = sview(mesh.vert_coords, :, :, i)
    apf.getAllEntityCoords(mesh.m_ptr, el_i, coords_tmp)
    copy!(coords_i, coords_tmp)
  end

  return nothing
end

"""
  This function calculates the fields of the mesh that hold coordinates of the
  face nodes for boundaries, interfaces, and sharedfaces.  This function uses
  smart allocators to allocate the arrays if needed
"""
function getFaceCoordinatesAndNormals(mesh::PumiMeshDG{Tmsh}, sbp::AbstractSBP) where Tmsh

  allocateFaceCoordinates(mesh)

  allocateNormals(mesh, sbp)

  if length(mesh.bndryfaces) > 0  # debugging: don't call if unneeded
    calcFaceCoordinatesAndNormals(mesh, sbp, mesh.bndryfaces,
                                  mesh.coords_bndry, mesh.nrm_bndry)
  end

  calcFaceCoordinatesAndNormals(mesh, sbp, mesh.interfaces,
                               mesh.coords_interface, mesh.nrm_face)
  for i=1:mesh.npeers
    calcFaceCoordinatesAndNormals(mesh, sbp, mesh.bndries_local[i],
                               mesh.coords_sharedface[i], mesh.nrm_sharedface[i])
  end

  return nothing
end

"""
  This function back propigates mesh.nrm_*_bar and mesh.coords_*_bar to
  mesh.vert_coords_bar.  mesh.vert_coords_bar is updated (not overwritten)
  with the results.
"""
function getFaceCoordinatesAndNormals_rev(mesh::PumiMeshDG{Tmsh},
                                          sbp::AbstractSBP) where Tmsh
  if length(mesh.bndryfaces) > 0  # debugging: don't call if unneeded
    calcFaceCoordinatesAndNormals_rev(mesh, sbp, mesh.bndryfaces,
                                      mesh.coords_bndry,
                                      mesh.coords_bndry_bar,
                                      mesh.nrm_bndry,
                                      mesh.nrm_bndry_bar)
  end

  coords_face_bar = zeros(Tmsh, mesh.dim, mesh.numNodesPerFace, mesh.numInterfaces)
  calcFaceCoordinatesAndNormals_rev(mesh, sbp, mesh.interfaces,
                                mesh.coords_interface, coords_face_bar,
                                mesh.nrm_face, mesh.nrm_face_bar)

  for i=1:mesh.npeers
    coords_sharedface_bar = zeros(Tmsh, mesh.dim, mesh.numNodesPerFace, length(mesh.bndries_local[i]))
    calcFaceCoordinatesAndNormals_rev(mesh, sbp, mesh.bndries_local[i],
                                  mesh.coords_sharedface[i],
                                  coords_sharedface_bar,
                                  mesh.nrm_sharedface[i],
                                  mesh.nrm_sharedface_bar[i])
  end

  return nothing
end

"""
  This is the inner function used by getFaceCoordinatesandNormals.  It
  gets the coordinates of the face nodes and their normal vectors for
  a given array of Interfaces/Boundaries

  Inputs:
    mesh: a DG mesh object
    faces: an array of Boundaries or Interfaces

  Inputs/Outputs
    coords_face: array Tdim x numNodesPerFace x length(faces)
    nrm_face: array, same size as coords_face

  Aliasing restrictions: none, although it would be weird if coords_face and
                         nrm_face aliased
"""
function calcFaceCoordinatesAndNormals(
                    mesh::PumiMeshDG{Tmsh}, sbp::AbstractSBP,
                    faces::AbstractArray{I, 1},
                    coords_face::AbstractArray{Tmsh, 3},
                    nrm_face::AbstractArray{Tmsh, 3}) where {Tmsh, I <: Union{Boundary, Interface}}

  blocksize = 1000  # magic parameter: number of faces to do in a group
  nfaces = length(faces)

  # calculate number of blocks
  nblocks_full = div(nfaces, blocksize)
  nrem = nfaces % blocksize

  numNodesPerElement = mesh.coord_numNodesPerElement
  numNodesPerFace = mesh.coord_numNodesPerType[mesh.dim]

  # some temporary arrays
  down_faces = Array{Ptr{Void}}(12)
  coords_lag_face = Array{Tmsh}(mesh.dim, mesh.coord_numNodesPerFace, blocksize)

  # get the parametic coordinates of the face nodes
  face_xi = mesh.coord_facexi
  ref_verts = baryToXY(face_xi, mesh.sbpface.vtx)

  face_idx = 1  # index in faces array
  for block=1:nblocks_full
    start_idx = (block - 1)*blocksize + 1
    end_idx = block*blocksize
    coords_face_block = sview(coords_face, :, :, start_idx:end_idx)
    nrm_face_block = sview(nrm_face, :, :, start_idx:end_idx)

    # load up data for current block
    for i=1:blocksize
      el_i = getElementL(faces[face_idx])
      face_i = getFaceL(faces[face_idx])
      el_ptr = mesh.elements[el_i]

      coords_i = sview(coords_lag_face, :, :, i)
      getMeshFaceCoordinates(mesh, el_i, face_i, coords_i)
      face_idx += 1
    end

    # populate output array for current block
    calcFaceNormals!(mesh.sbpface, mesh.coord_order, ref_verts, coords_lag_face,
                     coords_face_block, nrm_face_block)
    fill!(coords_lag_face, 0.0)

    fixOutwardNormal(mesh, sview(faces, start_idx:end_idx), nrm_face_block)
  end  # end loop over full blocks

  # do remainder loop
  start_idx = nblocks_full*blocksize + 1
  end_idx = nfaces
  @assert end_idx - start_idx + 1 <= blocksize
  @assert end_idx - start_idx + 1 == nrem

  coords_face_block = sview(coords_face, :, :, start_idx:end_idx)
  nrm_face_block = sview(nrm_face, :, :, start_idx:end_idx)
  coords_lag_face_block = sview(coords_lag_face, :, :, 1:nrem)

  for i=1:nrem  # loop over remaining faces
    el_i = getElementL(faces[face_idx])
    face_i = getFaceL(faces[face_idx])
    el_ptr = mesh.elements[el_i]

    coords_i = sview(coords_lag_face, :, :, i)
    getMeshFaceCoordinates(mesh, el_i, face_i, coords_i)
    face_idx += 1

  end

  calcFaceNormals!(mesh.sbpface, mesh.coord_order, ref_verts,
                   coords_lag_face_block, coords_face_block, nrm_face_block)

  # make sure the normal vectors point outwards
  fixOutwardNormal(mesh, sview(faces, start_idx:end_idx), nrm_face_block)

  return nothing
end

"""
  Reverse mode of calcFaceCoordinatesAndNormals_rev, back propigates
  coords_face_bar and nrm_face_bar to mesh.vertcoords_bar.

  This function also recalculates coords_face and nrm_face in the course
  of doing the reverse mode.
"""
function calcFaceCoordinatesAndNormals_rev(
                    mesh::PumiMeshDG{Tmsh}, sbp::AbstractSBP,
                    faces::AbstractArray{I, 1},
                    coords_face::AbstractArray{Tmsh, 3},
                    coords_face_bar::AbstractArray{Tmsh, 3},
                    nrm_face::AbstractArray{Tmsh, 3},
                    nrm_face_bar::AbstractArray{Tmsh, 3}; print=false) where {Tmsh, I <: Union{Boundary, Interface}}

  blocksize = 1000  # magic parameter: number of faces to do in a group
  nfaces = length(faces)

  # calculate number of blocks
  nblocks_full = div(nfaces, blocksize)
  nrem = nfaces % blocksize

  numNodesPerElement = mesh.coord_numNodesPerElement
  numNodesPerFace = mesh.coord_numNodesPerType[mesh.dim]

  # some temporary arrays
  down_faces = Array{Ptr{Void}}(12)
  coords_lag_face = Array{Tmsh}(mesh.dim, mesh.coord_numNodesPerFace, blocksize)
  coords_lag_face_bar = zeros(coords_lag_face)

  # get the parametic coordinates of the face nodes
  face_xi = mesh.coord_facexi
  ref_verts = baryToXY(face_xi, mesh.sbpface.vtx)

  for block=1:nblocks_full
    start_idx = (block - 1)*blocksize + 1
    end_idx = block*blocksize
    faces_block = sview(faces, start_idx:end_idx)
    coords_face_block = sview(coords_face, :, :, start_idx:end_idx)
    coords_face_bar_block = sview(coords_face_bar, :, :, start_idx:end_idx)
    nrm_face_block = sview(nrm_face, :, :, start_idx:end_idx)
    nrm_face_bar_block = sview(nrm_face_bar, :, :, start_idx:end_idx)

    # load up data for current block
    for i=1:blocksize
      el_i = getElementL(faces_block[i])
      face_i = getFaceL(faces_block[i])
      el_ptr = mesh.elements[el_i]

      coords_i = sview(coords_lag_face, :, :, i)
      getMeshFaceCoordinates(mesh, el_i, face_i, coords_i)
    end

    # forward sweep
    # need the *original* face normals for fix_outward_normal, so recalculate
    # them here
    calcFaceNormals!(mesh.sbpface, mesh.coord_order, ref_verts, coords_lag_face,
                     coords_face_block, nrm_face_block)

    # reverse sweep
    fixOutwardNormal_rev(mesh, sview(faces, start_idx:end_idx), nrm_face_block,
                         nrm_face_bar_block)

    fill!(coords_lag_face_bar, 0.0)
    calcFaceNormals_rev!(mesh.sbpface, mesh.coord_order, ref_verts,
                         coords_lag_face, coords_lag_face_bar,
                         coords_face_bar_block, nrm_face_bar_block)

    #TODO: unecessary?
    fill!(coords_lag_face, 0.0)

    for i=1:blocksize
      el_i = getElementL(faces_block[i])
      face_i = getFaceL(faces_block[i])
      el_ptr = mesh.elements[el_i]

      coords_bar_i = sview(coords_lag_face_bar, :, :, i)
      getMeshFaceCoordinates_rev(mesh, el_i, face_i, coords_bar_i)
    end
  end  # end loop over full blocks

  # do remainder loop
  start_idx = nblocks_full*blocksize + 1
  end_idx = nfaces
  @assert end_idx - start_idx + 1 <= blocksize
  @assert end_idx - start_idx + 1 == nrem

  faces_block = sview(faces, start_idx:end_idx)
  coords_face_block = sview(coords_face, :, :, start_idx:end_idx)
  coords_face_bar_block = sview(coords_face_bar, :, :, start_idx:end_idx)
  nrm_face_block = sview(nrm_face, :, :, start_idx:end_idx)
  nrm_face_bar_block = sview(nrm_face_bar, :, :, start_idx:end_idx)
  coords_lag_face_block = sview(coords_lag_face, :, :, 1:nrem)
  coords_lag_face_bar_block = sview(coords_lag_face_bar, :, :, 1:nrem)

  for i=1:nrem  # loop over remaining faces
    el_i = getElementL(faces_block[i])
    face_i = getFaceL(faces_block[i])

    coords_i = sview(coords_lag_face, :, :, i)
    getMeshFaceCoordinates(mesh, el_i, face_i, coords_i)
  end

  # forward sweep
  # we need to use the *original* face normals, not the already reversed
  # ones for fixOutwardNormal_rev
  calcFaceNormals!(mesh.sbpface, mesh.coord_order, ref_verts,
                   coords_lag_face_block, coords_face_block, nrm_face_block)

  # reverse sweep
  fixOutwardNormal_rev(mesh, sview(faces, start_idx:end_idx), nrm_face_block,
                    nrm_face_bar_block)

  fill!(coords_lag_face_bar, 0.0)

  calcFaceNormals_rev!(mesh.sbpface, mesh.coord_order, ref_verts,
                   coords_lag_face_block, coords_lag_face_bar_block,
                   coords_face_bar_block, nrm_face_bar_block)

  for i=1:nrem
    el_i = getElementL(faces_block[i])
    face_i = getFaceL(faces_block[i])

    coords_bar_i = sview(coords_lag_face_bar_block, :, :, i)
    getMeshFaceCoordinates_rev(mesh, el_i, face_i, coords_bar_i)
  end

  return nothing
end

"""
  This function check to make sure each face normal vector is oriented
  outwards, and flips it if needed.

  Inputs:
    mesh
    faces: array of Boundary of Interfaces to check for normal orientation

  Inputs/Outputs:
    nrm_face: array of size mesh.dim x mesh.numNodesPerFace x length(faces)
              containing the normal vector at each node of each face.
              Updated in place to make normal vector point outward
"""
function fixOutwardNormal(mesh,
faces::AbstractArray{I, 1},
nrm_face::AbstractArray{Tmsh, 3}) where {I <: Union{Boundary, Interface}, Tmsh}

  nfaces = length(faces)
  should_flip_node = Array{Bool}(mesh.numNodesPerFace)
  for i=1:nfaces
    is_inward_normal(mesh, faces[i], sview(nrm_face, :, :, i), should_flip_node)

    for j=1:mesh.numNodesPerFace
      if should_flip_node[j]
        for p=1:mesh.dim
          nrm_face[p, j, i] = -nrm_face[p, j, i]
        end
      end
    end  # end j

  end  # end loop i

   return nothing
end

"""
  Reverse mode of fixOutwardNormal, reverses the primal normal vectors.
  On entry nrm_face should have the normal vectors as calculated by
  SBP.  On exit, they will point outwards.

  Inputs:
    mesh
    faces

  Inputs/Outputs:
    nrm_face
    nrm_face_bar: adjoint part of nrm_face
"""
function fixOutwardNormal_rev(mesh,
                          faces::AbstractArray{I, 1},
                          nrm_face::AbstractArray{Tmsh, 3},
                          nrm_face_bar::AbstractArray{Tmsh, 3}) where {Tmsh, I <: Union{Boundary, Interface}}

  nfaces = length(faces)
  should_flip_node = Array{Bool}(mesh.numNodesPerFace)
  for i=1:nfaces
    is_inward_normal(mesh, faces[i], sview(nrm_face, :, :, i), should_flip_node)

    for j=1:mesh.numNodesPerFace
      if should_flip_node[j]
        for p=1:mesh.dim
          nrm_face_bar[p, j, i] = -nrm_face_bar[p, j, i]
          nrm_face[p, j, i] = -nrm_face[p, j, i]
        end
      end
    end  # end j

  end  # end loop i

  return nothing
end

"""
  Check if the normal vector on each node of a face is pointing inward

  The algorithm determines orientation by comparing the normal vector against
  the vector along an edge of the simplex (from the vertex not on the face to
  the vertex on the face).  For highly skewed elements, using different
  vertices on the face to compute the vector can give different results, so
  only the vector with the largest magnitude dot product with the normal vector
  is considered.

  Inputs:
    mesh: the mesh
    iface: either a Boundary or an Interface
    nrm_face: mesh.dim x mesh.numNodesPerFace array containing the normal
              vectors for each face node

  Inputs/Outputs:
    should_flip_node: an Bool array of length numNodesPerFace specifying whether
                      the normal vector at each node is pointing inward
"""
function is_inward_normal(mesh, iface::Union{Boundary, Interface},
                    nrm_face::AbstractMatrix{Tmsh},
                    should_flip_node::AbstractVector{Bool}) where Tmsh

  tmp = zeros(3)  # temporary vector to hold coordinates
  topo = mesh.topo
  numVertPerElement = mesh.numTypePerElement[1]
  numVertPerFace = numVertPerElement - 1

  # temporary arrays
#  el_verts = Array{Ptr{Void}}(numVertPerElement)
  other_vert_coords = zeros(Tmsh, mesh.dim)
#  face_verts = Array{Ptr{Void}}(numVertPerElement - 1)
  face_vert_coords = zeros(Tmsh, mesh.dim, numVertPerFace)

  elnum = getElementL(iface)
  facenum_local = getFaceL(iface)

#  el_i = mesh.elements[elnum]
#  apf.getDownward(mesh.m_ptr, el_i, 0, el_verts)

  for j=1:numVertPerFace
    v_j = topo.face_verts[j, facenum_local]

    for p=1:mesh.dim
      face_vert_coords[p, j] = mesh.vert_coords[p, v_j, elnum]
    end
  end

  # get the vert not on the face
  other_vert = 0
  for j=1:numVertPerElement
     if !(j in sview(topo.face_verts, :, facenum_local))
       other_vert = j
     end
  end

  for p=1:mesh.dim
    other_vert_coords[p] = mesh.vert_coords[p, other_vert, elnum]
  end
  # check that the face normal is in the opposite direction as the
  # vectors from a vertex on the face to the vertex not on the face

  # in some cases the normal vector can be nearly orthoogonal to vertex
  # vectors, so use the one with the greatest magnitude dot product
  should_flip = false
  max_mag = 0.0  # maximum dot product
  for j=1:mesh.numNodesPerFace
    outward_count = 0  # count number of calculations that showed outward
    for k=1:numVertPerFace
      val = zero(Tmsh)
      for p=1:mesh.dim
        r1_p = other_vert_coords[p] - face_vert_coords[p, k]
        val += nrm_face[p, j]*r1_p  # accumulate dot product
      end

      if abs(val) > max_mag
        max_mag = abs(val)
        # flip if the value is greater than 0
        should_flip = real(val) > 0
        #=
        if val < 0
          should_flip = false
        else
          should_flip = true
        end  # end if
        =#

      end
    end  # end k

    should_flip_node[j] = should_flip
  end  # end loop j

  return nothing
end

"""
  This function calculates the node coordinates, scaled mapping jacobian, and
  mapping jacobian determinant for a curvilinear mesh and stores them to
  the fields of the mesh object.  This function uses smart allocators to
  allocate the arrays if needed

  Inputs:
    mesh: a DG mesh object
    sbp: an SBP operator
"""
function getCurvilinearCoordinatesAndMetrics(mesh::PumiMeshDG{Tmsh},
                                            sbp::AbstractSBP) where Tmsh

  allocateCurvilinearCoordinateAndMetricArrays(mesh, sbp)

  ref_vtx = baryToXY(mesh.coord_xi, sbp.vtx)
#  if mesh.dim == 2
#    calcMappingJacobian!(sbp, mesh.coord_order, ref_vtx, mesh.vert_coords,
#                         mesh.coords, mesh.dxidx, mesh.jac)
#  else  # need to calculate Eone

    # block format
    blocksize = 1000  # number of elements per block
    nblocks_full = div(mesh.numEl, blocksize)
    nrem = mesh.numEl % blocksize

    Eone = zeros(Tmsh, mesh.numNodesPerElement, mesh.dim, blocksize)

    for block=1:nblocks_full
      start_idx = (block - 1)*blocksize + 1
      end_idx = block*blocksize
      element_range = start_idx:end_idx

      getCurvilinearMetricsAndCoordinates_inner(mesh, sbp, element_range, Eone)

    end  # end loop over blocks

    if nrem != 0
      # remainder block
      start_idx = nblocks_full*blocksize + 1
      end_idx = mesh.numEl
      @assert end_idx - start_idx + 1 <= blocksize
      @assert end_idx - start_idx + 1 == nrem

      element_range = start_idx:end_idx

      # make sure dimensions of Eone is correct (number of elements might be
      # less than before)
      Eone_rem = sview(Eone, :, :, 1:nrem)

      getCurvilinearMetricsAndCoordinates_inner(mesh, sbp, element_range, Eone_rem)
    end
#  end  # end if dim == 2

  return nothing
end

"""
  Back propigate dxidx to mesh.vert_coords and mesh.nrm.  See the primal method
  for details.

  Inputs:
    mesh: vert_coords_bar, nrm_face_bar, nrm_bndry_bar, nrm_sharedface are
          updated with the results
    sbp:
"""
function getCurvilinearCoordinatesAndMetrics_rev(mesh::PumiMeshDG{Tmsh},
                                                 sbp::AbstractSBP) where Tmsh

  # we have to compute E1_bar for both 2d and 3D
  blocksize = 1000  # number of elements per block
  nblocks_full = div(mesh.numEl, blocksize)
  nrem = mesh.numEl % blocksize

  Eone_bar = zeros(Tmsh, mesh.numNodesPerElement, mesh.dim, blocksize)

  for block=1:nblocks_full
    start_idx = (block - 1)*blocksize + 1
    end_idx = block*blocksize
    element_range = start_idx:end_idx

    fill!(Eone_bar, 0.0)
    getCurvilinearMetricsAndCoordinates_inner_rev(mesh, sbp, element_range,
                                                  Eone_bar)
  end

 if nrem != 0
    # remainder block
    start_idx = nblocks_full*blocksize + 1
    end_idx = mesh.numEl
    @assert end_idx - start_idx + 1 <= blocksize
    @assert end_idx - start_idx + 1 == nrem

    element_range = start_idx:end_idx

    # make sure dimensions of Eone is correct (number of elements might be
    # less than before)
    Eone_bar_rem = sview(Eone_bar, :, :, 1:nrem)

    fill!(Eone_bar_rem, 0.0)
    getCurvilinearMetricsAndCoordinates_inner_rev(mesh, sbp, element_range,
                                                  Eone_bar_rem)
  end

  return Eone_bar
end

"""
  This function calculates the metrics and coordinates for one block of
  elements.  Used by getCurvilinearMetricsAndCoordinates
"""
function getCurvilinearMetricsAndCoordinates_inner(mesh, sbp,
                             element_range::UnitRange, Eone::AbstractArray{T, 3}) where T

  # calculate Eone for current range

  vert_coords_block = sview(mesh.vert_coords, :, :, element_range)
  coords_block = sview(mesh.coords, :, :, element_range)
  dxidx_block = sview(mesh.dxidx, :, :, :, element_range)
  jac_block = sview(mesh.jac, :, element_range)
  ref_vtx = baryToXY(mesh.coord_xi, sbp.vtx)

  calcEone(mesh, sbp, element_range, Eone)

  #=
  println("checking Eone")
  for el=1:size(Eone, 3)
    println("el = ", el)
    println("Eone = \n", Eone[:, :, el])
    for d=1:mesh.dim
      println("  d = ", d)
      val = abs(sum(Eone[:, d, el]))
      println("  val = ", val)
      @assert val < 1e-6
    end
  end
  =#

#=
  println("sbp.numnodes = ", sbp.numnodes)
  println("size(coords) = ", size(coords_block))
  println("size(dxidx) = ", size(dxidx_block))
  println("size(jac) = ", size(jac_block))
  println("size(Eone) = ", size(Eone))
=#
  calcMappingJacobian!(sbp, mesh.coord_order, ref_vtx, vert_coords_block,
                       coords_block, dxidx_block, jac_block, Eone)

  fill!(Eone, 0.0)
  return nothing
end

"""
  This function calculates the metrics and coordinates for one block of
  elements.  Used by getCurvilinearMetricsAndCoordinates
"""
function getCurvilinearMetricsAndCoordinates_inner_rev(mesh, sbp,
                             element_range::UnitRange, Eone_bar::AbstractArray{T, 3}) where T

  vert_coords_block = sview(mesh.vert_coords, :, :, element_range)
  # we don't allow perturbing mesh coordinate directly (yet)
  coords_bar_block = zeros(T, mesh.dim, mesh.numNodesPerElement, length(element_range))
  dxidx_block = sview(mesh.dxidx, :, :, :, element_range)
  dxidx_bar_block = sview(mesh.dxidx_bar, :, :, :, element_range)
#  jac_block = sview(mesh.jac, :, element_range)
  jac_bar_block = sview(mesh.jac_bar, :, element_range)
  #@assert vecnorm(jac_bar_block) < 1e-13  # this is currently broken in SBP

  # outputs
  vert_coords_bar_block = sview(mesh.vert_coords_bar, :, :, element_range)
  fill!(Eone_bar, 0.0)

  ref_vtx = baryToXY(mesh.coord_xi, sbp.vtx)

  # back propigate dxidx to vert_coords, E1
  calcMappingJacobian_rev!(sbp, mesh.coord_order, ref_vtx, vert_coords_block,
                           vert_coords_bar_block, coords_bar_block,
                           dxidx_bar_block, jac_bar_block, Eone_bar)

  # back propigate E1 to the face normals
  calcEone_rev(mesh, sbp, element_range, Eone_bar)

  return nothing
end

function calcEone(mesh::PumiMeshDG{Tmsh}, sbp, element_range,
                  Eone::AbstractArray{Tmsh, 3}) where Tmsh

  # search interfaces and bndryfaces

  first_el = element_range[1]
  last_el = element_range[end]

  offset = first_el - 1

  # R times vector of ones
  # the permutation doesn't matter because it is being multiplied by a constant
  # vector.
  # also R1 = 1 by definition
  Rone = ones(Float64, mesh.numNodesPerFace)
#  Rone = vec(sum(mesh.sbpface.interp.', 2))
  sbpface = mesh.sbpface
  tmp = zeros(Tmsh, length(Rone))
  nrmL = zeros(Tmsh, mesh.dim, sbpface.numnodes)

  # accumulate E1 for a given element
  Eone_el = zeros(Tmsh, size(sbpface.perm, 1), mesh.dim)

  for i=1:mesh.numInterfaces
    iface_i = mesh.interfaces[i]

    if iface_i.elementL >= first_el && iface_i.elementL <= last_el
      elnum = iface_i.elementL
      facenum_local = iface_i.faceL
      nrm = sview(mesh.nrm_face, :, :, i)

      # call inner functions
      calcEoneElement(sbpface, nrm, Rone, tmp, Eone_el)
      assembleEone(sbpface, elnum - offset, facenum_local, Eone_el, Eone)

    end

    if iface_i.elementR >= first_el && iface_i.elementR <= last_el
      # do same as above, negating and permuting nrm
      elnum = iface_i.elementR
      facenum_local = iface_i.faceR
      orient = iface_i.orient
      for j=1:sbpface.numnodes
        for d=1:mesh.dim
          nrmL[d, j] = -mesh.nrm_face[d, sbpface.nbrperm[j, orient], i]
        end
      end

      calcEoneElement(sbpface, nrmL, Rone, tmp, Eone_el)
      assembleEone(sbpface, elnum - offset, facenum_local, Eone_el, Eone)
    end  # end if/else

  end  # end loop over interfaces

  for i=1:mesh.numBoundaryFaces
    bface_i = mesh.bndryfaces[i]

    if bface_i.element >= first_el && bface_i.element <= last_el
      elnum = bface_i.element
      facenum_local = bface_i.face
      nrm = sview(mesh.nrm_bndry, :, :, i)

      # call inner functions
      calcEoneElement(sbpface, nrm, Rone, tmp, Eone_el)
      assembleEone(sbpface, elnum - offset, facenum_local, Eone_el, Eone)
    end
  end  # end loop over boundary faces

  # check shared faces
  for peer=1:mesh.npeers
    bndryfaces_peer = mesh.bndries_local[peer]
    nrm_peer = mesh.nrm_sharedface[peer]
    for i=1:mesh.peer_face_counts[peer]
      bface_i = bndryfaces_peer[i]

      if bface_i.element >= first_el && bface_i.element <= last_el
        elnum = bface_i.element
        facenum_local = bface_i.face
        nrm = sview(nrm_peer, :, :, i)

        calcEoneElement(sbpface, nrm, Rone, tmp, Eone_el)
        assembleEone(sbpface, elnum - offset, facenum_local, Eone_el, Eone)
      end
    end
  end

  return nothing
end

"""
  Back propigates Eone_bar to the various normal vectors stored in the
  mesh objects (bndry, face, sharedface)

  Inputs:
    mesh: mesh.nrm_face_bar, mesh.nrm_bndry_bar, mesh.nrm_sharedface_bar are
          updated
    sbp
    element range: range of elements to compute
    Eone_bar: the adjoint part of Eone, see the primal method for details
"""
function calcEone_rev(mesh::PumiMeshDG{Tmsh}, sbp, element_range,
                      Eone_bar::AbstractArray{Tmsh, 3}) where Tmsh

  # search interfaces and bndryfaces

  first_el = element_range[1]
  last_el = element_range[end]

  offset = first_el - 1

  # R times vector of ones
  # the permutation doesn't matter because it is being multiplied by a constant
  # vector.
  # also R1 = 1 by def.inition
  Rone = ones(Float64, mesh.numNodesPerFace)
#  Rone = vec(sum(mesh.sbpface.interp.', 2))
  sbpface = mesh.sbpface
  tmp = zeros(Rone)
  nrmL_bar = zeros(Tmsh, mesh.dim, sbpface.numnodes)

  # accumulate E1 for a given element
  Eone_el_bar = zeros(Tmsh, size(sbpface.perm, 1), mesh.dim)

  for i=1:mesh.numInterfaces
    iface_i = mesh.interfaces[i]

    if iface_i.elementL >= first_el && iface_i.elementL <= last_el
      elnum = iface_i.elementL
      facenum_local = iface_i.faceL
      nrm_bar = sview(mesh.nrm_face_bar, :, :, i)

      # call inner functions
      fill!(Eone_el_bar, 0.0)
      assembleEone_rev(sbpface, elnum - offset, facenum_local, Eone_el_bar,
                       Eone_bar)
      calcEoneElement_rev(sbpface, nrm_bar, Rone, tmp, Eone_el_bar)

    end

    if iface_i.elementR >= first_el && iface_i.elementR <= last_el
      # do same as above, negating and permuting nrm
      elnum = iface_i.elementR
      facenum_local = iface_i.faceR
      orient = iface_i.orient

      fill!(Eone_el_bar, 0.0)
      fill!(nrmL_bar, 0.0)
      assembleEone_rev(sbpface, elnum - offset, facenum_local, Eone_el_bar,
                       Eone_bar)
      calcEoneElement_rev(sbpface, nrmL_bar, Rone, tmp, Eone_el_bar)

      for j=1:sbpface.numnodes
        for d=1:mesh.dim
          mesh.nrm_face_bar[d, sbpface.nbrperm[j, orient], i] -= nrmL_bar[d, j]
        end
      end

    end  # end if/else

  end  # end loop over interfaces

  for i=1:mesh.numBoundaryFaces
    bface_i = mesh.bndryfaces[i]

    if bface_i.element >= first_el && bface_i.element <= last_el
      elnum = bface_i.element
      facenum_local = bface_i.face
      nrm_bar = sview(mesh.nrm_bndry_bar, :, :, i)

      # call inner functions
      fill!(Eone_el_bar, 0.0)
      assembleEone_rev(sbpface, elnum - offset, facenum_local, Eone_el_bar,
                       Eone_bar)
      calcEoneElement_rev(sbpface, nrm_bar, Rone, tmp, Eone_el_bar)
    end
  end  # end loop over boundary faces

  # check shared faces
  for peer=1:mesh.npeers
    bndryfaces_peer = mesh.bndries_local[peer]
    nrm_peer_bar = mesh.nrm_sharedface_bar[peer]
    for i=1:mesh.peer_face_counts[peer]
      bface_i = bndryfaces_peer[i]

      if bface_i.element >= first_el && bface_i.element <= last_el
        elnum = bface_i.element
        facenum_local = bface_i.face
        nrm_bar = sview(nrm_peer_bar, :, :, i)

        fill!(Eone_el_bar, 0.0)
        assembleEone_rev(sbpface, elnum - offset, facenum_local, Eone_el_bar,
                         Eone_bar)
        calcEoneElement_rev(sbpface, nrm_bar, Rone, tmp, Eone_el_bar)
      end
    end
  end

  return nothing
end

"""
  Calculates E1 for a given interface.  Used by calcEone.

  Actually what it computes is:
      (R^T)*N*B*R*P*1

  Note that there should be a factor of P^T on the left.  That factor is
  applied in assembleEone().

  Inputs:
    sbpface: an AbstractFace
    nrm: the normal vectors for each node of the face, dim x numFaceNodes
    Rone: the interpolation operator R times the vector of ones
    tmp: a temporary vector of length numFaceNodes, overwritten

  Inputs/Outputs:
    Eone_el: sbp.stencilsize x dim matrix to be populated with the E
             contribution for this face.  Overwritten.

"""
function calcEoneElement(sbpface::AbstractFace, nrm::AbstractMatrix,
                         Rone::AbstractVector, tmp::AbstractVector,
                         Eone_el::AbstractMatrix)

  dim = size(Eone_el, 2)
  numFaceNodes = length(Rone)
  for d=1:dim
    for i=1:numFaceNodes
      tmp[i] = Rone[i]*nrm[d, i]*sbpface.wface[i]
    end

    Eone_dim = sview(Eone_el, :, d)
    smallmatvec!(sbpface.interp, tmp, Eone_dim)
  end

  return nothing
end

"""
  This methods works for SparseFaces (ex. diagonlE operators).
  It takes in all the same arguments as the other method for compatability
  but does not need them
"""
function calcEoneElement(sbpface::SparseFace, nrm::AbstractMatrix,
                         Rone::AbstractVector, tmp::AbstractVector,
                         Eone_el::AbstractMatrix)

  dim = size(Eone_el, 2)
  numFaceNodes = length(Rone)
  for d=1:dim
    for i=1:numFaceNodes
      Eone_el[i, d] = Rone[i]*nrm[d, i]*sbpface.wface[i]
    end
  end

  return nothing
end

# back propigate Eone_el_bar to nrm_bar
"""
  This function uses reverse mode to back-propigate Eone_el_bar to nrm_bar

  Input:
    sbpface: an AbstractFace
    Rone: the interpolation operator R times a vector of ones
    tmp: a temporary vector, overwritten
    Eone_el_bar: the adjoint part of E1

  Inputs/Outputs:
    nrm_bar: the adjoint part of the scaled face normal vector in x-y space
             updated with the contribution from this function

"""
function calcEoneElement_rev(sbpface::AbstractFace,
                         nrm_bar::AbstractMatrix,
                         Rone::AbstractVector, tmp_bar::AbstractVector,
                         Eone_el_bar::AbstractMatrix)

  dim = size(Eone_el_bar, 2)
  numFaceNodes = length(Rone)

  for d=1:dim
    # this function is linear, so no need for a forward sweep

    # reverse sweep
    Eone_dim_bar = sview(Eone_el_bar, :, d)
    # only back propigate to tmp_bar (sbpface.interp is unimportant)
    fill!(tmp_bar, 0.0)
    smallmatvec_revv!(sbpface.interp, tmp_bar, Eone_dim_bar)

    for i=1:numFaceNodes
      nrm_bar[d, i] += tmp_bar[i]*Rone[i]*sbpface.wface[i]
    end
  end

  return nothing
end

"""
  This method works for SparseFace sbp face objects.
"""
function calcEoneElement_rev(sbpface::SparseFace,
                         nrm_bar::AbstractMatrix,
                         Rone::AbstractVector, tmp_bar::AbstractVector,
                         Eone_el_bar::AbstractMatrix)

  dim = size(Eone_el_bar, 2)
  numFaceNodes = length(Rone)

  for d=1:dim
    for i=1:numFaceNodes
      nrm_bar[d, i] += Eone_el_bar[i, d]*Rone[i]*sbpface.wface[i]
    end
  end

  return nothing
end

"""
  Takes an Eone_el matrix from calcEoneElement and assemble it into the big
  Eone.

  Inputs:
    sbpface: an SBP face
    elnum: the element index of Eone to put the values into.  Note that if
           Eone is for the entire mesh, then this is the element number.  If
           Eone is for a range of elements, then elnum is the index of the
           element within the range.
    Eone_el: the E1 contribution of an element

  Inputs/Output
    Eone: a numNodesPerElement x dim x blocksize array to store E1 for a
          range of elements, updated with the new contribution
"""
function assembleEone(sbpface::AbstractFace, elnum::Integer,
                facenum_local::Integer, Eone_el::AbstractMatrix{Tmsh},
                Eone::AbstractArray{Tmsh, 3}) where Tmsh

  dim = size(Eone, 2)
  for d=1:dim
    for i=1:size(sbpface.perm, 1)
      p_i = sbpface.perm[i, facenum_local]
      Eone[p_i, d, elnum] += Eone_el[i, d]
    end
  end

  return nothing
end

"""
  Back propigates Eone_bar to Eone_el_bar.

  Inputs:
    sbpface: an AbstractFace
    elnum: the element number
    facenum_local: the local number of the face that Eone_el is calculated for
    Eone_bar: the adjoint part

  Inputs/Outputs:
    Eone_el_bar: the adjoint part
"""
function assembleEone_rev(sbpface::AbstractFace, elnum::Integer,
                      facenum_local::Integer,
                      Eone_el_bar::AbstractMatrix{Tmsh},
                      Eone_bar::AbstractArray{Tmsh, 3}) where Tmsh

  dim = size(Eone_bar, 2)
  for d=1:dim  # TODO: switch loops: turn an indexed store into a strided store
    for i=1:size(sbpface.perm, 1)
      p_i = sbpface.perm[i, facenum_local]
      # reverse mode step
      Eone_el_bar[i, d] += Eone_bar[p_i, d, elnum]
    end
  end

  return nothing
end

# get the coordinates of the nodes on a given face (in the coordinate field, not
# the solution field)
# this only works up to second order, and uses the ElementTopology to figure
# out the orientation of the edge
# elnum is the global element number
# facenum is the local face number
"""
  This function gets the coordinates of the coordinate field nodes of the
  face of an element, in the orientation specified by mesh.topo.  This only
  works up to second order.  It uses the SBP element topology to determine
  the ordering of the nodes.  I guess this makes sense because calcFaceNormals
  is an SBP function and requires its definition of the vertices that define the
  face.  Because ref_vtx is passed into calcFaceNormals, it should be ok to
  use the Pumi definition of the coordinates of the reference nodes (precisely
  because we use the SBP definition of which vertices define the face, the
  face node coordinates are relative to those vertices).

  Inputs:
    mesh: a 2D mesh
    elnum: the global element number
    facenum: the local face number

  Inputs/Outputs:
    coords: array to be populated with the coordinates, Tdim x
            number of coordinate nodes on a face (2 for linear 2D,
            3 for quadratic 2D)
"""
function getMeshFaceCoordinates(mesh::PumiMesh2DG, elnum::Integer,
                                facenum::Integer,
                                coords::AbstractMatrix)

# coords = a 2 x number of nodes on the face array to be populated with the
# coordinates

  topo = mesh.topo

  # equivalent to topo.face_verts[:, facenum]
  v1_idx = topo.face_verts[1, facenum]
  v2_idx = topo.face_verts[2, facenum]

  coords[1, 1] = mesh.vert_coords[1, v1_idx, elnum]
  coords[2, 1] = mesh.vert_coords[2, v1_idx, elnum]

  coords[1, 2] = mesh.vert_coords[1, v2_idx, elnum]
  coords[2, 2] = mesh.vert_coords[2, v2_idx, elnum]

  if mesh.coord_numNodesPerType[2] > 0
    edge_idx = 3 + topo.face_edges[1, facenum]
    coords[1, 3] = mesh.vert_coords[1, edge_idx, elnum]
    coords[2, 3] = mesh.vert_coords[2, edge_idx, elnum]
  end

  return nothing
end

"""
  This function is the reverse mode of getMeshFaceCoordinates
  This function stores the adjoint part of the vert_coords to mesh.vert_coords

  2nd order coordinate fields only.

  This function *accumulates* into mesh.vertcoords_bar.

  Inputs
    mesh::PumiMesh2DG
    elnum: the element number
    facenum: the local face number
    coords_bar:  the adjoint part of the coordinate field for the given face,
                 2 x coord_numNodesPerFace.
                 The data should be ordered vertices, then mid edge nodes.
"""
function getMeshFaceCoordinates_rev(mesh::PumiMesh2DG, elnum::Integer,
                                facenum::Integer,
                                coords_bar::AbstractMatrix)

#  vertmap = mesh.topo_pumi.face_verts
  topo = mesh.topo_pumi

  # equivalent to topo.face_verts[:, facenum]
  v1_idx = topo.face_verts[1, facenum]
  v2_idx = topo.face_verts[2, facenum]

  mesh.vert_coords_bar[1, v1_idx, elnum] += coords_bar[1, 1]
  mesh.vert_coords_bar[2, v1_idx, elnum] += coords_bar[2, 1]

  mesh.vert_coords_bar[1, v2_idx, elnum] += coords_bar[1, 2]
  mesh.vert_coords_bar[2, v2_idx, elnum] += coords_bar[2, 2]

  if mesh.coord_numNodesPerType[2] > 0
    edge_idx = facenum + 3  # for simplex elements
    mesh.vert_coords_bar[1, edge_idx, elnum] += coords_bar[1, 3]
    mesh.vert_coords_bar[2, edge_idx, elnum] += coords_bar[2, 3]
  end

  return nothing
end

function getMeshFaceCoordinates(mesh::PumiMesh3DG, elnum::Integer,
                                facenum::Integer, coords::AbstractMatrix)

  topo = mesh.topo
  # get the face vertices
  face_vert_idx = sview(topo.face_verts, :, facenum)

  for i=1:3  # 3 vertices per face
    for j=1:3  # 3 coordinates at each vertex
      coords[j, i] = mesh.vert_coords[j, face_vert_idx[i], elnum]
    end
  end

  if mesh.coord_numNodesPerType[2] > 0
    for i=1:3  # 3 edges per face
      edge_idx = topo.face_edges[i, facenum] + 4 # 4 = numVerts
      for j=1:3  # 3 coordinates per face
        coords[j, 3 + i] = mesh.vert_coords[j, edge_idx, elnum]
      end
    end

  end

  return nothing
end

"""
  Similar to the 2d method.

  coords_bar should be ordered vertices, edge nodes, then face nodes
"""
function getMeshFaceCoordinates_rev(mesh::PumiMesh3DG, elnum::Integer,
                                facenum::Integer,
                                coords_bar::AbstractMatrix)

#  vertmap = mesh.topo.face_verts
  topo = mesh.topo

  # get the face vertices
  face_vert_idx = sview(topo.face_verts, :, facenum)

  for i=1:3  # 3 vertices per face
    for j=1:3  # 3 coordinates at each vertex
      mesh.vert_coords_bar[j, face_vert_idx[i], elnum] += coords_bar[j, i]
    end
  end

  if mesh.coord_numNodesPerType[2] > 0
    offset = 3 + (facenum - 1)*3  # 3 vertices + 1 edge node per face

    for i=1:3  # 3 edges per face
      edge_idx = topo.face_edges[i, facenum] + 4  # 4 = numVerts
      for j=1:3  # 3 coordinates per face
        mesh.vert_coords_bar[j, edge_idx, elnum] += coords_bar[j, 3 + i]  # 3 verts
      end
    end

  end

  return nothing
end