# MIT license
# Copyright (c) Microsoft Corporation. All rights reserved.
# See LICENSE in the project root for full license information.

"""
    AcceleratedParametricSurface{T,N,S} <: ParametricSurface{T,N}

Wrapper class for [`ParametricSurface`](@ref)s where analytical intersection isn't feasible (e.g. [`ZernikeSurface`](@ref), [`ChebyshevSurface`](@ref)).
The surface is instead triangulated and an iterative (newton raphson) process carried out to determine precise ray intersection points.
`S` is the type of the ParametricSurface being wrapped.

```julia
AcceleratedParametricSurface(surf::ParametricSurface{T,N}, numsamples::Int = 17; interface::NullOrFresnel{T} = nullinterface(T))
```
"""
struct AcceleratedParametricSurface{T,N,S<:ParametricSurface{T,N}} <: ParametricSurface{T,N}
    surface::S
    triangles::Vector{Triangle{T}}
    sidelengths::Vector{T}
    triangles_bbox::BoundingBox{T}
    interface::NullOrFresnel{T}

    function AcceleratedParametricSurface(surf::S, numsamples::Int = 17; interface::NullOrFresnel{T} = NullInterface(T)) where {T<:Real,N,S<:ParametricSurface{T,N}}
        a = AcceleratedParametricSurface(surf, triangulate(surf, numsamples, true, true), interface = interface)
        emptytrianglepool!(T)
        return a
    end

    function AcceleratedParametricSurface(surf::S, triangles::Vector{Triangle{T}}; interface::NullOrFresnel{T} = NullInterface(T)) where {T<:Real,N,S<:ParametricSurface{T,N}}
        sidelengths = Vector{T}(undef, length(triangles))
        @inbounds @simd for i in 1:length(triangles)
            t = triangles[i]
            sidelengths[i] = max(norm(t.BA), norm(t.CA))
        end
        return new{T,N,S}(surf, copy(triangles), sidelengths, BoundingBox(triangles), interface)
    end
end
export AcceleratedParametricSurface

uvrange(::Type{AcceleratedParametricSurface{T,N,S}}) where {T<:Real,N,S<:ParametricSurface{T,N}} = uvrange(S)
uvrange(surf::AcceleratedParametricSurface{T,N,S}) where {T<:Real,N,S<:ParametricSurface{T,N}} = uvrange(surf.surface)
point(surf::AcceleratedParametricSurface{T,N,S}, u::T, v::T) where {T<:Real,N,S<:ParametricSurface{T,N}} = point(surf.surface, u, v)
partials(surf::AcceleratedParametricSurface{T,N,S}, u::T, v::T) where {T<:Real,N,S<:ParametricSurface{T,N}} = partials(surf.surface, u, v)
normal(surf::AcceleratedParametricSurface{T,N,S}, u::T, v::T) where {T<:Real,N,S<:ParametricSurface{T,N}} = normal(surf.surface, u, v)
inside(surf::AcceleratedParametricSurface{T,N,S}, x::T, y::T, z::T) where {T<:Real,N,S<:ParametricSurface{T,N}} = inside(surf.surface, x, y, z)
onsurface(surf::AcceleratedParametricSurface{T,N,S}, x::T, y::T, z::T) where {T<:Real,N,S<:ParametricSurface{T,N}} = onsurface(surf.surface, x, y, z)
"""
    interface(surf::Surface{T}) -> OpticalInterface{T}

Return the [`OpticalInterface`](@ref) associated with `surf`.
"""
interface(a::AcceleratedParametricSurface{T}) where {T<:Real} = a.interface
uv(surf::AcceleratedParametricSurface{T,3}, x::T, y::T, z::T) where {T<:Real} = uv(surf.surface, x, y, z)

function BoundingBox(surf::AcceleratedParametricSurface{T,3}) where {T<:Real}
    # the stored bbox is a tight one around the triangulated surface only, in reality the half-space created is an infinite prism capped by the triangulated surface
    # this bounding box is not accurate to the actual surface, but it is accurate to the triangulated surface
    # if a ray hits the accurate bounding box but not the triangulated bounding box then it would miss the triangulated surface
    # anyway (erroneously) and not return an intersection - so we don't lose anything by making this approximation here
    tbox = surf.triangles_bbox
    return BoundingBox(tbox.xmin, tbox.xmax, tbox.ymin, tbox.ymax, typemin(T), tbox.zmax)
end

# function makemesh(surface::AcceleratedParametricSurface{S,N,T}, ::Int) where {S<:Real,N,T<:ParametricSurface{S,N}}
#     # If we've already triangulated the surface then use this directly as it will be the most
#     # accurate representation of the surface being intersected with
#     return TriangleMesh(surface.triangles)
# end

# "computes all the intersections of the ray with the surface. Each intersection is represented as an Interval that encloses the interior of the object. If the ray is entering the surface, dot(ray,normal) < 0, then the interval is (intersection,infinity) where α is the parametric value of the line surface intersection. If the ray is leaving the surface, dot(ray,normal) > 0, then the interval is (rayorigin, α). If there are two intersections with the surface then returns an Interval(intersection,intersection). If there are multiple segments enclosing the object then will return a list that will contain some combination of the above types of Interval."
# fallback method for any accelerated surface which doesn't have an overridden method

"""
    surfaceintersection(surf::Surface{T}, r::AbstractRay{T}) where {T}

Calculates the intersection of `r` with a surface of any type, `surf`.
Note that some surfaces cannot be intersected analytically so must be wrapped in an [`AcceleratedParametricSurface`](@ref) in order to be intersected.

Returns an [`EmptyInterval`](@ref) if there is no [`Intersection`](@ref), an [`Interval`](@ref) if there is one or two intersections and a [`DisjointUnion`](@ref) if there are more than two intersections.
"""
function surfaceintersection(surf::AcceleratedParametricSurface{T,N}, r::AbstractRay{T,N}) where {T,N}
    if doesintersect(surf.triangles_bbox, r)
        i = triangulatedintersection(surf, r)
        if i isa EmptyInterval{T}
            if inside(surf, origin(r))
                return rayorigininterval(Infinity(T))
            else
                return EmptyInterval(T)
            end
        else
            return i
        end
    else
        if inside(surf, origin(r))
            return rayorigininterval(Infinity(T))
        else
            return EmptyInterval(T)
        end
    end
end

"""
    triangulatedintersection(surf::AcceleratedParametricSurface{T,N,S}, r::AbstractRay{T,N})

Intersection of a ray, `r`, with a triangulated surface, `surf`, no concept of inside so never returns a [`RayOrigin`](@ref) [`Interval`](@ref).
"""
function triangulatedintersection(surf::AcceleratedParametricSurface{T,N,S}, r::AbstractRay{T,N}) where {T<:Real,N,S}
    result = newinintervalpool!(T)
    offset_ray = Ray(origin(r) - ACCEL_SURF_RAY_OFFSET * direction(r), direction(r))
    for (i, tri) in enumerate(surf.triangles)
        # test if the ray is within the maximum side length from one of the points, if not then skip it
        w = vertex(tri, 1) - origin(r)
        dist = norm(w - dot(w, direction(r)) * direction(r))
        if dist > surf.sidelengths[i]
            continue
        end
        # find the triangle intersection - it's quite possible that for rays starting near the surface we will we miss
        # an intersection because of imprecise triangulation, so introduce an offset
        intv = surfaceintersection(tri, offset_ray)
        if !(intv isa EmptyInterval{T})
            ints = halfspaceintersection(intv)
            if α(ints) <= ACCEL_SURF_RAY_OFFSET
                # skip any intersection behind the true start point (with some wiggle room)
                continue
            end
            let dup = false
                # check if a point with the same alpha had already been included, if so ignore this one
                # doing this check now avoids executing newton many times unnecessarily
                for eintv in result
                    eints = halfspaceintersection(eintv)
                    if samepoint(α(eints), α(ints) + ACCEL_SURF_RAY_OFFSET)
                        dup = true
                        break
                    end
                end
                if !dup
                    # evaluate the precise point of the intersection with the surface
                    intuv = newton(surf.surface, r, uv(ints))
                    # check that the evaluated uv point lies on the (correct side of the) ray
                    if intuv !== nothing
                        u, v = intuv
                        surfpt = point(surf, u, v)
                        t = α(r, surfpt) # this is the 'true' α again (i.e. not offset)
                        if t > zero(T)
                            # check again that this point hasn't already been included (α might be slightly different, i.e. more accurate, now)
                            for eintv in result
                                eints = halfspaceintersection(eintv)
                                if samepoint(α(eints), t)
                                    dup = true
                                    break
                                end
                            end
                            if !dup && samepoint(point(r, t), surfpt)
                                # intersection is good so calc the normal and add to list
                                surfnormal = normal(surf, u, v)
                                intsct = Intersection(t, surfpt, surfnormal, u, v, interface(surf))
                                if dot(direction(r), surfnormal) < zero(T)
                                    intvl = positivehalfspace(intsct)
                                else
                                    intvl = rayorigininterval(intsct)
                                end
                                push!(result, intvl)
                            end
                        end
                    end
                end
            end
        end
    end

    nr = length(result)
    if nr === 0
        return EmptyInterval(T)
    elseif nr === 1
        return result[1]
    end

    # sort!(result, lt = (a, b) -> α(halfspaceintersection(a)) < α(halfspaceintersection(b))) #this will sort the intervals by α
    # alloc free sort of result
    @inbounds for i in 2:nr
        value = result[i]
        j = i - 1
        while j > 0 && α(halfspaceintersection(result[j])) > α(halfspaceintersection(value))
            result[j + 1] = result[j]
            j = j - 1
        end
        result[j + 1] = value
    end

    let start = 1
        temp = newinintervalpool!(T)

        if lower(result[1]) isa RayOrigin{T}
            # Starting "inside" the surface, i.e., n̂⋅d̂, > 0.
            # Can't truly define inside for Bezier patches because they don't define a half-space.
            start = 2
            push!(temp, result[1])
        end

        @inbounds for i in start:2:(nr - 1)
            intvlintsct = intervalintersection(result[i], result[i + 1])
            if !(intvlintsct isa EmptyInterval{T})
                push!(temp, intvlintsct)
            else
                # this can happen in some edge cases, seems like the best thing to do is juse ignore it...
                @warn "Funny behavior in triangulated intersection" maxlog = 1
                # throw(ErrorException("Triangulated intersection error - this should never happen."))
            end
        end

        if mod(nr - (start - 1), 2) === 1 #one unpaired interval
            push!(temp, last(result))
        end

        return DisjointUnion(temp)
    end
end

"""
    jacobian(surf::ParametricSurface{T,N}, u::T, v::T, P1::SVector{M,T}, P2::SVector{M,T})

Computes Jacobian of `f(t,u,v) = ( dot(P1,[surf(u,v),1],P2,[surf(u,v),1]) )`.
`P1`, `P2` are orthogonal planes that pass through the ray.
`J = [ ∂f1/∂u ∂f1/∂v ; ∂f2/∂u ∂f2/∂v]`
"""
function jacobian(surf::ParametricSurface{T,N}, u::T, v::T, P1::SVector{M,T}, P2::SVector{M,T}) where {T<:Real,N,M}
    (du, dv) = partials(surf, u, v)
    return SMatrix{2,2,T}(dot(view(P1, 1:(M - 1)), du), dot(view(P2, 1:(M - 1)), du), dot(view(P1, 1:(M - 1)), dv), dot(view(P2, 1:(M - 1)), dv))
end

"""
    newton(surf::ParametricSurface{T,N}, r::AbstractRay{T,N}, startingpoint::SVector{2,T})

Newton iteration to find the precise intersection of a parametric surface with a ray given a starting point (in uv space) on the surface.
"""
function newton(surf::ParametricSurface{T,N}, r::AbstractRay{T,N}, startingpoint::SVector{2,T}) where {T,N}
    tolerance = 1e-12
    maxiterations = 6

    urange, vrange = uvrange(surf)

    dx, dy, dz = direction(r)
    if abs(dx) > abs(dy) && abs(dx) > abs(dz)
        N1 = SVector{3,T}(dy, -dx, zero(T))
    else
        N1 = SVector{3,T}(zero(T), dz, -dy)
    end

    N2 = cross(N1, direction(r))
    P1 = SVector{4,T}(N1[1], N1[2], N1[3], -dot(N1, origin(r)))
    P2 = SVector{4,T}(N2[1], N2[2], N2[3], -dot(N2, origin(r)))

    @inline function f(uv::SVector{2,T})::SVector{2,T}
        psurf = point(surf, uv[1], uv[2])
        res1 = zero(T)
        res2 = zero(T)
        @inbounds for i in 1:3
            res1 += P1[i] * psurf[i]
            res2 += P2[i] * psurf[i]
        end
        res1 += P1[4]
        res2 += P2[4]
        # return [dot(P1, vcat(psurf, 1)), dot(P2, vcat(psurf, 1))]
        return SVector{2,T}(res1, res2)
    end

    xilast = startingpoint
    xi = zeros(SVector{2,T})
    error = typemax(T)
    i = 0
    finalpass = false

    while i < maxiterations
        if any(isnan.(xilast))
            # if something is NaN then just return the starting point and hope that the surface point matches
            xilast = startingpoint
            error = zero(T)
            break
        end

        j = jacobian(surf, xilast[1], xilast[2], P1, P2) # xi has values u,v

        if det(j) == zero(T)
            # if the jacobian is singular then break at the current iteration and return whatever we have
            error = zero(T)
            break
        end

        lstf = f(xilast)
        xi = xilast - (j \ lstf)
        error = sum(abs.(xi - xilast))

        if !finalpass && error < tolerance
            i = maxiterations - 3 # iterate a few more times after the xi and xi-1 are the same value, at least to 64 bit precision. Seems to give more accurate solutions. Perhaps because of higher precision for intermediate results?
            finalpass = true
        end

        i += 1

        xilast = xi
    end

    if error > tolerance
        return nothing
    else
        if xi[1] < urange[1] || xi[1] > urange[2] || xi[2] < vrange[1] || xi[2] > vrange[2]
            return nothing
        else
            return xilast
        end
    end
end

# Only show the underlying Bezier surface control points, not the approximating triangle mesh which is generally huge
Base.show(io::IO, a::AcceleratedParametricSurface{S,N,T}) where {S<:Real,N,T<:ParametricSurface{S,N}} = println(io, "Accelerated$(string(a.surface))")


__precompile__()

module HumanoidLCMSim

export # types
    HumanoidRobotInfo,
    LCMControlReceiver,
    LCMControlPublisher,
    Actuator,
    Side

export # functions
    parse_actuators,
    handle

export # submodules
    Sides,
    AtlasSim,
    AtlasControl

using RigidBodySim
using RigidBodyDynamics
using OrdinaryDiffEq
using DiffEqCallbacks
using StaticArrays
using Rotations
using LCMCore
using FastIOBuffers
using LightXML
using BotCoreLCMTypes

using Dates: Second

include("gains.jl")
include("sides.jl")

using .Sides

include("util.jl")
include("humanoid_robot_info.jl")
include("lcm_interop.jl")
include("controlreceiver.jl")
include("controlpublisher.jl")

include("atlassim.jl")
include("atlascontrol.jl")

end # module


## LocalRule
export NagyRule

struct NagyRule{T<:Vector} <: MCMCRule
    adjacency_list::T
end

function NagyRule(H::AbsLinearOperator)
    N_sites = nsites(basis(H))

    all_sites = sites(H)
    if !(eltype(all_sites) <: Vector)
        all_sites = [all_sites]
    end

    couplings = Vector{Tuple{Int,Int}}()
    for coupling=all_sites
        length(coupling) == 1 && continue
        if length(coupling) > 2
            @warn "Can't exchange between 3-site couplings. This coupling is ignored"
            continue
        end
        push!(couplings, (coupling[1], coupling[2]))
    end

    return NagyRule{typeof(couplings)}(couplings)
end

struct NagyRuleCache
end

function RuleSamplerCache(rule::NagyRule, s, v, net, par_cache)
    return NagyRuleCache()
end

function propose_step!(σp::ADoubleState, s::MetropolisSampler{<:NagyRule},
                       net::Union{MatrixNet,KetNet}, c, rc)
    adj_list = s.rule.adjacency_list
    hilb     = physical(c.hilb)

    # determine move type
    move_type = rand(c.rng, 1:8)#1:n_moves)
    sites_to_flip = 1:nsites(hilb)

    # 1- Flip a spin
    if move_type == 1 || move_type == 2                 # hopping in σ
        flipat_r1 = rand(c.rng, sites_to_flip)
        flipat_r2 = rand(c.rng, adj_list[flipat_r1])

        @inbounds flipat!(c.rng, row(σp), hilb, flipat_r1)
        @inbounds flipat!(c.rng, row(σp), hilb, flipat_r2)

    elseif move_type == 3 || move_type == 4             # hopping in η
        flipat_c1 = rand(c.rng, sites_to_flip)
        flipat_c2 = rand(c.rng, adj_list[flipat_c1])

        @inbounds flipat!(c.rng, col(σp), hilb, flipat_c1)
        @inbounds flipat!(c.rng, col(σp), hilb, flipat_c2)

    elseif move_type == 5                               # exctitation in σ
        flipat_r1 = rand(c.rng, sites_to_flip)

        @inbounds flipat!(c.rng, row(σp), hilb, flipat_r1)

    elseif move_type == 6                               # excitations in η
        flipat_r1 = rand(c.rng, sites_to_flip)

        @inbounds flipat!(c.rng, col(σp), hilb, flipat_r1)

    elseif move_type == 7                                # Dissipator
        flipat_r  = rand(c.rng, sites_to_flip)
        old_val_r = row(σp)[flipat_r]

        if old_val_r == 0
            if rand(c.rng, 1:10) == 1 # excite with 10% chance
                @inbounds flipat!(c.rng, row(σp), hilb, flipat_r)
            end
        else # always dissipate
            @inbounds flipat!(c.rng, row(σp), hilb, flipat_r)
        end

        flipat_c  = flipat_r #rand(c.rng, sites_to_flip)
        old_val_c = col(σp)[flipat_c]
        if old_val_c == 0
            if rand(c.rng, 1:10) == 1 # excite with 10% chance
                @inbounds flipat!(c.rng, col(σp), hilb, flipat_c)
            end
        else # always dissipate
            @inbounds flipat!(c.rng, col(σp), hilb, flipat_c)
        end

    elseif move_type == 8                                # jumper
        flipat_r = rand(c.rng, sites_to_flip)
        flipat_c = rand(c.rng, sites_to_flip)

        @inbounds old_val_r, _ =flipat!(c.rng, row(σp), hilb, flipat_r)
        @inbounds old_val_c, _ =flipat!(c.rng, col(σp), hilb, flipat_c)
    end
#    elseif move_type == 9
#        if rand(c.rng, 1:100) == 1
#            zero!(σ)
#        end
        #prob = exp(2*real(net(σ)) - logψ_σ)
        #rv = rand(c.rng)
        #if !(rv < prob)
        #    setat!(row(σ), flipat_r, old_val_r)
        #    setat!(col(σ), flipat_c, old_val_c)
        #else
        #end
#    end

    return true
end


# This file is a part of BAT.jl, licensed under the MIT License (MIT).

"""
    abstract type AbstractVariateTransform <: Function

Abstract type for change-of-variables transformations.

Subtypes (e.g. `SomeTrafo <: AbstractVariateTransform`) must support (with
`trafo::SomeTrafo`):

```julia
    (trafo)(v_prev::SomeVariate) == v_new
    (trafo)(v_prev::SomeVariate, ladj_prev::Real)) == (v = v_new, ladj = ladj_new)
    (trafo)(s_prev::DensitySample)::DensitySample
    ((trafo2 ∘ trafo1)(v)::AbstractVariateTransform)(v) == trafo2(trafo1(v))
    inv(trafo)(trafo(v)) == v
    inv(inv(trafo)) == trafo

    ValueShapes.varshape(trafo)::ValueShapes.AbstractValueShape
```

with `varshape(v_prev) == varshape(trafo)`.

`ladj` must be `logabsdet(jacobian(trafo, v))`.
"""
abstract type AbstractVariateTransform end
export AbstractVariateTransform

"""
    ladjof(r::NamedTuple{(...,:ladj,...)})::Real

Extract the `log(abs(det(jacobian)))` value that is part of a result `r`.

Examples:

```julia
ladjof((..., ladj = some_ladj, ...)) == some_ladj
ladjof(trafo)(v) = trafo(v, )
```
"""
function ladjof end
export ladjof

ladjof(x::NamedTuple) = x.ladj

struct LADJOfVarTrafo{T<:AbstractVariateTransform} <: Function
    trafo::T
end

(ladjof_trafo::LADJOfVarTrafo)(v::Any) = ladjof(trafo(v, 1))
(ladjof_trafo::LADJOfVarTrafo)(v::Any, prev_ladj::Real) = ladjof(trafo(v, prev_ladj))

"""
    ladjof(trafo::AbstractVariateTransform)::Function

Returns a function that computes the `log(abs(det(jacobian)))` of `trafo` for
a given variate `v`:

```julia
    ladjof(trafo)(v) == ladjof(trafo(v, 1))
    ladjof(trafo)(v, prev_ladj) == ladjof(trafo(v, prev_ladj))
```
"""
ladjof(trafo::AbstractVariateTransform) = LADJOfVarTrafo(trafo)

"""
    abstract type VariateSpace <: Function

*BAT-internal, not part of stable public API.*

Abstract type for variate spaces.
"""
abstract type VariateSpace end

struct UnitSpace <: VariateSpace end
struct InfiniteSpace <: VariateSpace end
struct MixedSpace <: VariateSpace end

product_varspace(s::VariateSpace) = s

product_varspace(::UnitSpace, ::UnitSpace) = MixedSpace
product_varspace(::InfiniteSpace, ::InfiniteSpace) = MixedSpace
product_varspace(::VariateSpace, ::VariateSpace) = MixedSpace

function product_varspace(s1::VariateSpace, s2::VariateSpace, spcs::VariateSpace...)
    product_varspace(product_varspace(s1, s2), spcs...)
end

"""
    abstract type VariateTransform{VF:<VariateForm,ST<:VariateSpace,SF<:VariateSpace}

*BAT-internal, not part of stable public API.*

Abstract parameterized type for change-of-variables transformations.

Subtypes (e.g. `SomeTrafo <: VariateTransform`) must implement:

* `BAT.target_space(trafo::SomeTrafo, v)`
* `BAT.source_space(trafo::SomeTrafo, v)`
* `BAT.apply_vartrafo_impl(trafo::SomeTrafo, v)`
* `BAT.apply_vartrafo_impl(inv_trafo::InverseVT{SomeTrafo}, v)`
* `ValueShapes.varshape(trafo::SomeTrafo)`

for real values and/or real-valued vectors `v`.
"""
abstract type VariateTransform{
    VF<:VariateForm,ST<:VariateSpace,SF<:VariateSpace
} <: AbstractVariateTransform end

function target_space end

function source_space end

function apply_vartrafo end

function apply_vartrafo_impl end

apply_vartrafo(trafo::VariateTransform{Univariate}, v::Real, prev_ladj::Real) =
    apply_vartrafo_impl(trafo, v, prev_ladj)

function apply_vartrafo(trafo::VariateTransform{Univariate}, v::AbstractArray{<:Real,0}, prev_ladj::Real)
    r = apply_vartrafo_impl(trafo, v[], prev_ladj)
    (v = fill(r.v), ladj = r.ladj)
end

apply_vartrafo(trafo::VariateTransform{Multivariate}, v::AbstractVector{<:Real}, prev_ladj::Real) =
    apply_vartrafo_impl(trafo, v, prev_ladj)

apply_vartrafo(trafo::VariateTransform{Matrixvariate}, v::AbstractMatrix{<:Real}, prev_ladj::Real) =
    apply_vartrafo_impl(trafo, v, prev_ladj)

apply_vartrafo(trafo::VariateTransform{ValueShapes.NamedTupleVariate{names}}, v::NamedTuple{names}, prev_ladj::Real) where names =
    apply_vartrafo_impl(trafo, v, prev_ladj)

apply_vartrafo(trafo::VariateTransform{ValueShapes.NamedTupleVariate{names}}, v::ShapedAsNT{<:NamedTuple{names}}, prev_ladj::Real) where names =
    apply_vartrafo_impl(trafo, v, prev_ladj)

(trafo::VariateTransform)(v::Any) = apply_vartrafo(trafo, v, Float32(NaN)).v
(trafo::VariateTransform)(v::Any, prev_ladj::Real) = apply_vartrafo(trafo, v, prev_ladj)

# ToDo: Move to ValueShapes.jl:
_variate_form(shape::ScalarShape{<:Real}) = Univariate
_variate_form(shape::ArrayShape{<:Real,1}) = Multivariate
_variate_form(shape::ArrayShape{<:Real,2}) = Matrixvariate
_variate_form(shape::NamedTupleShape{names}) where names = ValueShapes.NamedTupleVariate{names}

function var_trafo_result(trg_v::Real, src_v::Real, trafo_ladj::Real, prev_ladj::Real)
    R = float(typeof(src_v))
    ladj_sum = convert(R, trafo_ladj + prev_ladj)
    trg_ladj = if !isnan(ladj_sum)
        ladj_sum
    else
        # Should be safe to assume that target dist goes to zero at infinity, should win out over infinite prev_ladj:
        ladjs_should_cancel = (trafo_ladj == R(-Inf) && prev_ladj == R(+Inf) && isinf(trg_v))
        ladjs_should_cancel ? zero(R) : ladj_sum
    end
    (v = convert(R, trg_v), ladj = trg_ladj)
end

function var_trafo_result(trg_v::Real, src_v::Real)
    R = float(typeof(src_v))
    (v = convert(R, trg_v), ladj = convert(R, NaN))
end

function var_trafo_result(trg_v::AbstractVector{<:Real}, src_v::AbstractVector{<:Real}, trafo_ladj::Real, prev_ladj::Real)
    R = float(eltype(src_v))
    ladj_sum = convert(R, trafo_ladj + prev_ladj)
    trg_ladj = if !isnan(ladj_sum)
        ladj_sum
    else
        # Should be safe to assume that target dist goes to zero at infinity, should win out over infinite prev_ladj:
        ladjs_should_cancel = (trafo_ladj == R(-Inf) && prev_ladj == R(+Inf) && any(isinf, trg_v))
        ladjs_should_cancel ? zero(R) : ladj_sum
    end
    (v = convert_eltype(R, trg_v), ladj = trg_ladj)
end

function var_trafo_result(trg_v::AbstractVector{<:Real}, src_v::AbstractVector{<:Real})
    R = float(eltype(src_v))
    (v = convert_eltype(R, trg_v), ladj = convert(R, NaN))
end

function (trafo::VariateTransform)(s::DensitySample)
    r = trafo(s.v, zero(Float32))
    v = stripscalar(r.v)  # ToDo: Do we want to use stripscalar here?
    logd = s.logd - r.ladj
    DensitySample(v, logd, s.weight, s.info, s.aux)
end

# Custom broadcast(::VariateTransform, DensitySampleVector), multithreaded:
function Base.copy(
    instance::Base.Broadcast.Broadcasted{
        <:Base.Broadcast.AbstractArrayStyle{1},
        <:Any,
        <:VariateTransform,
        <:Tuple{DensitySampleVector}
    }
)
    trafo = instance.f
    s_src = instance.args[1]
    vs_trg = valshape(trafo(first(s_src.v)))
    s_src_us = unshaped.(s_src)

    n = length(eachindex(s_src_us))
    s_trg_unshaped = DensitySampleVector((
        nestedview(similar(flatview(s_src_us.v), totalndof(vs_trg), n)),
        zero(s_src_us.logd),
        deepcopy(s_src_us.weight),
        deepcopy(s_src_us.info),
        deepcopy(s_src_us.aux),
    ))
    @assert axes(s_trg_unshaped) == axes(s_src)
    @assert s_trg_unshaped.v isa ArrayOfSimilarArrays
    @threads for i in eachindex(s_trg_unshaped, s_src)
        r = trafo(s_src.v[i], zero(Float32))
        s_trg_unshaped.v[i] .= unshaped(r.v)
        s_trg_unshaped.logd[i] -= r.ladj
    end
    vs_trg.(s_trg_unshaped)
end

struct InvVT{
    VF <: VariateForm,
    ST <: VariateSpace,
    SF <: VariateSpace,
    FT <: VariateTransform{VF,ST,SF}
} <: VariateTransform{VF,SF,ST}
    orig::FT
end

Base.inv(trafo::VariateTransform) = InvVT(trafo)
Base.inv(trafo::InvVT) = trafo.orig

ValueShapes.varshape(trafo::InvVT) = varshape(trafo.orig)

target_space(trafo::InvVT) = source_space(trafo.orig)

source_space(trafo::InvVT) = target_space(trafo.orig)

const InverseVT{FT} = InvVT{VF,ST,SF,FT} where {VF,ST,SF}

struct IdentityVT{
    VF <: VariateForm,
    S <: VariateSpace,
    VS <: AbstractValueShape
} <: VariateTransform{VF,S,S}
    space::S
    varshape::VS
end

function IdentityVT(space::VariateSpace, shape::AbstractValueShape)
    VF = _variate_form(shape)
    S = typeof(space)
    VS = typeof(shape)
    IdentityVT{VF,S,VS}(space, shape)
end

Base.inv(trafo::IdentityVT) = trafo

ValueShapes.varshape(trafo::IdentityVT) = trafo.varshape

function apply_vartrafo_impl(trafo::Union{IdentityVT,InvVT{<:IdentityVT}}, v::Any, prev_ladj::Real)
    (v = v, ladj = prev_ladj)
end

(trafo::Union{IdentityVT,InvVT{<:IdentityVT}})(s::DensitySample) = s

# Custom broadcast(::IdentityVT, DensitySampleVector), multithreaded:
function Base.copy(
    instance::Base.Broadcast.Broadcasted{
        <:Base.Broadcast.AbstractArrayStyle{1},
        <:Any,
        <:Union{IdentityVT,InvVT{<:IdentityVT}},
        <:Tuple{DensitySampleVector}
    }
)
    instance.args[1]
end


# Unit tests for integrations/dftk_integration.jl

@testset "integrations/dftk_integration.jl" begin
    pspkey = list_psp(:Ar, functional = "lda")[1].identifier
    particles = [Atom(:Ar, [i & 1 == 0 ? 7 : 21, i & 2 == 0 ? 7 : 21, i & 4 == 0 ? 7 : 21]u"bohr"; pseudopotential = pspkey) for i ∈ 0:7]
    box = [[28.0, 0, 0], [0, 28.0, 0], [0, 0, 28.0]]u"bohr"
    boundary_conditions = [Periodic(), Periodic(), Periodic()]
    system1 = FlexibleSystem(particles, box, boundary_conditions)

    potential1 = DFTKPotential(5, [1, 1, 1]; damping = 0.7, tol = 1e-4)
    potential2 = DFTKPotential(136.05693123044495u"eV", [1, 1, 1]; damping = 0.7, tol = 1e-4)

    @test potential1.Ecut ≈ potential2.Ecut

    @test Atomistic.calculate_scf(system1, potential1) == potential1.previous_scfres[]
    @test InteratomicPotentials.potential_energy(system1, potential1) isa Float64
    @test InteratomicPotentials.force(system1, potential1) isa AbstractVector{<:StaticVector{3,<:Float64}}

    system2 = DynamicSystem(system1, 0u"s")
    system3 = DynamicSystem(system1, 1u"ħ_au / hartree")

    @test InteratomicPotentials.potential_energy(system2, potential1) == potential1.potential_energy_cache[0]
    @test InteratomicPotentials.force(system3, potential1) isa AbstractVector{<:StaticVector{3,<:Float64}}
    @test keys(potential1.potential_energy_cache) == Set([0.0, 1.0])
    pe2 = potential1.potential_energy_cache[0]
    pe3 = potential1.potential_energy_cache[1]

    # workaround way to validate that cached results are used
    modified_particles = [Atom(:Ar, [i & 1 == 0 ? 5 : 15, i & 2 == 0 ? 5 : 15, i & 4 == 0 ? 5 : 15]u"bohr"; pseudopotential = pspkey) for i ∈ 0:7]
    system4 = FlexibleSystem(modified_particles, box, boundary_conditions)
    system5 = DynamicSystem(system4, 0u"s")
    system6 = DynamicSystem(system4, 1u"ħ_au / hartree")

    @test InteratomicPotentials.potential_energy(system5, potential1) == pe2
    @test InteratomicPotentials.potential_energy(system6, potential1) == pe3
    @test keys(potential1.potential_energy_cache) == Set([0.0, 1.0])
end


using Revise
using SpineOpt
#using Gurobi
using JuMP
using Cbc

optimizer = optimizer_with_attributes(Cbc.Optimizer, "logLevel" => 0)
m = run_spineopt(ARGS...; mip_solver=optimizer, lp_solver=optimizer)

#optimizer = optimizer_with_attributes(Cbc.Optimizer, "logLevel" => 0)
#optimizer = optimizer_with_attributes(Gurobi.Optimizer, "OutputFlag" => 0, "MIPGap" => 1e-2)
#m = run_spineopt(ARGS...)


const cube_edges = (0x00, 0x01, 0x00, 0x02, 0x00, 0x04, 0x01, 0x03, 0x01, 0x05, 0x02, 0x03,
                    0x02, 0x06, 0x03, 0x07, 0x04, 0x05, 0x04, 0x06, 0x05, 0x07, 0x06, 0x07)

const sn_edge_table = (0x0007, 0x0019, 0x001e, 0x0062, 0x0065, 0x007b, 0x007c, 0x00a8,
                       0x00af, 0x00b1, 0x00b6, 0x00ca, 0x00cd, 0x00d3, 0x00d4, 0x0304,
                       0x0303, 0x031d, 0x031a, 0x0366, 0x0361, 0x037f, 0x0378, 0x03ac,
                       0x03ab, 0x03b5, 0x03b2, 0x03ce, 0x03c9, 0x03d7, 0x03d0, 0x0510,
                       0x0517, 0x0509, 0x050e, 0x0572, 0x0575, 0x056b, 0x056c, 0x05b8,
                       0x05bf, 0x05a1, 0x05a6, 0x05da, 0x05dd, 0x05c3, 0x05c4, 0x0614,
                       0x0613, 0x060d, 0x060a, 0x0676, 0x0671, 0x066f, 0x0668, 0x06bc,
                       0x06bb, 0x06a5, 0x06a2, 0x06de, 0x06d9, 0x06c7, 0x06c0, 0x0a40,
                       0x0a47, 0x0a59, 0x0a5e, 0x0a22, 0x0a25, 0x0a3b, 0x0a3c, 0x0ae8,
                       0x0aef, 0x0af1, 0x0af6, 0x0a8a, 0x0a8d, 0x0a93, 0x0a94, 0x0944,
                       0x0943, 0x095d, 0x095a, 0x0926, 0x0921, 0x093f, 0x0938, 0x09ec,
                       0x09eb, 0x09f5, 0x09f2, 0x098e, 0x0989, 0x0997, 0x0990, 0x0f50,
                       0x0f57, 0x0f49, 0x0f4e, 0x0f32, 0x0f35, 0x0f2b, 0x0f2c, 0x0ff8,
                       0x0fff, 0x0fe1, 0x0fe6, 0x0f9a, 0x0f9d, 0x0f83, 0x0f84, 0x0c54,
                       0x0c53, 0x0c4d, 0x0c4a, 0x0c36, 0x0c31, 0x0c2f, 0x0c28, 0x0cfc,
                       0x0cfb, 0x0ce5, 0x0ce2, 0x0c9e, 0x0c99, 0x0c87, 0x0c80, 0x0c80,
                       0x0c87, 0x0c99, 0x0c9e, 0x0ce2, 0x0ce5, 0x0cfb, 0x0cfc, 0x0c28,
                       0x0c2f, 0x0c31, 0x0c36, 0x0c4a, 0x0c4d, 0x0c53, 0x0c54, 0x0f84,
                       0x0f83, 0x0f9d, 0x0f9a, 0x0fe6, 0x0fe1, 0x0fff, 0x0ff8, 0x0f2c,
                       0x0f2b, 0x0f35, 0x0f32, 0x0f4e, 0x0f49, 0x0f57, 0x0f50, 0x0990,
                       0x0997, 0x0989, 0x098e, 0x09f2, 0x09f5, 0x09eb, 0x09ec, 0x0938,
                       0x093f, 0x0921, 0x0926, 0x095a, 0x095d, 0x0943, 0x0944, 0x0a94,
                       0x0a93, 0x0a8d, 0x0a8a, 0x0af6, 0x0af1, 0x0aef, 0x0ae8, 0x0a3c,
                       0x0a3b, 0x0a25, 0x0a22, 0x0a5e, 0x0a59, 0x0a47, 0x0a40, 0x06c0,
                       0x06c7, 0x06d9, 0x06de, 0x06a2, 0x06a5, 0x06bb, 0x06bc, 0x0668,
                       0x066f, 0x0671, 0x0676, 0x060a, 0x060d, 0x0613, 0x0614, 0x05c4,
                       0x05c3, 0x05dd, 0x05da, 0x05a6, 0x05a1, 0x05bf, 0x05b8, 0x056c,
                       0x056b, 0x0575, 0x0572, 0x050e, 0x0509, 0x0517, 0x0510, 0x03d0,
                       0x03d7, 0x03c9, 0x03ce, 0x03b2, 0x03b5, 0x03ab, 0x03ac, 0x0378,
                       0x037f, 0x0361, 0x0366, 0x031a, 0x031d, 0x0303, 0x0304, 0x00d4,
                       0x00d3, 0x00cd, 0x00ca, 0x00b6, 0x00b1, 0x00af, 0x00a8, 0x007c,
                       0x007b, 0x0065, 0x0062, 0x001e, 0x0019, 0x0007)


importall MathProgBase.SolverInterface

###############################################################################
# Solver objects
export KnitroSolver
immutable KnitroSolver <: AbstractMathProgSolver
    options
end
KnitroSolver(;kwargs...) = KnitroSolver(kwargs)

type KnitroMathProgModel <: AbstractNonlinearModel
    options
    inner::KnitroProblem

    numVar::Int
    numConstr::Int
    nnzJ::Int
    nnzH::Int

    varLB::Vector{Float64}
    varUB::Vector{Float64}
    constrLB::Vector{Float64}
    constrUB::Vector{Float64}

    jac_con::Vector{Int32}
    jac_var::Vector{Int32}
    hess_row::Vector{Int32}
    hess_col::Vector{Int32}

    varType::Vector{Int32}
    objType::Int32
    objFnType::Int32
    constrType::Vector{Int32}
    constrFnType::Vector{Int32}

    initial_x
    sense::Int32
    d::AbstractNLPEvaluator

    hasbeenrestarted::Bool

    solve_time::Float64

    function KnitroMathProgModel(;options...)
        new(options)
    end
end

NonlinearModel(s::KnitroSolver) = KnitroMathProgModel(;s.options...)
LinearQuadraticModel(s::KnitroSolver) = NonlinearToLPQPBridge(NonlinearModel(s))

###############################################################################
# Begin interface implementation

function sparse_merge_hess_duplicates(I, J, m, n)
    V = [Int[i] for i in 1:length(I)]
    for i in 1:length(I) # make upper triangular
        if I[i] > J[i]
            I[i],J[i] = J[i],I[i]
        end
    end
    findnz(sparse(I,J,V,m,n,vcat))
end

function sparse_merge_jac_duplicates(I, J, m, n)
    V = [Int[i] for i in 1:length(I)]
    findnz(sparse(I,J,V,m,n,vcat))
end

# generic nonlinear interface
function loadproblem!(m::KnitroMathProgModel,
                      numVar::Int,
                      numConstr::Int,
                      x_l, x_u, g_lb, g_ub,
                      sense::Symbol,
                      d::AbstractNLPEvaluator)
    features = features_available(d)
    has_hessian = (:Hess in features)
    if has_hessian
        initialize(d, [:Grad, :Jac, :Hess])
        Ihess, Jhess = hesslag_structure(d)
    else
        initialize(d, [:Grad, :Jac])
        Ihess = Int[]
        Jhess = Int[]
    end

    Ijac, Jjac = jac_structure(d)
    m.nnzJ = length(Ijac)
    m.nnzH = length(Ihess)
    jac_tmp = Array(Float64, m.nnzJ)
    hess_tmp = Array(Float64, m.nnzH)
    @assert length(Ijac) == length(Jjac)
    @assert length(Ihess) == length(Jhess)
    m.jac_con, m.jac_var, jac_indices = sparse_merge_jac_duplicates(Ijac, Jjac,
                                                                numConstr,
                                                                numVar)
    m.jac_con = m.jac_con .- 1
    m.jac_var = m.jac_var .- 1
    m.hess_row, m.hess_col, hess_indices = sparse_merge_hess_duplicates(Ihess,
                                                                    Jhess,
                                                                    numVar,
                                                                    numVar)
    m.hess_row = m.hess_row .- 1
    m.hess_col = m.hess_col .- 1
    n_jac_indices = length(jac_indices)
    n_hess_indices = length(hess_indices)

    m.varLB = x_l
    m.varUB = x_u
    m.constrLB = g_lb
    m.constrUB = g_ub
    m.numVar = length(m.varLB)
    m.numConstr = length(m.constrLB)
    @assert m.numVar == length(m.varUB)
    @assert m.numConstr == length(m.constrUB)

    for i in 1:m.numVar
        if m.varLB[i] == -Inf
            m.varLB[i] = -KTR_INFBOUND
        end
        if m.varUB[i] == Inf
            m.varUB[i] = KTR_INFBOUND
        end
    end

    for i in 1:m.numConstr
        if m.constrLB[i] == -Inf
            m.constrLB[i] = -KTR_INFBOUND
        end
        if m.constrUB[i] == Inf
            m.constrUB[i] = KTR_INFBOUND
        end
    end
    m.initial_x = C_NULL

    @assert sense == :Min || sense == :Max
    m.sense = (sense == :Min) ? KTR_OBJGOAL_MINIMIZE : KTR_OBJGOAL_MAXIMIZE
    # allow for the possibility of specializing to LINEAR or QUADRATIC?
    if isobjlinear(d)
        m.objType = KTR_OBJTYPE_LINEAR
        m.objFnType = KTR_FNTYPE_CONVEX
    elseif isobjquadratic(d)
        m.objType = KTR_OBJTYPE_QUADRATIC
        m.objFnType = KTR_FNTYPE_UNCERTAIN
    else
        m.objType = KTR_OBJTYPE_GENERAL
        m.objFnType = KTR_FNTYPE_UNCERTAIN
    end

    m.constrType = fill(KTR_CONTYPE_GENERAL, m.numConstr)
    m.constrFnType = fill(KTR_FNTYPE_UNCERTAIN, m.numConstr)
    for i=1:m.numConstr
        if isconstrlinear(d, i)
            # Bug in Knitro, this is unsafe until 10.1.2
            # See https://groups.google.com/d/msg/knitro/H9ck55HjY0s/U0r2-f6zAwAJ
            #m.constrType[i] = KTR_CONTYPE_LINEAR
            m.constrFnType[i] = KTR_FNTYPE_CONVEX
        end
    end

    m.varType = fill(KTR_VARTYPE_CONTINUOUS, m.numVar)

    # Objective callback
    eval_f_cb(x) = eval_f(d,x)

    # Objective gradient callback
    eval_grad_f_cb(x, grad_f) = eval_grad_f(d, grad_f, x)

    # Constraint value callback
    eval_g_cb(x, g) = eval_g(d, g, x)

    # Jacobian callback
    function eval_jac_g_cb(x, jac)
        eval_jac_g(d, jac_tmp, x)
        for i in 1:n_jac_indices
            jac[i] = sum(jac_tmp[jac_indices[i]])
        end
    end

    # Hessian callback
    function eval_h_cb(x, lambda, sigma, hess)
        eval_hesslag(d, hess_tmp, x, sigma, lambda)
        for i in 1:n_hess_indices
            hess[i] = sum(hess_tmp[hess_indices[i]])
        end
    end

    # Hessian-vector callback
    eval_hv_cb(x, lambda, sigma, hv) = eval_hesslag_prod(d, hv, x, sigma,
                                                         lambda)

    m.inner = createProblem()
    defined_hessopt = false; hessopt_value = 0
    for (param,value) in m.options
        param = string(param)
        if param == "KTR_PARAM_HESSOPT"
            defined_hessopt = true; hessopt_value = value
        end
        if param == "options_file"
            loadOptionsFile(m.inner, value)
        elseif param == "tuner_file"
            loadTunerFile(m.inner, value)
        else
            if haskey(paramName2Indx, param) # KTR_PARAM_*
                setOption(m.inner, paramName2Indx[param], value)
            else # string name
                setOption(m.inner, param, value)
            end
        end
    end

    # check and define default hessian option
    if (!has_hessian && !defined_hessopt) || (!has_hessian && !in(hessopt_value,2:6))
        setOption(m.inner,paramName2Indx["KTR_PARAM_HESSOPT"],6)
    end

    setCallbacks(m.inner, eval_f_cb, eval_g_cb, eval_grad_f_cb, eval_jac_g_cb,
                 eval_h_cb, eval_hv_cb)
end

getsense(m::KnitroMathProgModel) = m.sense
numvar(m::KnitroMathProgModel) = m.numVar
numconstr(m::KnitroMathProgModel) = m.numConstr

function optimize!(m::KnitroMathProgModel)
    if applicationReturnStatus(m.inner) == :Uninitialized
        if all(x->x==KTR_VARTYPE_CONTINUOUS, m.varType)
            initializeProblem(m.inner, m.sense, m.objType, m.varLB, m.varUB,
                              m.constrType, m.constrLB, m.constrUB, m.jac_var,
                              m.jac_con, m.hess_row, m.hess_col;
                              initial_x = m.initial_x)
        else
            initializeProblem(m.inner, m.sense, m.objType, m.objFnType,
                              m.varType, m.varLB, m.varUB, m.constrType,
                              m.constrFnType, m.constrLB, m.constrUB,
                              m.jac_var, m.jac_con, m.hess_row, m.hess_col;
                              initial_x = m.initial_x)
        end
    elseif !m.hasbeenrestarted
        restartProblem(m.inner)
    end
    t = time()
    solveProblem(m.inner)
    m.solve_time = time() - t
    m.hasbeenrestarted = false
end

function status(m::KnitroMathProgModel)
    applicationReturnStatus(m.inner)
end

getobjval(m::KnitroMathProgModel) = m.inner.obj_val[1]
getobjbound(m::KnitroMathProgModel) = get_mip_relaxation_bnd(m.inner)
getsolution(m::KnitroMathProgModel) = m.inner.x
getconstrsolution(m::KnitroMathProgModel) = m.inner.g
getreducedcosts(m::KnitroMathProgModel) =
    -1 .* m.inner.lambda[m.numConstr + 1:end]
getconstrduals(m::KnitroMathProgModel) = -1 .* m.inner.lambda[1:m.numConstr]
getrawsolver(m::KnitroMathProgModel) = m.inner
getsolvetime(m::KnitroMathProgModel) = m.solve_time

function warmstart(m::KnitroMathProgModel, x)
    m.initial_x = [Float64(i) for i in x]
    if applicationReturnStatus(m.inner) != :Uninitialized
        restartProblem(m.inner, m.initial_x, m.inner.lambda)
        m.hasbeenrestarted = true
    end
end

setwarmstart!(m::KnitroMathProgModel, x) = warmstart(m,x)
setvartype!(m::KnitroMathProgModel, typ::Vector{Symbol}) =
    (m.varType = map(t->rev_var_type_map[t], typ))

freemodel!(m::KnitroMathProgModel) = freeProblem(m.inner)


#=
The text needs to be uploaded into a 2D texture, with 1D alignement, as there is no way to vary the row length, which would be a big waste of memory.
This is why there is the need, to prepare offsets information about where exactly new lines reside.
If the string doesn't contain a new line, the text is restricted to one line, which is uploaded into one 1D texture.
This is important for differentiation between multi-line and single line text, which might need different treatment
=#
function visualize(style::Style{:Default}, text::String, data::Dict{Symbol, Any})
  glypharray          = toglypharray(text)
  data[:style_group]  = Texture([data[:color]])
  data[:textlength]   = length(text) # needs to get remembered, as glypharray is usually bigger than the text
  data[:lines]        = count(x->x=='\n', text)
  textGPU             = Texture(glypharray)
  # To make things simple for now, checks if the texture is too big for the GPU are done by 'Texture' and an error gets thrown there.
  return visualize(style, textGPU, data)
end

# This is the low-level text interface, which simply prepares the correct shader and cameras
function visualize(::Style{:Default}, text::Texture{GLGlyph{Uint16}, 4, 2}, data::Dict{Symbol, Any})
  screen             = data[:screen]
  camera             = screen.orthographiccam
  renderdata         = merge(data, data[:font].data) # merge font texture and uv informations -> details @ GLFont/src/types.jl

  view = @compat Dict(
    "GLSL_EXTENSIONS" => "#extension GL_ARB_draw_instanced : enable"
  )
  renderdata[:text]           = text
  renderdata[:projectionview] = camera.projectionview
  shader = TemplateProgram(
    Pkg.dir("GLText", "src", "textShader.vert"), Pkg.dir("GLText", "src", "textShader.frag"),
    view=view, attributes=renderdata, fragdatalocation=[(0, "fragment_color"),(1, "fragment_groupid")]
  )
  obj = instancedobject(renderdata, data[:textlength], shader, GL_TRIANGLES, textboundingbox)
  prerender!(obj, enabletransparency, glDisable, GL_DEPTH_TEST, glDisable, GL_CULL_FACE,)
  return obj
end
function visualize(style::Style{:Default}, text::Vector{GLGlyph{Uint16}}, data::Dict{Symbol, Any})
  textstride = data[:stride]
  data[:textlength] = length(text) # remember text
  if length(text) % textstride != 0
    append!(text, Array(GLGlyph{Uint16}, textstride-(length(text)%textstride))) # append if can't be reshaped with 1024
  end
  data[:style_group]  = Texture([data[:color]])
  # To make things simple for now, checks if the texture is too big for the GPU are done by 'Texture' and an error gets thrown there.
  return visualize(style, Texture(reshape(text, textstride, div(length(text), textstride))), data)
end
function textboundingbox(obj)
  glypharray  = data(obj[:text])
  advance     = obj[:advance]
  newline     = obj[:newline]

  maxv = Vector3(typemin(Float32))
  minv = Vector3(typemax(Float32))
  glyphbox = Vec3(12,24,0)
  for elem in glypharray[1:obj.alluniforms[:textlength]]

    currentpos = elem.row*advance + elem.line*newline

    maxv = maxper(maxv, currentpos + glyphbox)
    minv = minper(minv, currentpos)
  end
  AABB(minv+newline, maxv)
end


# Control script for calculating CPCP for multiple timesteps.
# Using Channels for parallel tasks.
#
# To run this on a server without x-server, you must switch to the headless
# version of Paraview.
#
# Hongyang Zhou, hyzhou@umich.edu 03/19/2020

## Import packages
using Distributed, Glob
@everywhere using DelimitedFiles, ParallelDataTransfer
if !isdefined(Main, :ReconnectionRate)
   @everywhere include("utility.jl")
   @everywhere using .ReconnectionRate
end

## Function and global definitions

# The first parameter of a channel is the Type (e.g.: Int) and the second one,
# the maximum number of elements allowed in the channel.
const jobs    = RemoteChannel(()->Channel{Int}(32))
const results = RemoteChannel(()->Channel{Tuple}(32))

@everywhere function process_file(i)
   file = files[i]
   @info "processing file $file ..."
   # Create Python script with the correct input filename for tracing
   txt = read(filePy_orig[1], String)
   tracePy = filePy_orig[1][1:end-3]*string(i)*".py"
   fileTrace = "streamline"*lpad(i,4,"0")*".txt"
   fileStream = joinpath(fileDir, fileTrace)
   fileBoundaryXYZ = "boundary"*lpad(i,4,"0")*".txt"

   if isfile(fileDir*fileBoundaryXYZ)
      return # skip the finished snapshots
   end

   open(tracePy, "w") do f
      txt = replace(txt, "test.vtu"=>file)
      txt = replace(txt, "streamline.txt"=>fileTrace)
      write(f, txt)
   end

   # Step 2: field line tracing (in Paraview)
   run(`$run_pvbatch $tracePy`)
   rm(tracePy, force=true)

   # Step 3: find magnetosphere boundary
   x, y, z = findBoundary(fileStream, method=1, nSmooth=100, DoPlot=false)
   saveBoundary(x, y, z, joinpath(fileDir,fileBoundaryXYZ))
   rm(fileDir*fileTrace, force=true)

   # Create Python script with the correct input filename for interpolation
   txt = read(filePy_orig[2], String)
   interpPy = filePy_orig[2][1:end-3]*string(i)*".py"
   fileBoundary = "boundary_value"*lpad(i,4,"0")*".txt"
   open(interpPy, "w") do f
      txt = replace(txt, "test.vtu" => file)
      txt = replace(txt, "boundary.txt" => fileBoundaryXYZ)
      txt = replace(txt, "boundary_value.txt" => fileBoundary)
      write(f, txt)
   end

   # Step 4: interpolate onto the boundary points (in Paraview)
   run(`$run_pvbatch $interpPy`)
   rm(interpPy)
end

function write_CPCP(filenames, fileDir="./")

   ϕ = Vector{Float32}(undef,length(filenames))

   for (i,file) in enumerate(filenames)
      fileBoundary = "boundary_value"*lpad(i,4,"0")*".txt"
      # Step 5: 2D line integral
      file_integration = joinpath(fileDir, fileBoundary)
      ϕ[i] = integrate_along_boundary_hall(file_integration, false)

      open(joinpath(fileDir,"CPCP.txt"), "a") do io
         i_end  = findfirst("_n",file)[1] - 1
         second = parse(Int32, file[i_end-1:i_end])
         minute = parse(Int32, file[i_end-3:i_end-2])
         timestep = 60*minute + second
         writedlm(io, [timestep round(ϕ[i],digits=4)])
      end
   end

end

@everywhere function do_work(jobs, results) # Define work function everywhere.
   while true
      job_id = take!(jobs)
      exec_time = @elapsed process_file(job_id)
      put!(results, (job_id, exec_time, myid()))
   end
end

function make_jobs(n)
   for i in 1:n
      put!(jobs, i)
   end
end

## Main
if Sys.isapple() # Sys.MACHINE might be better
   run_pvbatch = "pvbatch"
else
   run_pvbatch = "run_pvbatch"
end
# Step 1: create seeding points (same for all timesteps)
@everywhere dataDir = "../../GM/"
@everywhere fileDir = "data/"

!isdir(fileDir) && mkdir(fileDir)

createSeeds("seeds.txt", fileDir)
# Loop over multiple snapshots
files = basename.(glob("cut*.vtu", dataDir))

sendto(workers(), files=files)

@everywhere filePy_orig = ["streamtrace_save.py", "Interpolate_On_Boundary_batch.py"]

nFile = length(files)

@async make_jobs(nFile) # Feed the jobs channel with "n" jobs.

for p in workers() # Start tasks on the workers to process requests in parallel.
   @async remote_do(do_work, p, jobs, results) # Similar to remotecall.
end

@elapsed while nFile > 0 # Print out results.
   job_id, exec_time, wid = take!(results)
   println("$job_id finished $(round(exec_time; digits=2)) seconds "*"
   on worker $wid")
   global nFile -= 1
end

write_CPCP(files, fileDir)


# *********************************************************************************
# REopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this list
# of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
# *********************************************************************************
"""
    run_mpc(m::JuMP.AbstractModel, fp::String)

Solve the model predictive control problem using the `MPCScenario` defined in the JSON file stored at the file path `fp`.

Returns a Dict of results with keys matching those in the `MPCScenario`.
"""
function run_mpc(m::JuMP.AbstractModel, fp::String)
	s = MPCScenario(JSON.parsefile(fp))
	run_mpc(m, MPCInputs(s))
end

"""
    run_mpc(m::JuMP.AbstractModel,  d::Dict)

Solve the model predictive control problem using the `MPCScenario` defined in the dict `d`.

Returns a Dict of results with keys matching those in the `MPCScenario`.
"""
function run_mpc(m::JuMP.AbstractModel, d::Dict)
	s = MPCScenario(d)
	run_mpc(m, MPCInputs(s))
end

"""
    run_mpc(m::JuMP.AbstractModel, p::MPCInputs)

Solve the model predictive control problem using the `MPCInputs`.

Returns a Dict of results with keys matching those in the `MPCScenario`.
"""
function run_mpc(m::JuMP.AbstractModel, p::MPCInputs)
    build_mpc!(m, p)

    if !p.s.settings.add_soc_incentive
		@objective(m, Min, m[:Costs])
	else # Keep SOC high
		@objective(m, Min, m[:Costs] - sum(m[:dvStoredEnergy]["ElectricStorage", ts] for ts in p.time_steps) /
									   (8760. / p.hours_per_time_step)
		)
	end

	@info "Model built. Optimizing..."
	tstart = time()
	optimize!(m)
	opt_time = round(time() - tstart, digits=3)
	if termination_status(m) == MOI.TIME_LIMIT
		status = "timed-out"
    elseif termination_status(m) == MOI.OPTIMAL
        status = "optimal"
    else
		status = "not optimal"
		@warn "MPC solved with " termination_status(m), ", returning the model."
		return m
	end
	@info "MPC solved with " termination_status(m)
	@info "Solving took $(opt_time) seconds."

	tstart = time()
	results = mpc_results(m, p)
	time_elapsed = time() - tstart
	@info "Results processing took $(round(time_elapsed, digits=3)) seconds."
	results["status"] = status
	results["solver_seconds"] = opt_time
	return results
end

"""
    build_mpc!(m::JuMP.AbstractModel, p::MPCInputs)

Add variables and constraints for model predictive control model.
Similar to a REopt model but with any length of horizon (instead of one calendar year),
and the DER sizes must be provided.
"""
function build_mpc!(m::JuMP.AbstractModel, p::MPCInputs)
    add_variables!(m, p)

	for ts in p.time_steps_without_grid

		fix(m[:dvGridPurchase][ts], 0.0, force=true)

		for t in p.s.storage.types.elec
			fix(m[:dvGridToStorage][t, ts], 0.0, force=true)
		end

		for t in p.techs.elec, u in p.export_bins_by_tech[t]
			fix(m[:dvProductionToGrid][t, u, ts], 0.0, force=true)
		end
	end

	for b in p.s.storage.types.all
		if p.s.storage.attr[b].size_kw == 0 || p.s.storage.attr[b].size_kwh == 0
			@constraint(m, [ts in p.time_steps], m[:dvStoredEnergy][b, ts] == 0)
			@constraint(m, [t in p.techs.elec, ts in p.time_steps_with_grid],
						m[:dvProductionToStorage][b, t, ts] == 0)
			@constraint(m, [ts in p.time_steps], m[:dvDischargeFromStorage][b, ts] == 0)
			if b in p.s.storage.types.elec
				@constraint(m, [ts in p.time_steps], m[:dvGridToStorage][b, ts] == 0)
			end
		else
			add_general_storage_dispatch_constraints(m, p, b)
			if b in p.s.storage.types.elec
				add_elec_storage_dispatch_constraints(m, p, b)
			elseif b in p.s.storage.types.hot
				add_hot_thermal_storage_dispatch_constraints(m, p, b)
			elseif b in p.s.storage.types.cold
				add_cold_thermal_storage_dispatch_constraints(m, p, b)
			else
				@error("Invalid storage does not fall in a thermal or electrical set")
			end
		end
	end

	if any(size_kw->size_kw > 0, (p.s.storage.attr[b].size_kw for b in p.s.storage.types.all))
		add_storage_sum_constraints(m, p)
	end

	add_production_constraints(m, p)

	if !isempty(p.techs.no_turndown)
		@constraint(m, [t in p.techs.no_turndown, ts in p.time_steps],
            m[:dvRatedProduction][t,ts] == m[:dvSize][t]
        )
	end

	add_elec_load_balance_constraints(m, p)

	if !isempty(p.s.limits.grid_draw_limit_kw_by_time_step)
		add_grid_draw_limits(m, p)
	end

	if !isempty(p.s.electric_tariff.export_bins)
		add_export_constraints(m, p)
		if !isempty(p.s.limits.export_limit_kw_by_time_step)
			add_export_limits(m, p)
		end
	end

	if !isempty(p.s.electric_tariff.monthly_demand_rates)
		add_monthly_peak_constraint(m, p)
	end

	if !isempty(p.s.electric_tariff.tou_demand_ratchet_time_steps)
		add_tou_peak_constraint(m, p)
	end

	if !(p.s.electric_utility.allow_simultaneous_export_import) & !isempty(p.s.electric_tariff.export_bins)
		add_simultaneous_export_import_constraint(m, p)
	end

    m[:TotalFuelCosts] = 0.0
    m[:TotalPerUnitProdOMCosts] = 0.0

    if !isempty(p.techs.gen)
        add_gen_constraints(m, p)
		m[:TotalPerUnitProdOMCosts] += @expression(m,
			sum(p.s.generator.om_cost_per_kwh * p.hours_per_time_step *
			m[:dvRatedProduction][t, ts] for t in p.techs.gen, ts in p.time_steps)
		)
        m[:TotalGenFuelCosts] = @expression(m,
            sum(m[:dvFuelUsage][t,ts] * p.s.generator.fuel_cost_per_gallon for t in p.techs.gen, ts in p.time_steps)
        )
        m[:TotalFuelCosts] += m[:TotalGenFuelCosts]
	end

	add_elec_utility_expressions(m, p)
    add_previous_monthly_peak_constraint(m, p)
    add_previous_tou_peak_constraint(m, p)

    # TODO: random outages in MPC?
	if !isempty(p.s.electric_utility.outage_durations)
		add_dv_UnservedLoad_constraints(m,p)
		add_outage_cost_constraints(m,p)
		add_MG_production_constraints(m,p)
		if !isempty(p.s.storage.types.elec)
			add_MG_storage_dispatch_constraints(m,p)
		else
			fix_MG_storage_variables(m,p)
		end
		add_cannot_have_MG_with_only_PVwind_constraints(m,p)
		add_MG_size_constraints(m,p)

		if !isempty(p.techs.gen)
			add_MG_fuel_burn_constraints(m,p)
			add_binMGGenIsOnInTS_constraints(m,p)
		else
			m[:ExpectedMGFuelUsed] = 0
			m[:ExpectedMGFuelCost] = 0
			@constraint(m, [s in p.s.electric_utility.scenarios, tz in p.s.electric_utility.outage_start_time_steps, ts in p.s.electric_utility.outage_time_steps],
				m[:binMGGenIsOnInTS][s, tz, ts] == 0
			)
		end

		if p.s.site.min_resil_time_steps > 0
			add_min_hours_crit_ld_met_constraint(m,p)
		end
	end

	#################################  Objective Function   ########################################
	@expression(m, Costs,

		# Variable O&M
		m[:TotalPerUnitProdOMCosts] +

		# Total Generator Fuel Costs
        m[:TotalFuelCosts] +

		# Utility Bill
		m[:TotalElecBill]
	);
	if !isempty(p.s.electric_utility.outage_durations)
		add_to_expression!(Costs, m[:ExpectedOutageCost] + m[:mgTotalTechUpgradeCost] + m[:dvMGStorageUpgradeCost] + m[:ExpectedMGFuelCost])
	end
    #= Note: 0.9999*MinChargeAdder in Objective b/c when TotalMinCharge > (TotalEnergyCharges + TotalDemandCharges + TotalExportBenefit + TotalFixedCharges)
		it is arbitrary where the min charge ends up (eg. could be in TotalDemandCharges or MinChargeAdder).
		0.0001*MinChargeAdder is added back into LCC when writing to results.  =#
	nothing
end

function add_variables!(m::JuMP.AbstractModel, p::MPCInputs)
    @variables m begin
		# dvSize[p.techs.all] >= 0  # System Size of Technology t [kW]
		# dvPurchaseSize[p.techs.all] >= 0  # system kW beyond existing_kw that must be purchased
		dvGridPurchase[p.time_steps] >= 0  # Power from grid dispatched to meet electrical load [kW]
		dvRatedProduction[p.techs.all, p.time_steps] >= 0  # Rated production of technology t [kW]
		dvCurtail[p.techs.all, p.time_steps] >= 0  # [kW]
		dvProductionToStorage[p.s.storage.types.all, p.techs.all, p.time_steps] >= 0  # Power from technology t used to charge storage system b [kW]
		dvDischargeFromStorage[p.s.storage.types.all, p.time_steps] >= 0 # Power discharged from storage system b [kW]
		dvGridToStorage[p.s.storage.types.elec, p.time_steps] >= 0 # Electrical power delivered to storage by the grid [kW]
		dvStoredEnergy[p.s.storage.types.all, 0:p.time_steps[end]] >= 0  # State of charge of storage system b
		dvStoragePower[p.s.storage.types.all] >= 0   # Power capacity of storage system b [kW]
		dvStorageEnergy[p.s.storage.types.all] >= 0   # Energy capacity of storage system b [kWh]
		# TODO rm dvStoragePower/Energy dv's
		dvPeakDemandTOU[p.ratchets, 1:1] >= 0  # Peak electrical power demand during ratchet r [kW]
		dvPeakDemandMonth[p.months] >= 0  # Peak electrical power demand during month m [kW]
		# MinChargeAdder >= 0
	end
	# TODO: tiers in MPC tariffs and variables?

	if !isempty(p.s.electric_tariff.export_bins)
		@variable(m, dvProductionToGrid[p.techs.elec, p.s.electric_tariff.export_bins, p.time_steps] >= 0)
	end

    m[:dvSize] = p.existing_sizes

	for b in p.s.storage.types.all
		fix(m[:dvStoragePower][b], p.s.storage.attr["ElectricStorage"].size_kw, force=true)
		fix(m[:dvStorageEnergy][b], p.s.storage.attr["ElectricStorage"].size_kwh, force=true)
	end

	# not modeling min charges since control does not affect them
    m[:MinChargeAdder] = 0

	if !isempty(p.techs.gen)  # Problem becomes a MILP
		@warn """Adding binary variable to model gas generator.
				 Some solvers are very slow with integer variables"""
		@variables m begin
			dvFuelUsage[p.techs.gen, p.time_steps] >= 0 # Fuel burned by technology t in each time step [kWh]
			binGenIsOnInTS[p.techs.gen, p.time_steps], Bin  # 1 If technology t is operating in time step h; 0 otherwise
		end
	end

	if !(p.s.electric_utility.allow_simultaneous_export_import)
		@warn """Adding binary variable to prevent simultaneous grid import/export.
				 Some solvers are very slow with integer variables"""
		@variable(m, binNoGridPurchases[p.time_steps], Bin)
	end

	if !isempty(p.s.electric_utility.outage_durations) # add dvUnserved Load if there is at least one outage
		@warn """Adding binary variable to model outages.
				 Some solvers are very slow with integer variables"""
		max_outage_duration = maximum(p.s.electric_utility.outage_durations)
		outage_time_steps = p.s.electric_utility.outage_time_steps
		tZeros = p.s.electric_utility.outage_start_time_steps
		S = p.s.electric_utility.scenarios
		# TODO: currently defining more decision variables than necessary b/c using rectangular arrays, could use dicts of decision variables instead
		@variables m begin # if there is more than one specified outage, there can be more othan one outage start time
			dvUnservedLoad[S, tZeros, outage_time_steps] >= 0 # unserved load not met by system
			dvMGProductionToStorage[p.techs.all, S, tZeros, outage_time_steps] >= 0 # Electricity going to the storage system during each time_step
			dvMGDischargeFromStorage[S, tZeros, outage_time_steps] >= 0 # Electricity coming from the storage system during each time_step
			dvMGRatedProduction[p.techs.all, S, tZeros, outage_time_steps]  # MG Rated Production at every time_step.  Multiply by ProdFactor to get actual energy
			dvMGStoredEnergy[S, tZeros, 0:max_outage_duration] >= 0 # State of charge of the MG storage system
			dvMaxOutageCost[S] >= 0 # maximum outage cost dependent on number of outage durations
			# dvMGTechUpgradeCost[p.techs.all] >= 0
			# dvMGStorageUpgradeCost >= 0
			# dvMGsize[p.techs.all] >= 0

			dvMGFuelUsed[p.techs.all, S, tZeros] >= 0
			dvMGMaxFuelUsage[S] >= 0
			dvMGMaxFuelCost[S] >= 0
			dvMGCurtail[p.techs.all, S, tZeros, outage_time_steps] >= 0

			# binMGStorageUsed, Bin # 1 if MG storage battery used, 0 otherwise
			# binMGTechUsed[p.techs.all], Bin # 1 if MG tech used, 0 otherwise
			binMGGenIsOnInTS[S, tZeros, outage_time_steps], Bin
		end
	end
end


#=

Proposal functions for joint
Biogeographic competition model

Ignacio Quintero Mächler

t(-_-t)

May 16 2017

=#

#=
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
`Y` IID proposal functions
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
=#

"""
    upnode!(λ1     ::Float64,
            λ0     ::Float64,
            triad  ::Array{Int64,1},
            Y      ::Array{Int64,3},
            stemevs::Array{Array{Float64,1},1},
            bridx_a::Array{Array{UnitRange{Int64},1},1},
            brδt   ::Vector{Vector{Float64}},
            brl    ::Vector{Float64},
            brs    ::Array{Int64,3},
            narea  ::Int64,
            nedge  ::Int64)

Update node and incident branches using discrete
Data Augmentation for all areas using a non-competitive
mutual-independence Markov model.
"""
function upnode!(λ1     ::Float64,
                 λ0     ::Float64,
                 triad  ::Array{Int64,1},
                 Y      ::Array{Int64,3},
                 stemevs::Array{Array{Float64,1},1},
                 bridx_a::Array{Array{UnitRange{Int64},1},1},
                 brδt   ::Vector{Vector{Float64}},
                 brl    ::Vector{Float64},
                 brs    ::Array{Int64,3},
                 narea  ::Int64,
                 nedge  ::Int64)

  @inbounds begin

    # define branch triad
    pr, d1, d2 = triad

    # sample
    samplenode!(λ1, λ0, pr, d1, d2, brs, brl, narea)

    # save extinct
    ntries = 1
    while iszero(sum(view(brs,pr,2,:)))
      samplenode!(λ1, λ0, pr, d1, d2, brs, brl, narea)

      ntries += 1
      if ntries == 500
        return false
      end
    end

    # set new node in Y
    @simd for k in Base.OneTo(narea)
      Y[bridx_a[k][d1][1]] = Y[bridx_a[k][d2][1]] = brs[pr,2,k]
    end

    # sample a consistent history
    createhists!(λ1, λ0, Y, pr, d1, d2, brs, brδt, bridx_a, narea, nedge,
                 stemevs, brl[nedge])

    ntries = 1
    while ifextY(Y, stemevs, triad, brs, brl[nedge], narea, bridx_a, nedge)
      createhists!(λ1, λ0, Y, pr, d1, d2, brs, brδt, bridx_a, narea, nedge,
                   stemevs, brl[nedge])

      ntries += 1
      if ntries == 500
        return false
      end
    end

  end

  return true
end

"""
    samplenode!(λ1   ::Float64,
                λ0   ::Float64,
                pr   ::Int64,
                d1   ::Int64,
                d2   ::Int64,
                brs  ::Array{Int64,3},
                brl  ::Array{Float64,1},
                narea::Int64)

Sample one internal node according to
mutual-independence model transition probabilities.
"""
function samplenode!(λ1   ::Float64,
                     λ0   ::Float64,
                     pr   ::Int64,
                     d1   ::Int64,
                     d2   ::Int64,
                     brs  ::Array{Int64,3},
                     brl  ::Array{Float64,1},
                     narea::Int64)
  @inbounds begin

    # estimate transition probabilities
    pr0_1, pr0_2 = Ptrfast_start(λ1, λ0, brl[pr], Val{0})
    pr1_1, pr1_2 = Ptrfast_start(λ1, λ0, brl[pr], Val{1})
    d10_1, d10_2 = Ptrfast_end(  λ1, λ0, brl[d1], Val{0})
    d11_1, d11_2 = Ptrfast_end(  λ1, λ0, brl[d1], Val{1})
    d20_1, d20_2 = Ptrfast_end(  λ1, λ0, brl[d2], Val{0})
    d21_1, d21_2 = Ptrfast_end(  λ1, λ0, brl[d2], Val{1})

    for k = Base.OneTo(narea)

      if iszero(brs[pr,1,k])
        ppr_1, ppr_2 = pr0_1, pr0_2
      else
        ppr_1, ppr_2 = pr1_1, pr1_2
      end

      if iszero(brs[d1,2,k])
        pd1_1, pd1_2 = d10_1, d10_2
      else
        pd1_1, pd1_2 = d11_1, d11_2
      end

      if iszero(brs[d2,2,k])
        pd2_1, pd2_2 = d20_1, d20_2
      else
        pd2_1, pd2_2 = d21_1, d21_2
      end

      tp = normlize(*(ppr_1, pd1_1, pd2_1),
                    *(ppr_2, pd1_2, pd2_2))::Float64

      # sample the node's character
      brs[pr,2,k] = brs[d1,1,k] = brs[d2,1,k] = coinsamp(tp)::Int64
    end
  end

  return nothing
end

"""
    createhists!(λ::Array{Float64,1}, Y::Array{Int64,3}, pr::Int64, d1::Int64, d2::Int64, brs::Array{Int64,3}, brδt::Array{Array{Float64,1},1}, bridx_a::Array{Array{Array{Int64,1},1},1}, narea::Int64)

Create bit histories for all areas for the branch trio.
"""
function createhists!(λ1     ::Float64,
                      λ0     ::Float64,
                      Y      ::Array{Int64,3},
                      pr     ::Int64,
                      d1     ::Int64,
                      d2     ::Int64,
                      brs    ::Array{Int64,3},
                      brδt   ::Array{Array{Float64,1},1},
                      bridx_a::Array{Array{UnitRange{Int64},1},1},
                      narea  ::Int64,
                      nedge  ::Int64,
                      stemevs::Array{Array{Float64,1},1},
                      stbrl  ::Float64)

  @inbounds begin

    if pr == nedge
      # if stem branch do continuous DA
      mult_rejsam!(stemevs, brs, λ1, λ0, stbrl, narea, nedge)
      for j = Base.OneTo(narea), idx = (d1,d2)
        bit_rejsam!(Y, bridx_a[j][idx], brs[idx,2,j],
                    λ1, λ0, brδt[idx])
      end
    else
      for j = Base.OneTo(narea), idx = (pr,d1,d2)
        bit_rejsam!(Y, bridx_a[j][idx], brs[idx,2,j],
                    λ1, λ0, brδt[idx])
      end
    end
  end

  return nothing
end

"""
    mult_rejsam!(evs  ::Array{Array{Float64,1},1},
                 brs  ::Array{Int64,3},
                 λ1   ::Float64,
                 λ0   ::Float64,
                 t    ::Float64,
                 narea::Int64,
                 nedge::Int64)

  Multi-area branch rejection independent model sampling.
"""
function mult_rejsam!(evs  ::Array{Array{Float64,1},1},
                      brs  ::Array{Int64,3},
                      λ1   ::Float64,
                      λ0   ::Float64,
                      t    ::Float64,
                      narea::Int64,
                      nedge::Int64)

  @simd for k = Base.OneTo(narea)
    rejsam!(evs[k], brs[nedge,1,k], brs[nedge,2,k], λ1, λ0, t)
  end

  return nothing
end

"""
    ifextY(Y      ::Array{Int64,3},
           triad  ::Array{Int64,1},
           narea  ::Int64,
           bridx_a::Array{Array{UnitRange{Int64},1},1})

Return `true` if at some point the species
goes extinct and/or more than one change is
observed after some **δt**, otherwise returns `false`.
"""
function ifextY(Y      ::Array{Int64,3},
                stemevs::Array{Array{Float64,1},1},
                triad  ::Array{Int64,1},
                brs    ::Array{Int64,3},
                stbrl  ::Float64,
                narea  ::Int64,
                bridx_a::Array{Array{UnitRange{Int64},1},1},
                nedge  ::Int64)

  @inbounds begin

    if triad[1] == nedge
      ifext_cont(stemevs, brs, stbrl, narea, nedge) && return true::Bool

      for k in (triad[2],triad[3])
        ifext_disc(Y, k, narea, bridx_a) && return true::Bool
      end
    else

      for k in triad
        ifext_disc(Y, k, narea, bridx_a) && return true::Bool
      end
    end
  end

  return false::Bool
end

"""
    ifext_cont(t_hist::Array{Array{Float64,1},1},
               brs   ::Array{Int64,3},
               t     ::Float64,
               narea ::Int64,
               nedge ::Int64)

Return true if lineage goes extinct.
"""
function ifext_cont(t_hist::Array{Array{Float64,1},1},
                    brs   ::Array{Int64,3},
                    t     ::Float64,
                    narea ::Int64,
                    nedge ::Int64)

  @inbounds begin

    ioc = 0
    # initial occupancy time
    for k in Base.OneTo(narea)
      if brs[nedge,1,k] == 1
        ioc = k
        break
      end
    end

    ioct = t_hist[ioc][1]

    ntries = 0
    while !isapprox(ioct, t, atol = 1.0e-12)

      if ioc == narea
        ioc = 1
      else
        ioc += 1
      end

      tc = 0.0
      cs = brs[nedge,1,ioc]
      for ts in t_hist[ioc]
        tc += ts
        if ioct < tc
          if cs == 1
            ioct  = tc
            ntries = 0
            break
          else
            ntries += 1
            if ntries > narea
              return true
            end
            break
          end
        end
        cs = 1 - cs
      end

    end
  end

  return false::Bool
end

"""
    ifext_disc(Y      ::Array{Int64,3},
               br     ::Int64,
               narea  ::Int64,
               bridx_a::Array{Array{UnitRange{Int64},1},1})

Return `true` if at some point the species
goes extinct and/or more than one change is
observed after some **δt**, otherwise returns `false`.
This specific method is for single branch updates.
"""
function ifext_disc(Y      ::Array{Int64,3},
                    br     ::Int64,
                    narea  ::Int64,
                    bridx_a::Array{Array{UnitRange{Int64},1},1})

  @inbounds begin

    for i = Base.OneTo(length(bridx_a[1][br]::UnitRange{Int64})-1)
      s_e::Int64 = 0            # count current areas
      s_c::Int64 = 0            # count area changes

      for k = Base.OneTo(narea)
        s_e += Y[bridx_a[k][br][i]]::Int64
        if Y[bridx_a[k][br][i]]::Int64 != Y[bridx_a[k][br][i+1]]::Int64
          s_c += 1
        end
      end

      if s_e == 0 || s_c > 1
        return true::Bool
      end
    end

  end

  return false::Bool
end

#=
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
Y stem node proposal function (continuous DA)
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
=#

"""
    upstemnode!(λ1   ::Float64,
                λ0   ::Float64,
                nedge::Int64,
                stemevs::Array{Array{Float64,1},1},
                brs  ::Array{Int64,3},
                brl  ::Array{Float64,1},
                narea::Int64)

Update stem node.
"""
function upstemnode!(λ1     ::Float64,
                     λ0     ::Float64,
                     nedge  ::Int64,
                     stemevs::Array{Array{Float64,1},1},
                     brs    ::Array{Int64,3},
                     stbrl  ::Float64,
                     narea  ::Int64)

  @inbounds begin

    # sample
    samplestem!(λ1, λ0, nedge, brs, stbrl, narea)

    # save extinct
    ntries = 1
    while sum(view(brs,nedge,1,:)) < 1
      samplestem!(λ1, λ0, nedge, brs, stbrl, narea)
      ntries += 1
      if ntries == 500
        return false::Bool
      end
    end

    # sample a congruent history
    mult_rejsam!(stemevs, brs, λ1, λ0, stbrl, narea, nedge)

    ntries = 1
    # check if extinct

    while ifext_cont(stemevs, brs, stbrl, narea, nedge)
      mult_rejsam!(stemevs, brs, λ1, λ0, stbrl, narea, nedge)

      ntries += 1
      if ntries == 500
        return false::Bool
      end
    end
  end

  return true::Bool
end

"""
    samplestem!(λ1   ::Float64,
                λ0   ::Float64,
                nedge::Int64,
                brs  ::Array{Int64,3},
                brl  ::Array{Float64,1},
                narea::Int64)

Sample stem node.
"""
function samplestem!(λ1   ::Float64,
                     λ0   ::Float64,
                     nedge::Int64,
                     brs  ::Array{Int64,3},
                     stbrl::Float64,
                     narea::Int64)

 @inbounds begin

    # estimate transition probabilities
    p0 = normlize(Ptrfast_end(λ1, λ0, stbrl, Val{0}))::Float64
    p1 = normlize(Ptrfast_end(λ1, λ0, stbrl, Val{1}))::Float64

    for k = Base.OneTo(narea)
      # sample the node's character
      if iszero(brs[nedge,2,k])
        brs[nedge,1,k] = coinsamp(p0)::Int64
      else
        brs[nedge,1,k] = coinsamp(p1)::Int64
      end
    end
  end

  return nothing
end

#=
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
Y branch proposal functions
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
=#

"""
upbranchY!(λ1     ::Float64,
           λ0     ::Float64,
           ω1     ::Float64,
           ω0     ::Float64,
           avg_Δx ::Array{Float64,2},
           br     ::Int64,
           Y      ::Array{Int64,3},
           stemevc::Array{Array{Float64,1},1},
           wareas ::Array{Int64,1},
           bridx_a::Array{Array{UnitRange{Int64},1},1},
           brδt   ::Vector{Vector{Float64}},
           brl    ::Vector{Float64},
           brs    ::Array{Int64,3},
           narea  ::Int64,
           nedge  ::Int64)

Update one branch using discrete Data Augmentation
for all areas with independent
proposals taking into account `Δx` and `ω1` & `ω0`.
"""
function upbranchY!(λ1     ::Float64,
                    λ0     ::Float64,
                    br     ::Int64,
                    Y      ::Array{Int64,3},
                    stemevs::Array{Array{Float64,1},1},
                    bridx_a::Array{Array{UnitRange{Int64},1},1},
                    brδt   ::Vector{Vector{Float64}},
                    stbrl  ::Float64,
                    brs    ::Array{Int64,3},
                    narea  ::Int64,
                    nedge  ::Int64)

  ntries = 1

  # if stem branch
  if br == nedge
    mult_rejsam!(stemevs, brs, λ1, λ0, stbrl, narea, nedge)

    # check if extinct
    while ifext_cont(stemevs, brs, stbrl, narea, nedge)
      mult_rejsam!(stemevs, brs, λ1, λ0, stbrl, narea, nedge)

      ntries += 1
      if ntries == 500
        return false
      end
    end

  else
    createhists!(λ1, λ0, Y, br, brs, brδt, bridx_a, narea)

    # check if extinct
    while ifext_disc(Y, br, narea, bridx_a)
      createhists!(λ1, λ0, Y, br, brs, brδt, bridx_a, narea)

      ntries += 1
      if ntries == 500
        return false
      end
    end
  end

  return true
end

"""
    createhists!(λ1     ::Float64,
                 λ0     ::Float64,
                 ω1     ::Float64,
                 ω0     ::Float64,
                 avg_Δx ::Array{Float64,2},
                 Y      ::Array{Int64,3},
                 br     ::Int64,
                 brs    ::Array{Int64,3},
                 brδt   ::Array{Array{Float64,1},1},
                 bridx_a::Array{Array{UnitRange{Int64},1},1},
                 narea  ::Int64)

Create bit histories for all areas for one single branch
taking into account `Δx` and `ω1` & `ω0` for all areas.
"""
function createhists!(λ1     ::Float64,
                      λ0     ::Float64,
                      Y      ::Array{Int64,3},
                      br     ::Int64,
                      brs    ::Array{Int64,3},
                      brδt   ::Array{Array{Float64,1},1},
                      bridx_a::Array{Array{UnitRange{Int64},1},1},
                      narea  ::Int64)
  @inbounds begin
    for j = Base.OneTo(narea)
      bit_rejsam!(Y, bridx_a[j][br], brs[br,2,j], λ1, λ0, brδt[br])
    end
  end

  return nothing
end

#=
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
X proposal functions
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
=#

"""
    uptrioX!(pr   ::Int64,
             d1   ::Int64,
             d2   ::Int64,
             X    ::Array{Float64,2},
             bridx::Array{UnitRange{Int64},1},
             brδt ::Array{Array{Float64,1},1},
             σ²c  ::Float64)

Update the node and adjoining branhces of `trio` using Brownian bridges.
"""
function uptrioX!(pr   ::Int64,
                  d1   ::Int64,
                  d2   ::Int64,
                  X    ::Array{Float64,2},
                  bridx::Array{UnitRange{Int64},1},
                  brδt ::Array{Array{Float64,1},1},
                  brl  ::Array{Float64,1},
                  σ²ϕ  ::Float64,
                  nedge::Int64)
  @inbounds begin

    # if not root
    if pr != nedge

      ipr = bridx[pr]
      id1 = bridx[d1]
      id2 = bridx[d2]

      # update node
      X[id1[1]] =
      X[id2[1]] = trioupd(X[ipr[1]],
                          X[id1[end]],
                          X[id2[end]],
                          brl[pr], brl[d1], brl[d2], σ²ϕ)

      #update branches
      bbX!(X, ipr, brδt[pr], σ²ϕ)
      bbX!(X, id1, brδt[d1], σ²ϕ)
      bbX!(X, id2, brδt[d2], σ²ϕ)

    else
      id1 = bridx[d1]
      id2 = bridx[d2]

      # update node
      X[id1[1]] =
      X[id2[1]] = duoupd(X[id1[end]],
                         X[id2[end]],
                         brl[d1], brl[d2], σ²ϕ)

      # update branches
      bbX!(X, id1, brδt[d1], σ²ϕ)
      bbX!(X, id2, brδt[d2], σ²ϕ)
    end

  end

  return nothing
end

"""
    upbranchX!(j    ::Int64,
               X    ::Array{Float64,2},
               bridx::Array{UnitRange{Int64},1},
               brδt ::Array{Array{Float64,1},1},
               σ²c  ::Float64)

Update a branch j in X using a Brownian bridge.
"""
function upbranchX!(j    ::Int64,
                    X    ::Array{Float64,2},
                    bridx::Array{UnitRange{Int64},1},
                    brδt ::Array{Array{Float64,1},1},
                    σ²ϕ  ::Float64)

  @inbounds bbX!(X, bridx[j], brδt[j], σ²ϕ)

  return nothing
end

"""
    bbX!(X::Array{Float64,2}, idx::UnitRange, t::Array{Float64,1}, σ::Float64)

Brownian bridge simulation function for updating a branch in X in place.
"""
function bbX!(X  ::Array{Float64,2},
              idx::UnitRange,
              t  ::Array{Float64,1},
              σ²ϕ::Float64)

  @inbounds begin

    xf::Float64 = X[idx[end]]

    for i = Base.OneTo(lastindex(t)-1)
      X[idx[i+1]] = (X[idx[i]] + randn()*sqrt((t[i+1] - t[i])*σ²ϕ))::Float64
    end

    invte::Float64 = 1.0/t[end]
    xdif ::Float64 = (X[idx[end]] - xf)

    @simd for i = Base.OneTo(lastindex(t))
      X[idx[i]] = (X[idx[i]] - t[i] * invte * xdif)::Float64
    end
  end

  return nothing
end


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct RLFilter <: Filter
        rf::Float64
        lf::Float64
        ext::Dict{String, Any}
        states::Vector{Symbol}
        n_states::Int
    end

Parameters of RL series filter in algebraic representation

# Arguments
- `rf::Float64`: Series resistance in p.u. of converter filter to the grid, validation range: `(0, nothing)`
- `lf::Float64`: Series inductance in p.u. of converter filter to the grid, validation range: `(0, nothing)`
- `ext::Dict{String, Any}`
- `states::Vector{Symbol}`: RLFilter has zero states
- `n_states::Int`: RLFilter has zero states
"""
mutable struct RLFilter <: Filter
    "Series resistance in p.u. of converter filter to the grid"
    rf::Float64
    "Series inductance in p.u. of converter filter to the grid"
    lf::Float64
    ext::Dict{String, Any}
    "RLFilter has zero states"
    states::Vector{Symbol}
    "RLFilter has zero states"
    n_states::Int
end

function RLFilter(rf, lf, ext=Dict{String, Any}(), )
    RLFilter(rf, lf, ext, Vector{Symbol}(), 0, )
end

function RLFilter(; rf, lf, ext=Dict{String, Any}(), states=Vector{Symbol}(), n_states=0, )
    RLFilter(rf, lf, ext, states, n_states, )
end

# Constructor for demo purposes; non-functional.
function RLFilter(::Nothing)
    RLFilter(;
        rf=0,
        lf=0,
        ext=Dict{String, Any}(),
    )
end

"""Get [`RLFilter`](@ref) `rf`."""
get_rf(value::RLFilter) = value.rf
"""Get [`RLFilter`](@ref) `lf`."""
get_lf(value::RLFilter) = value.lf
"""Get [`RLFilter`](@ref) `ext`."""
get_ext(value::RLFilter) = value.ext
"""Get [`RLFilter`](@ref) `states`."""
get_states(value::RLFilter) = value.states
"""Get [`RLFilter`](@ref) `n_states`."""
get_n_states(value::RLFilter) = value.n_states

"""Set [`RLFilter`](@ref) `rf`."""
set_rf!(value::RLFilter, val) = value.rf = val
"""Set [`RLFilter`](@ref) `lf`."""
set_lf!(value::RLFilter, val) = value.lf = val
"""Set [`RLFilter`](@ref) `ext`."""
set_ext!(value::RLFilter, val) = value.ext = val


module PerformanceProfilingHttpEndpoints

import HTTP
import Profile
import PProf

using Serialization: serialize

#----------------------------------------------------------
#  Description
#
# It would be nice if visiting the `/profile` page from a web browser gives you a form where you can:
# - configure the Profiling parameters:
#    - Set julia Profile configuration, either by:
#       - Manually setting `n` and `delay`, for `Profile.init(n, delay)`, or
#       - provide a `duration` and a `delay`, and we will compute suggested `n` and `delay` for you. (`n` is the number of stack _frames_  the profiling will hold, so it also depends on the expected stack _depth_ for each sample, and the number of threads currently running. It's good to leave a wide buffer.)
# - Export the profiling data (either with or without C-lang functions - default is `C=true`)
#   - Download the raw Julia profile data (`Profile.retrieve()`)
#   - Convert the profile data to PProf, and download it (`PProf.pprof()`)
#   - Upload the resulting profile to DataDog (as PProf profile) -- see #4937.
#
#----------------------------------------------------------

default_n() = "1e8"
default_delay() = "0.01"
default_duration() = "10.0"
default_pprof() = "true"
default_alloc_sample_rate() = "0.0001"

cpu_profile_error_message() = """Need to provide query params:
    - duration=$(default_duration())
    - delay=$(default_delay())
    - n=$(default_n())
    - pprof=$(default_pprof())

Hint: A good goal is to shoot for around 1,000 to 10,000 samples. So if you know what
duration you want to profile for, you can pick a delay via e.g. `delay = duration / 1,000`.

For example, for a 30 second profile:
30 / 1_000 = 0.03
for `duration=30&delay=0.03`

But note that if your delay gets too small, you can really slow down the program you're
profiling, and thus end up with an inaccurate profile.

If you wanted to get an extremely large number of samples (or you have very very deep stack
traces), you'll have to start worrying about filling up your julia profiling buffer,
controlled by `n=`. If you assume an average stack depth of 100, and you were aiming to get
50,000 samples, you'd need a buffer at least 50,000 * 100 big, or at least 5e6.

The default `n` is 1e8, which should be big enough for most profiles.
"""
allocs_profile_error_message() = """Need to provide query params:
    - duration=$(default_duration())
    - sample_rate=$(default_alloc_sample_rate())

Hint: A good goal is to shoot for around 1,000 to 10,000 samples. So if you know what
duration you want to profile for, and you *already have an expectation for how much your
program will allocate,* you can pick a sample_rate via `sample_rate = 1,000 / expected_allocations`.

For example, if you expect your program will actually perform 1 million allocations:
1_000 / 1_000_000 = 0.001
for `duration=30&sample_rate=0.001`

Note that if your sample_rate gets too large, you can really slow down the program you're
profiling, and thus end up with an inaccurate profile.

Finally, if you think your program only allocates a small amount, you can capture *all*
allocations by passing sample_rate=1.
"""

function cpu_profile_endpoint(req::HTTP.Request)
    uri = HTTP.URI(HTTP.Messages.uri(req))
    qp = HTTP.queryparams(uri)
    if isempty(qp)
        @info "TODO: interactive HTML input page"
        return HTTP.Response(400, cpu_profile_error_message())
    end

    # Run the profile
    n = convert(Int, parse(Float64, get(qp, "n", default_n())))
    delay = parse(Float64, get(qp, "delay", default_delay()))
    duration = parse(Float64, get(qp, "duration", default_duration()))
    with_pprof = parse(Bool, get(qp, "pprof", default_pprof()))

    return _do_cpu_profile(n, delay, duration, with_pprof)
end

function _do_cpu_profile(n, delay, duration, with_pprof)
    @info "Starting CPU Profiling from PerformanceProfilingHttpEndpoints with configuration:" n delay duration

    Profile.clear()

    Profile.init(n, delay)

    Profile.@profile sleep(duration)

    data = Profile.retrieve()
    if with_pprof
        prof_name = tempname()
        PProf.pprof(out=prof_name, web=false)
        prof_name = "$prof_name.pb.gz"
        return _http_response(read(prof_name),
            "cpu_profile-duration=$duration&delay=$delay&n=$n.pb.gz")
    else
        iobuf = IOBuffer()
        serialize(iobuf, data)
        return _http_response(take!(iobuf),
            "cpu_profile&duration=$duration&delay=$delay&n=$n.prof.bin")
    end
end

function _http_response(binary_data, filename)
    return HTTP.Response(200, [
        "Content-Type" => "application/octet-stream"
        "Content-Disposition" => "attachment; filename=$(repr(filename))"
    ], body = binary_data)
end

function heap_snapshot_endpoint(req::HTTP.Request)
    # TODO: implement this once https://github.com/JuliaLang/julia/pull/42286 is merged
end

@static if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs))

function allocations_profile_endpoint(::HTTP.Request)
    return HTTP.Response(501, "You must use a build of Julia (1.8+) and PProf that support Allocations profiling.")
end

else

function allocations_profile_endpoint(req::HTTP.Request)

    uri = HTTP.URI(HTTP.Messages.uri(req))
    qp = HTTP.queryparams(uri)
    if isempty(qp)
        @info "TODO: interactive HTML input page"
        return HTTP.Response(400, allocs_profile_error_message())
    end

    sample_rate = convert(Float64, parse(Float64, get(qp, "sample_rate", default_alloc_sample_rate())))
    duration = parse(Float64, get(qp, "duration", default_duration()))

    return _do_alloc_profile(duration, sample_rate)
end

function _do_alloc_profile(duration, sample_rate)
    @info "Starting allocation Profiling from PerformanceProfilingHttpEndpoints with configuration:" duration sample_rate

    Profile.Allocs.clear()

    Profile.Allocs.@profile sample_rate=sample_rate sleep(duration)

    prof_name = tempname()
    PProf.Allocs.pprof(out=prof_name, web=false)
    prof_name = "$prof_name.pb.gz"
    return _http_response(read(prof_name),
            "allocs_profile-duration=$duration&sample_rate=$sample_rate.pb.gz")
end

end  # if isdefined

function serve_profiling_server(;addr="127.0.0.1", port=16825)
    @info "Starting HTTP profiling server on port $port"
    HTTP.serve(addr, port) do req
        # Invoke latest for easier development with Revise.jl :)
        Base.invokelatest(_server_handler, req)
    end
end

function _server_handler(req::HTTP.Request)
    @info "DEBUG REQUEST: $(HTTP.Messages.uri(req))"

    uri = HTTP.URI(HTTP.Messages.uri(req))
    segments = HTTP.URIs.splitpath(uri)
    @assert length(segments) >= 1
    path = segments[1]

    if path == "profile"
        return cpu_profile_endpoint(req)
    elseif path == "allocs_profile"
        return allocations_profile_endpoint(req)
    end

    @info "Unsupported Path: $path"

    return HTTP.Response(404)
end

# Precompile the endpoints as much as possible, so that your /profile attempt doesn't end
# up profiling compilation!
function __init__()
    precompile(serve_profiling_server, ()) || error("precompilation of package functions is not supposed to fail")
    precompile(cpu_profile_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
    precompile(_do_cpu_profile, (Int,Float64,Float64,Bool)) || error("precompilation of package functions is not supposed to fail")
    precompile(allocations_profile_endpoint, (HTTP.Request,)) || error("precompilation of package functions is not supposed to fail")
    if isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)
        precompile(_do_alloc_profile, (Float64,Float64,)) || error("precompilation of package functions is not supposed to fail")
    end
end

end # module PerformanceProfilingHttpEndpoints