# Krylov phiv with internal time-stepping
"""
    exp_timestep(ts,A,b[;adaptive,tol,kwargs...]) -> U

Evaluates the matrix exponentiation-vector product using time stepping

```math
u = \\exp(tA)b
```

`ts` is an array of time snapshots for u, with `U[:,j] ≈ u(ts[j])`. `ts` can
also be just one value, in which case only the end result is returned and `U`
is a vector.

The time stepping formula of Niesen & Wright is used [^1]. If the time step
`tau` is not specified, it is chosen according to (17) of Neisen & Wright. If
`adaptive==true`, the time step and Krylov subsapce size adaptation scheme of
Niesen & Wright is used, the relative tolerance of which can be set using the
keyword parameter `tol`. The delta and gamma parameter of the adaptation
scheme can also be adjusted.

Set `verbose=true` to print out the internal steps (for debugging). For the
other keyword arguments, consult `arnoldi` and `phiv`, which are used
internally.

Note that this function is just a special case of `phiv_timestep` with a more
intuitive interface (vector `b` instead of a n-by-1 matrix `B`).

[^1]: Niesen, J., & Wright, W. (2009). A Krylov subspace algorithm for
evaluating the φ-functions in exponential integrators. arXiv preprint
arXiv:0907.4631.
"""
function expv_timestep(ts::Vector{tType}, A, b; kwargs...) where {tType <: Real}
    U = Matrix{eltype(b)}(undef, size(A, 1), length(ts))
    expv_timestep!(U, ts, A, b; kwargs...)
end
function expv_timestep(t::tType, A, b; kwargs...) where {tType <: Real}
    u = Vector{eltype(b)}(undef, size(A, 1))
    expv_timestep!(u, t, A, b; kwargs...)
end
"""
    expv_timestep!(u,t,A,b[;kwargs]) -> u

Non-allocating version of `expv_timestep`.
"""
function expv_timestep!(u::AbstractVector{T}, t::tType, A, b::AbstractVector{T};
                        kwargs...) where {T <: Number, tType <: Real}
    expv_timestep!(reshape(u, length(u), 1), [t], A, b; kwargs...)
    return u
end
function expv_timestep!(U::AbstractMatrix{T}, ts::Vector{tType}, A, b::AbstractVector{T};
                        kwargs...) where {T <: Number, tType <: Real}
    B = reshape(b, length(b), 1)
    phiv_timestep!(U, ts, A, B; kwargs...)
end
"""
    phiv_timestep(ts,A,B[;adaptive,tol,kwargs...]) -> U

Evaluates the linear combination of phi-vector products using time stepping

```math
u = \\varphi_0(tA)b_0 + t\\varphi_1(tA)b_1 + \\cdots + t^p\\varphi_p(tA)b_p
```

`ts` is an array of time snapshots for u, with `U[:,j] ≈ u(ts[j])`. `ts` can
also be just one value, in which case only the end result is returned and `U`
is a vector.

The time stepping formula of Niesen & Wright is used [^1]. If the time step
`tau` is not specified, it is chosen according to (17) of Neisen & Wright. If
`adaptive==true`, the time step and Krylov subsapce size adaptation scheme of
Niesen & Wright is used, the relative tolerance of which can be set using the
keyword parameter `tol`. The delta and gamma parameter of the adaptation
scheme can also be adjusted.

Set `verbose=true` to print out the internal steps (for debugging). For the
other keyword arguments, consult `arnoldi` and `phiv`, which are used
internally.

[^1]: Niesen, J., & Wright, W. (2009). A Krylov subspace algorithm for
evaluating the φ-functions in exponential integrators. arXiv preprint
arXiv:0907.4631.
"""
function phiv_timestep(ts::Vector{tType}, A, B; kwargs...) where {tType <: Real}
    U = Matrix{eltype(B)}(undef, size(A, 1), length(ts))
    phiv_timestep!(U, ts, A, B; kwargs...)
end
function phiv_timestep(t::tType, A, B; kwargs...) where {tType <: Real}
    u = Vector{eltype(B)}(undef, size(A, 1))
    phiv_timestep!(u, t, A, B; kwargs...)
end
"""
    phiv_timestep!(U,ts,A,B[;kwargs]) -> U

Non-allocating version of `phiv_timestep`.
"""
function phiv_timestep!(u::AbstractVector{T}, t::tType, A, B::AbstractMatrix{T};
                        kwargs...) where {T <: Number, tType <: Real}
    phiv_timestep!(reshape(u, length(u), 1), [t], A, B; kwargs...)
    return u
end
function phiv_timestep!(U::AbstractMatrix{T}, ts::Vector{tType}, A, B::AbstractMatrix{T}; tau::Real=0.0,
                        m::Int=min(10, size(A, 1)), tol::Real=1e-7, opnorm=LinearAlgebra.opnorm, iop::Int=0,
                        correct::Bool=false, caches=nothing, adaptive=false, delta::Real=1.2,
                        gamma::Real=0.8, NA::Int=0, verbose=false) where {T <: Number, tType <: Real}
    # Choose initial timestep
    abstol = tol * opnorm(A, Inf)
    verbose && println("Absolute tolerance: $abstol")
    if iszero(tau)
        Anorm = opnorm(A, Inf)
        b0norm = norm(@view(B[:, 1]), Inf)
        tau = 10/Anorm * (abstol * ((m+1)/ℯ)^(m+1) * sqrt(2*pi*(m+1)) /
                          (4*Anorm*b0norm))^(1/m)
        verbose && println("Initial time step unspecified, chosen to be $tau")
    end
    # Initialization
    n = size(U, 1)
    sort!(ts); tend = ts[end]
    p = size(B, 2) - 1
    @assert length(ts) == size(U, 2) "Dimension mismatch"
    @assert n == size(A, 1) == size(A, 2) == size(B, 1) "Dimension mismatch"
    if caches == nothing
        u = Vector{T}(undef, n)              # stores the current state
        W = Matrix{T}(undef, n, p+1)         # stores the w vectors
        P = Matrix{T}(undef, n, p+2)         # stores output from phiv!
        Ks = KrylovSubspace{T}(n, m)  # stores output from arnoldi!
        phiv_cache = nothing         # cache used by phiv!
    else
        u, W, P, Ks, phiv_cache = caches
        @assert length(u) == n && size(W, 1) == n && size(P, 1) == n "Dimension mismatch"
        # W and P may be bigger than actually needed
        W = @view(W[:, 1:p+1])
        P = @view(P[:, 1:p+2])
    end
    copyto!(u, @view(B[:, 1])) # u(0) = b0
    coeffs = ones(tType, p);
    if adaptive # initialization step for the adaptive scheme
        if ishermitian(A)
            iop = 2 # does not have an effect on arnoldi!, just for flops estimation
        end
        if iszero(NA)
            _A = convert(AbstractMatrix, A)
            if isa(_A, SparseMatrixCSC)
                NA = nnz(_A)
            else
                NA = count(!iszero, _A) # not constant operation, should be best avoided
            end
        end
    end

    t = 0.0       # current time
    snapshot = 1  # which snapshot to compute next
    while t < tend # time stepping loop
        if t + tau > tend # last step
            tau = tend - t
        end
        # Part 1: compute w0...wp using the recurrence relation (16)
        copyto!(@view(W[:, 1]), u) # w0 = u(t)
        @inbounds for l = 1:p-1 # compute cl = t^l/l!
            coeffs[l+1] = coeffs[l] * t / l
        end
        @views @inbounds for j = 1:p
            mul!(W[:, j+1], A, W[:, j])
            for l = 0:p-j
                axpy!(coeffs[l+1], B[:, j+l+1], W[:, j+1])
            end
        end
        # Part 2: compute ϕp(tau*A)wp using Krylov, possibly with adaptation
        arnoldi!(Ks, A, @view(W[:, end]); tol=tol, m=m, opnorm=opnorm, iop=iop)
        _, epsilon = phiv!(P, tau, Ks, p + 1; cache=phiv_cache, correct=correct, errest=true)
        verbose && println("t = $t, m = $m, tau = $tau, error estimate = $epsilon")
        if adaptive
            omega = (tend / tau) * (epsilon / abstol)
            epsilon_old = epsilon; m_old = m; tau_old = tau
            q = m/4; kappa = 2.0; maxtau = tend - t
            while omega > delta # inner loop of Algorithm 3
                m_new, tau_new, q, kappa = _phiv_timestep_adapt(
                    m, tau, epsilon, m_old, tau_old, epsilon_old, q, kappa,
                    gamma, omega, maxtau, n, p, NA, iop, opnorm(getH(Ks), 1), verbose)
                m, m_old = m_new, m
                tau, tau_old = tau_new, tau
                # Compute ϕp(tau*A)wp using the new parameters
                arnoldi!(Ks, A, @view(W[:, end]); tol=tol, m=m, opnorm=opnorm, iop=iop)
                _, epsilon_new = phiv!(P, tau, Ks, p + 1; cache=phiv_cache, correct=correct, errest=true)
                epsilon, epsilon_old = epsilon_new, epsilon
                omega = (tend / tau) * (epsilon / abstol)
                verbose && println("  * m = $m, tau = $tau, error estimate = $epsilon")
            end
        end
        # Part 3: update u using (15)
        lmul!(tau^p, copyto!(u, @view(P[:, end - 1])))
        @inbounds for l = 1:p-1 # compute cl = tau^l/l!
            coeffs[l+1] = coeffs[l] * tau / l
        end
        @views @inbounds for j = 0:p-1
            axpy!(coeffs[j+1], W[:, j+1], u)
        end
        # Fill out all snapshots in between the current step
        while snapshot <= length(ts) && t + tau >= ts[snapshot]
            tau_snapshot = ts[snapshot] - t
            u_snapshot = @view(U[:, snapshot])
            phiv!(P, tau_snapshot, Ks, p + 1; cache=phiv_cache, correct=correct)
            lmul!(tau_snapshot^p, copyto!(u_snapshot, @view(P[:, end - 1])))
            @inbounds for l = 1:p-1 # compute cl = tau^l/l!
                coeffs[l+1] = coeffs[l] * tau_snapshot / l
            end
            @views @inbounds for j = 0:p-1
                axpy!(coeffs[j+1], W[:, j+1], u_snapshot)
            end
            snapshot += 1
        end

        t += tau
    end

return U
end
# Helper functions for phiv_timestep!
function _phiv_timestep_adapt(m, tau, epsilon, m_old, tau_old, epsilon_old, q, kappa,
                              gamma, omega, maxtau, n, p, NA, iop, Hnorm, verbose)
    # Compute new m and tau (Algorithm 4)
    if tau_old > tau
        q = log(tau/tau_old) / log(epsilon/epsilon_old) - 1
    end # else keep q the same
    tau_new = tau * (gamma / omega)^(1/(q + 1))
    tau_new = min(max(tau_new, tau/5), 2*tau, maxtau)
    if m_old < m
        kappa = (epsilon/epsilon_old)^(1/(m_old - m))
    end # else keep kappa the same
    m_new = m + ceil(Int, log(omega / gamma) / log(kappa))
    m_new = min(max(m_new, div(3*m, 4), 1), Int(ceil(4*m / 3)))
    verbose && println("  - Proposed new m: $m_new, new tau: $tau_new")
    # Compare costs of using new m vs new tau (23)
    cost_tau = _phiv_timestep_estimate_flops(m, tau_new, n, p, NA, iop, Hnorm, maxtau)
    cost_m = _phiv_timestep_estimate_flops(m_new, tau, n, p, NA, iop, Hnorm, maxtau)
    verbose && println("  - Cost to use new m: $cost_m flops, new tau: $cost_tau flops")
    if cost_tau < cost_m
        m_new = m
    else
        tau_new = tau
    end
    return m_new, tau_new, q, kappa
end
function _phiv_timestep_estimate_flops(m, tau, n, p, NA, iop, Hnorm, maxtau)
    # Estimate flops for the update of W and u
    flops_W = 2 * (p - 1) * (NA + n)
    flops_u = (2 * p + 1) * n
    # Estimate flops for arnoldi!
    if iop == 0
        iop = m
    end
    flops_matvec = 2 * m * NA
    flops_vecvec = 0
    for i = 1:m
        flops_vecvec += 3 * min(i, iop)
    end
    # Estimate flops for phiv! (7)
    MH = 44/3 + 2 * ceil(max(0.0, log2(Hnorm / 5.37)))
    flops_phiv = round(Int, MH * (m + p)^3)

    flops_onestep = flops_W + flops_u + flops_matvec + flops_vecvec + flops_phiv
    return flops_onestep * Int(ceil(maxtau / tau))
end
function _phiv_timestep_caches(u_prototype, maxiter::Int, p::Int)
    n = length(u_prototype); T = eltype(u_prototype)
    u = similar(u_prototype)                      # stores the current state
    W = Matrix{T}(undef, n, p+1)                  # stores the w vectors
    P = Matrix{T}(undef, n, p+2)                  # stores output from phiv!
    Ks = KrylovSubspace{T}(n, maxiter)            # stores output from arnoldi!
    phiv_cache = PhivCache{T}(maxiter, p+1)       # cache used by phiv! (need +1 for error estimation)
    return u, W, P, Ks, phiv_cache
end


using Nemo, GibbsTypePriors, RCall, JLD

grid_k = 1:100

n, β, σ = 100, 1.2, 0.6

exact_Pkn = GibbsTypePriors.Pkn_NGG_arb.(grid_k, n, β, σ)
approx_Pkn = GibbsTypePriors.Pkn_NGG_approx.(grid_k, n, β, σ, n, exact_Pkn[n])

R"library(tidyverse)"
R"p2 = tibble(k = $grid_k,
         exact_Pkn = $(Float64.(exact_Pkn)),
         approx_Pkn = $(Float64.(approx_Pkn))) %>%
         #mutate(abs_rel_error = abs(approx_Pkn-exact_Pkn)/exact_Pkn) %>%
         mutate(Error = approx_Pkn-exact_Pkn) %>%
         #ggplot(aes(x = k, y = abs_rel_error)) +
         ggplot(aes(x = k, y = Error)) +
         theme_bw() +
         geom_point() +
         geom_line()"

R"p1 = tibble(k = $grid_k,
        exact_Pkn = $(Float64.(exact_Pkn)),
        approx_Pkn = $(Float64.(approx_Pkn))) %>%
        gather(type, value, -k) %>%
        mutate(type = gsub('_Pkn', '', type)) %>%
        ggplot(aes(x = k, y = value, colour = type, group = type)) +
        theme_bw() +
        geom_point() +
        geom_line() +
        theme(legend.position = 'left') +
        scale_colour_discrete(name='') +
        ylab('Pkn')"

R"p = gridExtra::grid.arrange(p1, p2, nrow = 1)"

R"pdf('test/graphical_tests/figures_graphical_tests/Pkn_NGG_approx_quality.pdf', height = 5, width = 7)
        plot(p)
dev.off()"


### A Pluto.jl notebook ###
# v0.16.1

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 37146eed-cabd-4024-9e91-4b8fe15139a3
# Setup
using FFTW, Statistics, LinearAlgebra, PlutoUI, Plots, Random

# ╔═╡ ad88469d-06d5-4ec7-868f-611e999930b4
md"""
## Nonparametric Bayes and Fourier methods in Julia
##### Moritz Schauer `@mschauer`

This notebook takes a tour through non-parametric Bayesian regression in Fourier domain using Julia.

### Content:

* A non-parametric regression problem

* Changing perspective to Fourier domain

* Defining smoothing prior and computing the posterior

* Choosing the right smoothness

But before some notebook setup:
"""

# ╔═╡ 426085d1-1f7d-4186-a8dc-28ffd8fb5343
@bind resample Button("Resample observations!")

# ╔═╡ 0722b786-402d-4ba9-a427-8c87cbc3df24
md"Show hidden ground truth? $(@bind show_truth CheckBox())"

# ╔═╡ fb55fde1-7669-4f91-bc67-2a0172e5ae3e
md"""
## Fourier domain and prior

Taking the vector of observations ``y`` and ground truth vector ``f_i = f(x_i)``,
we can apply the discrete Fourier transformation ``\mathcal F`` to our linear equation
```math
\mathcal F y = \mathcal F (f +  \eta)
```
In Julia, ``\mathcal F`` is `fft(x)/sqrt(length(x))`, but you can think of it as a matrix
```math
\mathcal F_{kl} = \exp(2\pi\cdot i \cdot (k l/n)).
```

We obtain the equation in frequency domain with ``\tilde y = \mathcal F y``, ``\tilde f = \mathcal F f`` etc.
```math
\tilde y_i = \tilde f_i + \tilde\eta_i
```

``\mathcal F`` is an ``L_2`` isometry and maps independend and normal noise  on the observations to equivalent independent and normal noise in frequency domain,
so here
```math
\tilde\eta_i \sim N(0, \varsigma^2)
```

"""

# ╔═╡ 093506f2-5a52-4217-9dfb-d673ce8b4a29
# Fourier matrix
# ℱ(x) = [exp(2pi*im*i*j/n) for i in fftshift(-n÷2:n÷2-1), j in fftshift(-n÷2:n÷2-1)]*x
ℱ(x) = (fft(x)/sqrt(length(x)))

# ╔═╡ 7f5a4d79-1e64-4c99-a588-63bcca7267fe
md"""

We define our prior as

```math
\tilde f_i \sim N(0, \varpi_i^{-\frac{\alpha}{2}})
```
where ``\varpi_i`` is the frequency corresponding to Fourier coordinate ``\tilde f_i``. Julia's `fftfreq` helps with that.

Large values of ``\alpha`` imply higher smoothenss of prior samples.

For example:

* α = 3.0 for type integrated Brownian motion
* α = 2.0 for type Brownian motion prior
* α = 1.0 for a very rough prior (pink noise)

``\alpha`` can be translated into a Hurst parameter related to self similarity of the path under scaling of both axes
```math
H = (\alpha - 1)/2
```

Having a prior in frequency domains means we get samples from our prior for ``f`` via the inverse Fourier transform, either the matrix inverse ``{\mathcal F}^{-1}`` or `sqrt(n)*real(ifft(x))` in Julia for the real part.

"""

# ╔═╡ 012f54e1-01a3-4acd-b7db-d5d3f3bc7d87
# Define prior on unit interval in frequency domain
# by choosing power decay
function freq_decay(n, α)  #ifftshift(-n/2:n/2-1)
    fr = 1 ./ fftfreq(n)
    fr[1] = 2n # allow for intercept
    fr = abs.(fr) .^(α/2) # scale frequencies
    fr/norm(fr)*sqrt(n)
end

# ╔═╡ e7c6b81f-202a-43e0-9bc0-ac5eb64060db
md"""
Prior power decay of prior samples in frequency domain
$(@bind α1 Slider(0.5:0.5:4.0))
"""

# ╔═╡ 0e4ea7bf-4eb1-4923-a2e8-a7e2e8bc33a5
@bind resample_prior Button("Resample prior!")

# ╔═╡ 12c188f4-5042-4cc4-9de5-89ea87c08c1e
md"""
## Bayesian regression in Fourier domain

If we have prior, model and data ``\tilde y`` in Fourier domain, we can compute the posterior distribution of the Fourier coefficients
``\tilde f_i``.

This gives a Gaussian posterior distribution for the signal in Fourier domain
```math
\tilde f \sim N(\tilde \mu, \tilde \Sigma) \text{ (a posteriori) }
```
with mean ``\tilde \mu`` and covariance ``\tilde \Sigma``.

With the inverse transform we estimate of ``f \approx \mathcal F^{-1} \tilde\mu`` and quantify uncertainty of the estimate as ``\Sigma = \mathcal F^{-1} \tilde \Sigma \mathcal F^{-1}``.

The marginal uncertainty (diagonal entries of ``\Sigma`` ) can be computed very efficiently using `ifft` aswell.

"""

# ╔═╡ 517c3961-c104-4811-ab4c-7e189761709b
# Equivalent to
# Γ = ℱ*Diagonal(freq_decay(n, α).^(-2))*ℱ'/n/n # ≈ ifft(Diagonal(n. + freq_decay(n, α).^(-2)))
# σ = 1.96sqrt.(real(diag(inv((I + Γ)))))

# ╔═╡ c51271a7-0614-445f-87ec-a5bc2fee1768
@bind n Slider(10:10:1000) # number of observations

# ╔═╡ 64feb3c2-919b-4246-9fbf-e836a0c80ed0
# Design points
x = range(0, 1, length=n);

# ╔═╡ c502c779-e776-490d-9ec7-02a3379aaa28
# Ground truth
f = 3*x.*sin.(2pi*x); # periodic signal in x (time domain)
# f = 6*sqrt.(abs.(x .- 0.5)) # this one is difficult to estimate

# ╔═╡ b7c5ee08-5d2d-4775-a232-472a033b59a1
begin
	resample_prior
	z1 = sqrt(n)*real(ifft(randn(Complex{Float64},n).*abs.(freq_decay(n, α1))));
	plot(x, z1, label="prior sample")
end

# ╔═╡ a97a39ff-059e-4622-ae2e-219d8464ecb2
@bind ς Slider(0.5:0.5:2.0)  # noise level

# ╔═╡ c29066a9-574d-4ce1-ba9b-abb44aeb8fa0
md"""
## A nonparametric regression problem

We are interested in recovering the function ``f`` from ``n=`` $(n) noisy observations ``y_i`` at points ``x_i``

```math
y_i = f(x_i) + \eta_i, \qquad \eta_i \sim N(0,\varsigma^2),
```
where
``\varsigma =`` $ς is the noise level.

* You can choose the values for ``n`` and ``\varsigma`` with the sliders below!

"""

# ╔═╡ a4ff81fe-a9cf-4789-aa68-5838c58c1f69
# Model: Signal distorted by white noise
begin
	resample
	y = f + ς*randn(n)
end;

# ╔═╡ 873ba234-b4b1-4678-8c67-086cd3b149bc
begin
	p1 = scatter(x, y, markersize=10/sqrt(n), label="obs", title="Data")
	show_truth && plot!(p1, x, f, color=:green, label="truth")
	p1
end

# ╔═╡ 0071252a-42cb-4967-b426-7d28ce054ba1
begin
	p2 = scatter(x, real.(ℱ(y)), markersize=10/sqrt(n), color=:black, label="obs", title="Truth and observation in frequency domain")
	scatter!(p2, x, imag.(ℱ(y)), markersize=10/sqrt(n), color=:black, alpha=0.1, label="obs (imag)", title="Truth and observation in frequency domain")
	plot!(p2, x, real.(ℱ(f)), color=:green, label="truth")
	plot!(p2, x, imag.(ℱ(f)), color=:green, alpha=0.3, label="truth (im)")
	p2
end

# ╔═╡ af6df9cd-cb2f-48de-9697-db23bb7e3804
# Prior power decay of prior samples in frequency domain
@bind α Slider(0.5:0.5:2.0)

# ╔═╡ 5a475143-5078-4033-84cc-ec6abd3c86c4
md"Chosen prior smoothness α = $(α1), H = $(H = (α - 1)/2)"

# ╔═╡ 9c48ca1f-7ab3-4b72-a9bd-4b0d6c1ee8db
# posterior precision in frequency domain
γ = ς^(-2) .+ (freq_decay(n, α)).^(-2);

# ╔═╡ fb008be0-b823-4eaf-b2c7-eb189041f0c8
# posterior mean
μ̂ = real(ifft(γ.\(fft(ς^(-2)*y))))

# ╔═╡ e0e75144-aa8f-433e-90c5-bfa9bbdfac17
# We can even compute the posterior covariance

# Use circulant property of covariance in time domain, could be done with https://github.com/JuliaMatrices/ToeplitzMatrices.jl
σ = 1.96*sqrt.(abs.((ifft(diag(inv(Diagonal(γ)))))))[1]

# ╔═╡ 8ba316ed-2c77-4ddf-9e57-e673b4ef00ec
@bind resample_post Button("New posterior sample!")

# ╔═╡ 23bfdb98-1001-4f49-a841-292ebe4c0246
# posterior sample
begin
	resample_post
    m = μ̂ + sqrt(n)*real(ifft(sqrt.(γ).\randn(Complex{Float64},n)))
end;

# ╔═╡ 6615b4b2-26fc-4a80-9576-810c7f919270
md"Show truth? $(@bind show_truth2 CheckBox())"

# ╔═╡ 608fe8ea-227b-4a87-86fe-78eee54871b4
(;
	n,
	ς,
	α
)

# ╔═╡ c78bd411-c9f4-4e1f-8c96-ea97480d9cd9
begin
	global p = scatter(x, y, markersize=10/sqrt(n), label="obs")
	plot!(x, μ̂, color=:blue, ribbon=σ, fillalpha=0.2, label="posterior mean and band")
	show_truth2 && plot!(x, f, color=:green, alpha=0.8, label="truth")
    resample_post
	plot!(x, m, label="posterior sample")
end

# ╔═╡ 09af6b8a-73a1-4e87-8a89-ef7ac72c72b4
md"""

## What is the right choice of ``\alpha``

That will depend on the smoothness of the truth we want to estimate. If we know that apriori ``f`` is very smooth, we prefer larger values of ``\alpha``

Dutch Bayesians (and I should count myself as one) like to think about how to choose ``\alpha`` in an optimal way.

One way to reason about this, is to investigate the frequentist/asymptotic properties of Bayesian procedures (insert Galaxy brain meme).

For the example one finds that the posterior puts most of it's mass near the true function ``f`` with increasing probability if ``n`` becomes large.

This happens at a rate depending on ``\alpha`` as long as we match ``\alpha`` to the smoothness of the underlying truth.

```math
e = \varsigma n^{-H/(1 + 2H)}
```

Let's compute our ``L_2`` estimation error and compare with theoretical size

* `norm(μ̂ - f)/sqrt(n)` = $(norm(μ̂ - f)/sqrt(n))
* `e` =  $(ς*n^(-H/(1 + 2H)))

Harry van Zanten's summer school slides are a nice starting point:

* https://warwick.ac.uk/fac/sci/statistics/crism/workshops/masterclassapril/
* https://fdnss.files.wordpress.com/2016/03/vanzanten-slides.pdf

"""

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[compat]
FFTW = "~1.4.5"
Plots = "~1.22.3"
PlutoUI = "~0.7.14"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[AbstractFFTs]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0"
uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
version = "1.0.1"

[[Adapt]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
version = "3.3.1"

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[Bzip2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2"
uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
version = "1.0.8+0"

[[Cairo_jll]]
deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
git-tree-sha1 = "f2202b55d816427cd385a9a4f3ffb226bee80f99"
uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
version = "1.16.1+0"

[[ColorSchemes]]
deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"]
git-tree-sha1 = "a851fec56cb73cfdf43762999ec72eff5b86882a"
uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
version = "3.15.0"

[[ColorTypes]]
deps = ["FixedPointNumbers", "Random"]
git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597"
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
version = "0.11.0"

[[Colors]]
deps = ["ColorTypes", "FixedPointNumbers", "Reexport"]
git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40"
uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
version = "0.12.8"

[[Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "31d0151f5716b655421d9d75b7fa74cc4e744df2"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.39.0"

[[CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"

[[Contour]]
deps = ["StaticArrays"]
git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7"
uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
version = "0.5.7"

[[DataAPI]]
git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.9.0"

[[DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.10"

[[DataValueInterfaces]]
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
version = "1.0.0"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[EarCut_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "3f3a2501fa7236e9b911e0f7a588c657e822bb6d"
uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5"
version = "2.2.3+0"

[[Expat_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "b3bfd02e98aedfa5cf885665493c5598c350cd2f"
uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
version = "2.2.10+0"

[[FFMPEG]]
deps = ["FFMPEG_jll"]
git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8"
uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a"
version = "0.4.1"

[[FFMPEG_jll]]
deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
git-tree-sha1 = "d8a578692e3077ac998b50c0217dfd67f21d1e5f"
uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
version = "4.4.0+0"

[[FFTW]]
deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"]
git-tree-sha1 = "463cb335fa22c4ebacfd1faba5fde14edb80d96c"
uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
version = "1.4.5"

[[FFTW_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "c6033cc3892d0ef5bb9cd29b7f2f0331ea5184ea"
uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a"
version = "3.3.10+0"

[[FixedPointNumbers]]
deps = ["Statistics"]
git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
version = "0.8.4"

[[Fontconfig_jll]]
deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03"
uuid = "a3f928ae-7b40-5064-980b-68af3947d34b"
version = "2.13.93+0"

[[Formatting]]
deps = ["Printf"]
git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
version = "0.4.2"

[[FreeType2_jll]]
deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
git-tree-sha1 = "87eb71354d8ec1a96d4a7636bd57a7347dde3ef9"
uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7"
version = "2.10.4+0"

[[FriBidi_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91"
uuid = "559328eb-81f9-559d-9380-de523a88c83c"
version = "1.0.10+0"

[[GLFW_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"]
git-tree-sha1 = "dba1e8614e98949abfa60480b13653813d8f0157"
uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89"
version = "3.3.5+0"

[[GR]]
deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"]
git-tree-sha1 = "c2178cfbc0a5a552e16d097fae508f2024de61a3"
uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
version = "0.59.0"

[[GR_jll]]
deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
git-tree-sha1 = "ef49a187604f865f4708c90e3f431890724e9012"
uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
version = "0.59.0+0"

[[GeometryBasics]]
deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"]
git-tree-sha1 = "58bcdf5ebc057b085e58d95c138725628dd7453c"
uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
version = "0.4.1"

[[Gettext_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"]
git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046"
uuid = "78b55507-aeef-58d4-861c-77aaff3498b1"
version = "0.21.0+0"

[[Glib_jll]]
deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "7bf67e9a481712b3dbe9cb3dac852dc4b1162e02"
uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
version = "2.68.3+0"

[[Graphite2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011"
uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472"
version = "1.3.14+0"

[[Grisu]]
git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2"
uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe"
version = "1.0.2"

[[HTTP]]
deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
git-tree-sha1 = "14eece7a3308b4d8be910e265c724a6ba51a9798"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "0.9.16"

[[HarfBuzz_jll]]
deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"]
git-tree-sha1 = "8a954fed8ac097d5be04921d595f741115c1b2ad"
uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566"
version = "2.8.1+0"

[[HypertextLiteral]]
git-tree-sha1 = "72053798e1be56026b81d4e2682dbe58922e5ec9"
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
version = "0.9.0"

[[IOCapture]]
deps = ["Logging", "Random"]
git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.2.2"

[[IniFile]]
deps = ["Test"]
git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
version = "0.5.0"

[[IntelOpenMP_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "d979e54b71da82f3a65b62553da4fc3d18c9004c"
uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0"
version = "2018.0.3+2"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[IterTools]]
git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18"
uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
version = "1.3.0"

[[IteratorInterfaceExtensions]]
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"

[[JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.3.0"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.2"

[[JpegTurbo_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "d735490ac75c5cb9f1b00d8b5509c11984dc6943"
uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8"
version = "2.1.0+0"

[[LAME_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c"
uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d"
version = "3.100.1+0"

[[LZO_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6"
uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
version = "2.10.1+0"

[[LaTeXStrings]]
git-tree-sha1 = "c7f1c695e06c01b95a67f0cd1d34994f3e7db104"
uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
version = "1.2.1"

[[Latexify]]
deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"]
git-tree-sha1 = "a4b12a1bd2ebade87891ab7e36fdbce582301a92"
uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
version = "0.15.6"

[[LazyArtifacts]]
deps = ["Artifacts", "Pkg"]
uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"

[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[Libffi_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "761a393aeccd6aa92ec3515e428c26bf99575b3b"
uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490"
version = "3.2.2+0"

[[Libgcrypt_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"]
git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae"
uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4"
version = "1.8.7+0"

[[Libglvnd_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"]
git-tree-sha1 = "7739f837d6447403596a75d19ed01fd08d6f56bf"
uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29"
version = "1.3.0+3"

[[Libgpg_error_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9"
uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8"
version = "1.42.0+0"

[[Libiconv_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778"
uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
version = "1.16.1+1"

[[Libmount_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73"
uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9"
version = "2.35.0+0"

[[Libtiff_jll]]
deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"]
git-tree-sha1 = "340e257aada13f95f98ee352d316c3bed37c8ab9"
uuid = "89763e89-9b03-5906-acba-b20f662cd828"
version = "4.3.0+0"

[[Libuuid_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066"
uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
version = "2.36.0+0"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MKL_jll]]
deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"]
git-tree-sha1 = "5455aef09b40e5020e1520f551fa3135040d4ed0"
uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
version = "2021.1.1+2"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "5a5bc6bf062f0f95e62d0fe0a2d99699fed82dd9"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.8"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MbedTLS]]
deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe"
uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.0.3"

[[MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[Measures]]
git-tree-sha1 = "e498ddeee6f9fdb4551ce855a46f54dbd900245f"
uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e"
version = "0.3.1"

[[Missings]]
deps = ["DataAPI"]
git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f"
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
version = "1.0.2"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[NaNMath]]
git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "0.3.5"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[Ogg_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f"
uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
version = "1.3.5+0"

[[OpenSSL_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a"
uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
version = "1.1.10+0"

[[Opus_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720"
uuid = "91d4177d-7536-5919-b921-800302f37372"
version = "1.3.2+0"

[[OrderedCollections]]
git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.4.1"

[[PCRE_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "b2a7af664e098055a7529ad1a900ded962bca488"
uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc"
version = "8.44.0+0"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "a8709b968a1ea6abc2dc1967cb1db6ac9a00dfb6"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.0.5"

[[Pixman_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "b4f5d02549a10e20780a24fce72bea96b6329e29"
uuid = "30392449-352a-5448-841d-b1acce4e97dc"
version = "0.40.1+0"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[PlotThemes]]
deps = ["PlotUtils", "Requires", "Statistics"]
git-tree-sha1 = "a3a964ce9dc7898193536002a6dd892b1b5a6f1d"
uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a"
version = "2.0.1"

[[PlotUtils]]
deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"]
git-tree-sha1 = "2537ed3c0ed5e03896927187f5f2ee6a4ab342db"
uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043"
version = "1.0.14"

[[Plots]]
deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs"]
git-tree-sha1 = "cfbd033def161db9494f86c5d18fbf874e09e514"
uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
version = "1.22.3"

[[PlutoUI]]
deps = ["Base64", "Dates", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"]
git-tree-sha1 = "d1fb76655a95bf6ea4348d7197b22e889a4375f4"
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
version = "0.7.14"

[[Preferences]]
deps = ["TOML"]
git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.2"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[Qt5Base_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"]
git-tree-sha1 = "ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8"
uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1"
version = "5.15.3+0"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[RecipesBase]]
git-tree-sha1 = "44a75aa7a527910ee3d1751d1f0e4148698add9e"
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
version = "1.1.2"

[[RecipesPipeline]]
deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"]
git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8"
uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c"
version = "0.4.1"

[[Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.1.3"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Scratch]]
deps = ["Dates"]
git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda"
uuid = "6c6a2e73-6563-6170-7368-637461726353"
version = "1.1.0"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[SharedArrays]]
deps = ["Distributed", "Mmap", "Random", "Serialization"]
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"

[[Showoff]]
deps = ["Dates", "Grisu"]
git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de"
uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f"
version = "1.0.3"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SortingAlgorithms]]
deps = ["DataStructures"]
git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508"
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
version = "1.0.1"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[StaticArrays]]
deps = ["LinearAlgebra", "Random", "Statistics"]
git-tree-sha1 = "3c76dde64d03699e074ac02eb2e8ba8254d428da"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "1.2.13"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[StatsAPI]]
git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510"
uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
version = "1.0.0"

[[StatsBase]]
deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
git-tree-sha1 = "8cbbc098554648c84f79a463c9ff0fd277144b6c"
uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
version = "0.33.10"

[[StructArrays]]
deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"]
git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3"
uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
version = "0.6.3"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[TableTraits]]
deps = ["IteratorInterfaceExtensions"]
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
version = "1.0.1"

[[Tables]]
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
git-tree-sha1 = "1162ce4a6c4b7e31e0e6b14486a6986951c73be9"
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
version = "1.5.2"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[URIs]]
git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
version = "1.3.0"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[Wayland_jll]]
deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23"
uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89"
version = "1.19.0+0"

[[Wayland_protocols_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll"]
git-tree-sha1 = "2839f1c1296940218e35df0bbb220f2a79686670"
uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91"
version = "1.18.0+4"

[[XML2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a"
uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
version = "2.9.12+0"

[[XSLT_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"]
git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a"
uuid = "aed1982a-8fda-507f-9586-7b0439959a61"
version = "1.1.34+0"

[[Xorg_libX11_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"]
git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527"
uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc"
version = "1.6.9+4"

[[Xorg_libXau_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e"
uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec"
version = "1.0.9+4"

[[Xorg_libXcursor_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"]
git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd"
uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724"
version = "1.2.0+4"

[[Xorg_libXdmcp_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4"
uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05"
version = "1.1.3+4"

[[Xorg_libXext_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3"
uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3"
version = "1.3.4+4"

[[Xorg_libXfixes_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4"
uuid = "d091e8ba-531a-589c-9de9-94069b037ed8"
version = "5.0.3+4"

[[Xorg_libXi_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"]
git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246"
uuid = "a51aa0fd-4e3c-5386-b890-e753decda492"
version = "1.7.10+4"

[[Xorg_libXinerama_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"]
git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123"
uuid = "d1454406-59df-5ea1-beac-c340f2130bc3"
version = "1.1.4+4"

[[Xorg_libXrandr_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"]
git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631"
uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484"
version = "1.5.2+4"

[[Xorg_libXrender_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96"
uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa"
version = "0.9.10+4"

[[Xorg_libpthread_stubs_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb"
uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74"
version = "0.1.0+3"

[[Xorg_libxcb_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"]
git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6"
uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b"
version = "1.13.0+3"

[[Xorg_libxkbfile_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "926af861744212db0eb001d9e40b5d16292080b2"
uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a"
version = "1.1.0+4"

[[Xorg_xcb_util_image_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97"
uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b"
version = "0.4.0+1"

[[Xorg_xcb_util_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"]
git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1"
uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5"
version = "0.4.0+1"

[[Xorg_xcb_util_keysyms_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00"
uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7"
version = "0.4.0+1"

[[Xorg_xcb_util_renderutil_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e"
uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e"
version = "0.3.9+1"

[[Xorg_xcb_util_wm_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67"
uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361"
version = "0.4.1+1"

[[Xorg_xkbcomp_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxkbfile_jll"]
git-tree-sha1 = "4bcbf660f6c2e714f87e960a171b119d06ee163b"
uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4"
version = "1.4.2+4"

[[Xorg_xkeyboard_config_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xkbcomp_jll"]
git-tree-sha1 = "5c8424f8a67c3f2209646d4425f3d415fee5931d"
uuid = "33bec58e-1273-512f-9401-5d533626f822"
version = "2.27.0+4"

[[Xorg_xtrans_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845"
uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10"
version = "1.4.0+3"

[[Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[Zstd_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6"
uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
version = "1.5.0+0"

[[libass_jll]]
deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47"
uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
version = "0.15.1+0"

[[libfdk_aac_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55"
uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280"
version = "2.0.2+0"

[[libpng_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c"
uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f"
version = "1.6.38+0"

[[libvorbis_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"]
git-tree-sha1 = "c45f4e40e7aafe9d086379e5578947ec8b95a8fb"
uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a"
version = "1.3.7+0"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"

[[x264_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2"
uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a"
version = "2021.5.5+0"

[[x265_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9"
uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76"
version = "3.5.0+0"

[[xkbcommon_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"]
git-tree-sha1 = "ece2350174195bb31de1a63bea3a41ae1aa593b6"
uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd"
version = "0.9.1+5"
"""

# ╔═╡ Cell order:
# ╟─ad88469d-06d5-4ec7-868f-611e999930b4
# ╠═37146eed-cabd-4024-9e91-4b8fe15139a3
# ╟─c29066a9-574d-4ce1-ba9b-abb44aeb8fa0
# ╠═64feb3c2-919b-4246-9fbf-e836a0c80ed0
# ╠═c502c779-e776-490d-9ec7-02a3379aaa28
# ╠═a4ff81fe-a9cf-4789-aa68-5838c58c1f69
# ╠═426085d1-1f7d-4186-a8dc-28ffd8fb5343
# ╠═0722b786-402d-4ba9-a427-8c87cbc3df24
# ╠═873ba234-b4b1-4678-8c67-086cd3b149bc
# ╟─fb55fde1-7669-4f91-bc67-2a0172e5ae3e
# ╠═093506f2-5a52-4217-9dfb-d673ce8b4a29
# ╠═0071252a-42cb-4967-b426-7d28ce054ba1
# ╟─7f5a4d79-1e64-4c99-a588-63bcca7267fe
# ╠═012f54e1-01a3-4acd-b7db-d5d3f3bc7d87
# ╠═e7c6b81f-202a-43e0-9bc0-ac5eb64060db
# ╠═5a475143-5078-4033-84cc-ec6abd3c86c4
# ╠═0e4ea7bf-4eb1-4923-a2e8-a7e2e8bc33a5
# ╠═b7c5ee08-5d2d-4775-a232-472a033b59a1
# ╟─12c188f4-5042-4cc4-9de5-89ea87c08c1e
# ╠═9c48ca1f-7ab3-4b72-a9bd-4b0d6c1ee8db
# ╠═fb008be0-b823-4eaf-b2c7-eb189041f0c8
# ╠═23bfdb98-1001-4f49-a841-292ebe4c0246
# ╠═e0e75144-aa8f-433e-90c5-bfa9bbdfac17
# ╠═517c3961-c104-4811-ab4c-7e189761709b
# ╠═c51271a7-0614-445f-87ec-a5bc2fee1768
# ╠═a97a39ff-059e-4622-ae2e-219d8464ecb2
# ╠═af6df9cd-cb2f-48de-9697-db23bb7e3804
# ╠═8ba316ed-2c77-4ddf-9e57-e673b4ef00ec
# ╠═6615b4b2-26fc-4a80-9576-810c7f919270
# ╠═608fe8ea-227b-4a87-86fe-78eee54871b4
# ╠═c78bd411-c9f4-4e1f-8c96-ea97480d9cd9
# ╟─09af6b8a-73a1-4e87-8a89-ef7ac72c72b4
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


#
# conjgrad.jl -
#
# Linear conjugate-gradient.
#
#-------------------------------------------------------------------------------
#
# This file is part of LazyAlgebra (https://github.com/emmt/LazyAlgebra.jl)
# released under the MIT "Expat" license.
#
# Copyright (c) 2017-2020 Éric Thiébaut.
#

struct WrappedLeftHandSideMatrix{T}
    op::T
end
(obj::WrappedLeftHandSideMatrix)(dst, src) = apply!(dst, obj.op, src)

"""
```julia
conjgrad(A, b, x0=vzeros(b)) -> x
```

solves the symmetric linear system `A⋅x = b` starting at `x0` by means of the
iterative conjugate gradient method.  The returned solution `x` is a new object
similar to `b` and to `x0`.

Argument `A` implements the symmetric positive definite linear mapping `A`, it
can be provided as a Julia array (interpreted as a general matrix, see
[`GeneralMatrix`](@ref)), as an instance of [`LinearMapping`](@ref) or as a
callable object (like a function) which is used as:

```julia
A(dst, src)
```

to overwrite `dst` with `A⋅src`.  If `A` has been implemented as a callable
object, such that `A(x)` yields `A⋅x`, then call `conjgrad` with an
inline function:

```julia
conjgrad((dst,src) -> (dst .= A(src); return dst), b, ...)
```

See [`conjgrad!`](@ref) for accepted keywords and more details.

"""
conjgrad(A, b, x0; kwds...) =
    conjgrad!(vcreate(b), A, b, x0; kwds...)

function conjgrad(A, b; kwds...)
    x = vzeros(b)
    return conjgrad!(x, A, b, x; kwds...)
end

"""
# Linear conjugate gradient

```julia
conjgrad!(x, A, b, [x0=vfill!(x,0), p, q, r]) -> x
```

finds an approximate solution to the symmetric linear system `A⋅x = b` starting
at `x0` by means of the iterative conjugate gradient method.  The result is
stored in `x` which is returned.

Argument `A` implements the symmetric positive definite linear mapping `A`, it
can be provided as a Julia array (interpreted as a general matrix, see
[`GeneralMatrix`](@ref)), as an instance of [`LinearMapping`](@ref) or as a
callable object (like a function) which is used as:

```julia
A(dst, src)
```

to overwrite `dst` with `A⋅src`.  If `A` has been implemented as a callable
object, such that `A(x)` yields `A⋅x`, then call `conjgrad!` with an inline
function:

```julia
conjgrad!(x, (dst,src) -> (dst .= A(src); return dst), b, ...)
```

If no initial variables are specified, the default is to start with all
variables set to zero.

Optional arguments `p`, `q` and `r` are writable workspace *vectors*.  On
return, `p` is the last search direction, `q = A⋅p` and `r = b - A⋅xp` with
`xp` the previous or last solution.  If provided, these workspaces must be
distinct.  All *vectors* must have the same sizes.  If all workspace vectors
are provided, no other memory allocation is necessary (unless `A` needs
to allocate some temporaries).

Provided `A` be positive definite, the solution `x` of the equations `A⋅x = b`
is also the minimum of the quadratic function:

    f(x) = (1/2) x'⋅A⋅x - b'⋅x + ϵ

where `ϵ` is an arbitrary constant.  The variations of `f(x)` between
successive iterations, the norm of the gradient of `f(x)` or the variations of
`x` may be used to decide the convergence of the algorithm (see keywords
`ftol`, `gtol` and `xtol` below).

## Saving memory

To save memory, `x` and `x0` can be the same object.  Otherwise, if no
restarting occurs (see keyword `restart` below), `b` can also be the same as
`r` but this is not recommended.

## Keywords

There are several keywords to control the algorithm:

* Keyword `ftol` specifies the function tolerance for convergence.  The
  convergence is assumed as soon as the variation of the objective function
  `f(x)` between two successive iterations is less or equal `ftol` times the
  largest variation so far.  By default, `ftol = 1e-8`.

* Keyword `gtol` specifies the gradient tolerances for convergence, it is a
  tuple of two values `(gatol, grtol)` which are the absolute and relative
  tolerances.  Convergence occurs when the Euclidean norm of the residuals
  (which is that of the gradient of the associated objective function) is less
  or equal the largest of `gatol` and `grtol` times the Euclidean norm of the
  initial residuals.  By default, `gtol = (0.0, 0.0)`.

* Keyword `xtol` specifies the variables tolerance for convergence.  The
  convergence is assumed as soon as the Euclidean norm of the change of
  variables is less or equal `xtol` times the Euclidean norm of the variables
  `x`.  By default, `xtol = 0`.

* Keyword `maxiter` specifies the maximum number of iterations which is
  practically unlimited by default.

* Keyword `restart` may be set with the maximum number of iterations before
  restarting the algorithm.  By default, `restart` is set with the smallest of
  `50` and the number of variables.  Set `restart` to at least `maxiter` if you
  do not want that any restarts ever occur.

* Keyword `strict` can be set to a boolean value (default is `true`) to specify
  whether non-positive definite operator `A` throws a `NonPositiveDefinite`
  exception or just returns the best solution found so far (with a warning if
  `quiet` is false).

* Keyword `quiet` can be set to a boolean value (default is `false`) to specify
  whether or not to print warning messages.

See also: [`conjgrad`][@ref).

"""
conjgrad!(x, A::Union{LinearMapping,AbstractArray}, b, args...; kwds...) =
    conjgrad!(x, WrappedLeftHandSideMatrix(A), b, args...; kwds...)

function conjgrad!(x, A::Mapping, b, args...; kwds...)
    is_linear(A) || bad_argument("`A` must be a linear map")
    conjgrad!(x, WrappedLeftHandSideMatrix(A), b, args...; kwds...)
end

function conjgrad!(x, A, b, x0 = vfill!(x, 0),
                   p = vcreate(x), q = vcreate(x), r = vcreate(x);
                   ftol::Real = 1e-8,
                   gtol::NTuple{2,Real} = (0.0,0.0),
                   xtol::Real = 0.0,
                   maxiter::Integer = typemax(Int),
                   restart::Integer = min(50, length(b)),
                   verb::Bool = false,
                   io::IO = stdout,
                   quiet::Bool = false,
                   strict::Bool = true)
    # Initialization.
    0 ≤ ftol < 1 ||
        bad_argument("bad function tolerance (ftol = ", ftol, ")")
    gtol[1] ≥ 0 ||
        bad_argument("bad gradient absolute tolerance (gtol[1] = ",
                     gtol[1], ")")
    0 ≤ gtol[2] < 1 ||
        bad_argument("bad gradient relative tolerance (gtol[2] = ",
                     gtol[2], ")")
    0 ≤ xtol < 1 ||
        bad_argument("bad variables tolerance (xtol = ", xtol, ")")
    restart ≥ 1 ||
        bad_argument("bad number of iterations for restarting (restart = ",
                     restart,")")
    vcopy!(x, x0)
    if maxiter < 1 && quiet && !verb
        return x
    end
    if vnorm2(x) > 0 # cheap trick to check whether x is non-zero
        # Compute r = b - A⋅x.
        A(r, x)
        vcombine!(r, 1, b, -1, r)
    else
        # Save applying A since x = 0.
        vcopy!(r, b)
    end
    local rho    :: Float64 = vdot(r, r)
    local ftest  :: Float64 = ftol
    local gtest  :: Float64 = max(gtol[1], gtol[2]*sqrt(rho))
    local xtest  :: Float64 = xtol
    local psimax :: Float64 = 0
    local psi    :: Float64 = 0
    local oldrho :: Float64
    local gamma  :: Float64

    # Conjugate gradient iterations.
    k = 0
    while true
        if verb
            if k == 0
                @printf(io, "# %s\n# %s\n",
                        "Iter.    Δf(x)       ||∇f(x)||",
                        "-------------------------------")
            end
            @printf(io, "%6d %12.4e %12.4e\n", k, psi, sqrt(rho))
        end
        k += 1
        if sqrt(rho) ≤ gtest
            # Normal convergence.
            if verb
                @printf(io, "# %s\n", "Convergence (gtest statisfied).")
            end
            break
        elseif k > maxiter
            verb && @printf(io, "# %s\n", "Too many iteration(s).")
            quiet || warn("too many (", k, " conjugate gradient iteration(s)")
            break
        end
        if rem(k, restart) == 1
            # Restart or first iteration.
            if k > 1
                # Restart.
                A(r, x)
                vcombine!(r, 1, b, -1, r)
            end
            vcopy!(p, r)
        else
            beta = rho/oldrho
            vcombine!(p, beta, p, +1, r)
        end

        # Compute optimal step size.
        A(q, p)
        gamma = vdot(p, q)
        if gamma ≤ 0
            verb && @printf(io, "# %s\n", "Operator is not positive definite.")
            strict && throw(NonPositiveDefinite("in conjugate gradient"))
            quiet || warn("operator is not positive definite")
            break
        end
        alpha = rho/gamma

        # Update variables and check for convergence.
        vupdate!(x, +alpha, p)
        psi = alpha*rho/2      # psi = f(x_{k}) - f(x_{k+1})
        psimax = max(psi, psimax)
        if psi ≤ ftest*psimax
            # Normal convergence.
            verb && @printf(io, "# %s\n", "Convergence (ftest statisfied).")
            break
        end
        if xtest > 0 && alpha*vnorm2(p) ≤ xtest*vnorm2(x)
            # Normal convergence.
            verb && @printf(io, "# %s\n", "Convergence (xtest statisfied).")
            break
        end

        # Update residuals and related quantities.
        vupdate!(r, -alpha, q)
        oldrho = rho
        rho = vdot(r, r)
    end
    return x
end


# SimLynx.jl Event-Based Discrete-Event Simulation Example

using SimLynx
SimLynx.greet()

using Distributions: Exponential, Uniform
using Random

const N_TELLERS = 2
const MEAN_INTERARRIVAL_TIME = 4.0
const MIN_SERVICE_TIME = 2.0
const MAX_SERVICE_TIME = 10.0

struct Customer
    id::Int64
    Customer(id::Int64) = new(id)
end

Base.show(io::IO, customer::Customer) =
    print(io, "Customer($(customer.id))")

mutable struct Teller
    id::Int64
    serving::Union{Customer, Nothing}
    Teller(id::Int64) = new(id, nothing)
end

Base.show(io::IO, teller::Teller) =
    print(io, "Teller($(teller.id))")

tellers = nothing
teller_queue = nothing

function available_teller()::Union{Teller, Nothing}
    for teller in tellers
        if isnothing(teller.serving)
            return teller
        end
    end
    return nothing
end

generator_dist = Exponential(MEAN_INTERARRIVAL_TIME)

@event generate(i::Integer, n::Integer) begin
    @schedule now arrival(Customer(i))
    if i < n
        @schedule in rand(generator_dist) generate(i + 1, n)
    end
end

@event arrival(customer::Customer) begin
    teller = available_teller()
    if isnothing(teller)
        enqueue!(teller_queue, customer)
    else
        @schedule now service(teller, customer)
    end
end

@event service(teller::Teller, customer::Customer) begin
    dist = Uniform(MIN_SERVICE_TIME, MAX_SERVICE_TIME)
    teller.serving = customer
    @schedule in rand(dist) departure(teller, customer)
end

@event departure(teller::Teller, customer::Customer) begin
    if isempty(teller_queue)
        teller.serving = nothing
    else
        next_customer = dequeue!(teller_queue)
        teller.serving = next_customer
        @schedule now service(teller, next_customer)
    end
end

function run_simulation(n::Integer)
    @assert n > 0
    println("SimLynx.jl Event-Based Discrete-Event Simulation Example")
    println("$N_TELLERS Teller, Single Queue Bank Model")
    println("  Inter-arrival time = Exponential($MEAN_INTERARRIVAL_TIME)")
    println("  Service time = Uniform($MIN_SERVICE_TIME, $MAX_SERVICE_TIME)")
    println("  Number of customers = $n")
    @simulation begin
        global tellers = [Teller(i) for i = 1:N_TELLERS]
        global teller_queue = FifoQueue{Customer}()
        current_trace!(true)
        @schedule at 0.0 generate(1, n)
        start_simulation()
        print_stats(teller_queue.n, title="Teller Queue Length Statistics")
        plot_history(teller_queue.n, title="Teller Queue Length History")
    end
end

run_simulation(10)


using DiscriminantAnalysis
using Base.Test

MOD = DiscriminantAnalysis

FloatingPointTypes = (Float32, Float64)
IntegerTypes = (Int32, Int64)

function sampledata{U<:Integer}(n_k::Vector{U}, p::Integer)
    p >= 3 || error("need at least 3 dimensions")
    all(n_k .> p) || error("Need p+1 obs per group")
    k = length(n_k)
    Z = vcat([vcat(vcat(eye(Float64,p), Float64[ones(p-1); 0]')/2,
                   rand(Float64, n_k[i]-(p+1), p)) .+ i for i = 1:k]...)
    refs = vcat([Int64[i for j = 1:n_k[i]] for i = 1:k]...)
    σ = sortperm(rand(sum(n_k)))
    (MOD.RefVector(refs[σ]), Z[σ,:])
end

include("test_common.jl")
include("test_lda.jl")
include("test_qda.jl")


quatErr(q_err::UnitQuaternion{TD}) where {TD} = (0 <= q_err.w) ? SVector( q_err.x,  q_err.y,  q_err.z) : SVector(-q_err.x, -q_err.y, -q_err.z)

function quatErr(q1::UnitQuaternion{TD}, qRef::UnitQuaternion{TD}) where {TD}
  q_err = q1 * inv(qRef)
  return quatErr(q_err)
end

cheapRV(q::UnitQuaternion) = 2 * quatErr(q)
cheapRV(spq::MRP) = cheapRV(UnitQuaternion(spq))

components(q::UnitQuaternion{T}) where {T} = SVector{4,T}(q.w, q.x, q.y, q.z)
components(spq::MRP{T}) where {T} = SVector{3,T}(spq.x, spq.y, spq.z)


function plotqq!(
    ax::Axis,
    P::AbstractVector;
    xstep::Real = 1,
    ystep::Real = 2
)

    n = length(P)
    high = Array{Float64}(undef, n)
    low = Array{Float64}(undef, n)
    for i in 1:n
        high[i] = -log(10, quantile(Beta(i, n + 1 - i), 0.025))
        low[i] = -log(10, quantile(Beta(i, n + 1 - i), 0.975))
    end
    obs = -log.(10, sort(P))
    obs = clamp.(obs, 0.0, -log(10, floatmin(0.0)))
    expect = -log.(10, (collect(1:n) .- 0.5) ./ n)
    xmax = ceil(maximum(expect))
    ymax = ceil(maximum(obs))
    xticks = 0:xstep:xmax
    yticks = (ymax % 2 == 0) ? (0:ystep:ymax) : (0:ystep:(ymax + 1))
    λgc = cquantile(Chisq(1), median(P)) / quantile(Chisq(1), 0.5)

    band!(ax, expect, low, high, color = :gray90)
    scatter!(ax, expect, obs, markersize = 2, color = :black)
    lines!(ax, expect, expect, color = :red2, linewidth = 0.75)
    text!(ax, "λgc = $(round(λgc, digits = 2))", position = (xmax - 0.5, 1),
        textsize = 6, align = (:right, :bottom))
    xlims!(ax, 0, xmax)
    ylims!(ax, 0, ymax)
    hidedecorations!(ax, label = false, ticklabels = false, ticks = false)
    ax.aspect = AxisAspect(1)
    ax.xlabel = "Expected -log[p]"
    ax.ylabel = "Observed -log[p]"
    ax.xticklabelsize = 6
    ax.yticklabelsize = 6
    ax.xlabelsize = 8
    ax.ylabelsize = 8
    ax.xticksize = 3
    ax.yticksize = 3
    ax.xtickwidth = 0.75
    ax.ytickwidth = 0.75
    ax.spinewidth = 0.75
    ax.xticks = xticks
    ax.yticks = yticks
end

"""
    plotqq!(ax::Axis, gwas::DataFrame; kwargs)
    plotqq!(ax::Axis, P::AbstractVector; kwargs)

Plot QQ plot of `P` values where the expected distribution is the uniform distribution.

Keyword arguments include `xstep::Real` and `ystep::Real` for x and y axes ticks step sizes.
"""
plotqq!(ax::Axis, gwas::DataFrame; kwargs...) = plotqq!(ax, gwas.P; kwargs...)


module PhysicalPoissonTests

using Test
using Gridap
import Gridap: ∇
using LinearAlgebra

domain = (0,1,0,1)
partition = (4,4)
model = CartesianDiscreteModel(domain,partition)
order = 2

const h = (domain[2]-domain[1]) / partition[1]
const γ = 10

labels = get_face_labeling(model)
add_tag_from_tags!(labels,"dirichlet",[1,2,5])
add_tag_from_tags!(labels,"neumann",[7,8])
add_tag_from_tags!(labels,"nitsche",6)

trian = get_triangulation(model)
degree = order
quad = CellQuadrature(trian,degree)

ntrian = BoundaryTriangulation(model,labels,"neumann")
ndegree = order
nquad = CellQuadrature(ntrian,ndegree)
const nn = get_normal_vector(ntrian)

dtrian = BoundaryTriangulation(model,labels,"nitsche")
ddegree = order
dquad = CellQuadrature(dtrian,ddegree)
const dn = get_normal_vector(dtrian)

u_scal(x) = x[1]^2 + x[2]
∇u_scal(x) = VectorValue( 2*x[1], one(x[2]) )
Δu_scal(x) = 2
f_scal(x) = - Δu_scal(x)
∇(::typeof(u_scal)) = ∇u_scal

scalar_data = Dict{Symbol,Any}()
scalar_data[:valuetype] = Float64
scalar_data[:u] = u_scal
scalar_data[:f] = f_scal

u_vec(x) = VectorValue( x[1]^2 + x[2], 4*x[1] - x[2]^2 )
∇u_vec(x) = TensorValue( 2*x[1], one(x[2]), 4*one(x[1]), - 2*x[2] )
Δu_vec(x) = VectorValue( 2, -2 )
f_vec(x) = - Δu_vec(x)
∇(::typeof(u_vec)) = ∇u_vec

vector_data = Dict{Symbol,Any}()
vector_data[:valuetype] = VectorValue{2,Float64}
vector_data[:u] = u_vec
vector_data[:f] = f_vec

for data in [ vector_data, scalar_data ]

  T = data[:valuetype]
  u = data[:u]
  f = data[:f]

  V = TestFESpace(
   model=model,
   order=order,
   reffe=:Lagrangian,
   labels=labels,
   valuetype=T,
   dirichlet_tags="dirichlet",
   dof_space = :physical)
   # dirichlet_tags="dirichlet")
  # Juno.@enter TestFESpace( model=model, order=order, reffe=:Lagrangian, labels=labels, valuetype=T, dirichlet_tags="dirichlet", dof_space = :physical)

  U = TrialFESpace(V,u)

  uh = interpolate(u,U)

  a(u,v) = inner(∇(v),∇(u))
  l(v) = v⊙f
  t_Ω = AffineFETerm(a,l,trian,quad)

  uh_Γn = restrict(uh,ntrian)
  uh_Γd = restrict(uh,dtrian)

  l_Γn(v) = v⊙(nn⋅∇(uh_Γn))
  t_Γn = FESource(l_Γn,ntrian,nquad)

  a_Γd(u,v) = (γ/h)*v⊙u  - v⊙(dn⋅∇(u)) - (dn⋅∇(v))⊙u
  l_Γd(v) = (γ/h)*v⊙uh_Γd - (dn⋅∇(v))⊙u
  t_Γd = AffineFETerm(a_Γd,l_Γd,dtrian,dquad)

  op = AffineFEOperator(U,V,t_Ω,t_Γn,t_Γd)

  uh = solve(op)

  e = u - uh

  l2(u) = inner(u,u)
  sh1(u) = a(u,u)
  h1(u) = sh1(u) + l2(u)

  el2 = sqrt(sum( integrate(l2(e),trian,quad) ))
  eh1 = sqrt(sum( integrate(h1(e),trian,quad) ))
  ul2 = sqrt(sum( integrate(l2(uh),trian,quad) ))
  uh1 = sqrt(sum( integrate(h1(uh),trian,quad) ))

  @test el2/ul2 < 1.e-8
  @test eh1/uh1 < 1.e-7

end

end # module


import Combinatorics: combinations

## ---------- DISCRIMINANT MEASURES ----------
"""
Discriminant measure for Local Discriminant Basis. Current available subtypes
are:
- [`ProbabilityDensityDM`](@ref)
- [`SignaturesDM`](@ref)
"""
abstract type DiscriminantMeasure end

"""
Discriminant measure for Probability Density and Time Frequency based energy
maps. Current available measures are:
- [`AsymmetricRelativeEntropy`](@ref)
- [`SymmetricRelativeEntropy`](@ref)
- [`LpDistance`](@ref)
- [`HellingerDistance`](@ref)
"""
abstract type ProbabilityDensityDM <: DiscriminantMeasure end

"""
Discriminant measure for Signatures based energy maps. Current available
measures are:
- [`EarthMoverDistance`](@ref)
"""
abstract type SignaturesDM <: DiscriminantMeasure end

@doc raw"""
    AsymmetricRelativeEntropy <: ProbabilityDensityDM

Asymmetric Relative Entropy discriminant measure for the Probability Density and Time
Frequency based energy maps. This measure is also known as cross entropy and
Kullback-Leibler divergence.

Equation:

``D(p,q) = \sum p(x) \log \frac{p(x)}{q(x)}``

where ``\int_{-\infty}^{\infty} p(t) dt = \int_{-\infty}^{\infty} q(t) dt = 1`` (``p(t)``
and ``q(t)`` are probability density functions.)
"""
struct AsymmetricRelativeEntropy <: ProbabilityDensityDM end

@doc raw"""
    SymmetricRelativeEntropy <: ProbabilityDensityDM

Symmetric Relative Entropy discriminant measure for the Probability Density and Time
Frequency energy maps. Similar idea to the Asymmetric Relative Entropy, but this aims to
make the measure more symmetric.

Equation: Denote the Asymmetric Relative Entropy as ``D_A(p,q)``, then

``D(p,q) = D_A(p,q) + D_A(q,p) = \sum p(x) \log \frac{p(x)}{q(x)} + q(x) \log
\frac{q(x)}{p(x)}``

where ``\int_{-\infty}^{\infty} p(t) dt = \int_{-\infty}^{\infty} q(t) dt = 1`` (``p(t)``
and ``q(t)`` are probability density functions.)

**See also:** [`AsymmetricRelativeEntropy`](@ref)
"""
struct SymmetricRelativeEntropy <: ProbabilityDensityDM end

@doc raw"""
    LpDistance <: ProbabilityDensityDM

``\ell^p`` Distance discriminant measure for the Probability Density and Time
Frequency based energy maps. The default ``p`` value is set to 2.

Equation:

``W(q,r) = ||q-r||_p^p = \sum_{i=1}^n (q_i - r_i)^p``
"""
@with_kw struct LpDistance{T<:Real} <: ProbabilityDensityDM
    p::T = 2
end

@doc raw"""
    HellingerDistance <: ProbabilityDensityDM

Hellinger Distance discriminant measure for the Probability Density energy
map.

Equation:

``H(p,q) = \sum_{i=1}^n (\sqrt{p_i} - \sqrt{q_i})^2``
"""
struct HellingerDistance <: ProbabilityDensityDM end

@doc raw"""
    EarthMoverDistance <: SignaturesDM

Earth Mover Distance discriminant measure for the Signatures energy map.

Equation:

``E(P,Q) = \frac{\sum_{k=1}^{m+n+1} |\hat p_k - \hat q_k| (r_{k+1} - r_k)}{w_\Sigma}``

where
- ``r_1, r_2, \ldots, r_{m+n}`` is the sorted list of ``p_1, \ldots, p_m, q_1, \ldots, q_n``
- ``\hat p_k = \sum_{p_i \leq r_k} w_{p_i}``
- ``\hat q_k = \sum_{q_i \leq r_k} w_{q_i}``
"""
struct EarthMoverDistance <: SignaturesDM end

"""
    discriminant_measure(Γ, dm)

Computes the discriminant measure of each subspace calculated from the energy maps.

# Arguments
- `Γ`: Energy map computed from `energy_map` function. The data structures of `Γ`, depending
  on their corresponding energy map, should be:
    - `TimeFrequency()`: `AbstractArray{T,3}` for 1D signals or `AbstractArray{T,4}` for 2D
      signals.
    - `ProbabilityDensity()`: `AbstractArray{T,4}` for 1D signals or `AbstractArray{T,5}`
      for 2D signals.
    - `Signatures()`: `AbstractVector{NamedTuple{(:coef, :weight), Tuple{S₁,S₂}}}`.
- `dm::DiscriminantMeasure`: Type of Discriminant Measure. The type of `dm` must match the
  type of `Γ`.

# Returns
- `D::Array{T}`: Discriminant measure at each coefficient of the decomposed signals.

# Examples
```julia
using Wavelets, WaveletsExt

X, y = generateclassdata(ClassData(:tri, 5, 5, 5))
Xw = wpdall(X, wavelet(WT.haar))

Γ = energy_map(Xw, y, TimeFrequency()); discriminant_measure(Γ, AsymmetricRelativeEntropy())
Γ = energy_map(Xw, y, ProbabilityDensity()); discriminant_measure(Γ, LpDistance())
Γ = energy_map(Xw, y, Signatures()); discriminant_measure(Γ, EarthMoverDistance())
```

**See also:** [`pairwise_discriminant_measure`](@ref)
"""
function discriminant_measure(Γ::AbstractArray{T}, dm::ProbabilityDensityDM) where
                              T<:AbstractFloat
    # Basic summary of data
    N = ndims(Γ)
    nc = size(Γ,N)
    @assert 3 ≤ N ≤ 5
    @assert nc > 1
    #=======================================================================================
    Classifying the type of signals and energy maps based on Γ

    1. ndims(Γ)=3                           : Time frequency energy map on 1D signals
    2. ndims(Γ)=4 and size(Γ,3) small (<100): Time frequency energy map on 2D signals
    3. ndims(Γ)=4 and size(Γ,3) large (≥100): Probability density energy map on 1D signals
    4. ndims(Γ)=5                           : Probability density energy map on 2D signals
    =======================================================================================#
    if (N==3) || (N==4 && size(Γ,3)≥100)
        sz = [size(Γ,1)]
        L = size(Γ,2)
        pdf_len = N==3 ? 1 : size(Γ,3)
    elseif (N==4 && size(Γ,3)<100) || (N==5)
        sz = size(Γ)[1:2]
        L = size(Γ,3)
        pdf_len = N==5 ? size(Γ,4) : 1
    else
        throw(ArgumentError("Γ has uninterpretable dimensions/unknown size."))
    end

    # --- Compute pairwise discriminant measure ---
    D = zeros(T, (sz...,L))
    for (i,j) in combinations(1:nc,2)
        if N==3
            Γᵢ = @view Γ[:,:,i]         # Energy map for i-th class
            Γⱼ = @view Γ[:,:,j]         # Energy map for j-th class
        elseif N==4
            Γᵢ = @view Γ[:,:,:,i]       # Energy map for i-th class
            Γⱼ = @view Γ[:,:,:,j]       # Energy map for j-th class
        else # N==5
            Γᵢ = @view Γ[:,:,:,:,i]     # Energy map for i-th class
            Γⱼ = @view Γ[:,:,:,:,j]     # Energy map for j-th class
        end
        D += pairwise_discriminant_measure(Γᵢ, Γⱼ, dm)
    end
    return D
end

# discriminant measure for EMD
function discriminant_measure(Γ::AbstractVector{NamedTuple{(:coef, :weight), Tuple{S₁,S₂}}},
                              dm::SignaturesDM) where
                             {S₁<:Array{T} where T<:AbstractFloat,
                              S₂<:Union{T,Array{T}} where T<:AbstractFloat}
    # Basic summary of data
    nc = length(Γ)
    @assert nc > 1
    T = eltype(Γ[1][:coef])
    sz = size(Γ[1][:coef])[1:end-2]
    L = size(Γ[1][:coef])[end-1]

    D = zeros(T, (sz...,L))
    for (Γᵢ,Γⱼ) in combinations(Γ,2)
        D += pairwise_discriminant_measure(Γᵢ, Γⱼ, dm)
    end
    return D
end

# discriminant measure between 2 energy maps
"""
    pairwise_discriminant_measure(Γ₁, Γ₂, dm)

Computes the discriminant measure between 2 classes based on their energy maps.

# Arguments
- `Γ₁::AbstractArray{T} where T<:AbstractFloat`: Energy map for class 1.
- `Γ₂::AbstractArray{T} where T<:AbstractFloat`: Energy map for class 2.
- `dm::DiscriminantMeasure`: Type of discriminant measure.

# Returns
- `::Array{T}`: Discriminant measure between `Γ₁` and `Γ₂`.
"""
function pairwise_discriminant_measure(Γ₁::AbstractArray{T}, Γ₂::AbstractArray{T},
                                       dm::ProbabilityDensityDM) where T<:AbstractFloat
    # parameter checking and basic summary
    N = ndims(Γ₁)
    @assert 2 ≤ ndims(Γ₁) ≤ 4
    @assert size(Γ₁) == size(Γ₂)
    #=======================================================================================
    Classifying the type of signals and energy maps based on Γ

    1. ndims(Γ)=2                           : Time frequency energy map on 1D signals
    2. ndims(Γ)=3 and size(Γ,3) small (<100): Time frequency energy map on 2D signals
    3. ndims(Γ)=3 and size(Γ,3) large (≥100): Probability density energy map on 1D signals
    4. ndims(Γ)=4                           : Probability density energy map on 2D signals
    =======================================================================================#
    if (N==2) || (N==3 && size(Γ₁,3)≥100)
        sz = [size(Γ₁,1)]
        L = size(Γ₁,2)
    elseif (N==3 && size(Γ₁,3)<100) || (N==4)
        sz = size(Γ₁)[1:2]
        L = size(Γ₁,3)
    else
        throw(ArgumentError("Γ has uninterpretable dimensions/unknown size."))
    end

    # --- Pairwise discriminant measure for each element ---
    D = zeros(T, (sz...,L))
    slice_size = prod([sz...,L])        # Number of elements in each slice of the pdf
    map_size = prod(size(Γ₁))           # Number of elements in entire energy map
    for i in 1:slice_size
        for j in i:slice_size:map_size
            D[i] += pairwise_discriminant_measure(Γ₁[j], Γ₂[j], dm)
        end
    end
    return D
end

# discriminant measure between 2 nergy maps for EMD
function pairwise_discriminant_measure(Γ₁::NamedTuple{(:coef, :weight), Tuple{S₁,S₂}},
                                       Γ₂::NamedTuple{(:coef, :weight), Tuple{S₁,S₂}},
                                       dm::SignaturesDM) where
                                      {S₁<:Array{T} where T<:AbstractFloat,
                                       S₂<:Union{T,Array{T}} where T<:AbstractFloat}

    # parameter checking and basic summary
    N = ndims(Γ₁.coef)
    T = eltype(Γ₁.coef)
    sz = size(Γ₁.coef)[1:end-2]
    L = size(Γ₁.coef)[end-1]
    @assert isa(Γ₁.weight, AbstractFloat) || size(Γ₁.coef) == size(Γ₁.weight)
    @assert isa(Γ₂.weight, AbstractFloat) || size(Γ₂.coef) == size(Γ₂.weight)
    @assert typeof(Γ₁.weight) == typeof(Γ₂.weight)
    @assert sz == size(Γ₂.coef)[1:end-2]
    @assert L == size(Γ₂.coef)[end-1]

    D = Array{T,N-1}(undef, (sz...,L))
    slice_size = prod([sz...,L])                # Number of elements for each signal's coefficients
    for i in 1:slice_size
        # Signatures
        if isa(Γ₁.weight, AbstractFloat)        # Equal weight
            P = (coef = Γ₁.coef[i:slice_size:end], weight = Γ₁.weight)
            Q = (coef = Γ₂.coef[i:slice_size:end], weight = Γ₂.weight)
        else                                    # Probability density weight
            P = (coef = Γ₁.coef[i:slice_size:end], weight = Γ₁.weight[i:slice_size:end])
            Q = (coef = Γ₂.coef[i:slice_size:end], weight = Γ₂.weight[i:slice_size:end])
        end
        D[i] = pairwise_discriminant_measure(P, Q, dm)
    end
    return D
end

# Asymmetric Relative Entropy
"""
    pairwise_discriminant_measure(p, q, dm)

Computes the discriminant measure between 2 classes at an index ``i``, where `Γ₁[i] = p` and
`Γ₂[i] = q`.

# Arguments
- `p::T where T<:AbstractFloat`: Coefficient at index ``i`` from `Γ₁`.
- `q::T where T<:AbstractFloat`: Coefficient at index ``i`` from `Γ₂`.
- `dm::DiscriminantMeasure`: Type of discriminant measure.

# Returns
- `::T`: Discriminant measure between `p` and `q`.
"""
function pairwise_discriminant_measure(p::T, q::T, dm::AsymmetricRelativeEntropy) where
                                       T<:AbstractFloat
    @assert p ≥ 0 && q ≥ 0
    return (p==0 || q==0) ? 0 : p * log(p/q)
end

# Symmetric Relative Entropy
function pairwise_discriminant_measure(p::T, q::T, dm::SymmetricRelativeEntropy) where
                                       T<:AbstractFloat
    return pairwise_discriminant_measure(p, q, AsymmetricRelativeEntropy()) +
           pairwise_discriminant_measure(q, p, AsymmetricRelativeEntropy())
end

# Hellinger Distance
function pairwise_discriminant_measure(p::T, q::T, dm::HellingerDistance) where
                                       T<:AbstractFloat
    return (sqrt(p) - sqrt(q))^2
end

# Lᵖ Distance
function pairwise_discriminant_measure(p::T, q::T, dm::LpDistance) where T<:AbstractFloat
    return (p - q)^dm.p
end

# Earth Mover Distance
function pairwise_discriminant_measure(P::NamedTuple{(:coef, :weight), Tuple{S1,S2}},
                                       Q::NamedTuple{(:coef, :weight), Tuple{S1, S2}},
                                       dm::EarthMoverDistance) where
                                      {S1<:Vector{T} where T<:AbstractFloat,
                                       S2<:Union{T,Vector{T}} where T<:AbstractFloat}
    # assigning tuple signatures into coef and weight
    p, w_p = P
    q, w_q = Q

    # sort signature values
    p_order = sortperm(p)
    q_order = sortperm(q)
    p = p[p_order]
    q = q[q_order]
    w_p = isa(w_p, AbstractFloat) ? repeat([w_p], length(p)) : w_p[p_order]
    w_q = isa(w_q, AbstractFloat) ? repeat([w_q], length(q)) : w_q[q_order]

    # concatenate p and q, then sort them
    r = [p; q]
    sort!(r)

    # compute emd
    n = length(r)
    emd = 0
    for i in 1:(n-1)
        # get total weight of p and q less than or equal to r[i]
        ∑w_p = sum(w_p[p .≤ r[i]])
        ∑w_q = sum(w_q[q .≤ r[i]])
        # add into emd
        @inbounds emd += abs(∑w_p - ∑w_q) * (r[i+1] - r[i])
    end
    emd /= (sum(w_p) + sum(w_q))
    return emd
end

## ---------- DISCRIMINATION POWER ----------
"""
Discriminant Power measure for the Local Discriminant Basis. Current available
measures are
- [`BasisDiscriminantMeasure`](@ref)
- [`FishersClassSeparability`](@ref)
- [`RobustFishersClassSeparability`](@ref)
"""
abstract type DiscriminantPower end

"""
    BasisDiscriminantMeasure <: DiscriminantPower

This is the discriminant measure of a single basis function computed in a previous step to
construct the energy maps.
"""
struct BasisDiscriminantMeasure <: DiscriminantPower end

@doc raw"""
    FishersClassSeparability <: DiscriminantPower

The Fisher's class separability of the expansion coefficients in the basis function.

Equation: ``\frac{\sum_{c=1}^C \pi_c\{{\rm mean}_i(\alpha_{\lambda,i}^{(c)}) - {\rm mean}_c
\cdot {\rm mean}_i(\alpha_{\lambda,i}^{(c)})\}^2}{\sum_{c=1}^C \pi_c {\rm
var}_i(\alpha_{\lambda,i}^{(c)})}``
"""
struct FishersClassSeparability <: DiscriminantPower end

@doc raw"""
    RobustFishersClassSeparability <: DiscriminantPower

The robust version of Fisher's class separability of the expansion coefficients in the basis
function.

Equation: ``\frac{\sum_{c=1}^C \pi_c\{{\rm med}_i(\alpha_{\lambda,i}^{(c)}) - {\rm med}_c
\cdot {\rm med}_i(\alpha_{\lambda,i}^{(c)})\}^2}{\sum_{c=1}^C \pi_c {\rm
mad}_i(\alpha_{\lambda,i}^{(c)})}``
"""
struct RobustFishersClassSeparability <: DiscriminantPower end

"""
    discriminant_power(D, tree, dp)
    discriminant_power(coefs, y, dp)

Returns the discriminant power of each leaf from the local discriminant basis (LDB) tree.

# Arguments
- `D::AbstractArray{T} where T<:AbstractFloat`: Discriminant measures.
- `tree::BitVector`: Best basis tree for selecting coefficients with largest discriminant
  measures.
- `coefs::AbstractArray{T} where T<:AbstractFloat`: Best basis coefficients for the input
  signals.
- `y::AbstractVector{S} where S`: Labels corresponding to each signal in `coefs`.
- `dp::DiscriminantPower`: The measure for discriminant power.

!!! note
    `discriminant_power(D, tree, dp)` only works for `dp = BasisDiscriminantMeasure()`,
    whereas `discriminant_power(coefs, y, dp)` works for `dp = FishersClassSeparability()`
    and `dp = RobustFishersClassSeparability()`.

# Returns
- `power::Array{T}`: The discriminant power at each index of `D` or `coefs`.
- `order::Vector{T}`: The order of discriminant power in descending order.
"""
function discriminant_power(D::AbstractArray{T}, tree::BitVector,
                            dp::BasisDiscriminantMeasure) where T<:AbstractFloat
    @assert 2 ≤ ndims(D) ≤ 3
    N = ndims(D)
    sz = size(D)[1:end-1]
    tmp = Array{T,N-1}(undef, sz)
    @assert isvalidtree(tmp, tree)

    power = getbasiscoef(D, tree)
    order = sortperm(vec(power), rev = true)

    return (power, order)
end

function discriminant_power(coefs::AbstractArray{T}, y::AbstractVector{S},
                            dp::FishersClassSeparability) where {T<:AbstractFloat, S}
    N = ndims(coefs)
    @assert 2 ≤ N ≤ 3
    sz = size(coefs)[1:end-1]           # signal length

    C = unique(y)
    Nc = length(C)                      # number of classes

    Nᵢ = Array{T,1}(undef, Nc)
    Eαᵢ = Array{T,N}(undef, (sz...,Nc))                # mean of each entry
    Varαᵢ = Array{T,N}(undef, (sz...,Nc))              # variance of each entry
    for (i, c) in enumerate(C)
        idx = findall(yᵢ -> yᵢ==c, y)
        Nᵢ[i] = length(idx)
        if N==2
            coefsᵢ = coefs[:,idx]
            Eαᵢ[:,i] = mean(coefsᵢ, dims = N)
            Varαᵢ[:,i] = var(coefsᵢ, dims = N)
        elseif N==3
            coefsᵢ = coefs[:,:,idx]
            Eαᵢ[:,:,i] = mean(coefsᵢ, dims = N)
            Varαᵢ[:,:,i] = var(coefsᵢ, dims = N)
        end
    end
    Eα = mean(Eαᵢ, dims = N)                      # overall mean of each entry
    pᵢ = Nᵢ / sum(Nᵢ)                             # proportions of each class

    if N==2     # For 1D signals, can be done via matrix multiplication
        power = ((Eαᵢ - (Eα .* Eαᵢ)).^2 * pᵢ) ./ (Varαᵢ * pᵢ)
    else        # For 2D signals, requires some manual work
        pᵢ = reshape(pᵢ,1,1,:)
        power = sum((Eαᵢ-(Eα.*Eαᵢ)).^2 .* pᵢ, dims=N) ./ sum(Varαᵢ.*pᵢ, dims=N)
        power = reshape(power, sz...)
    end
    order = sortperm(vec(power), rev = true)

    return (power, order)
end

function discriminant_power(coefs::AbstractArray{T}, y::AbstractVector{S},
                            dp::RobustFishersClassSeparability) where {T<:AbstractFloat,S}
    N = ndims(coefs)
    @assert 2 ≤ N ≤ 3
    sz = size(coefs)[1:end-1]           # signal length

    C = unique(y)
    Nc = length(C)                      # number of classes

    Nᵢ = Array{T,1}(undef, Nc)
    Medαᵢ = Array{T,N}(undef, (sz...,Nc))              # mean of each entry
    Madαᵢ = Array{T,N}(undef, (sz...,Nc))              # variance of each entry
    for (i, c) in enumerate(C)
        idx = findall(yᵢ -> yᵢ==c, y)
        Nᵢ[i] = length(idx)
        if N==2
            coefsᵢ = coefs[:,idx]
            Medαᵢ[:,i] = median(coefsᵢ, dims = N)
            Madαᵢ[:,i] = mapslices(x -> mad(x, normalize=false), coefsᵢ, dims = N)
        elseif N==3
            coefsᵢ = coefs[:,:,idx]
            Medαᵢ[:,:,i] = median(coefsᵢ, dims = N)
            Madαᵢ[:,:,i] = mapslices(x -> mad(x, normalize=false), coefsᵢ, dims = N)
        end
    end
    Medα = median(Medαᵢ, dims = N)               # overall mean of each entry
    pᵢ = Nᵢ / sum(Nᵢ)                            # proportions of each class

    if N==2     # For 1D signals, can be done via matrix multiplication
        power = ((Medαᵢ - (Medα.*Medαᵢ)).^2 *pᵢ) ./ (Madαᵢ * pᵢ)
    else        # For 2D signals, requires some manual work
        pᵢ = reshape(pᵢ,1,1,:)
        power = sum((Medαᵢ-(Medα.*Medαᵢ)).^2 .* pᵢ, dims=N) ./ sum(Madαᵢ.*pᵢ, dims=N)
        power = reshape(power, sz...)
    end
    order = sortperm(vec(power), rev = true)

    return (power, order)
end


using Mosek

"""
    basicPortfolio(n::Int, x0 :: Vector{Float64})

Implements the simple Portfolio Selection example maximizing return subject to a limited risk.

    maximize
        mu' * x
    such that
        sum(x) = budget
        GT * x - s = 0
        s >= || t ||
        s <= gamma

Parameters:

- `w`     Initial uninvested wealth
- `x0`    Initial invested wealth for each asset
- `mu`    Expected returns for each asset
- `gamma` Risk limit as maximum standard deviation
- `GT`    Factorized covariance matrix
"""
function basicPortfolio(w :: Float64, x0 :: Vector{Float64}, mu :: Vector{Float64}, GT :: Array{Float64,2}, gamma :: Float64)
    n = length(mu)
    maketask() do t
        putstreamfunc(t,MSK_STREAM_LOG,msg -> print(msg))

        budget = w + sum(x0)

        # variables
        appendvars(t,2*n + 1)
        x_start  = 1
        x_end    = n
        s_start  = n+1
        s_end    = n+1
        t_start  = n+2
        t_end    = t_start+n-1

        putobjsense(t,MSK_OBJECTIVE_SENSE_MAXIMIZE)
        putclist(t,Int32[Int32(x_start):Int32(x_end) ...], mu)

        putvarboundslice(t,Int32(x_start), Int32(x_end+1),
                         fill(MSK_BK_LO,n),
                         zeros(Float64,n),
                         fill(Inf,n))

        putvarboundslice(t,Int32(t_start), Int32(t_end+1),
                         fill(MSK_BK_LO,n),
                         zeros(Float64,n),
                         fill(Inf,n))

        for j in 1:n
            putvarname(t,Int32(x_start+j-1), "x$j")
            putvarname(t,Int32(t_start+j-1), "t$j")
        end
        putvarname(t,Int32(s_start),"s")

        # constraints
        appendcons(t,Int32(1+n))

        # sum(x) = w+sum(x0)
        putconbound(t,Int32(1),MSK_BK_FX,w + sum(x0),w + sum(x0))
        putconname(t,Int32(1),"budget")
        putarow(t,Int32(1),Int32[Int32(x_start):Int32(x_end)...],ones(Float64,n))

        # GT * x - s = 0
        putconboundslice(t,2,n+2,
                         fill(MSK_BK_FX, n),
                         zeros(Float64,n),
                         zeros(Float64,n))

        for i in 1:n
            putconname(t,Int32(i+1),"GT[$i]")
            putarow(t,Int32(i+1),Int32[Int32(x_start):Int32(x_end)...],GT[i,:])
            putaij(t,Int32(i+1),s_start+i-1, -1.0)
        end

        # s > || t ||
        appendcone(t,MSK_CT_QUAD, 0.0, [ s_start, t_start:t_end... ])
        putconename(t,Int32(1),"stddev")

        # s = gamma
        putvarbound(t,Int32(s_start),MSK_BK_FX,gamma,gamma)

        # Turn all log output off.
        putintparam(t,MSK_IPAR_LOG, Int32(0))

        optimize(t,"mosek://solve.mosek.com:30080")

        # Display the solution summary for quick inspection of results.
        solutionsummary(t, MSK_STREAM_MSG)

        xxvector = getxx(t,MSK_SOL_ITR)

        #expectedReturn = sum(xxvector[x_start:x_end] .* mu)
        #standardDeviation = xxvector[s_start]

        xxvector[x_start:x_end],xxvector[s_start]
    end
end

# default data
gamma = 0.05
mu = [0.1073, 0.0737, 0.0627]
GT = [0.1667 0.0232  0.0013
      0.0000 0.1033 -0.0022
      0.0000 0.0000  0.0338]
x0 = [0.0, 0.0, 0.0]
w = 1.0

x,s = basicPortfolio(w,x0,mu,GT,gamma)

expectedReturn = sum(x .* mu)
println("------------------------------------------------------------")
println("Expected return $expectedReturn for gamma $s")
println("Optimal investment: $x")
println("------------------------------------------------------------")


using ChaosTools, DelayEmbeddings
using Test, OrdinaryDiffEq
using DelayEmbeddings: localextrema, exponential_decay_fit

println("\nTesting delay estimation...")

testval = (val, vmin, vmax) -> @test vmin ≤ val ≤ vmax

@testset "Estimate Delay" begin
    # test exponential decay fit
    x = 0:200
    y = @. exp(-x/5)
    @test exponential_decay_fit(x, y, :equal) ≈ 5
    @test exponential_decay_fit(x, y, :small) ≈ 5

    ds = Systems.henon()
    data = trajectory(ds,1000;dt=1)
    x = data[:,1]
    @test estimate_delay(x,"ac_zero", 0:10) ≤ 2
    @test estimate_delay(x,"ac_min", 0:10)  ≤ 2
    @test estimate_delay(x,"exp_extrema", 0:10)  ≤ 4
    @test estimate_delay(x,"mi_min", 0:10) == 10

    ds = Systems.roessler(ones(3))
    dt = 0.02
    data = trajectory(ds,500.0;dt=dt,diffeq...)
    x = data[:,1]
    @test 1.3 ≤ estimate_delay(x,"ac_zero", 1:2:500)*dt ≤ 1.7
    @test 2.6 ≤ estimate_delay(x,"ac_min", 1:2:500)*dt  ≤ 3.4
    @test 1.0 ≤ estimate_delay(x,"mi_min", 1:2:500)*dt ≤ 1.6

    dt = 0.1
    data = trajectory(ds,2000.0;dt=dt,diffeq...)
    x = data[:,1]
    @test 1.3 ≤ estimate_delay(x,"ac_zero", 1:1:50)*dt ≤ 1.7
    @test 2.6 ≤ estimate_delay(x,"ac_min", 1:1:50)*dt  ≤ 3.4
    @test 1.15 ≤ estimate_delay(x,"mi_min", 1:1:50)*dt ≤ 1.6

    ds = Systems.lorenz()
    dt = 0.1
    data = trajectory(ds,5000;dt=dt,diffeq...)
    x = data[500:end,1]
    @test 0 < estimate_delay(x,"exp_extrema", 0:2:200)  < 200

    # Issue #18
    ds = Systems.gissinger(ones(3)) # 3D continuous chaotic system, also shown in orbit diagrams tutorial
    dt = 0.05
    data = trajectory(ds, 1000.0, dt = dt)
    s = data[:, 1]

    τ = estimate_delay(s, "mi_min", 0:1:400) # this was the erroring line
    @test τ > 0
end


"""
    isprobability(P)

Test if `P` is a is a probability distribution.
"""
isprobability(P) = all(P .≥ 0.0) && sum(P) ≈ 1.0

"""
    marginals(P::AbstractMatrix)

Returns the marginals of a probabilistic matrix P.
"""
marginals(P::AbstractMatrix) = sum(P, dims=2)[:], sum(P, dims=1)[:]

"""
    marginals(Ps::AbstractMatrix)

Returns the marginals of as array of probabilistic matrices Ps.
"""
function marginals(Ps::Array{M, 1} where M <: AbstractMatrix)
    A = hcat([sum(P, dims=2) for P in Ps]...)
    B = hcat([sum(P, dims=1)' for P in Ps]...)
    return A, B
end

"""
    uniform(n::Int)

Returns the PMF of length n, i.e. a vector of length n filled with 1/n.
"""
uniform(n::Int) = ones(n) / n

normalize(x) = x / sum(x)
#=
struct UtilityMatrix <: AbstractMatrix
    M::AbstractMatrix
    bottom_species::AbstractVector
    top_species::AbstractVector
    UtilityMatrix(M::AbstractMatrix, bottom_species=nothing, top_species=nothing)
        if

    end
end
=#