### A Pluto.jl notebook ###
# v0.17.2

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el)
        el
    end
end

# ╔═╡ 0a171174-2db6-4077-a4af-1da84c9f5470
begin
	using Random
	using Plots
	using Distributions
	using PlutoUI

	# Plots.jl backend
	gr()
end;

# ╔═╡ 7b93882c-9ad8-11ea-0288-0941e163f9d5
md"""
# Likelihood function

A broad brush overview.
"""

# ╔═╡ 7110fe00-83a8-4a9d-975e-d59d0f351630
md"""
## Statistical estimation

_Informal definition_: An estimator gives a guess about the true state (parameter, $\boldsymbol{\theta}$) of nature.
"""

# ╔═╡ 13f0fefb-a926-4689-9518-a5d77e0582d7
begin
	# Control and settings
	pop_mean = 5
	pop_std = 1
	n_pop_row = 20
	Random.seed!(122)

	# Population
	pop_distr = Normal(pop_mean, pop_std)
	radius = rand(pop_distr, n_pop_row^2)
	x = 1:n_pop_row
	y = 1:n_pop_row
	x_mx = x' .* ones(n_pop_row)
	y_mx = ones(n_pop_row)' .* y
end;

# ╔═╡ 26c379e4-8480-4e1e-ae8e-933eadfd69f3
md"""
Population radius distribution: ``\mathcal{N}(``$(pop_mean), $(pop_std)``^2)``
"""

# ╔═╡ 016bf804-ca81-4300-8d5b-1e063834d4d0
md"""Hit this button to resample: $(@bind do_sample Button("Sample"))"""

# ╔═╡ 6465a01e-d0e9-4670-b9af-05f7a7a6a200
 md"Change the sample size by dragging the slider: $(@bind n_sample Slider(1:1:(n_pop_row^2), default=20, show_value=true))"

# ╔═╡ 8db552a0-1a45-401c-a579-c3ffc47b27c9
begin
	do_sample
	# Control and settings
	# n_sample = 20

	# ........................
	# Sample
	# ........................
	sample_idx = sample(1:(n_pop_row^2), n_sample, replace=false)
	sample_x = x_mx[:][sample_idx]
	sample_y = y_mx[:][sample_idx]
	sample_radius = radius[sample_idx]

	# with_terminal() do
	# 	println(sample_idx)
	# end

	# ........................
	# Plotting
	# ........................
	begin
		p_pop = scatter(x_mx[:], y_mx[:], markersize=radius[:],
			label=nothing, axis=([], false), title="Population",
		)
		scatter!(sample_x, sample_y, markersize=sample_radius,
			label=nothing, markercolor="red",
		)
		p_sample = scatter(sample_x, sample_y, markersize=sample_radius,
			label=nothing, axis=([], false), markercolor="red", title="Sample",
		)
		plot(p_pop, p_sample, layout = (1, 2), legend = false, link=:all)
	end
end

# ╔═╡ 414cfce1-1bbf-4ae9-9f06-c4a4d55b7ad3
begin
	vline([mean(radius)], lw=3, label="population", yaxis=([], false), xlabel="mean")
	vline!([mean(sample_radius)], color="red", label="sample")
	plot!(xlims=(pop_mean - 3*pop_std, pop_mean + 3*pop_std))
end

# ╔═╡ a090b42b-ee45-4a63-a780-b63df6ac3d11
md"""
## Data generating process

_Informal definition_: The mathematical model that is assumed to generated the data. In statistics it should contain a probabilistic component.
"""

# ╔═╡ 7a0961e0-d332-492e-bbd2-e2127e65f1a4
md"""
For example, considering the above example:

* dataset: ``\mathbf{x}``
* to be estimated model parameter: ``\theta``

$X_i\sim\mathcal{N}(\theta, \sigma^2)$,

where $\sigma$=$(pop_std).

Usually it is more complex, for example a common linear model:

$X_i\sim\mathcal{N}(a \cdot z_i + b, \sigma)$,

where ``\theta=[a, b]``, and ``a`` and ``b`` are the slope and the intersection of the linear model respectively.

Data generating model: Steps of generating random observations from a model (the model definition is part of this!).

__Data generating process/model $\rightarrow$ likelihood function__
"""

# ╔═╡ a9f04175-2d40-4cf6-be59-b24058a4b1ce
md"""
## Likelihood function

_Informal definition_: Tells us what is the likelihood of seeing a dataset under a particular model.

* dataset: ``\mathbf{x}`` (``n`` number of observations)
* model: ``\mathcal{M}``, parametrized with ``\boldsymbol{\theta}``
* particular model: a particular set of ``\boldsymbol{\theta}`` values

$L(\theta, \mathbf{x})$

$L(\theta, \mathbf{x}) = \prod_{i}^{n} p(x_i, \theta)$

where ``p(.)`` is a probability density or mass function.
"""

# ╔═╡ 461c107c-a40e-4fb9-a4ce-9d9c387be074
md"""
## Example: [Task 2](https://docs.google.com/document/d/1vrzjcl8k1RoBaWocgTvVfLLG6LDAHiMwh_MC9PcnfsQ/edit#heading=h.fwqlqf4cfis1)
"""

# ╔═╡ afdd513d-6995-4801-9bac-59d4a2287322
begin
	x_sample = [30.1 28.7 21.2 28.6 25.8 25.1 24.2 27.3 20.0 34.8]
	sigma_pop = 5
	mean_prior_mean = 40
	mean_prior_std = 100
end;

# ╔═╡ 3f467d2d-5053-4e1e-970b-8c61152f5757
md"""
We have a vector independent observations (``n=``$(length(x_sample))):

``\mathbf{x} =`` [$(join(x_sample, ", "))]

drawn from ``\mathcal{N}(\mu, \sigma^2=``$(sigma_pop)``^2)``.

Based on this, we can write down the likelihood function:

$L(\mu, \mathbf{x}) = \prod_{i}^{n} \varphi \left( \frac {x_i - \mu}{\sigma} \right)$

where ``\varphi(.)`` is the standard normal probability density function:

$\varphi(x)={\frac {1}{{\sqrt {2\pi }}}}e^{-x^{2}}$.

Visualizing the log of the likelihood function:
"""

# ╔═╡ 2dfee269-4539-4f96-afca-4e975c2fd5f0
begin
	log_likelihood(t) = sum(logpdf(Normal(t, sigma_pop), x_sample))

	tt = range(30-2.5*sigma_pop, stop=30+2.5*sigma_pop, length=40)
	log_ll = log_likelihood.(tt)

	min_log_l = minimum(log_ll)
end;

# ╔═╡ 9a7b43ec-5d43-43c5-aa35-b776de4a48c7
md"Change the value of ``\mu`` by dragging the slider: $(@bind mu Slider((30-2.5*sigma_pop):0.2:(30+2.5*sigma_pop), default=30))"

# ╔═╡ 39e69cb1-207a-42b6-920a-1912d50edb4e
md"""
``\mu_\mathrm{selected}=``$mu MPa

``\mathrm{log} \left( L(\mu_\mathrm{selected}) \right) =``$(round(log_likelihood(mu), digits=1))
"""

# ╔═╡ 5bd1d628-beaa-47c8-98a6-f8eb36444e93
begin
	log_l_mu = log_likelihood(mu)
	begin
		plot(tt, log_ll, lw=3, xlabel="\\mu", ylabel="log-likelihood", label=nothing)
		plot!([mu, mu], [min_log_l, log_l_mu], ls=:dash, color="red", label="selected value")
		scatter!([mu], [log_l_mu], label=nothing, color="red", markersize=6)
	end
end

# ╔═╡ 684b5e7b-6de1-42af-9497-28a957aa73ad
md"""The maximum likelihood estimate is the value that maximizes the likelihood function (the same that maximizes the log-likelihood function)."""

# ╔═╡ 422e4766-7a75-4403-a189-fbf89d41a8f0
md"""
## Example: Simple linear regression

* data generating process
* constructing the likelihood function
* maximum likelihood estimate

_Task:_ estimate the slope parameter of a linear model from a sample.
"""

# ╔═╡ 5578a9d5-7d7b-4582-91e8-7dab5ba2a572
begin
	# Control and settings
	n_sample_lin = 15
	x_lims = (-2.0, 4.0)
	a_ground_truth = 1.0
	b_ground_truth = 3.0

	# Model
	linear_model(x, a) = a .* x + b_ground_truth

	eps_sigma = 0.5
	ground_truth_model(x) = linear_model(x, a_ground_truth)

	# Generate synthetic data
	x_data = rand(Uniform(x_lims[1], x_lims[2]), n_sample_lin)
	y_data = ground_truth_model.(x_data) .+ rand(Normal(0.0, eps_sigma), n_sample_lin)

	y_lims = ground_truth_model.(x_lims)
	y_range = y_lims[2] - y_lims[1]
	y_lims_lb = minimum([y_lims[1], minimum(y_data)]) - 0.2*y_range
	y_lims_ub = maximum([y_lims[2], maximum(y_data)]) + 0.2*y_range
	y_lims = (y_lims_lb, y_lims_ub)
	# with_terminal() do
	# 	println(y_lims)
	# end
end;

# ╔═╡ 534e1a70-62f4-41fa-bd03-86bb9e4bf495
md"""
Parametrization of the linear model:

$y = a \cdot x + b$

where ``b`` is known, ``b=``$(b_ground_truth), and ``a`` is to be estimated.

We have an ``n=``$n_sample_lin sample of ``x`` and ``y`` pairs:

``\mathbf{x} =`` [$(join(round.(x_data, digits=2), ", "))]

``\mathbf{y} =`` [$(join(round.(y_data, digits=2), ", "))]

This is how our data looks like:
"""

# ╔═╡ b9b80541-571c-4401-b9c9-1ae7fe95beba
begin
	scatter(x_data, y_data, xlabel="x", ylabel="y", label="observation", xlims=x_lims, ylims=y_lims, legend=:topleft)
end

# ╔═╡ 9b63c009-7984-49a6-be7d-5ba52c78a855
md"""
They do not fall on a straight line, so if we want to use a linear model it is better to formulate it with a model error. Let's use the following additive model error formulation:

$y = a \cdot x + b + \epsilon$

where ``\epsilon`` is assumed to be a realization from ``\mathcal{N}(0, \sigma_{\epsilon}=``$(eps_sigma)``^2)``. For simplicity, we assume that this distribution is fully known but in general its standard deviation is not known but also estimated. Moreover, we assume that the ``\epsilon`` values are independent between realizations.

The above equation also shows us the data generating process and we can formulate the likelihood function by recognizing that

$y_i \sim \mathcal{N}(a \cdot x_i + b, \sigma_{\epsilon}^2)$

To make the independent, identically distributed (IID) assumption explicit:

$y_i \overset{\text{IID}}{\sim} \mathcal{N}(a \cdot x_i + b, \sigma_{\epsilon}^2)$

"""

# ╔═╡ bd4dbdc8-84d0-48f8-b4fb-69805b14aaff
md"Change the value of ``a`` by dragging the slider: $(@bind a_selected Slider(-1:0.1:3, default=-0.5, show_value=true))"

# ╔═╡ 32b13ec7-b976-494a-bfff-2efedfe16ea4
begin
	# Plot
	scatter(x_data, y_data, xlabel="x", ylabel="y", label="observation", legend=:topleft, xlims=x_lims, ylims=y_lims)
	plot!([x_lims[1], x_lims[2]], linear_model.([x_lims[1], x_lims[2]], a_selected), label="selected linear model", lw=3)
end

# ╔═╡ cf56ff3d-d9fd-4222-997f-750896ac0440
begin
	# Log-likelihood
	function log_like_fun(t)
		ll = 0
		for ii = 1:n_sample_lin
			ll += logpdf(Normal(linear_model.(x_data[ii], t), eps_sigma), y_data[ii])
		end
		return ll
	end

	tl = range(-1.0, stop=3.0, length=40)
	log_like = log_like_fun.(tl)

	min_log_like = minimum(log_like)
	log_like_a = log_like_fun(a_selected)

	# Plot
	plot(tl, log_like, lw=3, label=nothing, xlabel="a", ylabel="log-likelihood")
	plot!([a_selected, a_selected], [min_log_like, log_like_a], ls=:dash, color="red", label="selected model")
	scatter!([a_selected], [log_like_a], label=nothing, color="red", markersize=6)
end

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[compat]
Distributions = "~0.25.32"
Plots = "~1.23.6"
PlutoUI = "~0.7.20"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[AbstractPlutoDingetjes]]
deps = ["Pkg"]
git-tree-sha1 = "0bc60e3006ad95b4bb7497698dd7c6d649b9bc06"
uuid = "6e696c72-6542-2067-7265-42206c756150"
version = "1.1.1"

[[Adapt]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
version = "3.3.1"

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[Bzip2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2"
uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0"
version = "1.0.8+0"

[[Cairo_jll]]
deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"]
git-tree-sha1 = "f2202b55d816427cd385a9a4f3ffb226bee80f99"
uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a"
version = "1.16.1+0"

[[ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "f885e7e7c124f8c92650d61b9477b9ac2ee607dd"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.11.1"

[[ChangesOfVariables]]
deps = ["LinearAlgebra", "Test"]
git-tree-sha1 = "9a1d594397670492219635b35a3d830b04730d62"
uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
version = "0.1.1"

[[ColorSchemes]]
deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"]
git-tree-sha1 = "a851fec56cb73cfdf43762999ec72eff5b86882a"
uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4"
version = "3.15.0"

[[ColorTypes]]
deps = ["FixedPointNumbers", "Random"]
git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597"
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
version = "0.11.0"

[[Colors]]
deps = ["ColorTypes", "FixedPointNumbers", "Reexport"]
git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40"
uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
version = "0.12.8"

[[Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "dce3e3fea680869eaa0b774b2e8343e9ff442313"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.40.0"

[[CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"

[[Contour]]
deps = ["StaticArrays"]
git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7"
uuid = "d38c429a-6771-53c6-b99e-75d170b6e991"
version = "0.5.7"

[[DataAPI]]
git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.9.0"

[[DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.10"

[[DataValueInterfaces]]
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
version = "1.0.0"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[DensityInterface]]
deps = ["InverseFunctions", "Test"]
git-tree-sha1 = "80c3e8639e3353e5d2912fb3a1916b8455e2494b"
uuid = "b429d917-457f-4dbc-8f4c-0cc954292b1d"
version = "0.4.0"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[Distributions]]
deps = ["ChainRulesCore", "DensityInterface", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns", "Test"]
git-tree-sha1 = "380dab663ca4234c4ec91340c3211c5e73374612"
uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
version = "0.25.32"

[[DocStringExtensions]]
deps = ["LibGit2"]
git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.6"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[EarCut_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "3f3a2501fa7236e9b911e0f7a588c657e822bb6d"
uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5"
version = "2.2.3+0"

[[Expat_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "b3bfd02e98aedfa5cf885665493c5598c350cd2f"
uuid = "2e619515-83b5-522b-bb60-26c02a35a201"
version = "2.2.10+0"

[[FFMPEG]]
deps = ["FFMPEG_jll"]
git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8"
uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a"
version = "0.4.1"

[[FFMPEG_jll]]
deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"]
git-tree-sha1 = "d8a578692e3077ac998b50c0217dfd67f21d1e5f"
uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
version = "4.4.0+0"

[[FillArrays]]
deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
version = "0.12.7"

[[FixedPointNumbers]]
deps = ["Statistics"]
git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
version = "0.8.4"

[[Fontconfig_jll]]
deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03"
uuid = "a3f928ae-7b40-5064-980b-68af3947d34b"
version = "2.13.93+0"

[[Formatting]]
deps = ["Printf"]
git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
version = "0.4.2"

[[FreeType2_jll]]
deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
git-tree-sha1 = "87eb71354d8ec1a96d4a7636bd57a7347dde3ef9"
uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7"
version = "2.10.4+0"

[[FriBidi_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91"
uuid = "559328eb-81f9-559d-9380-de523a88c83c"
version = "1.0.10+0"

[[GLFW_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"]
git-tree-sha1 = "0c603255764a1fa0b61752d2bec14cfbd18f7fe8"
uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89"
version = "3.3.5+1"

[[GR]]
deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"]
git-tree-sha1 = "30f2b340c2fff8410d89bfcdc9c0a6dd661ac5f7"
uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71"
version = "0.62.1"

[[GR_jll]]
deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"]
git-tree-sha1 = "fd75fa3a2080109a2c0ec9864a6e14c60cca3866"
uuid = "d2c73de3-f751-5644-a686-071e5b155ba9"
version = "0.62.0+0"

[[GeometryBasics]]
deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"]
git-tree-sha1 = "58bcdf5ebc057b085e58d95c138725628dd7453c"
uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
version = "0.4.1"

[[Gettext_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"]
git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046"
uuid = "78b55507-aeef-58d4-861c-77aaff3498b1"
version = "0.21.0+0"

[[Glib_jll]]
deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "74ef6288d071f58033d54fd6708d4bc23a8b8972"
uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
version = "2.68.3+1"

[[Graphite2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011"
uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472"
version = "1.3.14+0"

[[Grisu]]
git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2"
uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe"
version = "1.0.2"

[[HTTP]]
deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"]
git-tree-sha1 = "0fa77022fe4b511826b39c894c90daf5fce3334a"
uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
version = "0.9.17"

[[HarfBuzz_jll]]
deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"]
git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3"
uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566"
version = "2.8.1+1"

[[Hyperscript]]
deps = ["Test"]
git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9"
uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91"
version = "0.0.4"

[[HypertextLiteral]]
git-tree-sha1 = "2b078b5a615c6c0396c77810d92ee8c6f470d238"
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
version = "0.9.3"

[[IOCapture]]
deps = ["Logging", "Random"]
git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.2.2"

[[IniFile]]
deps = ["Test"]
git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8"
uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
version = "0.5.0"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[InverseFunctions]]
deps = ["Test"]
git-tree-sha1 = "a7254c0acd8e62f1ac75ad24d5db43f5f19f3c65"
uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
version = "0.1.2"

[[IrrationalConstants]]
git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151"
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
version = "0.1.1"

[[IterTools]]
git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18"
uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
version = "1.3.0"

[[IteratorInterfaceExtensions]]
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"

[[JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.3.0"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.2"

[[JpegTurbo_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "d735490ac75c5cb9f1b00d8b5509c11984dc6943"
uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8"
version = "2.1.0+0"

[[LAME_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c"
uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d"
version = "3.100.1+0"

[[LZO_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6"
uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac"
version = "2.10.1+0"

[[LaTeXStrings]]
git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996"
uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
version = "1.3.0"

[[Latexify]]
deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"]
git-tree-sha1 = "a8f4f279b6fa3c3c4f1adadd78a621b13a506bce"
uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316"
version = "0.15.9"

[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[Libffi_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290"
uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490"
version = "3.2.2+1"

[[Libgcrypt_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"]
git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae"
uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4"
version = "1.8.7+0"

[[Libglvnd_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"]
git-tree-sha1 = "7739f837d6447403596a75d19ed01fd08d6f56bf"
uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29"
version = "1.3.0+3"

[[Libgpg_error_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9"
uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8"
version = "1.42.0+0"

[[Libiconv_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778"
uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
version = "1.16.1+1"

[[Libmount_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73"
uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9"
version = "2.35.0+0"

[[Libtiff_jll]]
deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"]
git-tree-sha1 = "340e257aada13f95f98ee352d316c3bed37c8ab9"
uuid = "89763e89-9b03-5906-acba-b20f662cd828"
version = "4.3.0+0"

[[Libuuid_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066"
uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700"
version = "2.36.0+0"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[LogExpFunctions]]
deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
git-tree-sha1 = "be9eef9f9d78cecb6f262f3c10da151a6c5ab827"
uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
version = "0.3.5"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.9"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MbedTLS]]
deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"]
git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe"
uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
version = "1.0.3"

[[MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[Measures]]
git-tree-sha1 = "e498ddeee6f9fdb4551ce855a46f54dbd900245f"
uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e"
version = "0.3.1"

[[Missings]]
deps = ["DataAPI"]
git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f"
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
version = "1.0.2"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[NaNMath]]
git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "0.3.5"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[Ogg_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f"
uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051"
version = "1.3.5+0"

[[OpenLibm_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "05823500-19ac-5b8b-9628-191a04bc5112"

[[OpenSSL_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a"
uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
version = "1.1.10+0"

[[OpenSpecFun_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
version = "0.5.5+0"

[[Opus_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720"
uuid = "91d4177d-7536-5919-b921-800302f37372"
version = "1.3.2+0"

[[OrderedCollections]]
git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.4.1"

[[PCRE_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "b2a7af664e098055a7529ad1a900ded962bca488"
uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc"
version = "8.44.0+0"

[[PDMats]]
deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
git-tree-sha1 = "ee26b350276c51697c9c2d88a072b339f9f03d73"
uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
version = "0.11.5"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "ae4bbcadb2906ccc085cf52ac286dc1377dceccc"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.1.2"

[[Pixman_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "b4f5d02549a10e20780a24fce72bea96b6329e29"
uuid = "30392449-352a-5448-841d-b1acce4e97dc"
version = "0.40.1+0"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[PlotThemes]]
deps = ["PlotUtils", "Requires", "Statistics"]
git-tree-sha1 = "a3a964ce9dc7898193536002a6dd892b1b5a6f1d"
uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a"
version = "2.0.1"

[[PlotUtils]]
deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"]
git-tree-sha1 = "b084324b4af5a438cd63619fd006614b3b20b87b"
uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043"
version = "1.0.15"

[[Plots]]
deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun"]
git-tree-sha1 = "0d185e8c33401084cab546a756b387b15f76720c"
uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
version = "1.23.6"

[[PlutoUI]]
deps = ["AbstractPlutoDingetjes", "Base64", "Dates", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"]
git-tree-sha1 = "1e0cb51e0ccef0afc01aab41dc51a3e7f781e8cb"
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
version = "0.7.20"

[[Preferences]]
deps = ["TOML"]
git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.2"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[Qt5Base_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"]
git-tree-sha1 = "ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8"
uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1"
version = "5.15.3+0"

[[QuadGK]]
deps = ["DataStructures", "LinearAlgebra"]
git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39"
uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
version = "2.4.2"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[RecipesBase]]
git-tree-sha1 = "a4425fe1cde746e278fa895cc69e3113cb2614f6"
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
version = "1.2.0"

[[RecipesPipeline]]
deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"]
git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8"
uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c"
version = "0.4.1"

[[Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.1.3"

[[Rmath]]
deps = ["Random", "Rmath_jll"]
git-tree-sha1 = "bf3188feca147ce108c76ad82c2792c57abe7b1f"
uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
version = "0.7.0"

[[Rmath_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "68db32dff12bb6127bac73c209881191bf0efbb7"
uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
version = "0.3.0+0"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Scratch]]
deps = ["Dates"]
git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda"
uuid = "6c6a2e73-6563-6170-7368-637461726353"
version = "1.1.0"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[SharedArrays]]
deps = ["Distributed", "Mmap", "Random", "Serialization"]
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"

[[Showoff]]
deps = ["Dates", "Grisu"]
git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de"
uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f"
version = "1.0.3"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SortingAlgorithms]]
deps = ["DataStructures"]
git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508"
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
version = "1.0.1"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[SpecialFunctions]]
deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
git-tree-sha1 = "f0bccf98e16759818ffc5d97ac3ebf87eb950150"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "1.8.1"

[[StaticArrays]]
deps = ["LinearAlgebra", "Random", "Statistics"]
git-tree-sha1 = "3c76dde64d03699e074ac02eb2e8ba8254d428da"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "1.2.13"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[StatsAPI]]
git-tree-sha1 = "0f2aa8e32d511f758a2ce49208181f7733a0936a"
uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
version = "1.1.0"

[[StatsBase]]
deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
git-tree-sha1 = "2bb0cb32026a66037360606510fca5984ccc6b75"
uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
version = "0.33.13"

[[StatsFuns]]
deps = ["ChainRulesCore", "InverseFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"]
git-tree-sha1 = "bedb3e17cc1d94ce0e6e66d3afa47157978ba404"
uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
version = "0.9.14"

[[StructArrays]]
deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"]
git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3"
uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a"
version = "0.6.3"

[[SuiteSparse]]
deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[TableTraits]]
deps = ["IteratorInterfaceExtensions"]
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
version = "1.0.1"

[[Tables]]
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
git-tree-sha1 = "fed34d0e71b91734bf0a7e10eb1bb05296ddbcd0"
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
version = "1.6.0"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[URIs]]
git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355"
uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
version = "1.3.0"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[UnicodeFun]]
deps = ["REPL"]
git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf"
uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1"
version = "0.4.1"

[[Wayland_jll]]
deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"]
git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23"
uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89"
version = "1.19.0+0"

[[Wayland_protocols_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "66d72dc6fcc86352f01676e8f0f698562e60510f"
uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91"
version = "1.23.0+0"

[[XML2_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"]
git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a"
uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a"
version = "2.9.12+0"

[[XSLT_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"]
git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a"
uuid = "aed1982a-8fda-507f-9586-7b0439959a61"
version = "1.1.34+0"

[[Xorg_libX11_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"]
git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527"
uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc"
version = "1.6.9+4"

[[Xorg_libXau_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e"
uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec"
version = "1.0.9+4"

[[Xorg_libXcursor_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"]
git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd"
uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724"
version = "1.2.0+4"

[[Xorg_libXdmcp_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4"
uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05"
version = "1.1.3+4"

[[Xorg_libXext_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3"
uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3"
version = "1.3.4+4"

[[Xorg_libXfixes_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4"
uuid = "d091e8ba-531a-589c-9de9-94069b037ed8"
version = "5.0.3+4"

[[Xorg_libXi_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"]
git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246"
uuid = "a51aa0fd-4e3c-5386-b890-e753decda492"
version = "1.7.10+4"

[[Xorg_libXinerama_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"]
git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123"
uuid = "d1454406-59df-5ea1-beac-c340f2130bc3"
version = "1.1.4+4"

[[Xorg_libXrandr_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"]
git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631"
uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484"
version = "1.5.2+4"

[[Xorg_libXrender_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96"
uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa"
version = "0.9.10+4"

[[Xorg_libpthread_stubs_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb"
uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74"
version = "0.1.0+3"

[[Xorg_libxcb_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"]
git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6"
uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b"
version = "1.13.0+3"

[[Xorg_libxkbfile_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"]
git-tree-sha1 = "926af861744212db0eb001d9e40b5d16292080b2"
uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a"
version = "1.1.0+4"

[[Xorg_xcb_util_image_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97"
uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b"
version = "0.4.0+1"

[[Xorg_xcb_util_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"]
git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1"
uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5"
version = "0.4.0+1"

[[Xorg_xcb_util_keysyms_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00"
uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7"
version = "0.4.0+1"

[[Xorg_xcb_util_renderutil_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e"
uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e"
version = "0.3.9+1"

[[Xorg_xcb_util_wm_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"]
git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67"
uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361"
version = "0.4.1+1"

[[Xorg_xkbcomp_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxkbfile_jll"]
git-tree-sha1 = "4bcbf660f6c2e714f87e960a171b119d06ee163b"
uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4"
version = "1.4.2+4"

[[Xorg_xkeyboard_config_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xkbcomp_jll"]
git-tree-sha1 = "5c8424f8a67c3f2209646d4425f3d415fee5931d"
uuid = "33bec58e-1273-512f-9401-5d533626f822"
version = "2.27.0+4"

[[Xorg_xtrans_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845"
uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10"
version = "1.4.0+3"

[[Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[Zstd_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6"
uuid = "3161d3a3-bdf6-5164-811a-617609db77b4"
version = "1.5.0+0"

[[libass_jll]]
deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47"
uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0"
version = "0.15.1+0"

[[libfdk_aac_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55"
uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280"
version = "2.0.2+0"

[[libpng_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]
git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c"
uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f"
version = "1.6.38+0"

[[libvorbis_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"]
git-tree-sha1 = "c45f4e40e7aafe9d086379e5578947ec8b95a8fb"
uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a"
version = "1.3.7+0"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"

[[x264_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2"
uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a"
version = "2021.5.5+0"

[[x265_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9"
uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76"
version = "3.5.0+0"

[[xkbcommon_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"]
git-tree-sha1 = "ece2350174195bb31de1a63bea3a41ae1aa593b6"
uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd"
version = "0.9.1+5"
"""

# ╔═╡ Cell order:
# ╟─7b93882c-9ad8-11ea-0288-0941e163f9d5
# ╟─7110fe00-83a8-4a9d-975e-d59d0f351630
# ╟─0a171174-2db6-4077-a4af-1da84c9f5470
# ╟─13f0fefb-a926-4689-9518-a5d77e0582d7
# ╟─26c379e4-8480-4e1e-ae8e-933eadfd69f3
# ╟─016bf804-ca81-4300-8d5b-1e063834d4d0
# ╟─6465a01e-d0e9-4670-b9af-05f7a7a6a200
# ╟─8db552a0-1a45-401c-a579-c3ffc47b27c9
# ╟─414cfce1-1bbf-4ae9-9f06-c4a4d55b7ad3
# ╟─a090b42b-ee45-4a63-a780-b63df6ac3d11
# ╟─7a0961e0-d332-492e-bbd2-e2127e65f1a4
# ╟─a9f04175-2d40-4cf6-be59-b24058a4b1ce
# ╟─461c107c-a40e-4fb9-a4ce-9d9c387be074
# ╟─afdd513d-6995-4801-9bac-59d4a2287322
# ╟─3f467d2d-5053-4e1e-970b-8c61152f5757
# ╟─2dfee269-4539-4f96-afca-4e975c2fd5f0
# ╟─9a7b43ec-5d43-43c5-aa35-b776de4a48c7
# ╟─39e69cb1-207a-42b6-920a-1912d50edb4e
# ╟─5bd1d628-beaa-47c8-98a6-f8eb36444e93
# ╟─684b5e7b-6de1-42af-9497-28a957aa73ad
# ╟─422e4766-7a75-4403-a189-fbf89d41a8f0
# ╟─5578a9d5-7d7b-4582-91e8-7dab5ba2a572
# ╟─534e1a70-62f4-41fa-bd03-86bb9e4bf495
# ╟─b9b80541-571c-4401-b9c9-1ae7fe95beba
# ╟─9b63c009-7984-49a6-be7d-5ba52c78a855
# ╟─bd4dbdc8-84d0-48f8-b4fb-69805b14aaff
# ╟─32b13ec7-b976-494a-bfff-2efedfe16ea4
# ╟─cf56ff3d-d9fd-4222-997f-750896ac0440
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


using Distributions

# Definitions of simulated environments
# and interaction of learners with the environment

# For the Categorical task (Section 2.3.2)
mutable struct EnvMultinomial
    ns::Int
    state::Int
    stochasticity::Float64
    changeprobability::Float64
    trans_probs::Array{Float64, 1}
    switchflag::Bool
    seed::Any
    rng::MersenneTwister # Used only for switches!
    nsteps::Int
end
function EnvMultinomial(; ns = 10, state = 1, stochasticity = 0.1,
                        changeprobability = .001, seed = 3,
                        nsteps = 10^5)
    rng = MersenneTwister(seed)
    trans_probs = rand(ENV_RNG, Dirichlet(ns, stochasticity))
    EnvMultinomial(ns, state, stochasticity, changeprobability,
                trans_probs, false, seed, rng, nsteps)
end
export EnvMultinomial

function interact!(env::EnvMultinomial)
    env.switchflag = false
    r = rand(env.rng)
     # @show r
    if r < env.changeprobability # Switch or not!
        #println("Switch!")
        env.trans_probs = rand(env.rng, Dirichlet(env.ns, env.stochasticity))
        env.switchflag = true
    end
    env.state = wsample(ENV_RNG, env.trans_probs)
end

# For the Gaussian task (Section 2.3.1)
mutable struct EnvGaussian
    mu_0::Float64
    sigma_0::Float64 # std of prior
    sigma::Float64 # std of observations
    mu_n::Float64
    state::Float64
    changeprobability::Float64
    switchflag::Bool
    seed::Any
    rng::MersenneTwister # Used only for switches!
    nsteps::Int
end
function EnvGaussian(; mu_0 = 0., sigma_0 = 1., sigma = 1.,
                        changeprobability = .001,
                        seed = 3,
                        nsteps = 10^5)
    rng = MersenneTwister(seed)
    mu_n = rand(ENV_RNG, Normal(mu_0, sigma_0))
    state = rand(ENV_RNG, Normal(mu_n, sigma))
    EnvGaussian(mu_0, sigma_0, sigma, mu_n, state, changeprobability,
                false, seed, rng, nsteps)
end
export EnvGaussian

function interact!(env::EnvGaussian)
    env.switchflag = false
    r = rand(env.rng)
     # @show r
    if r < env.changeprobability # Switch or not!
        # println("Switch!")
        env.mu_n = rand(env.rng, Normal(env.mu_0, env.sigma_0))
        env.switchflag = true
    end
    env.state = rand(ENV_RNG, Normal(env.mu_n, env.sigma))
end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    ExponentialVariogram(range=r, sill=s, nugget=n)
    ExponentialVariogram(ball; sill=s, nugget=n)

An exponential variogram with range `r`, sill `s` and nugget `n`.
Optionally, use a custom metric `ball`.
"""
struct ExponentialVariogram{T,B} <: Variogram
  sill::T
  nugget::T
  ball::B
end

ExponentialVariogram(ball; sill=1.0, nugget=0.0) =
  ExponentialVariogram(sill, nugget, ball)

ExponentialVariogram(; range=1.0, sill=1.0, nugget=0.0) =
  ExponentialVariogram(sill, nugget, MetricBall(range))

function (γ::ExponentialVariogram)(h)
  r = radius(γ.ball)
  s = γ.sill
  n = γ.nugget
  (s - n) * (1 - exp(-3(h/r))) + (h > 0) * n
end

isstationary(::Type{<:ExponentialVariogram}) = true


@registerN  Charlier AbstractCDOP1 μ
export  Charlier

"""
    Charlier{μ}

References: [Koekoek and Swarttouw §1.12](https://arxiv.org/pdf/math/9602214.pdf)
"""
Charlier

abcde(::Type{<:Charlier{μ}})  where {μ} = NamedTuple{(:a,:b,:c,:d,:e)}((0,1,0,-1,μ))

basis_symbol(::Type{<:Charlier{μ}}) where {μ} = "cᵘ"
Polynomials.domain(::Type{<:Charlier{μ}}) where {μ} = Polynomials.Interval(0, Inf)
weight_function(::Type{<:Charlier{μ}}) where {μ} = x  ->  μ^x/gamma(1+x)

function k1k0(P::Type{<:Charlier{μ}}, n::Int)  where {μ}
    -one(eltype(P))/μ
end

function classical_hypergeometric(P::Type{<:Charlier{μ}}, n::Int, x) where {μ}
    pFq((-n,-x),(),-1/μ)
end


export softmax,
    softmax!,
    ∇softmax,
    ∇softmax!,
    logsoftmax,
    logsoftmax!,
    ∇logsoftmax,
    ∇logsoftmax!,
    logsumexp

"""
    softmax(x; dims = 1)

[Softmax](https://en.wikipedia.org/wiki/Softmax_function) turns input array `x`
into probability distributions that sum to 1 along the dimensions specified by `dims`.
It is semantically equivalent to the following:

    softmax(x; dims = 1) = exp.(x) ./ sum(exp.(x), dims = dims)

with additional manipulations enhancing numerical stability.

For a matrix input `x` it will by default (`dims = 1`) treat it as a batch of vectors,
with each column independent. Keyword `dims = 2` will instead treat rows independently, and so on.

See also [`logsoftmax`](@ref).

# Examples

```jldoctest
julia> softmax([1, 2, 3])
3-element Vector{Float64}:
 0.09003057317038046
 0.24472847105479764
 0.6652409557748218

julia> softmax([1 2 3; 2 2 2])  # dims=1
2×3 Matrix{Float64}:
 0.268941  0.5  0.731059
 0.731059  0.5  0.268941

julia> softmax([1 2 3; 2 2 2]; dims=2)
2×3 Matrix{Float64}:
 0.0900306  0.244728  0.665241
 0.333333   0.333333  0.333333
```
"""
softmax(x; dims = 1) = softmax!(similar(x, (float ∘ eltype)(x)), x; dims = dims)

softmax!(x; dims = 1) = softmax!(x, x; dims = dims)

function softmax!(out::AbstractArray{T}, x::AbstractArray; dims = 1) where {T}
    max_ = maximum(x; dims = dims)
    if all(isfinite, max_)
        out .= exp.(x .- max_)
    else
        @. out = ifelse(isequal(max_,Inf), ifelse(isequal(x,Inf), 1, 0), exp(x - max_))
    end
    out ./= sum(out; dims = dims)  # could re-use max_ when dims != (:) and eltype(x) == T.
end

∇softmax(Δ::AbstractArray{T}, x::AbstractArray, y::AbstractArray{S}; dims = 1) where {T,S} =
    ∇softmax!(similar(y, promote_type(T, S)), Δ, x, y; dims = dims)
∇softmax(Δ, x, y; dims = 1) = ∇softmax(unthunk(Δ), x, y, dims = dims)

# Can introduce at the end of deprecation cycle of ∇softmax!(out, Δ, x; dims = 1)
# ∇softmax!(Δ, x, y; dims = 1) = ∇softmax!(Δ, Δ, x, y; dims = dims)

function ∇softmax!(out::AbstractArray, Δ::AbstractArray,
                    x::AbstractArray, y::AbstractArray; dims = 1)
    out .= Δ .* y
    out .= out .- y .* sum(out; dims = dims)
end

# Old 2-arg version recomputing forward
∇softmax(Δ, x; dims = 1) = ∇softmax(Δ, x, softmax(x, dims = dims); dims = dims)
∇softmax!(Δ, x; dims = 1) = ∇softmax!(Δ, Δ, x, softmax(x, dims = dims); dims = dims)
∇softmax!(out, Δ, x; dims = 1) = ∇softmax!(out, Δ, x, softmax(x, dims = dims); dims = dims)

function rrule(::typeof(softmax), xs; dims=1)
    y = softmax(xs; dims=dims)
    softmax_pullback(Δ) = (NoTangent(), ∇softmax(unthunk(Δ), xs, y, dims = dims))
    return y, softmax_pullback
end

"""
    logsoftmax(x; dims = 1)

Computes the log of softmax in a more numerically stable
way than directly taking `log.(softmax(xs))`. Commonly used in
computing cross entropy loss.

It is semantically equivalent to the following:

    logsoftmax(x; dims = 1) = x .- log.(sum(exp.(x), dims = dims))

See also [`softmax`](@ref).
"""
logsoftmax(x; dims = 1) = logsoftmax!(similar(x, (float ∘ eltype)(x)), x; dims = dims)

logsoftmax!(x; dims = 1) = logsoftmax!(x, x; dims = dims)

function logsoftmax!(out::AbstractArray{T}, x::AbstractArray; dims = 1) where {T}
    max_ = maximum(x; dims = dims)
    if all(isfinite, max_)
        out .= x .- max_
    else
        @. out = ifelse(isequal(max_,Inf), ifelse(isequal(x,Inf), 0, -Inf), x - max_)
    end
    log_ = log.(sum(exp, out; dims = dims))
    out .-= log_
end

∇logsoftmax(Δ::AbstractArray{T}, x::AbstractArray, y::AbstractArray{S}; dims = 1) where {T,S} =
    ∇logsoftmax!(similar(y, promote_type(T, S)), Δ, x, y; dims = dims)
∇logsoftmax(Δ, x, y; dims = 1)  = ∇logsoftmax(unthunk(Δ), x, y, dims = dims)

# Old 2-arg version recomputing forward
∇logsoftmax(Δ, x; dims = 1) = ∇logsoftmax(Δ, x, logsoftmax(x, dims = dims); dims = dims)
∇logsoftmax!(Δ, x; dims = 1) = ∇logsoftmax!(Δ, Δ, x, logsoftmax(x, dims = dims); dims = dims)
∇logsoftmax!(out, Δ, x; dims = 1) = ∇logsoftmax!(out, Δ, x, logsoftmax(x, dims = dims); dims = dims)

function ∇logsoftmax!(out::AbstractArray, Δ::AbstractArray,
                    x::AbstractArray, y::AbstractArray; dims = 1)
    out .= Δ .- sum(Δ, dims = dims) .* exp.(y)
end

function rrule(::typeof(logsoftmax), xs; dims=1)
    y = logsoftmax(xs; dims=dims)
    logsoftmax_pullback(Δ) = (NoTangent(), ∇logsoftmax(unthunk(Δ), xs, y, dims = dims))
    return y, logsoftmax_pullback
end

"""
    logsumexp(x; dims = :)

Computes `log.(sum(exp.(x); dims = dims))` in a numerically stable
way.

See also [`logsoftmax`](@ref).
"""
function logsumexp(x::AbstractArray; dims = :)
    max_ = maximum(x; dims = dims)
    max_ .+ log.(sum(exp.(x .- max_); dims = dims))
end

# Informative error message if any of the softmax variants is called with a number
for f in (:softmax, :logsoftmax, :softmax!, :logsoftmax!, :logsumexp)
    @eval $(f)(x::Number, args...) =
      error("`", $(string(f)), "(x)` called with a number, but it expects an array. Usually this is because a layer like `Dense(3,4,softmax)` is broadcasting it like an activation function; `softmax` needs to be outside the layer.")
end


function weighted_median{T<:Real,W<:Real}(data::Array{T,1}, weights::Array{W,1};
                                          checknan::Bool=true)
    isempty(data) && error("median of an empty array is undefined")
    if length(data) != length(weights)
        error("data and weight vectors must be the same size")
    end
    if checknan
        for (i, x) in enumerate(data)
            (isnan(x) || isnan(weights[i])) && return NaN
        end
    end
    mask = weights .!= 0
    if any(mask)
        data = data[mask]
        weights = weights[mask]
        maxval, maxind = findmax(weights)
        midpoint = 0.5 * sum(weights)
        if maxval > midpoint
            data[maxind]
        else
            permute = sortperm(data)
            cumulative_weight = zero(eltype(weights))
            i = 0
            for (i, p) in enumerate(permute)
                if cumulative_weight > midpoint
                    cumulative_weight -= weights[p]
                    break
                end
                cumulative_weight += weights[p]
            end
            if cumulative_weight == midpoint
                middle(data[permute[i-2]], data[permute[i-1]])
            else
                middle(data[permute[i-1]])
            end
        end
    else
        error("No positive weights found")
    end
end


##################################################
# Computation of the local velocity deflections
##################################################

### Todo
# change vmax by wmax
# for coherence with notes and paper

"""
    _vmax(r,vr,vt,sinth,costh,cosph)

Computes the maximum allowed velocity in Rosenbluth integrals, for a test star at position `r` and velocity `(vr,vt)`.
At fixed integration variable (over field) `(θ,φ)` such that `sin(θ)=sinth`, `cos(θ)=costh` and `cos(φ)=cosph`.
Returns false if no such vmax exists (i.e. non-positive discriminant).

# Remarks
- Integrands in Rosenbluth integrals vanishes for v > _vmax.

# Arguments
- `r::Float64`: Radial position of the test star.
- `vr::Float64`: Radial velocity of the test star.
- `vt::Float64`: Tangential velocity of the test star.
- `sinth::Float64`: Value `sin(θ)=sinth` corresponding to `θ` integration variable.
- `costh::Float64`: Value `cos(θ)=costh` corresponding to `θ` integration variable.
- `cosph::Float64`: Value `cos(φ)=cosph` corresponding to `φ` integration variable.
"""
function _vmax(r::Float64, vr::Float64, vt::Float64,
    sinth::Float64,costh::Float64,cosph::Float64)

    vSq = vr^2 + vt^2
    deltav = (vr*costh+vt*sinth*cosph)^2 - (2*psi(r)+vSq)

    if (deltav > 0.0)
        return (vr*costh+vt*sinth*cosph) + sqrt(deltav)
    else
        return false # in order to test whether the discriminant is positive or not
    end

end

"""
    _EaLa(r,var2,vat2)

Computes the energy and angular momentum used in Rosenbluth integrands.

# Arguments
- `r::Float64`: Radial position of the test star.
- `var2::Float64`: Square radial velocity in Rosenbluth integrand.
- `vat2::Float64`: Square tangential velocity in Rosenbluth integrand.
"""
function _EaLa(r::Float64, var2::Float64, vat2::Float64)
    Ea = psi(r) + (1/2)*(var2+vat2)
    La = r*sqrt(vat2)
    return Ea, La
end

"""
    _fa(r,vr,vt,vp,snth,csth,snph,csph)

Distribution function used in Rosenbluth integrand

# Arguments
- `r::Float64`: Radial position of the test star.
- `vr::Float64`: Radial velocity of the test star.
- `vt::Float64`: Tangential velocity of the test star.
- `vp::Float64`: Velocity integration variable.
- `snth::Float64`: Value `sin(θ)=snth` corresponding to `θ` integration variable.
- `csth::Float64`: Value `cos(θ)=csth` corresponding to `θ` integration variable.
- `snph::Float64`: Value `sin(φ)=snph` corresponding to `φ` integration variable.
- `csph::Float64`: Value `cos(φ)=csph` corresponding to `φ` integration variable.
"""
function _fa(r::Float64, vr::Float64, vt::Float64, vp::Float64,
                snth::Float64,csth::Float64,snph::Float64,csph::Float64)

    pref_r = vr-vp*csth
    pref_t = vt-vp*snth*csph
    pref_t2 = vp*snth*snph

    var2 = pref_r^2
    vat2 = pref_t^2+pref_t2^2
    Ea, La = _EaLa(r,var2,vat2)
    DF = _F(Ea,La)

    return DF
end

"""
    RosenbluthPotentials(r,vr,vt,[nbK=nbK_default])

Computation of the Rosenbluth potential and its derivatives.
Returns a tuple `(dh/dvr, dh/dvt, dg/dvt, d2g/dvr2, d2g/dvrdvt, d2g/dvt2)`.

# Arguments
- `r::Float64`: Radial position of the test star.
- `vr::Float64`: Radial velocity of the test star.
- `vt::Float64`: Tangential velocity of the test star.
- `nbK::Int64`: Sampling number of Rosenbluth integrals.
"""
function RosenbluthPotentials(r::Float64, vr::Float64, vt::Float64, nbK::Int64=nbK_default)
    #X = (v',x,phi)

    sumhr = 0
    sumht = 0
    sumgt = 0
    sumh = 0
    sumgrrInt = 0
    sumgrt = 0
    sumgttInt = 0
    for ith=1:nbK
        thtemp = (2*ith-1)/(2*nbK)
        sinthtemp,costhtemp = sincos(PI*thtemp)
        sumhr_ph = 0
        sumht_ph = 0
        sumgt_ph = 0
        sumh_ph = 0
        sumgrrInt_ph = 0
        sumgrt_ph = 0
        sumgttInt_ph = 0

        for iph=1:nbK
            phtemp = (2*iph-1)/(2*nbK)
            sinphtemp, cosphtemp = sincos(2*PI*phtemp)
            vmax = _vmax(r,vr,vt,sinthtemp,costhtemp,cosphtemp)

            if (vmax != false)
                # if discriminant > 0 (i.e there exists an acceptable w-range where E'<0)
                # this is always the case for bound stars

                if (vmax > 0)
                    # if acceptable range has some part where w>0
                    # always the case for bounds stars
                    # happens for some unbound stars (in that case, one can optimize this code as vmin>0)

                    sumhr_v = 0
                    sumht_v = 0
                    sumgt_v = 0
                    sumh_v = 0
                    sumgrrInt_v = 0
                    sumgrt_v = 0
                    sumgttInt_v = 0

                    for iv=1:nbK
                        vtemp = (2*iv-1)/(2*nbK)
                        floc = _fa(r,vr,vt,vmax*vtemp,sinthtemp,costhtemp,sinphtemp,cosphtemp)
                        sumhr_v += -floc
                        sumht_v += -floc
                        sumgt_v += vtemp^2*floc

                        # compute h and the other integrals separately
                        sumh_v += vtemp*floc
                        sumgrrInt_v += -vtemp*floc
                        sumgrt_v += -vtemp*floc
                        sumgttInt_v += -vtemp*floc
                    end
                    sumhr_v *= vmax
                    sumht_v *= vmax
                    sumgt_v *= vmax^3
                    sumh_v *= vmax^2
                    sumgrrInt_v *= vmax^2
                    sumgrt_v *= vmax^2
                    sumgttInt_v *= vmax^2

                    sumhr_ph += sumhr_v
                    sumht_ph += sumht_v*cosphtemp
                    sumgt_ph += sumgt_v*cosphtemp
                    sumh_ph += sumh_v
                    sumgrrInt_ph += sumgrrInt_v
                    sumgrt_ph += sumgrt_v*cosphtemp
                    sumgttInt_ph += sumgttInt_v*cosphtemp^2
                end

                # when (vmax <= 0), there is no positive w-range, hence w-integral yields 0
                # sometimes happens for unbound stars
                # in that case, add nothing

            end

            # if discriminant < 0 (i.e for all w, E'>=0)
            # sometimes happens for unbound stars
            # hence w-integral yields 0
            # in that case, add nothing

        end
        sumhr += sinthtemp*costhtemp*sumhr_ph
        sumht += sinthtemp^2*sumht_ph
        sumgt += sinthtemp^2*sumgt_ph
        sumh += sinthtemp*sumh_ph
        sumgrrInt += sinthtemp*costhtemp^2*sumgrrInt_ph
        sumgrt += sinthtemp^2*costhtemp*sumgrt_ph
        sumgttInt += sinthtemp^3*sumgttInt_ph
    end
    sumhr *= PI*2*PI/(nbK^3)
    sumht *= PI*2*PI/(nbK^3)
    sumgt *= PI*2*PI/(nbK^3)
    sumh *= PI*2*PI/(nbK^3)
    sumgrrInt *= PI*2*PI/(nbK^3)
    sumgrt *= PI*2*PI/(nbK^3)
    sumgttInt *= PI*2*PI/(nbK^3)

    return sumhr, sumht, sumgt, sumh+sumgrrInt, sumgrt, sumh+sumgttInt
end

"""
    localVelChange(r,vr,vt,m_field,[nbK=nbK_default,m_test=m_field])

Computation of the local velocity deflections along and perpendicular to the trajectory of the test star.
Returns a tuple `(dvPar, dvPar2, dvPerp2)`.

# Arguments
- `r::Float64`: Radial position of the test star.
- `vr::Float64`: Radial velocity of the test star.
- `vt::Float64`: Tangential velocity of the test star.
- `m_field::Float64`: Mass of field star.
- `nbK::Int64`: Sampling number of Rosenbluth integrals.
- `m_test::Float64`: Mass of test star.
"""
function localVelChange(r::Float64, vr::Float64, vt::Float64,
                        m_field::Float64, nbK::Int64=nbK_default,
                        m_test::Float64=m_field)

    dhdvr, dhdvt, dgdvt, d2gdvr2, d2gdvrdvt, d2gdvt2 = RosenbluthPotentials(r,vr,vt,nbK)

    cst       = 4.0*PI*_G^2*logCoulomb
    v         = sqrt(vr^2+vt^2)

    if (vt>0)
        vr_v   = vr/v
        vt_v   = vt/v

        dvPar   = cst*(m_field+m_test) *(vr_v*dhdvr+vt_v*dhdvt)
        dvPar2  = cst* m_field         *(vr_v^2*d2gdvr2+(2*vr_v*vt_v)*d2gdvrdvt
                                        +vt_v^2*d2gdvt2)
        dvPerp2 = cst* m_field         *(vt_v^2*d2gdvr2-(2*vr_v*vt_v)*d2gdvrdvt
                                        +vr_v^2*d2gdvt2+(1/vt)*dgdvt)
    else # vt = 0

        dvPar   = cst*(m_field+m_test) *dhdvr
        dvPar2  = cst* m_field         *d2gdvr2
        dvPerp2 = cst* m_field         *2*d2gdvt2

    end

    return dvPar, dvPar2, dvPerp2
end

"""
    localOrbitChange(r,E,L,m_field,[nbK=nbK_default,m_test=m_field])

Computation of the local energy and angular momentum deflections.
Returns a tuple `(dE, dE2, dL, dEdL, dL2)`.

# Arguments
- `r::Float64`: Radial position of the test star.
- `E::Float64`: Energy of the test star.
- `L::Float64`: Angular momentum of the test star.
- `m_field::Float64`: Mass of field star.
- `nbK::Int64`: Sampling number of Rosenbluth integrals.
- `m_test::Float64`: Mass of test star.
"""
function localOrbitChange(r::Float64, E::Float64, L::Float64,
                           m_field::Float64, nbK::Int64=nbK_default,
                           m_test::Float64=m_field)

    vr = radialVelocity(r,E,L)
    vt = tangentVelocity(r,E,L)
    v = sqrt(vr^2+vt^2)

    dvPar, dvPar2, dvPerp2 = localVelChange(r,vr,vt,m_field,nbK,m_test)

    dE   = dvPar*(v)   + dvPar2*(1/2)     + dvPerp2*(1/2)
    dE2  =               dvPar2*(v^2)
    dL   = dvPar*(L/v)                    + dvPerp2*(r^2/(4.0*L))
    dL2  =               dvPar2*(L^2/v^2) + dvPerp2*(1/2 * (r^2 - L^2/v^2))
    dEdL =               dvPar2*(L)

    return dE, dE2, dL, dEdL, dL2
end


using Test
using Clp
using LinearAlgebra
using SteinDiscrepancy:
    SteinInverseMultiquadricKernel,
    SteinRectangularDomainMetaKernel,
    stein_discrepancy,
    ksd,
    gsd,
    wassersteindiscrete

# import data
include("data/test_data.jl")

function uniform_gradlogp(x::Array{Float64,1})
    zeros(size(x))
end

function gaussian_gradlogp(x::Array{Float64,1})
    -x
end
# define the covariance terms for the Riemannian-Langevin diffusion
function volatility_covariance(x::Array{Float64,1})
    p = length(x)
    sqrt(1 + sum(x.^2)) * I
end
function grad_volatility_covariance(x::Array{Float64,1})
    x / sqrt(1 + sum(x.^2))
end

imqkernel = SteinInverseMultiquadricKernel()
imqrectkernel = SteinRectangularDomainMetaKernel(imqkernel, [0., 0.], [1., 1.])
solver = Clp.ClpSolver(LogLevel=4)

# Graph Stein discrepancy bounded test
@testset "Univariate Graph discrepancy test" begin
    res = gsd(points=UNIFORM_TESTDATA[:,1],
              gradlogdensity=uniform_gradlogp,
              solver=solver,
              supportlowerbounds=[0.0],
              supportupperbounds=[1.0])

    @test res.objectivevalue[1] ≈ 0.15 atol=1.0e-5
end
@testset "Multivariate Graph discrepancy test" begin
    res = stein_discrepancy(points=UNIFORM_TESTDATA,
                            gradlogdensity=uniform_gradlogp,
                            solver=solver,
                            method="graph",
                            supportlowerbounds=zeros(size(UNIFORM_TESTDATA,2)),
                            supportupperbounds=ones(size(UNIFORM_TESTDATA,2)))

    @test res.objectivevalue[1] ≈ 0.308 atol=1.0e-5
    @test res.objectivevalue[2] ≈ 0.31 atol=1.0e-5
end
@testset "Riemannian Graph discrepancy test" begin
    res = stein_discrepancy(points=GAUSSIAN_TESTDATA,
                            gradlogdensity=gaussian_gradlogp,
                            operator="riemannian-langevin",
                            solver=solver,
                            method="graph",
                            volatility_covariance=volatility_covariance,
                            grad_volatility_covariance=grad_volatility_covariance)

    @test res.objectivevalue[1] ≈ 6.19106 atol=1.0e-5
    @test res.objectivevalue[2] ≈ 7.05982 atol=1.0e-5
end

# Kernel Stein discrepancy test
@testset "Univariate Kernel discrepancy test" begin
    res = stein_discrepancy(points=GAUSSIAN_TESTDATA[:,1],
                            gradlogdensity=gaussian_gradlogp,
                            method="kernel",
                            kernel=imqkernel)

    @test res.discrepancy2 ≈ 0.269298 atol=1.0e-5
end
@testset "Multivariate Kernel discrepancy test" begin
    res = ksd(points=GAUSSIAN_TESTDATA,
              gradlogdensity=gaussian_gradlogp,
              kernel=imqkernel)

    @test res.discrepancy2 ≈ 0.807566 atol=1.0e-5
end

# Wassersteindiscrete test
@testset "Wasserstein bivariate discrete test" begin
    (emd, numnodes, numedges, status) =
        wassersteindiscrete(xpoints=GAUSSIAN_TESTDATA,
                            ypoints=UNIFORM_TESTDATA,
                            solver=solver)

    @test emd ≈ 1.56 atol=1e-5
    @test status == :Optimal
end

# Stein Meta Kernel test
@testset "Rectangular domain meta kernel test" begin
    res = ksd(points=UNIFORM_TESTDATA,
              gradlogdensity=uniform_gradlogp,
              kernel=imqrectkernel)

    @test res.discrepancy2 ≈ 0.0064761153 atol=1.0e-8
end


# -------------------------------------------------------------------------------------------------------------------------------------------------------------
# Computations
# -------------------------------------------------------------------------------------------------------------------------------------------------------------

using DataFrames, CSV, XLSX, LinearAlgebra, Statistics, StatFiles

dir = "X:/VIVES/1-Personal/Florian/git/OECD_ICIO/src/"
dir_raw = "C:/Users/u0148308/Dropbox/STORE/3. GVCs/tasks/task4_MRIO_to_WIOD/output/" # location of raw data

# other scripts
include(dir * "functions.jl") # Script with functions to import and transform raw data

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

N = 43 + 3 # 43 countries + 3 regions of BE
S = 55 # industries

# Notes:
#   - import data in long format and reshape => far quicker!

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Intermediate demand
df = DataFrame(StatFiles.load(dir_raw * "MRIO_WIOT_isic4_" * "IG" * ".dta"))
sort!(df)
df.ID = df.use_country .* "_" .* df.use_isic
df = unstack(df[:, Not([:use_country, :use_isic])], :ID, :value)
Z = Matrix(df[:, 3:end]) # NS×NS

# Notes
#   - missing values in Z => transform to zeros instead
any(ismissing.(Z)) # check for missing values
Z .= ifelse.(ismissing.(Z), 0.0, Z)
Z = Float64.(Z)

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Final demand
df = DataFrame(StatFiles.load(dir_raw * "MRIO_WIOT_isic4_" * "FD" * ".dta"))
sort!(df)
df.ID = df.use_country .* "_" .* df.use_isic
df = unstack(df[:, Not([:use_country, :use_isic])], :ID, :value)
Y = Matrix(df[:, 3:end])

any(ismissing.(Y)) # check for missing values

Y = [sum(Y[i, (j-1)*4+1:j*4]) for i in 1:N*S, j in 1:N] # NS×N, sum final demand components

# Notes:
#   - some entries of Y are negative due to inventories
#       + inventory adjustment as in Antras and Chor (2018)?
any(Y .< 0.0) # check for negative values

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Value added
df = DataFrame(StatFiles.load(dir_raw * "MRIO_WIOT_isic4_" * "VA" * ".dta"))
sort!(df)
df = unstack(df, :supply_isic, :value)
transform!(df, [:TAX,:VA] => ByRow((t, v) -> t + v) => :NET)
W = Float64.(df.VA) # NS×1

# Notes:
#   - check wether we should use VA or NET?

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Gross output
X_1 = W .+ [sum(Z[:, j]) for j in 1:N*S] # NS×1, gross output computed from intermediate demand imports and VA
X_2 = [sum(Z[i, :]) + sum(Y[i, :]) for i in 1:N*S] # NS×1, gross output computed from intermediate and final demand exports

# Notes:
#   - X_1 and X_2 are pretty different
X = copy(X_2)

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Gross domestic product
GDP = [sum(W[(i-1)*S+1:i*S]) for i in 1:N] # N×1, sum of value added per country

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Domar weights
DW = X ./ repeat(GDP, inner=S) # NS×1
DW_WORLD = X ./ sum(GDP) # NS×1

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Technical coefficients
A = Z ./ repeat(X', N*S) # NS×NS, input coefficients
any(isnan.(A))
A .= ifelse.(isnan.(A), 0.0, A) # in case X = 0 (otherwise we cannot compute Leontief inverse since NaN in A)

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Leontief inverse
L = inv(I - A) # NS×NS, Leontief inverse
any(isnan.(L))
any(isinf.(L))

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Total requirements matrix
L * Z # NS×NS

# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# Country-industry final demand multipliers
FD_multi = [sum(L[(i-1)*S+1:i*S, j]) for i in 1:N, j in 1:N*S] # N×NS
FD_multi = reshape(FD_multi', (S, N*N)) # S×NN

# -------------------------------------------------------------------------------------------------------------------------------------------------------------


δ(j::Integer,k::Integer) = j == k ? 1 : 0

function ladderops(M,a,basis="Hermite")
    if basis=="Hermite"
        N=0:M-1
        X  = [δ(n,m+1)*√n + δ(n,m-1)*√(n+1) for n in N, m in N]a/√2
        Px = [δ(n,m+1)*√n - δ(n,m-1)*√(n+1) for n in N, m in N]im/(√2a)
        return X,Px
    else error("basis not implemented")
    end
end


using Revise
using AdFem
using LinearAlgebra
using PyPlot
using DelimitedFiles

mmesh = Mesh(8, 8 , 1/8)
DOF = zeros(Int64, 2mmesh.ndof)
for i = 1:mmesh.ndof
    DOF[i] = 2*i-1
    DOF[i+mmesh.ndof] = 2*i
end
u = readdlm("fenics/u.txt")[:]
u = u[DOF]
# u = ones(2mmesh.ndof)

μ = 10.0*ones(get_ngauss(mmesh))
λ = 3.0*ones(get_ngauss(mmesh))
ψ, J = neo_hookean(u, μ, λ, mmesh)

F = readdlm("fenics/F.txt")
S = readdlm("fenics/S.txt")
@info maximum(abs.(ψ - F[DOF]))
@info maximum(abs.(J - S[DOF, DOF]))

ψ, J = neo_hookean(constant(u), μ, λ, mmesh)
sess = Session(); init(sess)
@info maximum(abs.(run(sess, ψ) - F[DOF]))
@info maximum(abs.(run(sess, J) - S[DOF, DOF]))


function ACQR_em_s_run(y,nx::Int;max_iter::Int=100,tol::Float64=1e-6,txo::Bool=true)
	#=
	estimate A, C, Q, R, m1, P1 using the EM algorithm
	and the subspace algorithm for the starting point
	for model

	x_{t+1} = A*x_{t} + w_{t}
	y_{t}   = C*x_{t} + v_{t}

	cov(w_{t},v_{t}) = [Q 0;0 R]
	x1 -> N(m1,P1)

	javier.cara@upm.es, 2107-05

	=#

	# data as a matrix and by rows
	y,ny,nt = byrow(y)

	# starting values from the ssi algorithm
	i = nx+1
	Ai,Ci,Qi,Ri,Si = ACQRS_sub(y,nx,i)
	# x11 = C^{-1}(y_1 - v_1)
	m1i = Ci\y[:,1:1]
	P1i = zeros(nx,nx)

	# em
	A,C,Q,R,m1,P1,loglikv,aic = ACQR_em_s(y,Ai,Ci,Qi,Ri,m1i,max_iter,tol,txo)

	# output
	m = Dict("A"=>A,"C"=>C,"Q"=>Q,"R"=>R,"m1"=>m1,"P1"=>P1,
	"loglikv"=>loglikv[end],"aic"=>aic)
	return m

end


using Test
using GrammianAngularFields
using StatsBase

@testset "trigonometric identities" begin
    θ = 2/3*π
    @test sin(-θ) ≈ -sin(θ)
    @test cos(-θ) ≈ cos(θ)
    @test sin(2θ) ≈ 2*sin(θ)*cos(θ)
    @test cos(2θ) ≈ cos(θ)^2 - sin(θ)^2
end;

@testset "transformation checks" begin
    x = GrammianAngularFields.normalize(collect(1:10))
    @test StatsBase.mean(x) ≈ 0
    @test StatsBase.stdm(x, 0) ≈ 1
    x = GrammianAngularFields.standardize(collect(1:10))
    @test maximum(x) ≈ 1
    @test minimum(x) ≈ 0
    x = GrammianAngularFields.standardize(collect(1:10), 3, 5)
    @test maximum(x) ≈ 5
    @test minimum(x) ≈ 3
    x = GrammianAngularFields.grammify(collect(1:3))
    @test x ≈ [1.0 -1.83697e-16 -1.0; -1.83697e-16 -1.0 6.12323e-17; -1.0 6.12323e-17 1.0]
end;


using RCall

# R"require(OpenMx)"
R"ToPSD <- function(distance, frobenius = FALSE){
  DistanceMatrix <- as.matrix(distance)
  N <- dim(DistanceMatrix)[1]
  D2 <- DistanceMatrix^2
  I <- diag(N)
  one <- as.matrix(rep(1, N))
  one.t <- t(one)
  K <- -0.5*(I - one %*% one.t/N) %*% D2 %*% (I - one%*%one.t/N)
	if (frobenius){
  	f = norm(K, type = 'F')
  	K = K/f
  }
  evecK = as.matrix(eigen(K)$vector)
  evalK = as.vector(abs(eigen(K)$values))
  evalK[evalK < 1e-4] = 1e-4
  diagK = diag(evalK)
  psdK = evecK %*% diagK %*% t(evecK)
  return(psdK)
}"

function KernelMatrix(arg...; tableFile::String = "", treeFile::String = "",
  out::String = "", format::String = "nwk", alpha::Array{Float64,1} = [1.0],
  Dtype::String = "d_UW", Kernel::Bool = true, frobenius::Bool = true)

  tree = readNewick(filepath = treeFile, format = format)
  if Dtype == "d_VAW"
    VAW = true
  else
    VAW = false
  end

  Distance = GUniFrac(otuTableFile = tableFile, phy = tree, alpha = alpha, VAW = VAW)

  if Dtype == "d_UW"
    D = Distance[Dtype]
  elseif Dtype == "d_alpha"
    D = Distance[[string("d_", alpha)]]
  elseif Dtype == "d_VAW"
    D = Distance[Dtype]
  end

  if Kernel
    @rput D
    @rput out

    if frobenius
      R"K <- ToPSD(distance = D, frobenius = TRUE)"
    else
      R"K <- ToPSD(distance = D)"
    end
    @rget K
    writecsv(string("K_",out,".csv"), K)
  #  R"write.table(K, paste0('Kernel_', out '.csv'), row.names = F, col.names = F, sep=',')"
    return K
  else
    writecsv(string("D_",out,".csv"), D)
    return D
  end
end


#---------------------------------------------------------------------------------------------------
# Low-level multiplication
#---------------------------------------------------------------------------------------------------
export tlmul!, trmul!, tcmul!, tcmul
function tlmul!(
    λ::AbstractVector{<:Number},
    Γ::AbstractArray
)
    α = size(Γ, 1)
    Γ_r = reshape(Γ, α, :)
    lmul!(Diagonal(λ), Γ_r)
end
#---------------------------------------------------------------------------------------------------
function trmul!(
    Γ::AbstractArray,
    λ::AbstractVector{<:Number}
)
    β = size(Γ)[end]
    Γ_r = reshape(Γ, :, β)
    rmul!(Γ_r, Diagonal(λ))
end
#---------------------------------------------------------------------------------------------------
function tcmul!(
    Γ_n::AbstractArray{<:Number, 3},
    cmat::AbstractMatrix,
    Γ::AbstractArray{<:Number, 3}
)
    @tensor Γ_n[:] = cmat[-2,1] * Γ[-1,1,-3]
end
#---------------------------------------------------------------------------------------------------
function tcmul(
    cmat::AbstractMatrix,
    Γ::AbstractArray{<:Number, 3}
)
    @tensor Γ_n[:] := cmat[-2,1] * Γ[-1,1,-3]
    Γ_n
end

#---------------------------------------------------------------------------------------------------
# Grouping
#---------------------------------------------------------------------------------------------------
export tgroup, tgroup!
function tgroup!(
    Γ::AbstractArray{<:Number, 3},
    ΓA::AbstractArray{<:Number, 3},
    ΓB::AbstractArray{<:Number, 3}
)
    α, β = size(ΓA, 1), size(ΓB, 3)
    d1, d2 = size(ΓA, 2), size(ΓB, 2)
    ΓA_r = reshape(ΓA, α * d1, :)
    ΓB_r = reshape(ΓB, :, d2 * β)
    Γ_r = reshape(Γ, α * d1, d2 * β)
    mul!(Γ_r, ΓA_r, ΓB_r)
end
#---------------------------------------------------------------------------------------------------
function tgroup(
    ΓA::AbstractArray{<:Number, 3},
    ΓB::AbstractArray{<:Number, 3}
)
    α, β = size(ΓA, 1), size(ΓB, 3)
    d1, d2 = size(ΓA, 2), size(ΓB, 2)
    ΓA_r = reshape(ΓA, α * d1, :)
    ΓB_r = reshape(ΓB, :, d2 * β)
    reshape(ΓA_r * ΓB_r, α, d1 * d2, β)
end
#---------------------------------------------------------------------------------------------------
function tgroup!(
    Γ::AbstractArray{<:Number, 3},
    ΓA::AbstractArray{<:Number, 3},
    ΓB::AbstractArray{<:Number, 3},
    ΓC::AbstractArray{<:Number, 3}
)
    α, β = size(ΓA, 1), size(ΓC, 3)
    d1, d2, d3 = size(ΓA, 2), size(ΓB, 2), size(ΓC, 2)
    Γ_r = reshape(Γ, α, d1, d2, d3, β)
    @tensor Γ_r[:] = ΓA[-1,-2,1] * ΓB[1,-3,2] * ΓC[2,-4,-5]
end
#---------------------------------------------------------------------------------------------------
function tgroup(
    ΓA::AbstractArray{<:Number, 3},
    ΓB::AbstractArray{<:Number, 3},
    ΓC::AbstractArray{<:Number, 3}
)
    @tensor Γ_r[:] := ΓA[-1,-2,1] * ΓB[1,-3,2] * ΓC[2,-4,-5]
    reshape(Γ_r, size(ΓA, 1), :, size(ΓB, 3))
end

#---------------------------------------------------------------------------------------------------
# Tensor SVD
#---------------------------------------------------------------------------------------------------
export svd_trim, tsvd
function svd_trim(
    mat::AbstractMatrix,
    bound::Integer,
    cutoff::Real,
    renormalize::Bool
)
    res = svd(mat)
    vals = res.S
    len = min(count(i -> (i > cutoff), vals), bound)
    U = res.U[:, 1:len]
    S = vals[1:len]
    V = res.Vt[1:len, :]
    if renormalize
        S ./= norm(S)
    end
    U, S, V
end
#---------------------------------------------------------------------------------------------------
function tsvd(
    T::AbstractArray,
    α::Integer,
    d1::Integer,
    d2::Integer,
    β::Integer;
    bound::Integer=1000,
    cutoff::Real=1e-7,
    renormalize::Bool=false
)
    T_r = reshape(T, α * d1, d2 * β)
    U, S, V = svd_trim(T_r, bound, cutoff, renormalize)
    ΓA = reshape(U, α, d1, :)
    ΓB = reshape(V, :, d2, β)
    ΓA, S, ΓB
end


#=
     canonical(n, d)    n, d   two integers
     canonical(n__d)    n//d   one rational

     Map an ordered pair of integers in their given ratio
     to an ordered pair of integers, preserving their ratio
     while proportional reduction finds the ordered pair of
     integers with the given ratio and of least magnitude.
=#

#=
    System Rationals are maintained in lowest terms, no additional work needed
    FastRational{T,IsReduced} is already in lowest terms, no additional work needed
    FastRational{T,MayReduce} may or may not be in lowest terms, must apply work
=#

@inline canonical(q::Rational{T}) where{T} = q
@inline canonical(q::FastRational{T,IsReduced}) where{T} = q
@inline canonical(q::FastRational{T,MayReduce}) where{T} = canonical(q.num, q.den)

"""
    canonical(num::T, den::T) where T<:Union{Signed, Unsigned}

presumes
  num, den are "normative": den != 0, both are finite
  num, den share an integer type, Signed or Unsigned

provides
  num, den where either
      1 == |num| == |den|
  or
           |num| != |den|
      and
           either both are odd or
           one is odd, one even

assures
  num, den have no common factors
  den is strictly positive
  num is negative onlyif num/den < 0

invariant
  The integer type given num, den is carried in this
  functional action and is carried out unchanged.
""" canonical

function canonical(num::T, den::T) where {T<:Unsigned}
    num, den = canonical_valued(num, den)
    return num, den
end

function canonical(num::T, den::T) where {T<:Signed}
    num, den = canonical_signed(num, den)
    num, den = canonical_valued(num, den)
    return num, den
end

@inline function canonical_signed(num::T, den::T) where {T<:Signed}
    return flipsign(num, den), abs(den)
end

@inline function canonical_valued(num::T, den::T) where {T<:Signed}
    gcdval = gcd(num, den)
    gcdval === one(T) && return num, den
    num = div(num, gcdval)
    den = div(den, gcdval)
    return num ÷ den
end


"""
    Burr(k, c, λ)
The *Burr distribution*  with shape1 `k`, shape2 `c` and scale `λ` has probability density function
```math
f(x; k, c, \\lambda) = \\frac{ck}{\\lambda} \\left(\\frac{x}{\\lambda}\\right)^{c-1} \\left[ 1+\\left(\\frac{x}{\\lambda}\\right)^{c} \\right]^{-k-1}, \\quad x > 0
```
```julia
Burr()                  # Burr distribution with k = 1, c = 1 and λ = 1, i.e. Burr(1, 1, 1)
Burr(k, c)              # Burr distribution with λ = 1, i.e. Burr(k, c, 1)
Burr(k, c, λ)           # Burr distribution with shape1 k, shape2 c and scale λ
params(d)        # Get the parameters, i.e. (k, c, λ)
```
External links
* [Burr distribution on Wikipedia](https://en.wikipedia.org/wiki/Burr_distribution)
"""

struct Burr{T<:Real} <: ContinuousUnivariateDistribution
    k::T # shape1
    c::T # shape2
    λ::T # scale
end

function Burr(k::T, c::T, λ::T; check_args=true) where {T}
    check_args && @check_args(Burr, k > zero(k) && c > zero(c) && λ > zero(λ))
    return Burr{T}(k, c, λ)
end

Burr(k::Real, c::Real, λ::Real) = Burr(promote(k, c, λ)...)
Burr(k::Integer, c::Integer, λ::Integer) = Burr(float(k), float(c), float(λ))
Burr(k::T, c::T) where {T <: Real} = Burr(k, c, one(T))
Burr() = Burr(1.0, 1.0, 1.0, check_args=false)

@distr_support Burr 0.0 Inf

#### Conversions

function convert(::Type{Burr{T}}, k::S, c::S, λ::S) where {T <: Real, S <: Real}
    Burr(T(k), T(c), T(λ))
end
function convert(::Type{Burr{T}}, d::Burr{S}) where {T <: Real, S <: Real}
    Burr(T(d.k), T(d.c), T(d.λ), check_args=false)
end

#### Parameters

# shape1 and shape2 not defined in the package -> ignore these. params is sufficient
# shape1(d::Burr) = d.k
# shape2(d::Burr) = d.c
# scale(d::Burr) = d.λ
params(d::Burr) = (d.k, d.c, d.λ)
partype(::Burr{T}) where {T<:Real} = T

#### Statistics

function m(d::Burr{T}, g::Real) where T<:Real # The calculating moments: this should not be exported.
    (k, c, λ) = params(d)

    if (g-(-c))*(g-k*c) < 0
        λ^g * gamma(1 + g/c) * gamma(k - g/c) / gamma(k)
    else
        return T(Inf)
    end

end

function median(d::Burr{T}) where T<:Real
    (k, c, λ) = params(d)

    return λ * (2^(1/k) - 1)^(1/c)
end

function mean(d::Burr{T}) where T<:Real
    return m(d, 1)
end

function mode(d::Burr{T}) where T<:Real
    (k, c, λ) = params(d)

    if c > 1.0
        return λ * ( (c-1) / (k*c+1) )^(1/c)
    else
        return 0.0
    end
end

function var(d::Burr{T}) where T<:Real
    if isinf(m(d, 1))
        return T(Inf)
    else
        return m(d, 2) - (m(d, 1))^2
    end
end

function skewness(d::Burr{T}) where T<:Real
    if isinf(m(d, 2))
        return T(Inf)
    else
        g1 = m(d, 1)
        g2 = m(d, 2)
        g3 = m(d, 3)
        return (g3 - 3g1 * g2 + 2g1^3) / (g2 - g1^2) ^ (3/2)
    end
end

function kurtosis(d::Burr{T}) where T<:Real
    if isinf(m(d, 3))
        return T(Inf)
    else
        g1 = m(d, 1)
        g2 = m(d, 2)
        g3 = m(d, 3)
        g4 = m(d, 4)
        return (g4 - 4g1 * g3 + 6g2 * g1^2 - 3 * g1^4) / (g2 - g1^2)^2 - 3
    end
end

# function entropy(d::Burr{T}) where T<:Real
# end

function quantile(d::Burr, p::Real)
    (k, c, λ) = params(d)

    return λ * ( (1-p)^(-1/k) - 1 )^(1/c)
end

#### Evaluation

function pdf(d::Burr{T}, x::Real) where T<:Real
    if isinf(x)
        return zero(T)
    elseif x > 0
        (k, c, λ) = params(d)
        return k*c/λ * (x/λ)^(c-1) * ( 1 + (x/λ)^c )^(-k-1)
    else
        return zero(T)
    end
end

function logpdf(d::Burr{T}, x::Real) where T<:Real
    if isinf(x)
        return -T(Inf)
    elseif x > 0
        (k, c, λ) = params(d)
        return log(k) + log(c) - log(λ) + (c-1)*log(x) - (c-1)*log(λ) - (k+1)*log1p( (x/λ)^c )
    else
        return -T(Inf)
    end
end

function cdf(d::Burr{T}, x::Real) where T<:Real
    if x > 0
        (k, c, λ) = params(d)
        return 1 - ( 1 + (x/λ)^c )^(-k)
    else
        return zero(T)
    end
end

function ccdf(d::Burr{T}, x::Real) where T<:Real
    if x > 0
        (k, c, λ) = params(d)
        return ( 1 + (x/λ)^c )^(-k)
    else
        return one(T)
    end
end

function logcdf(d::Burr{T}, x::Real) where T<:Real
    if x > 0
        (k, c, λ) = params(d)
        return log1p( - ( 1 + (x/λ)^c )^(-k) )
    else
        return -T(Inf)
    end
end

function logccdf(d::Burr{T}, x::Real) where T<:Real
    if x > 0
        (k, c, λ) = params(d)
        return -k * log1p( (x/λ)^c )
    else
        return zero(T)
    end
end

#### Sampling
function rand(rng::AbstractRNG, d::Burr)
    # Generate a Float64 random number uniformly in (0,1].
    u = 1 - rand(rng)

    return quantile(d, u)
end


## Julia program for Polar Simulation
## author: weiya <szcfweiya@gmail.com>
## date: 2018-08-20

using Statistics

function polarsim(x::Array)
    while true
        phi1 = rand()*2*pi
        phi2 = rand()*pi - pi/2
        u = rand()
        xdotxi = x[1]*cos(phi1) + x[2]*sin(phi1)*cos(phi2) + x[3]*sin(phi1)*sin(phi2)
        if u <= exp(xdotxi - sum(x.^2)/2)
            return(randn() + xdotxi)
        end
    end
end

# approximate E^pi
function Epi(m, x = [0.1, 1.2, -0.7])
    rhos = ones(m)
    for i = 1:m
        rhos[i] = 1/(2*polarsim(x)^2+3)
    end
    return(mean(rhos))
end

# example
Epi(10)


using HCubature
using Distributions
using LinearAlgebra

struct Fun
    f::Function
    n::Int
    y::Vector{Float64}
end

Fun( f::Function, n::Int ) = Fun( f, n, zeros(n) )

function (f::Fun)( x::AbstractVector{Float64} )
    factor = 1.0
    for i = 1:f.n
        y = x[i]
        denom = y*(1-y)
        f.y[i] = (2*y-1)/denom
        factor *= (2*y^2 - 2*y + 1)/denom^2
    end
    return f.f( f.y ) * factor
end

function invtransform( x::Float64 )
    if x == 0.0
        return 0.5
    elseif isinf(x)
        return x == -Inf ? 0.0 : 1.0
    else
        return (x-2+sqrt(x^2+4))/(2*x)
    end
end

function nintegrate( f::Fun, a::Vector{Float64}, b::Vector{Float64}; kwargs... )
    for i = 1:f.n
        a[i] = invtransform(a[i])
        b[i] = invtransform(b[i])
    end
    return hcubature( f, a, b; kwargs... )
end

function lenormalcubature( mean, cov, A, b; kwargs... )
    n = length(mean)
    mvn = MvNormal( mean, cov )
    f = Fun( x -> all(A*x .<= b) ? pdf( mvn, x ) : 0.0, n )
    return nintegrate( f, fill(-Inf,n), fill(Inf,n); kwargs... )
end


"""
This example learns to classify hand-written digits from the MNIST
dataset.  There are 60000 training and 10000 test examples. Each input
x consists of 784 pixels representing a 28x28 image.  The pixel values
have been normalized to [0,1]. Each output y is a ten-dimensional
one-hot vector (a vector that has a single non-zero component)
indicating the correct class (0-9) for a given image.

To run the demo, simply `include("mnist.jl")` and run `MNIST.train()`.
The dataset will be automatically downloaded.  You can provide the
initial weights as an optional argument to `train`, which should have
the form [w0,b0,w1,b1,...] where wi (with size = output x input) is
the weight matrix and bi (with size = output) is the bias vector for
layer i.  The function `MNIST.weights(h...)` can be used to create
random starting weights for a neural network with hidden sizes (h...).
If not specified, default weights are created using `MNIST.weights()`
which correspond to a 0 hidden layer network, i.e. a softmax model.
`train` also accepts the following keyword arguments: `lr` specifies
the learning rate, `epochs` gives number of epochs.  The cross entropy
loss and accuracy for the train and test sets will be printed at every
epoch and optimized parameters will be returned.
"""
module MNIST
using AutoGrad
using GZip
using Main
using Compat

function predict(w, x)
    i = 1
    while i+2 < length(w)
        x = max(0, w[i]*x .+ w[i+1])
        i += 2
    end
    return w[i]*x .+ w[i+1]
end

function loss(w, x, ygold)
    ypred = predict(w, x)
    ynorm = ypred .- log(sum(exp(ypred),1))
    -sum(ygold .* ynorm) / size(ygold,2)
end

function accuracy(w, x, ygold)
    ypred = predict(w, x)
    sum((ypred .== maximum(ypred,1)) & (ygold .== maximum(ygold,1))) / size(ygold,2)
end

function train(w=weights(); lr=.1, epochs=20)
    isdefined(MNIST,:dtrn) || loaddata()
    println((0, :ltrn, loss(w,xtrn,ytrn), :ltst, loss(w,xtst,ytst), :atrn, accuracy(w,xtrn,ytrn), :atst, accuracy(w,xtst,ytst)))
    gradfun = grad(loss)
    for epoch=1:epochs
        for (x,y) in dtrn
            g = gradfun(w, x, y)
            for i in 1:length(w)
                w[i] -= lr * g[i]
            end
        end
        println((epoch, :ltrn, loss(w,xtrn,ytrn), :ltst, loss(w,xtst,ytst), :atrn, accuracy(w,xtrn,ytrn), :atst, accuracy(w,xtst,ytst)))
    end
    return w
end

function weights(h...; seed=nothing)
    seed==nothing || srand(seed)
    w = Any[]
    x = 28*28
    for y in [h..., 10]
        push!(w, convert(Array{Float32}, 0.1*randn(y,x)))
        push!(w, zeros(Float32, y))
        x = y
    end
    return w
end

function loaddata()
    info("Loading data...")
    global xtrn, xtst, ytrn, ytst, dtrn
    xshape(a)=reshape(a./255f0,784,div(length(a),784))
    yshape(a)=(a[a.==0]=10; full(sparse(convert(Vector{Int},a),1:length(a),1f0)))
    xtrn = xshape(gzread("train-images-idx3-ubyte.gz")[17:end])
    xtst = xshape(gzread("t10k-images-idx3-ubyte.gz")[17:end])
    ytrn = yshape(gzread("train-labels-idx1-ubyte.gz")[9:end])
    ytst = yshape(gzread("t10k-labels-idx1-ubyte.gz")[9:end])
    dtrn = minibatch(xtrn, ytrn, 100)
    info("Loading done...")
end

function gzread(file; path=joinpath(AutoGrad.datapath,file), url="http://yann.lecun.com/exdb/mnist/$file")
    isfile(path) || download(url, path)
    f = gzopen(path)
    a = @compat read(f)
    close(f)
    return(a)
end

function minibatch(x, y, batchsize)
    data = Any[]
    nx = size(x,2)
    for i=1:batchsize:nx
        j=min(i+batchsize-1,nx)
        push!(data, (x[:,i:j], y[:,i:j]))
    end
    return data
end

end # module


using AlgebraPDF
using Test

@testset "Generate in range" begin
    d = Normalized(FGauss((a=0,b=1.1)), (2,5))
    data = generate(d, 100)
    sum(x->inrange(x,lims(d)), data) == length(data)
end


# A bunch of functions to perform a single iteration to minimize h(t)
# and/or compute a root of h'(t)
include("DividedDifference2.jl")
include("coefHIN.jl")

# basic iterations
base =  Function[]

"""One iteration of the secant method"""
function s_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    s = xm - x
    y = dhm - dh

    d = - (dh * s / y)

    xp = x + d
    return xp
end
push!(base, s_iteration)

"""One iteration of the Newton's method"""
function N_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    d = - (dh / d2h)

    xp = x + d
    return xp
end
push!(base, N_iteration)

"""One iteration of a cubic Taylor approximation
   may also be described as a quadratic Taylor approximation of h'
"""
function C_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,d3h,d3hm)
    # solution cubique...
    disc = d2h^2 - 2.0*d3h*dh
    d = disc < 0 ? -sign(d3h)*Inf : (-d2h + sqrt(disc)) / d3h

    xp = x + d
    return xp
end
push!(base, C_iteration)

# iterations on the quintic interpolant of h, h' and h''
quintic = Function[]

"""Two Newton iteration on the quintic interpolant of h
   at the current and previous point
"""
function N2Opt_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    d = - (dh / d2h)
    x2 = x + d
    # estimate dh2 from the hermite interpolant

    a = coefHIN5([x;xm],[h dh d2h; hm dhm d2hm])

    p = Polynomial(a)
    dp = Polynomials.derivative(p)
    d2p = Polynomials.derivative(dp)

    d2 = -(dp(d) / d2p(d))

    xp = x2 + d2

    return xp
end
push!(quintic, N2Opt_iteration)

"""Three Newton iteration on the quintic interpolant of h
   at the current and previous point
"""
function N3Opt_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    d = - (dh / d2h)
    x2 = x + d
    # estimate dh2 from the hermite interpolant

    a = coefHIN5([x;xm],[h dh d2h; hm dhm d2hm])

    p = Polynomial(a)
    dp = Polynomials.derivative(p)
    d2p = Polynomials.derivative(dp)

    d2 = -(dp(d) / d2p(d))

    x3 = x2 + d2
    d3 = -(dp(d+d2) / d2p(d+d2))

    xp = x3 + d3
    return xp
end
push!(quintic, N3Opt_iteration)

# iterations on the cubic interpolant of h' and h''
cubic1 = Function[]

"""Two Newton iteration on the cubic interpolant of h'
   at the current and previous point
"""
function N2_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)

    d = - (dh / d2h)
    x2 = x + d
    # estimate dh2 from the hermite interpolant

    a = coefHIN3([x;xm],[dh d2h; dhm d2hm])
    p = Polynomial(a)
    dp = Polynomials.derivative(p)

    d2 = -(p(d) / dp(d))

    xp = x2 + d2
    return xp
end
push!(cubic1, N2_iteration)

"""Three Newton iterations on the cubic interpolant of h'
   at the current and previous point
"""
function N3_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    d = - (dh / d2h)
    x2 = x + d
    # estimate dh2 from the hermite interpolant

    a = coefHIN3([x;xm],[dh d2h; dhm d2hm])

    p = Polynomial(a)
    dp = Polynomials.derivative(p)

    d2 = -(p(d) / dp(d))

    x3 = x2 + d2
    d3 = -(p(d+d2) / dp(d+d2))
    xp = x3 + d3

    return xp
end
push!(cubic1, N3_iteration)

"""Two secant iterations on the cubic interpolant of h'
   at the current and previous point
"""
function s2Opt_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    s = x - xm
    y = dh - dhm

    d = - (dh * s / y)

    x2 = x + d
    # estimate dh2 from the hermite interpolant
    a = coefHIN3([x;xm],[h dh; hm dhm])
    s2 = x2 - x

    p = Polynomial(a)
    dp = Polynomials.derivative(p)
    dh2 = dp( s2)

    # perform a secant iteration with this estimated dh2
    y2 = dh2 - dh
    d2 = - (dh2 * s2 / y2)

    xp = x2 + d2
    if ( xp == x ) xp = x2 end #safeguard
    return xp
end
push!(cubic1, s2Opt_iteration)

# iterations on the cubic interpolant of h and h'
cubic0 = Function[]

"""Minimizer of the cubic interpolant of h and dh"""
function sCOpt_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    # Hermite cubic interpolant
    a = coefHIN3([x;xm],[h dh; hm dhm])
    p = Polynomial(a)
    dp = Polynomials.derivative(p)
    dh  = a[2]
    d2h = 2.0*a[3]
    d3h = 3.0*a[4]
    disc = d2h^2 - 2.0*d3h*dh
    d = disc < 0 ? -sign(d3h)*Inf : (-d2h + sqrt(disc)) / d3h

    xp = x + d

    return xp
end
push!(cubic0, sCOpt_iteration)

"""Two Newton's iterations on the cubic interpolant of h
   at the current and previous point
"""
function sNOpt_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    # Hermite cubic interpolant
    a = coefHIN3([x;xm],[h dh; hm dhm])
    p = Polynomial(a)
    dp = Polynomials.derivative(p)
    d2p = Polynomials.derivative(dp)

    # Newton step on the interpolant
    # Do not forget that the interpolant is rooted at 0
    d = - dp(0.0) / d2p(0.0)
    d2 = - dp(d) / d2p(d)
    xp = x + d + d2

    return xp
end
push!(cubic0, sNOpt_iteration)

"""Three secant iterations on the cubic interpolant of h
   at the current and previous point; not efficient????
"""
function s3Opt_iteration(x,xm,h,hm,dh,dhm,d2h,d2hm,::Any,::Any)
    s = x - xm
    y = dh - dhm

    d = - (dh * s / y)

    x2 = x + d

    # estimate dh2 from the hermite interpolant
    a = coefHIN3([x;xm],[h dh; hm dhm])
    p = Polynomial(a)
    dp = Polynomials.derivative(p)

    s2 = x2 - x
    dh2 = dp(s2)
    # perform a secant iteration with this estimated dh2
    y2 = dh2 - dh
    d2 = - (dh2 * s2 / y2)
    x3 = x2 + d2

    s3 = x3 - x2
    dh3 = dp(s3)
    # perform a secant iteration with this estimated dh3
    y3 = dh3 - dh2
    d3 = - (dh3 * s3 / y3)

    xp = x3 + d3
    # not good, seems to fall back close to regular secant iteration
    if ( xp == x ) xp = x3 end#(x+xm)/2.0 end#@show a end
    return xp
end
push!(cubic0, s3Opt_iteration)

export base, quintic, cubic1, cubic0

for n in nameof.(base ∪ quintic ∪ cubic1 ∪ cubic0 )
    @eval export $n
end