module FV12_plate_examples
using Statistics
using FinEtools
using FinEtools.MeshImportModule
using FinEtools.MeshExportModule
using LinearAlgebra: dot
using Arpack
using LinearAlgebra
using SparseArrays
using PGFPlotsX
using Test

E = 200e3*phun("MPa");
nu = 0.3;
rho = 8000*phun("KG/M^3");
L = 10.00*phun("M"); t = 0.05*phun("M");
nL = 8; nt =2;
neigvs = 14                   # how many eigenvalues
OmegaShift = (10.0*2*pi)^2;
# Fundamental frequency
f_analytical =[0 0 0  0 0 0 1.622 2.360 2.922 4.190 4.190 7.356 7.356 7.668];

function FV12_plate_esnice()
    global a,b,h,na,nb,nh
    MR = DeforModelRed3D
    fens, fes = T4block(L,L,t,nL,nL,nt);

    geom = NodalField(fens.xyz)
    u = NodalField(zeros(size(fens.xyz,1),3)) # displacement field

    numberdofs!(u)

    material = MatDeforElastIso(MR, rho, E, nu, 0.0)

    femm = FEMMDeforLinearESNICET4(MR, IntegDomain(fes, NodalSimplexRule(3)), material)
    associategeometry!(femm,  geom)

    K  = stiffness(femm, geom, u)
    M = mass(femm, geom, u)
    d,v,nev,nconv = eigs(K+OmegaShift*M, M; nev=neigvs, which=:SM)
    d = d .- OmegaShift;
    fs = real(sqrt.(complex(d)))/(2*pi)
    println("Eigenvalues: $fs [Hz]")
    println("f/f_analytical = $(fs[7:10]  ./ f_analytical[7:10] .* 100) %")

    vectors = []
    for i = 7:length(fs)
        scattersysvec!(u, v[:, i])
        push!(vectors, ("Mode_$i", deepcopy(u.values)))
    end
    File  =   "rectangular_plate_esnice.vtk"
    vtkexportmesh(File, connasarray(fes), fens.xyz, FinEtools.MeshExportModule.T4; vectors = vectors)
    @async run(`"paraview.exe" $File`)

    true
end # function

function allrun()
    println("#####################################################")
    println("# FV12_plate_esnice ")
    FV12_plate_esnice()
    return true
end # function allrun

end # module FV12_plate_examples


"""
# Percent Change

```julia
pctchange(ts::TS, periods::Int = 1)
```

Return the percentage change between successive row elements.
Default is the element in the next row. `periods` defines the number
of rows to be shifted over. The skipped rows are rendered as `missing`.

`pctchange` returns an error if column type does not have the method `/`.

# Examples
```jldoctest; setup = :(using TSx, DataFrames, Dates, Random, Statistics)
julia> using Random, Statistics;

julia> random(x) = rand(MersenneTwister(123), x);

julia> dates = collect(Date(2017,1,1):Day(1):Date(2017,1,10));

julia> ts = TS(random(length(dates)), dates)
julia> show(ts)
(10 x 1) TS with Date Index

 Index       x1
 Date        Float64
───────────────────────
 2017-01-01  0.768448
 2017-01-02  0.940515
 2017-01-03  0.673959
 2017-01-04  0.395453
 2017-01-05  0.313244
 2017-01-06  0.662555
 2017-01-07  0.586022
 2017-01-08  0.0521332
 2017-01-09  0.26864
 2017-01-10  0.108871

# Pctchange over successive rows
julia> pctchange(ts)
(10 x 1) TS with Date Index

 Index       x1
 Date        Float64?
────────────────────────────
 2017-01-01  missing
 2017-01-02        0.223915
 2017-01-03       -0.283415
 2017-01-04       -0.413238
 2017-01-05       -0.207886
 2017-01-06        1.11514
 2017-01-07       -0.115511
 2017-01-08       -0.911039
 2017-01-09        4.15295
 2017-01-10       -0.594733

# Pctchange over the third row
julia> pctchange(ts, 3)
(10 x 1) TS with Date Index

 Index       x1
 Date        Float64?
─────────────────────────────
 2017-01-01  missing
 2017-01-02  missing
 2017-01-03  missing
 2017-01-04       -0.485387
 2017-01-05       -0.666944
 2017-01-06       -0.0169207
 2017-01-07        0.4819
 2017-01-08       -0.83357
 2017-01-09       -0.59454
 2017-01-10       -0.814221

```
"""

# Pctchange
function pctchange(ts::TS, periods::Int = 1)
    if periods <= 0
        error("periods must be a positive int")
    end
    ddf = (ts.coredata[:, Not(:Index)] ./ TSx.lag(ts, periods).coredata[:, Not(:Index)]) .- 1
    insertcols!(ddf, 1, "Index" => ts.coredata[:, :Index])
    TS(ddf, :Index)
end


# Tracing charged particle in the magnetic bottle.
# This example shows how to trace protons in a stationary magnetic field that
# corresponds to the magnetic bottle.
# Reference: https://en.wikipedia.org/wiki/Magnetic_mirror#Magnetic_bottles
#
# Hongyang Zhou, hyzhou@umich.edu

using TestParticle
using TestParticle: getB_bottle
using OrdinaryDiffEq
using StaticArrays
using PyPlot
using Statistics: mean
using Printf

## Obtain field

# Magnetic bottle parameters in SI units
const I1 = 80. # current in the solenoid
const N1 = 1500 # number of windings
const I2 = 100. # current in the central solenoid
const N2 = 8000 # number of windings
const distance = 10. # distance between solenoids
const a = 1.0 # radius of each coil
const b = 4.0 # radius of central coil

function getB(xu)
   SVector{3}(getB_Bottle(xu[1], xu[2], xu[3], distance, a, b, I1*N1, I2*N2))
end

function getE(xu)
   SA[0.0, 0.0, 0.0]
end

## Initialize particles
m = TestParticle.mₑ
q = TestParticle.qₑ
c = TestParticle.c

# initial velocity, [m/s]
v₀ = [0.8, 0.48, 0.3595] .* c
# initial position, [m]
r₀ = [0.0, 0.0, -0.8]
stateinit = [r₀..., v₀...]

# Theoretically we can take advantage of the fact that magnetic field does not
# accelerate particles, so that γ remains constant. However, we are not doing
# that here since it is not generally true in the EM field.

param = prepare(getE, getB; species=Electron)
tspan = (0.0, 3e-7)

prob = ODEProblem(trace_analytic_relativistic!, stateinit, tspan, param)

@printf "Speed = %6.4f %s\n" √(v₀[1]^2+v₀[2]^2+v₀[3]^2)/c*100 "% speed of light"
@printf "Energy = %6.4f MeV\n" (1/√(1-(v₀[1]/c)^2-(v₀[2]/c)^2-(v₀[3]/c)^2)-1)*m*c^2/abs(q)/1e6

# Default Tsit5() and many solvers does not work in this case!
sol = solve(prob, AB3(); dt=1.5e-12, save_idxs=[1,2,3])

## Visualization

using3D()
fig = plt.figure(figsize=(10,6))
ax = fig.gca(projection="3d")

ax.plot(sol[1,:], sol[2,:], sol[3,:], label="electron")

# Plot coils
θ = range(0, 2π, length=100)
x = a.*cos.(θ)
y = a.*sin.(θ)
z = fill(distance/2, size(x))
ax.plot(x, y, z, color="r", label="coil")

z = fill(-distance/2, size(x))
ax.plot(x, y, z, color="r")

x = b.*cos.(θ)
y = b.*sin.(θ)
z = fill(0.0, size(x))
ax.plot(x, y, z, color="r")

ax.legend()
title("Electron trajectory in the magnetic bottle")
xlabel("x [m]")
ylabel("y [m]")
zlabel("z [m]")

xMin, xMax = -5, 5
yMin, yMax = -5, 5
zMin, zMax = -5.2, 5.2

xlim(xMin, xMax)
ylim(yMin, yMax)
zlim(zMin, zMax)

TestParticle.set_axes_equal(ax)

#savefig("test.png", bbox_inches="tight")


function wavenumbers_mesh(ω::T, k_effs::Vector{Complex{T}}, medium::PhysicalMedium{Dim}, species::Species{T};
        symmetry::AbstractSymmetry = PlanarSymmetry{Dim}(),
        tol::T = 1e-5,
        mesh_refine::T = T(0.4),
        inner_optimizer = LBFGS(),
        verbose::Bool = false,
        max_Rek::T = maximum(real(k1) for k1 in k_effs),
        min_Rek::T = minimum(real(k1) for k1 in k_effs),
        max_Imk::T = maximum(imag(k1) for k1 in k_effs),
        min_Imk::T = minimum(imag(k1) for k1 in k_effs),
        kws...) where {T<:Number,Dim}

    eff_medium = effective_medium(medium, species)
    k0 = ω/eff_medium.c
    if isnan(k0) k0 = ω + T(0)*im end
    # abs(k0) can be used to non-dimensionlise k_vec
    kscale = abs(k0)

    low_tol = 1e-2*minimum( (k1 == k2) ? Inf : abs(k1-k2) for k1 in k_effs, k2 in k_effs) / kscale
    low_tol = max(low_tol, 1e-4, tol) # a tolerance used for a first pass with time_limit

    # the dispersion equation is given by: `dispersion(k1,k2) = 0` where k_eff = k1 + im*k2.
    dispersion_dim = dispersion_equation(ω, medium, species, symmetry; tol = low_tol, kws...)
    dispersion(vec::Vector{T}) = dispersion_dim((vec[1] + vec[2] * im) .* kscale)

    # non-dimensionalise
    min_Rek = min_Rek / kscale
    min_Imk = min_Imk / kscale
    max_Rek = max_Rek / kscale
    max_Imk = max_Imk / kscale

    kx = LinRange(min_Rek, max_Rek, Int(round(length(k_effs)/(2*mesh_refine))))# tree shape makes this division by 2 natural
    ky = LinRange(min_Imk, max_Imk, Int(round(length(k_effs)/(2*mesh_refine))))
    k_mesh = [[x,y] for x in kx, y in ky][:]

    # Include k_effs in the mesh, we do not assume these are solutions
    k_vecs = [[real(keff),imag(keff)] for keff in k_effs] ./ kscale
    k_mesh = [k_mesh; k_vecs]

    # make slightly bigger box to be constrained within
    min_Rek = (min_Rek < zero(T)) ? min_Rek*T(1.1) : min_Rek*T(0.9)
    min_Imk = (min_Imk < zero(T)) ? min_Imk*T(1.1) : min_Imk*T(0.9)
    max_Rek = (max_Rek > zero(T)) ? max_Rek*T(1.001) : max_Rek*T(0.999)
    max_Imk = (max_Imk > zero(T)) ? max_Imk*T(1.001) : max_Imk*T(0.999)

    filter!(keff -> min_Rek <= keff[1] <= max_Rek && min_Imk <= keff[2] <= max_Imk, k_mesh)

    # Find all wavenumbers
    lower = [min_Rek, min_Imk]
    upper = [max_Rek, max_Imk]

    new_ks = [
        optimize(dispersion, lower, upper, kvec,
            Fminbox(inner_optimizer), Optim.Options(x_tol=low_tol, g_tol = low_tol^3)).minimizer
    for kvec in k_mesh]
    deleteat!(new_ks, findall(dispersion.(new_ks) .> low_tol))
    new_ks = reduce_kvecs(new_ks, low_tol)

    filter!(keff -> lower[1] <= keff[1] <= upper[1] && lower[2] <= keff[2] <= upper[2], new_ks)

    # Here we refine the new roots
    new_ks = map(new_ks) do k_vec
        # res = optimize(dispersion, k_vec; g_tol = tol^2.0, f_tol = tol^4.0, x_tol=tol)
        res = optimize(dispersion, lower, upper, k_vec, Fminbox(inner_optimizer),
            Optim.Options(g_tol = tol^3.0, x_tol=tol))
        if res.minimum < T(100)*tol || (Optim.converged(res) && res.minimum < T(10)*low_tol)
            res.minimizer
        else
            [zero(T),-one(T)]
        end
    end

    if verbose println("New roots from mesh refiner:",new_ks) end

    # group together wavenumbers which are closer than tol
    deleteat!(k_vecs, findall(v-> v == [zero(T),-one(T)], k_vecs) )
    k_vecs = reduce_kvecs(new_ks, T(10)*tol)

    # Finally delete unphysical waves, including waves travelling backwards with almost no attenuation. This only is important in the limit of very low frequency or very weak scatterers.
    deleteat!(k_vecs, findall([k_vec[2] < -low_tol for k_vec in k_vecs]))
    deleteat!(k_vecs, findall([-low_tol < abs(k_vec[2])/k_vec[1] < zero(T) for k_vec in k_vecs]))
    # deleteat!(k_vecs, find(k_vec[2] < tol && k_vec[1] < tol for k_vec in k_vecs))

    k_effs = kscale .* [k_vec[1] + k_vec[2]*im for k_vec in k_vecs]
    k_effs = sort(k_effs, by=imag)

    return k_effs
end

# Alternative solvers
# res = nlsolve(detMM!,initial_k_eff; iterations = 10000, factor=2.0)
# k_eff_nl = res.zero[1] + im*res.zero[2]
# lower = [0.,0.]; upper = [T(2)*k0,k0]
# result = optimize(dispersion, initial_k0, lower, upper; g_tol = tol^2.0, f_tol = tol^4.0)


function discretise(dist::ContinuousUnivariateDistribution, n::Integer=10)
    F(t) = cdf(dist, t)
    lb = minimum(dist)
    ub = maximum(dist)
    nodes = range(lb, ub; length = n + 1)
    pmf = [F(nodes[i + 1]) - F(nodes[i]) for i in 1:n]
    return DiscreteNonParametric(nodes[2:end], pmf)
end

discretise(dist::ContinuousUnivariateDistribution, n::Number) = discretise(dist, Integer(n))


using MLJ, CategoricalArrays, PrettyPrinting
import DataFrames: DataFrame, nrow
using UrlDownload

LinearRegressor = @load LinearRegressor pkg=GLM
LinearBinaryClassifier = @load LinearBinaryClassifier pkg=GLM

baseurl = "https://raw.githubusercontent.com/tlienart/DataScienceTutorialsData.jl/master/data/glm/"

dfX = DataFrame(urldownload(baseurl * "X3.csv"))
dfYbinary = DataFrame(urldownload(baseurl * "Y3.csv"))
dfX1 = DataFrame(urldownload(baseurl * "X1.csv"))
dfY1 = DataFrame(urldownload(baseurl * "Y1.csv"));

first(dfX, 3)

first(dfY1, 3)

ms = models() do m
    AbstractVector{Count} <: m.target_scitype
end
foreach(m -> println(m.name), ms)

ms = models() do m
    Vector{Continuous} <: m.target_scitype
end
foreach(m -> println(m.name), ms)

X = copy(dfX1)
y = copy(dfY1)

coerce!(X, autotype(X, :string_to_multiclass))
yv = Vector(y[:, 1])

LinearRegressorPipe = Pipeline(
    Standardizer(),
    OneHotEncoder(drop_last = true),
    LinearRegressor()
)

LinearModel = machine(LinearRegressorPipe, X, yv)
fit!(LinearModel)
fp = fitted_params(LinearModel)

ŷ = MLJ.predict(LinearModel, X)
yhatResponse = [ŷ[i,1].μ for i in 1:nrow(y)]
residuals = y .- yhatResponse
r = report(LinearModel)

k = collect(keys(fp.fitted_params_given_machine))[3]
println("\n Coefficients:  ", fp.fitted_params_given_machine[k].coef)
println("\n y \n ", y[1:5,1])
println("\n ŷ \n ", ŷ[1:5])
println("\n yhatResponse \n ", yhatResponse[1:5])
println("\n Residuals \n ", y[1:5,1] .- yhatResponse[1:5])
println("\n Standard Error per Coefficient \n", r.linear_regressor.stderror[2:end])

round(rms(yhatResponse, y[:,1]), sigdigits=4)

X = copy(dfX)
y = copy(dfYbinary)

coerce!(X, autotype(X, :string_to_multiclass))
yc = CategoricalArray(y[:, 1])
yc = coerce(yc, OrderedFactor)

LinearBinaryClassifierPipe = Pipeline(
    Standardizer(),
    OneHotEncoder(drop_last = true),
    LinearBinaryClassifier()
)

LogisticModel = machine(LinearBinaryClassifierPipe, X, yc)
fit!(LogisticModel)
fp = fitted_params(LogisticModel)

ŷ = MLJ.predict(LogisticModel, X)
residuals = [1 - pdf(ŷ[i], y[i,1]) for i in 1:nrow(y)]
r = report(LogisticModel)

k = collect(keys(fp.fitted_params_given_machine))[3]
println("\n Coefficients:  ", fp.fitted_params_given_machine[k].coef)
println("\n y \n ", y[1:5,1])
println("\n ŷ \n ", ŷ[1:5])
println("\n residuals \n ", residuals[1:5])
println("\n Standard Error per Coefficient \n", r.linear_binary_classifier.stderror[2:end])

yMode = [mode(ŷ[i]) for i in 1:length(ŷ)]
y = coerce(y[:,1], OrderedFactor)
yMode = coerce(yMode, OrderedFactor)
confusion_matrix(yMode, y)

# This file was generated using Literate.jl, https://github.com/fredrikekre/Literate.jl


# %% Basin stability of magnetic pendulum
using DrWatson
@quickactivate "NonlinearDynamicsTextbook"
include(srcdir("style.jl"))
using DynamicalSystems, PyPlot
using LinearAlgebra, Statistics

d, α, ω = 0.3, 0.2, 0.5
xg = yg = range(-3, 3, length = 300)
ma = Systems.magnetic_pendulum(;γs=[1,1,0.2], d, α, ω)

# This function is faster than basins, because it only picks some random points
function random_statespace_fraction(∫, xg, yg, T = 10000)
    c = 0 # count of states converged to attractor of interest
    for i in 1:T
        x = rand()*(maximum(xg)-minimum(xg)) + minimum(xg)
        y = rand()*(maximum(yg)-minimum(yg)) + minimum(yg)
        reinit!(∫, SVector(x, y, 0, 0))
        step!(∫, 100.0)
        while ∫.u[3]^2 + ∫.u[4]^2 > 1e-3
            step!(∫)
        end
        s = SVector(∫.u[1], ∫.u[2])
        dmin, k = findmin([(s-m)⋅(s-m) for m in ma.f.magnets])
        if k == 3; c += 1; end
    end
    return c/T
end

γs = 0:0.01:1
Fs = zero(γs)
Fσs = zero(γs)
λs = zero(γs)
∫ = integrator(ma)

# TODO: Use `produce_or_load` here.
for (i, γ) in enumerate(γs)
    @show (i, γ)
    ∫.p.γs[3] = γ
    allfracs = [random_statespace_fraction(∫, xg, yg, 1000) for i in 1:5]
    Fs[i] = mean(allfracs)
    Fσs[i] = std(allfracs)
    # TODO: This is slightly inaccurate; the fixed point is not exactly
    # on the magnetic. But it shouldn't have much of an impact for
    # final figure.
    J = Array(ma.jacobian(SVector(ma.f.magnets[3]..., 0, 0), ma.p, 0))
    eee = eigvals(J)
    λs[i] = maximum(real.(eee))
end

savedict = @strdict λs γs Fs
wsave(datadir("magnetic_basin_stability.jld2"), savedict)

# %%
data = wload(datadir("magnetic_basin_stability.jld2"))
@unpack Fs, λs, γs = data
fig, axs = subplots(1, 3)
axs[1].plot(γs, Fs; label = "\$F\$")
axs[1].fill_between(γs, Fs .- Fσs, Fs .+ Fσs; color = "C0", alpha = 0.5)
# axs[1].errorbar(γs, Fs; label = "\$F\$", yerr = Fσs)
# axs[1].plot(γs, Fs; label = "\$F\$")
axs[1].plot(γs, λs; label = "\$\\lambda_1\$")
axs[1].legend()
axs[1].set_xlabel("\$\\gamma_3\$"; labelpad = -20)
axs[1].set_xticks([0, 1])
axs[1].set_xlim(0,1)
axs[1].set_yticks(-0.1:0.1:0.3)

# perform 2 nice detailed plots
LC =  matplotlib.colors.ListedColormap
cmap = LC([matplotlib.colors.to_rgb("C$k") for k in 0:2])
γplots = (0.7, 0.25)
basins_pre = att_pre = nothing
for (i, γ) in enumerate(γplots)
    ma.p.γs[3] = γ
    basins, attractors = basins_of_attraction((xg, yg), ma)
    if i == 2
        match_attractors!(basins_pre, att_pre, basins, attractors)
    end
    axs[i+1].pcolormesh(xg, xg, basins'; cmap, shading = "gouraud")
    axs[i+1].set_xlabel("\$x\$")
    axs[i+1].set_ylabel("\$y\$")
    axs[i+1].set_xticks([])
    axs[i+1].set_yticks([])
    basins_pre = basins
    att_pre = attractors
end

add_identifiers!(fig)
fig.tight_layout(;pad = 0.33)
wsave(plotsdir("12", "basin_stability"), fig)


using Tikz

colors = Dict( "redred" => [150.,50.,50.] )

style = tikzStyle( "style1", lw = 2.0 )

obj = tikzObj( "example2.tex", standalone = true, colors = colors, styles = [style] )

x = linspace( 0., 3., 50).^1.
y = sin.( 2*π * x) .* cos.( 1./6 * π * x )

axis1 = axisObj( obj )

plot( obj, axis1, x, y )#, smooth=false, linestyle="--", color="blueblue", style=style )

closeObj( obj, axis1 )

closeObj( obj )


function partition_interval(nbins::Integer, minval::Real, maxval::Real)
    edges = (range(0,step=(maxval - minval) / nbins,length=nbins)) .+ minval
end

function partition_interval(nbins::Integer, minval::AbstractGray, maxval::AbstractGray)
    partition_interval(nbins, minval.val, maxval.val)
end

"""
```
edges, count = build_histogram(img, nbins)
edges, count = build_histogram(img, nbins; minval, maxval)
edges, count = build_histogram(img, edges)
```

Generates a histogram for the image over `nbins` spread between `[minval, maxval]`.
Color images are automatically converted to grayscale.

# Output
Returns `edges` which is a `AbstractRange` type that specifies how the  interval
`[minval, maxval]` is divided into bins, and an array `count` which records the
concomitant bin frequencies. In particular, `count` has the following
properties:

* `count[0]` is the number satisfying `x < edges[1]`
* `count[i]` is the number of values `x` that satisfy `edges[i] <= x < edges[i+1]`
* `count[end]` is the number satisfying `x >= edges[end]`.
* `length(count) == length(edges)+1`.

# Details

One can consider a histogram as a piecewise-constant model of a probability
density function ``f`` [1]. Suppose that ``f`` has support on some interval ``I =
[a,b]``.  Let ``m`` be an integer and ``a = a_1 < a_2 < \\ldots < a_m < a_{m+1} =
b`` a sequence of real numbers. Construct a sequence of intervals

```math
I_1 = [a_1,a_2], I_2 = (a_2, a_3], \\ldots, I_{m} = (a_m,a_{m+1}]
```

which partition ``I`` into subsets ``I_j`` ``(j = 1, \\ldots, m)`` on which
``f`` is constant. These subsets satisfy ``I_i \\cap I_j = \\emptyset, \\forall
i \\neq j``, and are commonly referred to as *bins*. Together they encompass the
entire range of data values such that ``\\sum_j |I_j | = | I |``. Each bin has
width ``w_j = |I_j| = a_{j+1} - a_j`` and height ``h_j`` which is the constant
probability density over the region of the bin. Integrating the constant
probability density over the width of the bin ``w_j`` yields a probability mass
of ``\\pi_j = h_j w_j`` for the bin.

For a sample ``x_1, x_2, \\ldots, x_N``, let

```math
n_j = \\sum_{n = 1}^{N}\\mathbf{1}_{(I_j)}(x_n),
\\quad \\text{where} \\quad
\\mathbf{1}_{(I_j)}(x) =
\\begin{cases}
 1 & \\text{if} \\; x \\in I_j,\\\\
 0 & \\text{otherwise},
\\end{cases},
```
represent the number of samples falling into the interval ``I_j``. An estimate
for the probability mass of the ``j``th bin is given by the relative frequency
``\\hat{\\pi} = \\frac{n_j}{N}``, and the histogram estimator of the probability
density function is defined as
```math
\\begin{aligned}
\\hat{f}_n(x)  & = \\sum_{j = 1}^{m}\\frac{n_j}{Nw_j} \\mathbf{1}_{(I_j)}(x) \\\\
& = \\sum_{j = 1}^{m}\\frac{\\hat{\\pi}_j}{w_j} \\mathbf{1}_{(I_j)}(x) \\\\
& = \\sum_{j = 1}^{m}\\hat{h}_j \\mathbf{1}_{(I_j)}(x).
\\end{aligned}
```

The function ``\\hat{f}_n(x)`` is a genuine density estimator because ``\\hat{f}_n(x)  \\ge 0`` and
```math
\\begin{aligned}
\\int_{-\\infty}^{\\infty}\\hat{f}_n(x) \\operatorname{d}x & = \\sum_{j=1}^{m} \\frac{n_j}{Nw_j} w_j \\\\
& = 1.
\\end{aligned}
```

# Options
Various options for the parameters of this function are described in more detail
below.

## Choices for `nbins`
You can specify the number of discrete bins for the histogram. When specifying
the number of bins consider the maximum number of graylevels that your image
type supports. For example, with an image of type `N0f8` there is a maximum
of 256 possible graylevels. Hence, if you request more than 256 bins for
that type of image you should expect to obtain zero counts for numerous bins.

## Choices for `minval`
You have the option to specify the lower bound of the interval over which the
histogram will be computed.  If `minval` is not specified then the minimum
value present in the image is taken as the lower bound.

## Choices for `maxval`
You have the option to specify the upper bound of the interval over which the
histogram will be computed.  If `maxval` is not specified then the maximum
value present in the image is taken as the upper bound.

## Choices for `edges`
If you do not designate the number of bins, nor the lower or upper bound of the
interval, then you have the option to directly stipulate how the intervals will
be divided by specifying a `AbstractRange` type.

# Example

Compute the histogram of a grayscale image.
```julia

using TestImages, FileIO, ImageView

img =  testimage("mandril_gray");
edges, counts  = build_histogram(img, 256, minval = 0, maxval = 1)
```

Given a color image, compute the histogram of the red channel.
```julia
img = testimage("mandrill")
r = red.(img)
edges, counts  = build_histogram(r, 256, minval = 0, maxval = 1)
```

# References
[1] E. Herrholz, "Parsimonious Histograms," Ph.D. dissertation, Inst. of Math. and Comp. Sci., University of Greifswald, Greifswald, Germany, 2011.
"""
function build_histogram(img::AbstractArray, nbins::Integer = 256; minval::Union{Real,AbstractGray}, maxval::Union{Real,AbstractGray})
    edges = partition_interval(nbins, minval, maxval)
    build_histogram(img, edges)
end

# Warning is due to: https://github.com/JuliaLang/julia/issues/9498
# Might resolve this based on this idea: https://discourse.julialang.org/t/fun-with-kwargs-methods/15711/8
function build_histogram(img::AbstractArray, nbins::Integer = 256)
    build_histogram(img, nbins, minval =  minfinite(img), maxval = maxfinite(img))
end

function build_histogram(img::AbstractArray{T}, edges::AbstractRange) where {T<:Color3}
    build_histogram(Gray.(img),edges)
end

function build_histogram(img::AbstractArray, edges::AbstractRange)
    Base.has_offset_axes(edges) && throw(ArgumentError("edges must be indexed starting with 1"))
    lb = first(axes(edges,1))-1
    ub = last(axes(edges,1))
    first_edge = first(edges)
    step_size = step(edges)
    counts = fill(0, lb:ub)
    for val in img
         if isnan(val)
             continue
         else
            if val >= edges[end]
                counts[end] += 1
                continue
            elseif val < first(edges)
                counts[lb] += 1
            else
                index = Int(Base.div(val-first_edge,step_size)) + 1
                counts[index] += 1
            end
        end
    end
    edges, counts
end


using LinearAlgebra

mutable struct Nystrom{T}
	Ω::AbstractMatrix{T}
	S::AbstractMatrix{T}
	function Nystrom{T}(n, R) where {T}
		new(randn(T, n, R), zeros(n, R))
	end
end

function update!(sketch, v, η)
	sketch.S .*= (1 - η)
	sketch.S .+= η * v * (v' * sketch.Ω)
end

function reconstruct(sketch)
	n = size(sketch.S, 1)

	σ = sqrt(n) * eps() * norm(sketch.S)
	Sσ = sketch.S + σ * sketch.Ω
	L = cholesky(Hermitian(sketch.Ω' * Sσ); check=false)
	U, Σ, = svd(Sσ / L.U)
	Λ = max.(0, Σ .^ 2 .- σ)

	U, Diagonal(Λ)
end


using ZonalFlow
using Test

domain  = Domain(extent=(2π,2π),res=(5,5));
coeffs  = Coefficients(Ω=2π,θ=0.0,μ=0.0,ν=0.0,ν₄=1.0,linear=true);
forcing = Stochastic(kf=3,dk=1,ε=0.0);
prob    = BetaPlane(domain,coeffs,forcing);

tspan   = (0.0,1000.0);
tsargs  = (
            dt=0.001,
            adaptive=false,
            progress=true,
            progress_steps=100000,
            save_everystep=false,
            saveat=500
           );

eqs     = [NL(),CE2()];
eqs     = append!(eqs,[GQL(Λ=l) for l=0:prob.d.nx-1])
eqs     = append!(eqs,[GCE2(Λ=l) for l=0:prob.d.nx-1])
sols    = integrate(prob,eqs,tspan;tsargs...);

@testset "Linear Equations" begin
        for sol in sols
                E,Z = energy(length(prob.d)...,size(prob.d)...,sol.t,sol.u);
                @test E[1] == E[end]
                @test Z[1] == Z[end]
        end
end

coeffs  = Coefficients(Ω=2π,θ=0.0,μ=0.0,ν=0.0,ν₄=1.0,linear=false);
prob    = BetaPlane(domain,coeffs,forcing);
sols    = integrate(prob,eqs,tspan;tsargs...);

@testset "Nonlinear Equations" begin
        for sol in sols
                E,Z = energy(length(prob.d)...,size(prob.d)...,sol.t,sol.u);
                @test E[1] == E[end]
                @test Z[1] == Z[end]
        end
end


using FractionalDiffEq, Plots
tspan = (0, 30); h = 0.01
rightfun(x, y) = 172/125*cos(4/5*x)
prob = MultiTermsFODEProblem([1, 1/16, 4/5, 3/2, 1/25, 6/5], [3, 2.5, 2, 1, 0.5, 0], rightfun, [0, 0, 0, 0, 0, 0], tspan)
sol = solve(prob, h, PIEX())
plot(sol, legend=:bottomright)


@test simplify(:x/:x,Div)==1

@test simplify(@equ a=c*c/c).rhs==:c
@test simplify(@equ a=c^2/c).rhs==:c
@test simplify(:a^3.5/:a)==:a^2.5

E=@equ E=sqrt(p^2*c^2+m^2*c^4)
vars=@equs p=sqrt(2)*1e6/c m=0.5e6/c^2
r=E&vars[2]
@test r.rhs==Equations.Sqrt(:p*:p*:c*:c+2.5e11)


"Problem 6 in the Hock-Schittkowski suite"
function hs6()

  nlp = Model()

  @variable(nlp, x[1:2])
  setvalue(x[1], -1.2)
  setvalue(x[2],  1.0)

  @NLobjective(
    nlp,
    Min,
    (1 - x[1])^2
  )

  @NLconstraint(
    nlp,
    10 * (x[2] - x[1]^2) == 0
  )

  return nlp
end


using JuMP, EAGO

                     m = Model()

                     EAGO.register_eago_operators!(m)

                     @variable(m, -1 <= x[i=1:2] <= 1)

                     @variable(m, -4.572700962608518 <= q <= 2.439967378874661)

                     add_NL_constraint(m, :(swish(-0.9934015186054679 + 0.8375762785867722*$(x[1]) + 0.668300383494036*$(x[2])) + swish(0.08494880908446678 + -0.07388095057292743*$(x[1]) + -0.9863741327473892*$(x[2])) + swish(-0.1579140823459273 + 0.7305802491259836*$(x[1]) + -0.2096221762144812*$(x[2])) - $q <= 0.0))

                     @objective(m, Min, q)

                     return m


"""
The normal form of pitchfork bifurcations.

https://en.wikipedia.org/wiki/Pitchfork_bifurcation
"""
module Pitchfork

using DiffEqBase: ODEProblem
using Setfield: @lens

using ...Bifurcations: BifurcationProblem

f(u::Real, p, t) = p * u - u^3

u0 = 0.0
tspan = (0.0, 1.0)
p = -1.0
ode = ODEProblem(f, u0, tspan, p)

param_axis = @lens _
prob = BifurcationProblem(ode, param_axis, (-1.0, 1.0))

using StaticArrays: @SVector

function closest_analytic(u, p)
    if p < 0
        return [abs(u), p]
    else
        analytic = @SVector [-sqrt(p), 0, sqrt(p)]
        _, i = findmin(abs.(u .- analytic))
        return [analytic[i], p]
    end
end

end  # module


module MatrixLMnet

using MatrixLM
import MatrixLM.calc_preds, MatrixLM.calc_preds!,
    MatrixLM.calc_resid, MatrixLM.calc_resid!

using DataFrames
using Distributed
using LinearAlgebra
using LinearAlgebra.BLAS
import LinearAlgebra.mul!, LinearAlgebra.norm, LinearAlgebra.dot,
     LinearAlgebra.diagm, LinearAlgebra.eigmin, LinearAlgebra.eigmax
using Random
using Statistics
using MLBase

export Response, Predictors, RawData, get_X, get_Z, get_Y, contr,
    add_intercept, remove_intercept, shuffle_rows, shuffle_cols,
    cd!, cd_active!, ista!, fista!, fista_bt!, admm!,
    Mlmnet, mlmnet, coef, coef_2d, predict, fitted, resid,
    mlmnet_perms,
    make_folds, make_folds_conds, Mlmnet_cv, mlmnet_cv,
    avg_mse, lambda_min, avg_prop_zero, mlmnet_cv_summary

# Helper functions
include("std_helpers.jl")
include("mlmnet_helpers.jl")

# L1 algorithms
include("cd.jl")
include("ista.jl")
include("fista.jl")
include("fista_bt.jl")
include("admm.jl")

# Top level functions that call L1 algorithms using warm starts
include("mlmnet.jl")
# Predictions and residuals
include("predict.jl")

# Permutations
include("mlmnet_perms.jl")

# Cross-validation
include("mlmnet_cv_helpers.jl")
include("mlmnet_cv.jl")
include("mlmnet_cv_summary.jl")

end


using Distributed

if nworkers()==1
	addprocs(2);
end

@everywhere begin
	using jInv.Mesh
	using jInv.LinearSolvers
	using Multigrid.ParallelJuliaSolver
	using Multigrid.DomainDecomposition
	using Multigrid
	using KrylovMethods

	using SparseArrays
	using LinearAlgebra
	using Test
	using jInv.Mesh;
end
@everywhere begin
	include(string(pwd(),"/DomainDecomposition/DDPoissonFuncs.jl"));
end

println("************************************************* Dirichlet DomainDecomposition for Poisson 2D ******************************************************");

domain = [0.0, 1.0, 0.0, 1.0];
n = [128,128];
Mr = getRegularMesh(domain,n)
G = getNodalGradientMatrix(Mr);
Ar = G'*G;
Ar = Ar + 1e-5*opnorm(Ar,1)*sparse(1.0I,size(Ar,2),size(Ar,2));

NumCells = [4,4];
overlap = [1,1];
n_tup = tuple(n...);
dim = length(n);

numCores 	= 2;
maxIter     = 10;
relativeTol = 1e-6;

q = randn(prod(n.+1));
x = zeros(size(q));
Ainv = getParallelJuliaSolver(Float64,Int64,numCores=numCores,backend=1);

DDparam = getDomainDecompositionParam(Mr,NumCells,overlap,getNodalIndicesOfCell,Ainv);

println("workers are: ",workers())

setupDDParallel(Ar,DDparam,workers());

Prec = r->solveDDParallel(Ar,r,zeros(eltype(q),size(q)),DDparam,workers(),1,0)[1];
x = zeros(size(q));

x, flag,rnorm,iter = KrylovMethods.fgmres(getAfun(Ar,zeros(eltype(q),size(q)),4),q,10,tol = 1e-10,maxIter = 2,M = Prec, x = x,out=2,flexible=true);

println(norm(Ar'*x - q)/norm(q))

println("************************************************* Neumann DomainDecomposition for Poisson 2D ******************************************************");

Ctor = DomainDecompositionOperatorConstructor(Mr,getSubParams,getLap,getDirichletMassNodalMesh);

setupDDParallel(Ctor,DDparam,workers());

Prec = r->solveDDParallel(Ar,r,zeros(eltype(q),size(q)),DDparam,workers(),1,0)[1];
x = zeros(size(q));

x, flag,rnorm,iter = KrylovMethods.fgmres(getAfun(Ar,zeros(eltype(q),size(q)),4),q,10,tol = 1e-10,maxIter = 2,M = Prec, x = x,out=2,flexible=true);


######## Sampler for pure fBm and related processes ########

#### Cholesky ####
"""
Cholesky sampler for general Gaussian process.

# Notes
- This method generates a sample trajectory on any type of sampling grid (regular or arbitrary) adapted to the underlying process.
"""
struct CholeskySampler{T, P, G} <: Sampler{T, P, G}
    proc::P  # instance of the stochastic process
    grid::G  # grid in use for sampling
    cmat::AbstractMatrix  # covariance matrix
    lmat::AbstractMatrix  # lower-triangular matrix: lmat * lmat' = cmat

    function CholeskySampler{T, P, G}(p::P, g::G) where {T, P, G}
        # check the grid for fBM
        if (P<:FractionalBrownianMotion) && (0 in g)
            error("The sampling grid must not contain the origin.")
        end
        # construct the auto-covariance matrix
        cmat = covmat(p, g)
        # cholesky decomposition yields lower and upper triangular matrices L, U such that L*U = cmat, and L=U'
        # lmat = cholesky(cmat).U'
        lmat = cholesky(cmat).L
        return new(p, g, cmat, lmat)
    end
end

# outer constructor as a shortcut
# CholeskySampler(p::P, g::G) where {P, G} = CholeskySampler{typeof(p), typeof(g)}(p, g)
CholeskySampler(p::StochasticProcess{T}, g::AbstractVector{<:T}) where T<:TimeStyle = CholeskySampler{T, typeof(p), typeof(g)}(p, g)

function rand!(x::Vector{<:AbstractFloat}, s::CholeskySampler)
    return copyto!(x, s.lmat * randn(length(s)))
end

#### Circulant Embedding ####
"""
Circulant embedding sampler for stationary Gaussian process.

# Notes
- This method generates a sample trajectory on any sampling grid adapted to the underlying process.
"""
struct CircSampler{T, P<:StationaryProcess{T}, G<:AbstractVector{<:T}} <: Sampler{T, P, G}
    proc::P  # instance of the stochastic process
    grid::G  # grid in use for sampling
    cseq::AbstractVector  # covariance sequence
    fseq::AbstractVector  # square-root of the Fourier coefficients of cseq

    function CircSampler{T, P, G}(p::P, g::G) where {T, P, G}
        @assert isregulargrid(g)  # only works on regular (continuous or discrete) grid

        # Nf = 1 << ceil(Integer, log2(length(p)))  # length of FFT, equivalent to 2^()
        c = covseq(p, g)
        # minimal circulant embedding
        cm = vcat(c, c[end-1:-1:2])  # mirror vector
        M = length(cm)
        # fft() in Julia is not normalized
        # cf = real(fft(cm)/sqrt(M))  # the imaginary part is close to zero
        cf = real(fft(cm))  # the imaginary part is close to zero
        # check the non-negative constraint
        idx = cf .< 0.
        any(idx) && @warn("Negative eigenvalues encountered, using Wood-Chan approximation.")
        cf[idx] .= 0.
        new(p, g, c, sqrt.(cf))
    end
end

CircSampler(p::StationaryProcess{T}, g::AbstractVector{<:T}) where T = CircSampler{T, typeof(p), typeof(g)}(p, g)

# function rand!(x::Vector{Complex128}, s::CircSampler{P, G}) where {P, G}
function rand!(x::Vector{<:AbstractFloat}, s::CircSampler)
    # @assert length(x) <= length(s)
    M = length(s.fseq)
    z = randn(M) + im * randn(M)
    # y = real(ifft(s.fseq .* fft(z)))
    y = real(ifft(s.fseq .* z) * sqrt(M))
    return copyto!(x, y[1:length(x)])
end

"""
Functional implementation of the circulant embedding method. It takes the sequence of covariance and return a sample path of the same length.
"""
function circulant_embedding(c::Vector{<:AbstractFloat})
    cm = vcat(c, c[end-1:-1:2])  # mirror vector
    M = length(cm)
    # fft() in Julia is not normalized
    # cf = real(fft(cm)/sqrt(M))  # the imaginary part is close to zero
    cf = real(fft(cm))  # the imaginary part is close to zero
    # check the non-negative constraint
    idx = cf .< 0.
#     any(idx) && warn("Negative eigenvalues encountered, using Wood-Chan approximation.")
#         println("length(idx)=$(length(idx)), cf[idx]=$(cf[idx])")
    cf[idx] = 0.
    fseq = sqrt.(cf)
    z = randn(M) + im * randn(M)
    # y = real(ifft(s.fseq .* fft(z)))
    return real(ifft(fseq .* z) * sqrt(M))[1:length(c)]
end

#### Hosking ####
"""
Hosking (Levinson-Durbin) sampler for stationary Gaussian process.

# Notes
* This method draws the sample trajectory on any sampling grid adapted to the underlying process.
"""
struct HoskingSampler{T, P<:StationaryProcess{T}, G<:AbstractVector{<:T}} <: Sampler{T, P, G}
    proc::P  # instance of the stochastic process
    grid::G  # grid in use for sampling
    cseq::AbstractVector  # covariance sequence
    rseq::AbstractVector  # sequence of partial correlation
    pseq::AbstractVector{<:AbstractVector}  # the coefficients of consecutive projections
    sseq::AbstractVector  # sequence of variance of residual of projections
    # pmat::SparseMatrixCSC{AbstractFloat}  # upper triangular matrix diagonalizing the covariance matrix
    # cmat::Matrix{<:AbstractFloat}  # matrix of covariance

    function HoskingSampler{T, P, G}(p::P, g::G) where {T, P, G}
        @assert isregulargrid(g)  # only works on regular (continuous or discrete) grid

        cseq = covseq(p, g)
        pseq, sseq, rseq = LevinsonDurbin(cseq)

        # N = length(g)
        # pmat = 1 * speye(N,N)
        # for c = 2:N
        #     pmat[1:c-1, c] = -pseq[c-1][end:-1:1]
        # end
        # cmat = covmat(p, g)
        return new(p, g, cseq, rseq, pseq, sseq) #, pmat, cmat)
    end
end

HoskingSampler(p::StationaryProcess{T}, g::AbstractVector{<:T}) where T = HoskingSampler{T, typeof(p), typeof(g)}(p, g)

# function rand!(x::Vector{<:AbstractFloat}, s::HoskingSampler{P, G}) where {P,G}
function rand!(x::Vector{<:AbstractFloat}, s::HoskingSampler)
    # length(x) == length(s) || error()
    # generate the first sample
    x[1] = sqrt(s.sseq[1]) * randn()
    # recursive conditional sampling
    for n = 1:length(x)-1
        μ = s.pseq[n]' * x[n:-1:1]
        σ = sqrt(s.sseq[n+1])
        x[n+1] = μ + σ * randn()
    end
    return x
end

"""
    rand_otf!(x::Vector{<:AbstractFloat}, s::HoskingSampler{P, G}) where {P, G}

On-the-fly sampling using Levinson-Durbin algorithm. The partial auto-correlation function must be known for the process, for example, the FARIMA{0,d,0} process.
"""
function rand_otf!(x::Vector{<:AbstractFloat}, p::StationaryProcess{T}, g::AbstractVector{<:T}) where T
    # println("Stationary otf")
    # check dimension
    # @assert length(x) <= length(g)

    # generate the first sample
    γ0 = autocov(p, 0)
    x[1] = sqrt(γ0) * randn()

    σ = sqrt(γ0)
    ϕ = zeros(length(x)); ϕ[1] = partcorr(p, 1)

    # recursive conditional sampling
    for n = 1:length(x)-1
        μ = ϕ[1:n]' * x[n:-1:1]
        σ *= sqrt(1-ϕ[n]^2)
        x[n+1] = μ + σ * randn()
        ϕ[n+1] = partcorr(p, n+1)
        ϕ[1:n] -= ϕ[n+1] * ϕ[n:-1:1]
    end
    return x
end

# function rand_otf!(x::Vector{<:AbstractFloat}, s::HoskingSampler{P, G}) where {P,G}
#     # println("Stationary otf")
#     # length(x) == length(s) || error()

#     # generate the first sample
#     γ0 = autocov(s.proc, 0)
#     x[1] = sqrt(γ0) * randn()
#     # recursive conditional sampling
#     σ = sqrt(γ0)
#     ϕ = [partcorr(s.proc, 1)]
#     for n = 1:length(x)-1
#         μ = ϕ' * x[n:-1:1]
#         σ *= sqrt(1-ϕ[end]^2)
#         x[n+1] = μ + σ * randn()
#         r = partcorr(s.proc, n+1)
#         ϕ = [ϕ - r*ϕ[end:-1:1]; r]  # <- memory reallocation!
#     end
#     return x
# end

"""
Sampling a general FARIMA process via a FARIMA{0,d,0} process.
"""
function rand(p::FARIMA, s::HoskingSampler{T, <:FractionalIntegrated, G}) where {T, G}
    p.d == s.proc.d || error("The order of fractional differential in FARIMA{P,Q} and FARIMA{0,0} must be identical.")
    ar_len, ma_len = length(p.ar), length(p.ma)
    x = zeros(length(s))

    # step 1: sample an FARIMA{0,d,0} process
    rand!(x, s)
    # or equivalently, invoke(rand!, Tuple{Vector{<:AbstractFloat}, HoskingSampler}, x, s)

    # step 2: generate an ARMA process
    # REVIEW 15/11/2018: NOT SURE about this implementation!!
    x = conv(x, [1; p.ma])[1:end-ma_len]
    if ar_len > 0
        for t = (ar_len+1):length(s)
            # singular case: Float64[]' * Float64[] = 0.0
            x[t] += p.ar' * x[(t-1):-1:(t-ar_len)]
        end
    end
    return x
end

#### CRMD ####
"""
Conditionalized random midpoint displacement (CRMD) sampler for the increment process of SSSI process, e.g. a fGn.

# Notes
- The step of increment process (e.g. a fGn) must be equal to 1, this is crucial since the Yule-Walker equations in our implementation is de-scaled.
- The original CRMD method draws the sample trajectory on the interval [0,1]. However in our implementation a scaling factor is applied at the end so that the final trajectory are samples of a discrete fGn process (i.e. of unit step).
- The sampling grid used in this method corresponds to the index of the sampling points but not their physical position in [0,1].
"""
struct CRMDSampler{P<:IncrementProcess} <: DiscreteTimeSampler{P}
    proc::P  # instance of the stochastic process
    grid::DiscreteTimeGrid  # grid in use for sampling
    jmin::Integer  # coarse scale index
    coarse_sampler::Sampler  # exact sampler for the coarse scale

    # sclrng::RegularGrid  # range of dyadic scales jmin..jmax
    wsize::Integer  # window size for the conditionalized sampling
    init_coef::AbstractMatrix  # coefficients of the initial sampler
    init_lmat::AbstractMatrix  # lower triangular matrix of the initial sampler
    rfn_coef::AbstractVector  # coefficients of the refinement sampler
    rfn_std::AbstractFloat  # square-root of variance of the refinement (scale independant)
    # cmat::AbstractMatrix  # system matrix of the refinement sampler
    # cvec::AbstractVector  # RHS vector of the refinement sampler
    # init_pos::Symbol  # window position of initial samples, :left, :right, :center

    """
    Constructor of CRMDSampler.

    # Args
    - p: object of `IncrementSSSIProcess`.
    - g: discrete-time regular sampling grid, the step can be >= 1.
    - w: moving window size.
    - jmin: index of the coarsest scale.

    # Notes
    - The step of increment process `p` (e.g. a fGn) must be equal to 1, this is crucial since the Yule-Walker equations in our implementation is de-scaled.
    - the grid `g` is used only at the end: the generated trajectory with unit step is restricted on `g` to obtain the (down-sampled) trajectory of desired length.
    """
    function CRMDSampler{P}(p::P, g::DiscreteTimeGrid, w::Integer, jmin::Integer) where {P}
        @assert isregulargrid(g)  # only works on regular (continuous or discrete) grid

        step(p) == 1 || error("Step of increment of the underlying process must be 1.")
        @assert 1 <= w <= 2^jmin
        # 2^(Int(round(log2(length(g))))) == length(g) || error("Length of the sampling grid must be power of 2.")
        # jmin = 10  # coarse scale length = 2^jmin
        # jmax = max(jmin, ceil(Int, log2(length(G))))

        H = ss_exponent(p)

        # full covariance matrix
        # Note: without explicite conversion RegularGrid(...) the autocov function
        # `autocov(X::IncrementProcess{T, P}, t::T, s::T) where {T, P}` will be called.
        Cf = covmat(p, 1:2*w)
        Ch = Cf[1:w, 1:w]  # half matrix
        Cd = Cf[1:2:end, 1:2:end]  # down-sampled
        Af = [autocov(p, 2*(l-m)) + autocov(p, 2*(l-m)+1) for l=0:2w-1, m=0:2w-1]

        # initial sampler
        Ah = Af[1:w, 1:w] # [autocov(p, 2*(l-m)) + autocov(p, 2*(l-m)+1) for l=0:w-1, m=0:w-1]
        ξ = (2^(2*H) * Ch) \ Ah
        M = Symmetric(Cd - ξ' * Ah)  # reinforce symmetry
        L = cholesky(M).U'

        # system matrix
        b11 = 2^(2*H) * Cf
        b12 = Af[:, 1:w]
        # b12 = [autocov(p, 2*(l-m)) + autocov(p, 2*(l-m)+1) for l=0:2*w-1, m=0:w-1]
        # b21 = [autocov(p, 2*(m-l)) + autocov(p, 2*(m-l)-1) for m=0:w-1, l=0:2*w-1]
        b22 = Cd
        Γ = [b11 b12; b12' b22]

        # RHS vector
        b1 = [autocov(p, 2*(l-w)) + autocov(p, 2*(l-w)+1) for l=0:2w-1]
        b2 = [autocov(p, 2*(m-w)) for m=0:w-1]
        γ = [b1; b2]

        φ = Γ \ γ  # coefficients of prediction
        σ = sqrt(autocov(p, 0) - φ' * γ)  # sqaure-root of variance

        coarse_sampler = CholeskySampler(p, 1:2^jmin)
        return new(p, g, jmin, coarse_sampler, w, ξ, L, φ, σ)
    end
end

CRMDSampler(p::P, g::DiscreteTimeGrid, w::Integer=10, jmin::Integer=10) where {P} = CRMDSampler{P}(p, g, w, jmin)

function rand!(x::Vector{<:AbstractFloat}, s::CRMDSampler)
    # @assert length(x) <= length(s)

    H = ss_exponent(s.proc)
    # sampling at the coarse scale using an exact sampler
    x0 = 2^(-s.jmin * H) * rand(s.coarse_sampler)

    # dyadic refinement, until the trajectory grows to the end of the sampling grid
    while length(x0) < s.grid[end]
        x0 = rand_rfn(x0, s)
    end
    # x0 is a sample trajectory of ${X^\delta_H(n)}_{n=0...N-1} $ with $\delta=1/N$ on the interval $[0,1]$. To obtain a sample trajectory of $X^1_H$, apply the scaling factor $N^H$.
    x0 .*= length(x0)^H

    return copyto!(x, x0[s.grid][1:length(x)])
end

function rand_rfn(x0::Vector{<:AbstractFloat}, s::CRMDSampler)
    j = ceil(Int, log2(length(x0)))  # scale index
    2^j == length(x0) || error("Length of the input vector must be a power of 2.")

    H = ss_exponent(s.proc)
    α = 2^(-j * H)  # scaling factor
    w = s.wsize

    x1 = zeros(2*length(x0))  # refined samples
    xo = view(x1, 1:2:length(x1))  # refined samples of odd positions
    xe = view(x1, 2:2:length(x1))  # refined samples of even positions
    xz = zeros(length(x0) + w); xz[1:length(x0)] = x0  # zero-padding to avoid boundary check
    wn = randn(length(x0))  # white noise vector

    # initialization: generate the w samples at the leftmost
    xo[1:w] = s.init_coef' * xz[1:w] + α * s.init_lmat * wn[1:w]

    # left to right (forward) propagation
    for t = (w+1):length(x0)
        xo[t] = s.rfn_coef' * [xz[t-w:t+(w-1)]; xo[t-w:t-1]] + α * s.rfn_std * wn[t]
    end
    xe[:] = x0 - xo  # [:] creates a reference, without [:] there will be re-allocation of memory.

    return x1
end

#### Wavelet ####
"""
Wavelet sampler for fractional integrated (fIt) process with d ∈ (-1/2, 1/2).

# Notes
- This method can generated FARIMA(0,H-1/2,0) as well as FARIMA(0, H+1/2,0) process for H in (0,1). The later corresponds to the partial sum of the former. The sampling grid corresponds to the index of the discrete samples. In the case of FARIMA(0, H+1/2, 0) the trajectory can approximate a fBm on the interval [0,1] by proper rescaling.
"""
struct WaveletSampler <: DiscreteTimeSampler{FractionalIntegrated}
    proc::FractionalIntegrated  # instance of the stochastic process
    grid::DiscreteTimeGrid  # grid in use for sampling
    coarse_sampler::Sampler  # exact sampler for the coarse scale
    psflag::Bool  # flag of partial sum
    jmin::Integer  # coarse scale index
    qmf_ori::Tuple{AbstractVector, AbstractVector}  # original qmf filter (lo, hi)
    # qmf_mod::Tuple{AbstractVector, AbstractVector}  # modified qmf filter (lo, hi)
    qmf_fra::Tuple{AbstractVector, AbstractVector}  # fractionnal qmf filter  (lo, hi)

    """
    Constructor of WaveletSampler.

    # Args
    - p: fractional integrated process, with the parameters d=H-1/2 and H the Hurst exponent
    - r: regularity of the wavelet function, must be strictly larger than s=H+1/2
    - psflag: if true generate a trajectory of FARIMA(0, H+1/2, 0), otherwise generate FARIMA(0, H-1/2, 0).
    """
    function WaveletSampler(p::FractionalIntegrated, g::DiscreteTimeGrid; r=5, jmin=10, psflag=true)
        @assert isregulargrid(g)  # only works on regular (continuous or discrete) grid

        # jmin = 10  # coarse scale index
        trunc_eps = 1e-8  # precision of truncation
        H = p.d + 1/2  # Hurst exponent

        r > H+1/2 || error("Regularity of the wavelet must be larger than H+1/2!")
        v = psflag ? H+1/2 : H-1/2  # fractional exponent for H+1/2 or H-1/2

        # s = H + 1/2; d = H - 1/2   #  or simply: s = p.d + 1; d = p.d
        # r > s || error("Regularity of the wavelet must be larger than H+1/2!")
        # v = psflag ? s : d  # fractional exponent for H+1/2 or H-1/2

        # original qmf filters
        lo_ori = Wavelets.WT.daubechies(2*r)
        hi_ori = qmf(lo_ori)  # or: hi_ori = reverse(mirror(lo_ori))

        # modified qmf filters
        lo_mod = copy(lo_ori)
        hi_mod = copy(hi_ori)
        knl = (-1).^(0:length(lo_mod)-1)
        for n=1:r
            lo_mod = conv(knl, lo_mod)[1:length(lo_mod)]
            hi_mod = cumsum(hi_mod)
        end

        lmax = 1000  # maximum length of truncation
        f0 = [binomial(v+r, k) for k in 0:lmax]
        g0 = [(-1)^k * binomial(r-v, k) for k in 0:lmax]
        lt = max(sum(abs.(f0).>trunc_eps), sum(abs.(g0).>trunc_eps))
        lo_fra = conv(f0[1:lt], lo_mod)
        hi_fra = conv(g0[1:lt], hi_mod)

        coarse_sampler = CholeskySampler(p, 1:2^jmin)
        return new(p, g, coarse_sampler, psflag, jmin, (lo_ori, hi_ori), (lo_fra, hi_fra))
    end
end

# binomial(n::Real, k::Real) = gamma(n+1)/gamma(n-k+1)/gamma(k+1)
binomial(n::Complex, k::Complex) = exp(lgamma(n+1)-lgamma(n-k+1)-lgamma(k+1))
# binomial(n::Complex, k::Complex) = exp(lgamma(k-n)-lgamma(-n)-lgamma(k+1))
binomial(n::Real, k::Real) = real(binomial(Complex(n), Complex(k)))
# binomial(n::Number, k::Number) = binomial(Complex(n), Complex(k))

"""
Dyadic refinement sampling.
"""
function rand_rfn(x0::Vector{<:AbstractFloat}, s::WaveletSampler)
    x = zeros(2*length(x0))  # up-sampled samples
    w = zeros(2*length(x0))  # white noise
    x[1:2:end] = x0
    w[1:2:end] = randn(length(x0))

    return conv(s.qmf_fra[1], x) + conv(s.qmf_fra[2], w)
end

# function rand_rfn_ori(x0::Vector{<:AbstractFloat}, s::WaveletSampler)
#     x = zeros(2*length(x0))  # up-sampled samples
#     w = zeros(2*length(x0))  # white noise
#     x[1:2:end] = x0
#     w[1:2:end] = randn(length(x0))
#     return conv(s.qmf_ori[1], x) + conv(s.qmf_ori[2], w)
# end

function rand!(x::Vector{<:AbstractFloat}, s::WaveletSampler)
    # @assert length(x) <= length(s)

    # sampling at the coarse scale using an exact sampler
    x0 = rand(s.coarse_sampler)
    if s.psflag
        x0 = cumsum(x0)
    end

    # dyadic refinement
    while length(x0) < 2 * s.grid[end]
        x0 = rand_rfn(x0, s)
    end

    Ng = s.grid[end]-s.grid[1]  # full range of grid
    # keep only the central part and force to start from 0
    n = max((length(x0)-Ng)÷2, 1)
    x1 = x0[n:(n+Ng)] .- x0[n]
    return copyto!(x, x1[s.grid][1:length(x)])
end


module Surrogates

using LinearAlgebra
using DataFrames
using GLM
using Distributions
using Sobol
using LatinHypercubeSampling
using XGBoost
using LIBSVM
using Flux

abstract type AbstractSurrogate <: Function end
include("Radials.jl")
include("Kriging.jl")
include("Sampling.jl")
include("Optimization.jl")
include("Lobachesky.jl")
include("LinearSurrogate.jl")
include("RandomForestSurrogate.jl")
include("SVMSurrogate.jl")
include("NeuralSurrogate.jl")

export AbstractSurrogate, SamplingAlgorithm
export Kriging, RadialBasis, add_point!, current_estimate, std_error_at_point
export sample, GridSample, UniformSample, SobolSample, LatinHypercubeSample, LowDiscrepancySample
export SRBF,LCBS,EI,DYCORS,surrogate_optimize
export LobacheskySurrogate, lobachesky_integral, lobachesky_integrate_dimension
export LinearSurrogate
export RandomForestSurrogate
export SVMSurrogate
export NeuralSurrogate

end


# L2EpsilonLoss

using LaTeXStrings, LossFunctions, Measures, Latexify

Plots.plot(L2EpsilonInsLoss(1.5), c=:olive, linewidth=2, annotation=(0, 1.5, L"L\left( r \right) = max\{0, |r| - \epsilon \}^2", 15))
Plots.plot!(L2EpsilonInsLoss(1), c=:orangered4, linewidth=2.5)
Plots.plot!(L2EpsilonInsLoss(.5), c=:springgreen4, linewidth=2.5)


"""
$(SIGNATURES)

Return whether `vertices` are ordered according to vertex order type `O` (a subtype of [`VertexOrder`](@ref)),
and as a result *strongly* convex (see e.g. [CGAL documentation](https://doc.cgal.org/latest/Convex_hull_2/index.html)
for a definition of strong convexity).
"""
function is_ordered_and_strongly_convex(vertices::AbstractVector{<:PointLike}, order::Type{O}) where {O<:VertexOrder}
    op = orientation_comparator(O)
    n = length(vertices)
    n <= 2 && return true
    @inbounds begin
        δprev = vertices[n] - vertices[n - 1]
        δnext = vertices[1] - vertices[n]
        for i in Base.OneTo(n - 1)
            op(cross2(δprev, δnext), 0) || return false
            δprev = δnext
            δnext = vertices[i + 1] - vertices[i]
        end
        return op(cross2(δprev, δnext), 0)
    end
end


#=
solves a simple random linear optimization problem (LP):
    min  c'x
    s.t. Ax = b, x >= 0
=#

using SparseArrays

struct LinearOptNative{T <: Real} <: ExampleInstanceNative{T}
    m::Int
    n::Int
    nz_frac::Float64
end

function build(inst::LinearOptNative{T}) where {T <: Real}
    (m, n, nz_frac) = (inst.m, inst.n, inst.nz_frac)
    @assert 0 < nz_frac <= 1

    # generate random data
    A = (nz_frac >= 1) ? rand(T, m, n) : sprand(T, m, n, nz_frac)
    A .*= 10
    b = vec(sum(A, dims = 2))
    c = rand(T, n)
    G = Diagonal(-one(T) * I, n) # TODO uniformscaling
    h = zeros(T, n)
    cones = Cones.Cone{T}[Cones.Nonnegative{T}(n)]

    model = Models.Model{T}(c, A, b, G, h, cones)
    return model
end


@testset "Regression" begin
    @test has_fit_methods(LinearRegression)

    # Test for 10 random points
    for _ in 1:10
        X = rand(100, 5)
        y = rand(100)
        model = LinearRegression(X, y)
        fit!(model)
        β = X \ y
        for i in 1:size(X, 2)
            @test get_constrained_value(model, "β_$i") .≈ β[i] atol = 1e-5
        end
    end
    X = [
        1 1
        1 2
        1 3
        1 4
        1 5.0
    ]
    y = [
        6
        7
        8
        9
        10.0
    ]
    model = LinearRegression(X, y)
    fit!(model)
    forec = forecast(model, [1 6.0])
    forec.expected_value[1] == [11.0]
end


using ReservoirComputing
using ParameterizedFunctions
using OrdinaryDiffEq
using Plots

#lorenz system parameters
u0 = [1.0,0.0,0.0]
tspan = (0.0,1000.0)
p = [10.0,28.0,8/3]

#define lorenz system
function lorenz(du,u,p,t)
    du[1] = p[1]*(u[2]-u[1])
    du[2] = u[1]*(p[2]-u[3]) - u[2]
    du[3] = u[1]*u[2] - p[3]*u[3]
end

#solve system
prob = ODEProblem(lorenz, u0, tspan, p)
sol = solve(prob, ABM54(), dt=0.02)
v = sol.u
data = Matrix(hcat(v...))

#model parameters
shift = 300
approx_res_size = 300
radius = 1.2
activation = tanh
degree = 6
sigma = 0.1
train_len = 5000
predict_len = 1250
beta = 0.0
alpha = 1.0
nla_type = NLAT2()
extended_states = false

#get data
train = data[:, shift:shift+train_len-1]
test = data[:, shift+train_len:shift+train_len+predict_len-1]

#create echo state network
esn = ESN(approx_res_size,
    train,
    degree,
    radius,
    activation = activation,
    sigma = sigma,
    alpha = alpha,
    nla_type = nla_type,
    extended_states = extended_states)

#training and prediction
W_out = ESNtrain(esn, beta)
output = ESNpredict(esn, predict_len, W_out)

#plots and images
comp = plot(transpose(output),layout=(3,1), label="predicted")
comp = plot!(transpose(test),layout=(3,1), label="actual")
savefig(comp, "lorenz_coord")
sf = plot(transpose(output)[:,1], transpose(output)[:,2], transpose(output)[:,3], label="predicted")
sf2 = plot(transpose(test)[:,1], transpose(test)[:,2], transpose(test)[:,3], label="actual")
final = plot(sf, sf2, layout=(1,2), label = ["predicted" "actual"])
savefig(final, "lorenz_attractor")


function run_TGV(parts,n::Int,order::Int,dt::Real,tf::Real)

  # Parameters
  L = 2π
  Re = 800.0
  ν = 1/Re

  # TGV initial solution
  u₀(x) = VectorValue(cos(x[1])*sin(x[2])*sin(x[3]),-sin(x[1])*cos(x[2])*sin(x[3]),0.0)
  p₀(x) = 1/16*(cos(2*x[1])+cos(2*x[2]))*(cos(2*x[3])+2)

  # Discretization
  domain = (0,L,0,L,0,L)
  cells = (n,n,n)
  𝒯 = CartesianDiscreteModel(parts,domain,cells;isperiodic=(true,true,true))

  # Triangulation and Integration measure
  Ω = Interior(𝒯)
  dΩ = Measure(Ω,2*order)

  # FE spaces
  refFEᵤ = ReferenceFE(lagrangian,VectorValue{3,Float64},order)
  refFEₚ = ReferenceFE(lagrangian,Float64,order-1)
  V = TestFESpace(𝒯,refFEᵤ)
  Q = TestFESpace(𝒯,refFEₚ)
  U = TransientTrialFESpace(V)
  P = TrialFESpace(Q)
  Y = MultiFieldFESpace([V,Q])
  X = TransientMultiFieldFESpace([U,P])

  # Explicit FE functions
  global ηₙₕ = interpolate(VectorValue(0.0,0.0,0.0),U(0.0))
  global uₙₕ = interpolate(u₀,U(0.0))
  global fv_u = zero_free_values(U(0.0))

  # Stabilization Parameters
  c₁ = 12.0
  c₂ = 2.0
  cc = 4.0
  h = L/(n*order)
  τₘ = 1/(c₁*ν/h^2 + c₂*(meas∘uₙₕ)/h)
  τc = cc *(h^2/(c₁*τₘ))

  # Weak form
  c(a,u,v) = 0.5*((∇(u)'⋅a)⋅v - u⋅(∇(v)'⋅a))
  res(t,(u,p),(v,q)) = ∫( ∂t(u)⋅v  + c(u,u,v) + ν*(∇(u)⊙∇(v)) - p*(∇⋅v) + (∇⋅u)*q +
                          τₘ*((∇(u)'⋅u - ηₙₕ)⋅(∇(v)'⋅u)) + τc*((∇⋅u)*(∇⋅v)) )dΩ
  jac(t,(u,p),(du,dp),(v,q)) = ∫( c(du,u,v) + c(u,du,v) + ν*(∇(du)⊙∇(v)) - dp*(∇⋅v) + (∇⋅du)*q +
                                  τₘ*((∇(u)'⋅u - ηₙₕ)⋅(∇(v)'⋅du) + (∇(du)'⋅u + ∇(u)'⋅du)⋅(∇(v)'⋅u)) +
                                  τc*((∇⋅du)*(∇⋅v)) )dΩ
  jac_t(t,(u,p),(dut,dpt),(v,q)) = ∫( dut⋅v )dΩ
  op = TransientFEOperator(res,jac,jac_t,X,Y)

  # Orthogonal projection
  a(η,κ) = ∫( τₘ*(η⋅κ) )dΩ
  b(κ) = ∫( τₘ*((∇(uₙₕ)'⋅uₙₕ)⋅κ) )dΩ
  op_proj = AffineFEOperator(a,b,U,V)
  ls = PETScLinearSolver()

  # Transient solver
  xₕ₀ = interpolate([u₀,p₀],X(0.0))
  nls = NLSolver(ls,show_trace=true,method=:newton,iterations=10)
  #nls = PETScNonlinearSolver()
  ode_solver = ThetaMethod(nls,dt,0.5)

  # Solution (lazy)
  xₕₜ = solve(ode_solver,op,xₕ₀,0,tf)

  K = Float64[]
  E = Float64[]
  T = Float64[]

  # Iterate over steps
  createpvd(parts,"results/TGV") do pvd
    for (xₕ,t) in xₕₜ
      println("--------------------")
      println("Start of time: $t")
      uₕ,pₕ=xₕ
      ωₕ = ∇×uₕ
      push!(K,0.5*(∑(∫(uₕ⋅uₕ)dΩ))/(2π^3))
      push!(E,ν*(∑(∫(ωₕ⋅ωₕ)dΩ))/(2π^3))
      push!(T,t)
      println("updating global variables")
      uₙₕ = interpolate!(uₕ,fv_u,U(t))
      ηₙₕ = solve(ls,op_proj)
      pvd[t] = createvtk(Ω,"results/TGV_$t";cellfields=["u"=>uₕ,"p"=>pₕ,"eta"=>ηₙₕ,"w"=>ωₕ],nsubcells=10)
      println("End of time: $t")
    end
 end

 return (T,K,E)
end


__precompile__()

module Tools
using LinearAlgebra
using Distributed

#################### utilities
function discount_sum(x::Vector{DT}, gamma::DT, terminal::DT) where DT<:AbstractFloat
   y = similar(x)
   runsum = terminal
   for t=length(x):-1:1
      runsum = x[t] + gamma*runsum
      y[t] = runsum
   end
   return y
end

function discount_sum!(y::Matrix{DT}, x::Matrix{DT},
                       t::Int, gamma::DT, terminal::DT) where DT<:AbstractFloat
   runsum = terminal
   for i=size(x, 1):-1:1
      runsum = x[i,t] + gamma*runsum
      y[i,t] = runsum
   end
end

function discount_sum!(y::Matrix{T}, x::Matrix{DT},
                       t::Int, gamma::DT, terminal::DT) where {T<:AbstractFloat,
                                                             DT<:AbstractFloat}
   runsum = terminal
   for i=size(x, 1):-1:1
      runsum = x[i,t] + gamma*runsum
      y[i,t] = runsum
   end
end

# reward: T x K
function compute_returns!(returns::Vector{DT},
                          rewards::Matrix{Float64},
                          gamma::Float64) where DT<:AbstractFloat
   T, K = size(rewards)
   mat_ret = reshape(returns, T, K)
   for t=1:K
      #mat_ret[:,t] = discount_sum(rewards[:,t], gamma, 0.0)
      discount_sum!(mat_ret, rewards, t, gamma, 0.0)
   end
   #return reshape(returns, T*K)
end

function compute_advantages(returns::Vector{Float64},
                            baseline::Vector{Float64},
                            obs::Matrix{Float64},
                            gamma::Float64)
   return returns - baseline
end

function computeGAEadvantages!(adv::Vector{DT},
                               baseline::Vector,
                               rewards::Matrix{Float64},
                               obs::Matrix{Float64},
                               gae::Float64,
                               gamma::Float64) where DT<:AbstractFloat
   T, K = size(rewards)
   tdsum = Array{DT}(undef, size(rewards))
   mat_adv = reshape(adv, T, K)

   nthread = min(K, Threads.nthreads()) # cant have more threads than data
   Threads.@threads for tid=1:nthread # WHY IS MT SO MUCH SLOWER??
      #nthread = 1
      #for tid=1:nthread # WHY IS MT SO MUCH SLOWER??
      thread_range = Distributed.splitrange(K, nthread)[tid]

      for t=thread_range
         base = view(baseline, (t-1)*T+1:t*T)
         for k=1:T-1
            tdsum[k,t] = rewards[k,t] + gamma * base[k+1] - base[k]
         end
         tdsum[T,t] = rewards[T,t] - base[T]
         #mat_adv[:,t] = discount_sum(tdsum[:,t], DT(gae*gamma), DT(0.0))
         discount_sum!(mat_adv, tdsum, t, DT(gae*gamma), DT(0.0))
      end
   end
end

# solver algorithms
function cpu_cg_solve(fim::Matrix{T}, # ((n+1)*m + m) X T*K
                      vpg::Vector{T}, # ((n+1)*m + m)
                      cg_iters::Integer=10,
                      reg::Float64=1e-4,
                      tol::Float64=1e-10) where T<:AbstractFloat
   # Initialize cg variables
   r = copy(vpg)
   p = copy(vpg)
   x = zeros(T, size(vpg))  # I want x to be same shape as vpg but full of zeros
   rdr = dot(vpg, vpg)  # outputs a scalar
   z = fim*p
   #z += p*reg

   iters = 1
   for i=1:cg_iters
      v = rdr/dot(p, z)      # scalar

      x .+= v.*p
      r .-= v.*z

      rdr_new = dot(r, r)    # scalar
      ratio = rdr_new/rdr
      rdr = rdr_new

      iters = i
      if rdr < tol
         break   # this should break the for loop
      end

      p .*= ratio
      p .+= r

      #println(norm(z))
      mul!(z, fim, p)
   end
   println("    Used $iters cojugate-gradient iterations.");
   return x
end

function hvp_cg_solve(hvpfim,
                      vpg::Vector{DT}, # ((n+1)*m + m)
                      cg_iters::Integer=10,
                      reg::Float64=1e-4,
                      tol::Float64=1e-10) where DT<:AbstractFloat
   # Initialize cg variables
   r = copy(vpg)
   p = copy(vpg)
   x = zeros(DT, size(vpg))
   rdr = dot(vpg, vpg)  # outputs a scalar
   z = zeros(DT, size(vpg))
   hvpfim(z, p)

   iters = 1
   for i=1:cg_iters
      v = rdr/dot(p, z)      # scalar

      x += v*p
      r -= v*z
      #@. x += v*p
      #@. r -= v*z

      rdr_new = dot(r, r)    # scalar
      ratio = rdr_new/rdr
      rdr = rdr_new

      iters = i
      if rdr < tol
         break   # this should break the for loop
      end

      @. p = r + ratio*p
      #p[:] .*= ratio
      #p[:] += r

      #println(norm(z))
      hvpfim(z, p)
   end
   println("    Used $iters HVP-cg iterations.");
   return x
end

function par_cg_solver(fim::Matrix{T}, # ((n+1)*m + m) X T*K
                       vpg::Vector{DT}, # ((n+1)*m + m)
                       toworkers, #::Vector{RemoteChannel},
                       tomaster, #::RemoteChannel,
                       cg_iters::Integer=10,
                       reg::DT=1e-4) where {T<:AbstractFloat,
                                            DT<:AbstractFloat}
   put!(tomaster[myid()], vpg) # send vpg
   vpg[:] = take!(toworkers[myid()])

   z = fim*vpg
   put!(tomaster[myid()], z)
   p = similar(z)

   for i=1:cg_iters
      z[:] = take!( toworkers[myid()] )
      p[:] = take!( toworkers[myid()] )

      if isnan(z[1]) || i==cg_iters break end

      mul!(z, fim, p)
      put!(tomaster[myid()], z)
   end
   return p # p is NPG
end
function scatter(tomaster, val::Vector{T}) where T<:AbstractFloat
   @sync for w in workers()
      @async put!(tomaster[w], copy(val))
   end
end
function gather!(toworkers, val::Vector{T}) where T<:AbstractFloat
   val[:] .= T(0.0)
   for w in workers()
      val[:] += take!(toworkers[w])
   end
   val[:] /= nworkers()
end
function par_cg_manager(nparam::Integer,
                        toworkers,
                        tomaster,
                        cg_iters::Integer=10,
                        reg::T=1e-4,
                        tol::Float64=1e-10) where T<:AbstractFloat
   vpg = Array{T}(undef, nparam)
   z = Array{T}(undef, nparam)
   x = zeros(T, nparam)

   gather!(tomaster, vpg)
   scatter(toworkers, vpg)
   rdr = dot(vpg, vpg)  # outputs a scalar
   p = copy(vpg)
   r = copy(vpg)

   count = 0
   for i=1:cg_iters
      count += 1
      gather!(tomaster, z) # get Z vector
      v = rdr/dot(p, z)      # scalar

      x += v*p
      r -= v*z

      rdr_new = dot(r, r)    # scalar
      ratio = rdr_new/rdr
      rdr = rdr_new

      if rdr < tol || i == cg_iters
         z[1:5] .= NaN
         scatter(toworkers, z)
         scatter(toworkers, x) # send NPG
         break
      end

      p[:] .*= ratio
      p[:] += r

      scatter(toworkers, z)
      scatter(toworkers, p)
   end
   println("\tconverged in $count iterations.")
   return vpg, x # x == npg
end

# GPU solver
#=
function gpu_cg_solve{T<:AbstractFloat,
                      DT<:AbstractFloat}(fim::Knet.KnetArray{T,2}, # ((n+1)*m + m) X T*K
                                         vpg::Vector{DT}, # ((n+1)*m + m)
                                         cg_iters::Integer=10,
                                         reg::Float64=1e-4,
                                         tol::Float64=1e-10)
   # gradll (fat): num_params x T*K
   # vpg : vector num_params

   # Initialize cg variables
   if T == DT
      vpg32 = reshape(vpg, length(vpg), 1)
   else
      vpg32 = Array{T,2}(reshape(vpg, length(vpg), 1))
   end
   r = KnetArray{T}(vpg32)
   p = KnetArray{T}(vpg32)
   x = KnetArray{T}(zeros(size(vpg32)))
   rdr = dot(vpg, vpg) #(p' * p)[1] #dot(p, p)  # outputs a scalar
   z = fim * p

   @inbounds for i=1:cg_iters
      #v = rdr/dot(p, z)      # scalar
      v = rdr/((p'*z)[1])      # scalar

      x += v*p
      r -= v*z

      #rdr_new = dot(r, r)    # scalar
      rdr_new = (r' * r)[1] #dot(r, r)    # scalar
      ratio = rdr_new/rdr
      rdr = rdr_new

      if rdr < tol
         break   # this should break the for loop
      end

      p = r + ratio*p

      z[:] = fim*p
   end
   return x
end
=#

end


using Dates, DelimitedFiles, Plots, Random, ScoreDrivenModels

# Define dates and load historical Affluent Natural Energy data
dates = collect(Date(1961):Month(1):Date(2000, 12))
y = vec(readdlm("../test/data/nie_northeastern.csv"))

y_train = y[1:400]
y_test = y[401:460]

# Set RNG seed to guarantee consistent results
Random.seed!(123)

# Specify GAS model: a lognormal model with time-varying μ, constant σ, and lags 4 and 12
gas = Model([1,2,3,4], [1,2,3,10,11,12], LogNormal, 1.0; time_varying_params=[1])

# Obtain initial parameters to start the GAS recursion
initial_params = dynamic_initial_params(y_train, gas)

# Fit specified model to historical data using initial parameters
f = ScoreDrivenModels.fit!(gas, y_train; initial_params=initial_params)

# Print estimation statistics
results(f)
plot(f)

# Simulate 1000 future scenarios and obtain the 5% and 95% quantiles in each time period
forec = forecast(y_train, gas, 60; S=1000, initial_params=initial_params)

# Plot results
plot(dates[401:460], forec.observation_scenarios, color="grey", w=0.05, label="", ylims=(0, 70))
plot!(dates[360:460], y[360:460], label="NIE", color="black", xlabel="Months", ylabel="GWmed", legend=:topright)
plot!(dates[401:460], forec.observation_quantiles, label=["Quantiles" "" ""], color="red", line=:dash)


export Logit

"""
Description:

    Logit mapping between a real variable in1 ∈ R and a binary variable out ∈ {0, 1}.

    f(out, in1, xi) =  Ber(out | σ(in1))
                    >= exp(in1*out) σ(xi) exp[-(in1 + xi)/2 - λ(xi)(in1^2 - xi^2)], where
               σ(x) =  1/(1 + exp(-x))
               λ(x) =  (σ(x) - 1/2)/(2*x)

Interfaces:

    1. out (binary)
    2. in1 (real)
    3. xi (auxiliary variable)

Construction:

    Logit(out, in1, xi)
"""
mutable struct Logit <: SoftFactor
    id::Symbol
    interfaces::Vector{Interface}
    i::Dict{Symbol, Interface}

    function Logit(out, in1, xi; id=generateId(Logit))
        @ensureVariables(out, in1, xi)
        self = new(id, Vector{Interface}(undef, 3), Dict{Symbol,Interface}())
        addNode!(currentGraph(), self)
        self.i[:out] = self.interfaces[1] = associate!(Interface(self), out)
        self.i[:in1] = self.interfaces[2] = associate!(Interface(self), in1)
        self.i[:xi] = self.interfaces[3] = associate!(Interface(self), xi)

        return self
    end
end

slug(::Type{Logit}) = "σ"

logisticSigmoid(x::Float64) = 1/(1 + exp(-x))
logLogisticSigmoid(x::Float64) = -log(1 + exp(-x))
logisticLambda(x::Float64) = (logisticSigmoid(x) - 0.5)/(2*x)

# Average energy functionals
function averageEnergy(::Type{Logit}, marg_out::ProbabilityDistribution{Univariate}, marg_in1::ProbabilityDistribution{Univariate}, marg_xi::ProbabilityDistribution{Univariate})
    xi_hat = unsafeMode(marg_xi)

    logisticLambda(xi_hat)*(unsafeMean(marg_in1)^2 + unsafeCov(marg_in1) - xi_hat^2) +
    0.5*(unsafeMean(marg_in1) + xi_hat) -
    logLogisticSigmoid(xi_hat) -
    unsafeMean(marg_in1)*unsafeMean(marg_out)
end


@testset "asymptotic.jl" begin
    @testset "estimate, statistic, and kernel matrix" begin
        kernel = (ExponentialKernel() ∘ ScaleTransform(0.1)) ⊗ WhiteKernel()
        biasedskce = BiasedSKCE(kernel)
        unbiasedskce = UnbiasedSKCE(kernel)

        for nclasses in (2, 10, 100), nsamples in (10, 50, 100)
            # sample predictions and targets
            dist = Dirichlet(nclasses, 1)
            predictions = [rand(dist) for _ in 1:nsamples]
            targets_consistent = [
                rand(Categorical(prediction)) for prediction in predictions
            ]
            targets_onlyone = ones(Int, length(predictions))

            # compute calibration error estimate and test statistic
            for targets in (targets_consistent, targets_onlyone)
                estimate, statistic, kernelmatrix = CalibrationTests.estimate_statistic_kernelmatrix(
                    kernel, predictions, targets
                )

                @test estimate ≈ unbiasedskce(predictions, targets)
                @test statistic ≈
                      nsamples / (nsamples - 1) * unbiasedskce(predictions, targets) -
                      biasedskce(predictions, targets)
                @test kernelmatrix ≈
                      CalibrationErrors.unsafe_skce_eval.(
                    (kernel,),
                    predictions,
                    targets,
                    permutedims(predictions),
                    permutedims(targets),
                )
            end
        end
    end

    @testset "consistency" begin
        kernel1 = ExponentialKernel() ∘ ScaleTransform(0.1)
        kernel2 = WhiteKernel()
        kernel = kernel1 ⊗ kernel2
        αs = 0.05:0.1:0.95
        nsamples = 100

        pvalues_consistent = Vector{Float64}(undef, 100)
        pvalues_onlyone = similar(pvalues_consistent)

        for nclasses in (2, 10)
            dist = Dirichlet(nclasses, 1)
            predictions = [Vector{Float64}(undef, nclasses) for _ in 1:nsamples]
            targets_consistent = Vector{Int}(undef, nsamples)
            targets_onlyone = ones(Int, nsamples)

            for i in eachindex(pvalues_consistent)
                # sample predictions and targets
                for j in 1:nsamples
                    rand!(dist, predictions[j])
                    targets_consistent[j] = rand(Categorical(predictions[j]))
                end

                # define test
                test_consistent = AsymptoticSKCETest(
                    kernel, predictions, targets_consistent
                )
                test_onlyone = AsymptoticSKCETest(kernel, predictions, targets_onlyone)

                # estimate pvalues
                pvalues_consistent[i] = pvalue(test_consistent; bootstrap_iters=200)
                pvalues_onlyone[i] = pvalue(test_onlyone; bootstrap_iters=200)
            end

            # compute empirical test errors
            ecdf_consistent = ecdf(pvalues_consistent)
            @test all(ecdf_consistent(α) ≤ α + 0.1 for α in αs)
            @test all(p < 0.05 for p in pvalues_onlyone)
        end
    end
end


@with_kw mutable struct RMSpropParameters{T <: AbstractFloat} <: AbstractPolicyParameters
    ρ::T = 0.9
    ϵ::T = 1e-6
end

struct RMSprop <: AbstractSmoothing
    params::RMSpropParameters{Float64}
    grms::Vector{Float64}

    function (::Type{RMSprop})(; kwargs...)
        return new(RMSpropParameters(; kwargs...),Vector{Float64}())
    end
end

function initialize!(rmsprop::RMSprop,x₀::Vector{Float64})
    resize!(rmsprop.grms,length(x₀))
    rmsprop.grms .= zeros(length(x₀))
end

function smooth!(rmsprop::RMSprop,klocal::Integer,kglobal::Integer,x::AbstractVector,gprev::AbstractVector,gcurr::AbstractVector)
    @unpack ρ,ϵ = rmsprop.params
    rmsprop.grms .= ρ*rmsprop.grms + (1-ρ)*(gprev .* gprev)
    gcurr .= gprev ./ (.√rmsprop.grms .+ ϵ)
end


using Distributions, Turing, Random, Parameters
ProjDir = @__DIR__
cd(ProjDir)
include("LNR.jl")

@model model(data,Nr) = begin
    μ = Array{Real,1}(undef,Nr)
    μ ~ [Normal(0,3)]
    σ ~ Truncated(Cauchy(0, 1), 0, Inf)
    for i in 1:length(data)
        data[i] ~ LNR(μ, σ, 0.0)
    end
end

Random.seed!(343)
Nreps = 30
error_count = fill(0.0,Nreps)
Nr = 3

for i in 1:Nreps
    μ = rand(Normal(0 ,3), Nr)
    σ = rand(Uniform(.5, 2))
    dist = LNR(μ=μ, σ=σ, ϕ=0.0)
    data = rand(dist, 30)
    chain = sample(model(data, Nr), NUTS(1000, .8), 2000, discard_adapt = false, progress = false)
    error_count[i] = sum(get(chain,:numerical_error)[1])
end

describe(error_count)


# A modification of Tang's table-driven logarithm for different base.

const t_log2_64 = Array((Float64,Float64),129)
N=39 # (can be up to N=42, which appears to be what Apple's libm uses).
sN = 2.0^N
isN = 1.0/sN
s7 = 2.0^7
is7 = 1.0/s7

for j=0:128
    l_big = Base.log2(big(1.0+j*is7))
    l_hi = isN*float64(round(sN*l_big))
    l_lo = float64(l_big-l_hi)
    t_log2_64[j+1] = (l_hi,l_lo)
    # c_invF[j+1] = 1.0+is7*j
end

const inv_ln2 = 0x1.71547652b82fep+0
const inv_ln2_hi = 0x1.7154766p+0 # 28 bits
const inv_ln2_lo = -0x1.a8fa03d1105ep-29

# const t_log_32 = Array((Float32,Float32),129)
# N=16
# sN = 2f0^N
# isN = 1f0/sN
# for j=0:128
#     l_big = Base.log(big(1.0+j*is7))
#     l_hi = isN*float32(round(sN*l_big))
#     l_lo = float32(l_big-l_hi)
#     t_log_32[j+1] = (l_hi,l_lo)
# end

# Procedure 1
@inline function log2_proc1(y::Float64,mf::Float64,F::Float64,f::Float64,jp::Int)
    ## Steps 1 and 2
    @inbounds hi,lo = libm.t_log2_64[jp]
    l_hi = mf + hi
    l_lo = lo

    ## Step 3' (alternative)
    u = (2.0f)/(y+F)
    us = inv_ln2*u
    v = u*u
    q = us*v*@horner(v,
                    0x1.5_5555_5555_0286p-4,
                    0x1.9_99a0_bc71_2416p-7)

    ## Step 4
    l_hi + (us + (q + l_lo))
end

@inline function log2_proc2(f::Float64)
    ## Step 1
    g = 1.0/(2.0+f)
    u = 2.0*f*g
    us = 1.4426950408889634*u
    v = u*u

    ## Step 2
    q = us*v*@horner(v,
                    0x1.5_5555_5555_54e6p-4,
                    0x1.9_9999_99ba_c6d4p-7,
                    0x1.2_4923_07f1_519fp-9,
                    0x1.c_8034_c85d_fff0p-12)

    ## Step 3
    u1 = float64(float32(u)) # round to 24 bits
    f1 = float64(float32(f))
    f2 = f-f1
    u2 = ((2.0*(f-u1)-u1*f1)-u1*f2)*g

    ## Step 4
    u1*inv_ln2_hi + (u2*inv_ln2 + (u1*inv_ln2_lo + q))
end

function log2_tang(x::Float64)
    if x > 0.0
        x == Inf && return x

        # Step 2
        if 0x1.e_0fab_fbc7_02a3p-1 < x < 0x1.1_082b_577d_34eep0
            f = x-1.0
            return log2_proc2(f)
        end

        # Step 3
        xu = reinterpret(Uint64,x)
        m = int(xu >> 52) & 0x07ff
        if m == 0 # x is subnormal
            x *= 0x1p54 # normalise significand
            xu = reinterpret(Uint64,x)
            m = int(xu >> 52) & 0x07ff - 54
        end
        m -= 1023
        y = reinterpret(Float64,(xu & 0x000f_ffff_ffff_ffff) | 0x3ff0_0000_0000_0000)

        mf = Float64(m)
        F = (y + 0x1p45) - 0x1p45 # 0x1p-7*round(0x1p7*y)
        f = y-F
        jp = itrunc(0x1p7*F)-127

        log2_proc1(y,mf,F,f,jp)
    elseif x == 0.0
        -Inf
    elseif isnan(x)
        NaN
    else
        throw(DomainError())
    end
end

@vectorize_1arg Real log2_tang


function logit∂𝑙(y::AbstractFloat, ŷ::AbstractFloat)
    res = ŷ - y
    isnan(res) ? zero(res) : res
end

function logit∂²𝑙(ŷ::T) where {T<:AbstractFloat}
    res = max(ŷ * (one(T) - ŷ), eps(T))
    isnan(res) ? zero(T) : res
end

function logitraw(p::T) where {T<:AbstractFloat}
    -log(one(T) / p - one(T))
end

function sigmoid(x::T) where {T<:AbstractFloat}
    one(T) / (one(T) + exp(-x))
end

function logloss(y::AbstractFloat, ŷ::AbstractFloat)
    ϵ = eps(ŷ)
    if ŷ < ϵ
        -y * log(ϵ) - (one(y) - y)  * log(one(ϵ) - ϵ)
    elseif one(ŷ) - ŷ < ϵ
        -y * log(one(ϵ) - ϵ) - (one(y) - y)  * log(ϵ)
    else
        -y * log(ŷ) - (one(y) - y) * log(one(ŷ) - ŷ)
    end
end

function xgblogit(label::AbstractCovariate, factors::Vector{<:AbstractFactor};
                  selector::AbstractBoolVariate = BoolVariate("", BitArray{1}(0)),
                  η::Real = 0.3, λ::Real = 1.0, γ::Real = 0.0, maxdepth::Integer = 6, nrounds::Integer = 2,
                  minchildweight::Real = 1.0, caching::Bool = true, slicelength::Integer = 0, usefloat64::Bool = false,
                  singlethread::Bool = false)

    T = usefloat64 ? Float64 : Float32
    factors = caching ? map(cache, widenfactors(filter((f -> getname(f) != getname(label)), factors))) : filter((f -> getname(f) != getname(label)), factors)
    label = caching ? cache(label) : label
    slicelength = slicelength <= 0 ? length(label) : slicelength
    λ = T(λ)
    γ = T(γ)
    η = T(η)
    minchildweight = T(minchildweight)
    μ = T(0.5f0)
    f0 = Vector{T}(length(label))
    fill!(f0, T(logitraw(μ)))
    zerocov = ConstCovariate(zero(T), length(selector))
    fm, trees = fold((f0, Vector{XGTree}()), Seq(1:nrounds)) do x, m
        fm, trees = x
        ŷ = Covariate(sigmoid.(fm))
        ∂𝑙 = length(selector) == 0 ? Trans2Covariate(T, "∂𝑙", label, ŷ, logit∂𝑙) |> cache : IfElseCovariate(selector, Trans2Covariate(T, "∂𝑙", label, ŷ, logit∂𝑙), zerocov) |> cache
        ∂²𝑙 = length(selector) == 0 ? TransCovariate(T, "∂²𝑙", ŷ, logit∂²𝑙) |> cache : IfElseCovariate(selector, TransCovariate(T, "∂²𝑙", ŷ, logit∂²𝑙), zerocov) |> cache
        tree, predraw = growtree(factors, ∂𝑙, ∂²𝑙, maxdepth, λ, γ, minchildweight, slicelength, singlethread)
        fm .= muladd.(η, predraw, fm)
        push!(trees, tree)
        (fm, trees)
    end
    pred = sigmoid.(fm)
    XGModel{T}(trees, λ, γ, η, minchildweight, maxdepth, pred)
end

function cvxgblogit(label::AbstractCovariate, factors::Vector{<:AbstractFactor}, nfolds::Integer;
                    aucmetric::Bool = true, loglossmetric::Bool = true, trainmetric::Bool = false,
                    η::Real = 0.3, λ::Real = 1.0, γ::Real = 0.0, maxdepth::Integer = 6, nrounds::Integer = 2,
                    minchildweight::Real = 1.0, caching::Bool = true, slicelength::Integer = 0, usefloat64::Bool = false,
                    singlethread::Bool = false)

    cvfolds = getnfolds(nfolds, false, length(label))
    trainaucfold = Vector{Float64}(nfolds)
    trainloglossfold = Vector{Float64}(nfolds)
    testaucfold = Vector{Float64}(nfolds)
    testloglossfold = Vector{Float64}(nfolds)
    for i in 1:nfolds
        trainselector = cvfolds .!= UInt8(i)
        testselector = cvfolds .== UInt8(i)
        model = xgblogit(label, factors; selector = BoolVariate("", trainselector), η = η, λ = λ, γ = γ, maxdepth = maxdepth,
                         nrounds = nrounds, minchildweight = minchildweight,
                         caching = caching, slicelength = slicelength, usefloat64 = usefloat64, singlethread = singlethread)
        if aucmetric
            testaucfold[i] = getauc(model.pred, label; selector = testselector)
            if trainmetric
                trainaucfold[i] = getauc(model.pred, label; selector = trainselector)
            end
        end
        if loglossmetric
            testloglossfold[i] = getlogloss(model.pred, label; selector = testselector)
            if trainmetric
                trainloglossfold[i] = getlogloss(model.pred, label; selector = trainselector)
            end
        end
    end
    res = Dict{String, Float64}()
    if aucmetric
        if trainmetric
            res["train_auc_mean"] = mean(trainaucfold)
            res["train_auc_std"] = std(trainaucfold)
        end
        res["test_auc_mean"] = mean(testaucfold)
        res["test_auc_std"] = std(testaucfold)
    end
    if loglossmetric
        if trainmetric
            res["train_logloss_mean"] = mean(trainloglossfold)
            res["train_logloss_std"] = std(trainloglossfold)
        end
        res["test_logloss_mean"] = mean(testloglossfold)
        res["test_logloss_std"] = std(testloglossfold)
    end
    res
end

function predict(model::XGModel{T}, dataframe::AbstractDataFrame) where {T<:AbstractFloat}
    trees = model.trees
    μ = T(0.5f0)
    η = model.η
    f0 = Vector{T}(length(dataframe))
    fill!(f0, T(logitraw(μ)))
    for tree in trees
        predraw = predict(tree, dataframe)
        f0 .= muladd.(η, predraw, f0)
    end
    sigmoid.(f0)
end

function getauc(pred::Vector{T}, label::AbstractCovariate{S}; selector::BitArray{1} = BitArray{1}()) where {T <: AbstractFloat} where {S <: AbstractFloat}
    label = convert(Vector{S}, label)
    label = length(selector) == 0 ? label : label[selector]
    pred = length(selector) == 0 ? pred : pred[selector]
    perm = sortperm(pred; rev = true)
    len = length(label)
    sumlabel = sum(label)
    cumlabel = cumsum(label[perm])
    x = ones(S, len)
    cumcount = cumsum!(x, x)
    pred = pred[perm]
    diff = view(pred, 2:len) .- view(pred, 1:(len - 1))
    isstep = diff .!= zero(T)
    push!(isstep, true)
    cumlabel = cumlabel[isstep]
    cumcount = cumcount[isstep]
    tpr = (cumlabel ./ sumlabel)
    fpr = ((cumcount .- cumlabel) ./ (length(label) .- sumlabel))
    len = length(tpr)
    fpr1 = view(fpr, 1:(len-1))
    fpr2 = view(fpr, 2:len)
    tpr1 = view(tpr, 1:(len - 1))
    tpr2 = view(tpr, 2:len)
    area0 = fpr[1] * tpr[1]
    0.5 * (sum((tpr1 .+ tpr2) .* (fpr2 .- fpr1)) + area0)
end

function getlogloss(pred::Vector{T}, label::AbstractCovariate{S}; selector::BitArray{1} = BitArray{1}()) where {T <: AbstractFloat} where {S <: AbstractFloat}
    label = convert(Vector{S}, label)
    label::Vector{S} = length(selector) == 0 ? label : label[selector]
    pred::Vector{T} = length(selector) == 0 ? pred : pred[selector]
    mean(logloss.(label, pred))
end


# Volume properties of an allipsoid
# =================================
# densite defines the mass density;
# ellipsoid is defined by the centre x, radii a, b, c, and orientation R;
# the subroutine outpuss mass, volume, Euler tensor E, and inertia tensor J;
# __________________________________________________________________________
function ellipsoid_properties(a, b, c, R, density)

  volume = (4.0/3.0) * pi * a * b * c;
  mass = density * volume;

  J = [mass*(b*b + c*c)/5.0 0.0 0.0;
       0.0 mass*(a*a + c*c)/5.0 0.0;
       0.0 0.0 mass*(b*b + a*a)/5.0];

  # note that Inertia = Trace(Euler)*Identity - Euler,
  # hence Euler = 0.5 * Trace (Inertia)*Identity - Inertia,
  # as Trace(Inertia) = 3*Trace(Euler) - Trace(Euler)

  Id = [1. 0. 0.; 0. 1. 0.; 0. 0. 1.]
  E0 = 0.5*(J[1,1]+J[2,2]+J[3,3])*Id - J;

  E = R*E0*R';

  return mass, volume, E, J
end


using JuMP, EAGO

                     m = Model()

                     EAGO.register_eago_operators!(m)

                     @variable(m, -1 <= x[i=1:4] <= 1)

                     @variable(m, -6.482349417201714 <= q <= 6.592803073996197)

                     add_NL_constraint(m, :(sigmoid(-0.7946398993851154 + 0.38713964342476537*sigmoid(0.024809189775459206 + 0.8902991643012754*$(x[1]) + 0.015536230415071905*$(x[2]) + -0.8172113269588737*$(x[3]) + 0.9746036973952403*$(x[4])) + -0.9438792568465639*sigmoid(-0.017899887293421735 + -0.33137455646820024*$(x[1]) + 0.5594894653083848*$(x[2]) + -0.34768764708208755*$(x[3]) + 0.43788143230656695*$(x[4]))) + sigmoid(0.8346867556655706 + -0.9900146410670576*sigmoid(0.024809189775459206 + 0.8902991643012754*$(x[1]) + 0.015536230415071905*$(x[2]) + -0.8172113269588737*$(x[3]) + 0.9746036973952403*$(x[4])) + -0.7397521454164151*sigmoid(-0.017899887293421735 + -0.33137455646820024*$(x[1]) + 0.5594894653083848*$(x[2]) + -0.34768764708208755*$(x[3]) + 0.43788143230656695*$(x[4]))) - $q <= 0.0))

                     @objective(m, Min, q)

                     return m


@testset "nodahilbert.jl" begin
    @test N(2) ≈ [0.0                0.3183098861837907;
                 -0.3183098861837907 0.0               ]
    @test N(3) ≈ [0.0                  0.3183098861837907 0.15915494309189535;
                 -0.3183098861837907   0.0                0.3183098861837907 ;
                 -0.15915494309189535 -0.3183098861837907 0.0                ]
    @test N(4) ≈ [0    1    1/2 1/3;
                 -1    0    1   1/2;
                 -1/2 -1    0   1  ;
                 -1/3 -1/2 -1   0  ] / π

    function f(i, j)
        if i == j
            return 0.0
        end
        return 1 / (π * (j - i))
    end
    for n in 5:10
        @test N(n) ≈ [f(i, j) for i in 1:n, j in 1:n]
    end

    @test N * [1, 2] ≈ [0.6366197723675814, -0.3183098861837907]
    @test N * [1, 2, 3] ≈ [1.1140846016432673, 0.6366197723675813, -0.7957747154594768]
end