"""
Power nonlinearity, with energy C ∫ρ^α where ρ is the density
"""
struct PowerNonlinearity
    C::Real
    α::Real
end
(P::PowerNonlinearity)(::AbstractBasis{T}) where {T} = TermPowerNonlinearity{T}(P.C, P.α)

struct TermPowerNonlinearity{T <: Real}<: TermNonlinear
    C::T
    α::T
end

function ene_ops(term::TermPowerNonlinearity, basis::PlaneWaveBasis, ψ, occ; ρ, kwargs...)
    E = term.C * sum(ρ .^ term.α) * basis.dvol
    potential = @. term.C * term.α * ρ^(term.α-1)

    # In the case of collinear spin, the potential is spin-dependent
    ops = [RealSpaceMultiplication(basis, kpt, potential[:, :, :, kpt.spin])
           for kpt in basis.kpoints]
    (E=E, ops=ops)
end

_pnl_kernel(C, α, ρ) = @. C * α * (α-1) * ρ^(α-2)

function compute_kernel(term::TermPowerNonlinearity, ::AbstractBasis; ρ, kwargs...)
    Diagonal(vec(_pnl_kernel(term.C, term.α, ρ)))
end

function apply_kernel(term::TermPowerNonlinearity, ::AbstractBasis, δρ; ρ, kwargs...)
    _pnl_kernel(term.C, term.α, ρ) .* δρ
end


function elementary_matrix!(A,el::AbstractElement, u::PolynomialBasis,
                           v::PolynomialBasis, q::AbstractQuadratureRule;
                           f=(u,v)->(i,j,x̂,_,_)->u(x̂)[j]*v(x̂)[i])
    x̂,ŵ = q()
    x,w = q(el)
    m, n = length(v), length(u)
    for k in 1:length(w)
        for i in 1:m
            for j in 1:n
                A[i,j] += f(u, v)(i, j, x̂[k], el, x[k]) * w[k]
            end
        end
    end
    return A
end

function elementary_matrix_new!(A,el::AbstractElement, u::PolynomialBasis,
                           v::PolynomialBasis, q::AbstractQuadratureRule;
                           f=(u,v)->(i,j,x̂,_,_)->u(x̂)[j]*v(x̂)[i])
    x̂,ŵ = q()
    x,w = q(el)
    m, n = length(v), length(u)
    for k in 1:length(w)
        for i in 1:m
            for j in 1:n
                A[i,j] += f(u, v)(i, j, x̂[k], el, x[k]) * w[k]
            end
        end
    end
    return A
end

function elementary_matrix(el::AbstractElement, u::PolynomialBasis,
    v::PolynomialBasis, q::AbstractQuadratureRule;
    f=(u,v)->(i,j,x̂,_,_)->u(x̂)[j]*v(x̂)[i])
    x̂,ŵ = q()
    x,w = q(el)
    m, n = length(v), length(u)
    A = zeros(Float64,m,n)
    elementary_matrix!(A,el,u,v,q,f=f)
end

function elementary_matrix(el::AbstractElement, v::PolynomialBasis,
                           q::AbstractQuadratureRule;
                           f=(v)->(i,x̂,_,_)->v(x̂)[i])
    x̂,ŵ = q()
    x,w = q(el)
    m = length(v)
    b = zeros(Float64,m)
    for k in 1:length(w)
        for i in 1:m
            b[i] += f(v)(i, x̂[k], el, x[k]) * w[k]
        end
    end
    return b
end

# NOTE: for testing purposes regarding speed differences between generic and a
# specific elementary matrix kernel
function (u::LagrangeBasis)(q::AbstractQuadratureRule)
    x̂,_ = q()
    mapreduce(x->u(x),hcat,x̂)
end

function mass_matrix(el::AbstractElement,u::PolynomialBasis,v::PolynomialBasis,q::AbstractQuadratureRule)
    x̂,ŵ = q()
    U = u(q)
    V = v(q)
    μ = map(u->measure(el,u),x̂)
    Σ = diagm(μ)*diagm(ŵ)
    return V*Σ*transpose(U)
end

function mass_matrix_unrolled!(A,el::AbstractElement,u::PolynomialBasis,v::PolynomialBasis,q::AbstractQuadratureRule)
    x̂,ŵ = q()
    U = u(q)
    V = v(q)
    U *= diagm(ŵ) # put precomputed weights here. Should be computed only once.
    for k in 1:length(ŵ)
        J = @inbounds jacobian(el,x̂[k])
        μ = abs(det(J))
        for j in 1:size(U,2)
            @inbounds uj = U[j,k]*μ
            for i in 1:size(V,2)
                @inbounds A[i,j] += uj*V[i,k]
            end
        end
    end
    return A
end

function (gu::gradLagrangeBasis)(q::AbstractQuadratureRule)
    x̂,_ = q()
    mapreduce(x->gu(x),hcat,x̂)
end

function stiffness_matrix_unrolled!(A,el::AbstractElement,u::PolynomialBasis,v::PolynomialBasis,q::AbstractQuadratureRule)
    x̂,ŵ = q()
    ∇U = grad(u)(q)
    ∇V = grad(v)(q)
    ∇U *= diagm(ŵ) # put the reference weights in one of the precomputed matrices
    for k in 1:length(ŵ)
        J  = jacobian(el,x̂[k])
        iJ = inv(J)
        μ  = abs(det(J))
        Σ  = iJ*transpose(iJ)*μ
        for i in 1:size(∇V,2)
            for j in 1:size(∇U,2)
                A[i,j] += transpose(∇V[i,k]) * Σ * ∇U[j,k]
            end
        end
    end
    return A
end


function mainpmap(argin)
    seed = argin[1]
    N    = argin[2]
    srand(seed)
    b=[2; 3]
    Z=[ones(N) randn(N)]
    y=Z*b+randn(N)
    m = Model( solver=NLoptSolver(algorithm=:LD_SLSQP))
    @variable(m, x[ i = 1 : size(Z)[2] ]  )
    @NLobjective(m, Min, sum{(y[i]- sum{ Z[i,j]*x[j], j=1:size(Z)[2]})^2, i=1:size(Z)[1]})
    solve(m)
    #println("beta= ",getvalue(x))
    #println("objective= ",getobjectivevalue(m))
    return getvalue(x), getobjectivevalue(m)
end


using SpatialFields
using Base.Test
using Iterators

points = SVector{3, Float64}[[1; 0; 0], [0; 1; 0], [-1; 0; 0], [0; -1; 0], [0; 0; 1], [0; 0; -1]]
normals = SVector{3, Float64}[[1; -1; 0], [0; 1; 0], [-1; 0; 0], [0; -1; 0], [1; 0; 1], [0; 0; -1]]

field = HermiteRadialField(points, normals)

X = linspace(-2, 2)
Y = linspace(-2, 2)
Z = linspace(-2, 2)

for (i, point) in enumerate(points)
    @test isapprox(field(point), 0, atol=1e-6)
    @test isapprox(SpatialFields.gradient(field)(point), normals[i], atol=1e-6)
end


module SCP_Exact

include("../Misc.jl")
using .Misc

using Distributions, SpecialFunctions, Einsum, Combinatorics
import Base.Iterators: product

export generate, log_marginal

function generate(I::Ƶ, J::Ƶ, K::Ƶ, R::Ƶ; μ::ℜ=3.5, γ::ℜ=0.1) where {ℜ<:Real, Ƶ<:Int}
    a::ℜ, b::ℜ = I*J*K*γ, γ/μ
    α_R, α_KR, α_IKR, α_JKR = fill(a/R,R), fill(a/(K*R),K,R), fill(a/(I*K*R),I,K,R), fill(a/(J*K*R),J,K,R)

    λ = rand(Gamma(a,1.0/b))
    θ_R = rand(Dirichlet(α_R))
    θ_KR = reshape([θ_kr for r=1:R for θ_kr=rand(Dirichlet(α_KR[:,r]))],K,R)
    θ_IKR = reshape([θ_ikr for k=1:K, r=1:R for θ_ikr=rand(Dirichlet(α_IKR[:,k,r]))],I,K,R)
    θ_JKR = reshape([θ_jkr for k=1:K, r=1:R for θ_jkr=rand(Dirichlet(α_JKR[:,k,r]))],J,K,R)

    Λ_IJK::Array{ℜ} = [λ * sum(θ_R .* θ_KR[k,:] .* θ_IKR[i,k,:] .* θ_JKR[j,k,:]) for i=1:I, j=1:J, k=1:K]

    X::Array{ℜ} = map.(λ_ijk -> rand(Poisson(λ_ijk)), Λ_IJK)
    X⁺::Array{ℜ} = sum(X,dims=3)[:,:]
    return X, X⁺, λ, (θ_R, θ_KR, θ_IKR, θ_JKR)
end

function generate(I::Ƶ, J::Ƶ, K::Ƶ, R::Ƶ, T::Ƶ; μ::ℜ=3.5, γ::ℜ=0.1) where {ℜ<:Real, Ƶ<:Int}
    a::ℜ, b::ℜ = I*J*K*γ, γ/μ
    α_R, α_KR, α_IKR, α_JKR = fill(a/R,R), fill(a/(K*R),K,R), fill(a/(I*K*R),I,K,R), fill(a/(J*K*R),J,K,R)

    θ_R = rand(Dirichlet(α_R))
    θ_KR = reshape([θ_kr for r=1:R for θ_kr=rand(Dirichlet(α_KR[:,r]))],K,R)
    θ_IKR = reshape([θ_ikr for k=1:K, r=1:R for θ_ikr=rand(Dirichlet(α_IKR[:,k,r]))],I,K,R)
    θ_JKR = reshape([θ_jkr for k=1:K, r=1:R for θ_jkr=rand(Dirichlet(α_JKR[:,k,r]))],J,K,R)

    θ_IJK::Array{ℜ} = [sum(θ_R .* θ_KR[k,:] .* θ_IKR[i,k,:] .* θ_JKR[j,k,:]) for i=1:I, j=1:J, k=1:K]
    θ_flat = reshape(θ_IJK, I*J*K)

    X::Array{ℜ} = reshape(rand(Multinomial(T,θ_flat)),I,J,K)
    X⁺::Array{ℜ} = sum(X,dims=3)[:,:,1]
    return X, X⁺, (θ_R, θ_KR, θ_IKR, θ_JKR)
end

function comb(X_ij::Array{ℜ,1}, X⁺_ij::Ƶ) where {ℜ <: Real, Ƶ <: Int}
    K::Ƶ = length(X_ij)
    if isnan(X_ij[1])
        return combinations(1:(X⁺_ij+K-1), K-1)
    else
        return combinations(cumsum(Int.(X_ij) .+ 1)[1:K-1], K-1)
    end
end

function log_marginal(X::Array{ℜ,3}, X⁺::Array{ℜ,2}, R::Ƶ; μ::ℜ=3.5, γ::ℜ=0.1) where {ℜ<:Real, Ƶ<:Int}
    I::Ƶ, J::Ƶ, K::Ƶ = size(X)

    a::ℜ, b::ℜ = I*J*K*γ, γ/μ
    α_KR, α_IKR, α_JKR = fill(a/(K*R),K,R), fill(a/(I*K*R),I,K,R), fill(a/(J*K*R),J,K,R)

    s, S_KR, S_IKR, S_JKR, S⁺ = zeros(ℜ,R), zeros(ℜ,K,R), zeros(ℜ,I,K,R), zeros(ℜ,J,K,R), sum(X⁺)
    log_PX, log_PS = -Inf, -Inf
    i::Ƶ, j::Ƶ , k::Ƶ = 0, 0, 0

    log_C::ℜ = a*log(b) - (a + S⁺)*log(b + 1.0)
    log_C += sum(lgamma.(α_KR)) - sum(lgamma.(α_IKR)) - sum(lgamma.(α_JKR))

    X_full = zeros(ℜ,I,J,K)

    for X_div = product([comb(X[i,j,:], Int(X⁺[i,j])) for i=1:I,j=1:J]...)
        if K==1
            X_full[:,:,1] .= X⁺
        else
            X_full .= 0.0
            for (ij, part) = enumerate(X_div) ## fix X
                i, j = mod(ij-1,I)+1, div(ij-1,I)+1
                X_full[i,j,1] = part[1]-1
                for k=2:K-1
                    X_full[i,j,k] = part[k] - part[k-1] - 1
                end
                X_full[i,j,K] = X⁺[i,j] + K-1 - part[K-1]
            end
        end

        if R==1
            S_KR[:,1] .= sum(X_full,dims=(1,2))[1,1,:]
            S_IKR[:,:,1] .= sum(X_full,dims=2)[:,1,:]
            S_JKR[:,:,1] .= sum(X_full,dims=1)[1,:,:]

            log_PS = log_C
            log_PS += sum(lgamma.(S_IKR .+ α_IKR)) + sum(lgamma.(S_JKR .+ α_JKR)) - sum(lgamma.(S_KR .+ α_KR))
            log_PS -= sum(lgamma.(X_full .+ 1.0))
            log_PX = logsumexp([log_PX,log_PS])
        else
            for S_div = product(map(X_ijk -> combinations(1:(X_ijk+R-1),R-1), X_full)...)
                S_KR .= α_KR
                S_IKR .= α_IKR
                S_JKR .= α_JKR

                log_PS = 0.0
                for (ijk, part) = enumerate(S_div)
                    i, j, k = mod(ijk-1,I)+1, mod(div(ijk-1,I),J)+1, div(ijk-1,I*J)+1
                    s[1] = part[1]-1
                    for r=2:R-1
                        s[r] = part[r] - part[r-1] - 1
                    end
                    s[R] = X_full[i,j,k] + R-1 - part[R-1]

                    S_KR[k,:] .+= s
                    S_IKR[i,k,:] .+= s
                    S_JKR[j,k,:] .+= s
                    log_PS -= sum(lgamma.(s .+ 1.0))
                end
                log_PS += sum(lgamma.(S_IKR)) + sum(lgamma.(S_JKR)) - sum(lgamma.(S_KR)) + log_C
                log_PX = logsumexp([log_PX,log_PS])
            end
        end
    end
    return log_PX
end

end


using Yao
using BitBasis
using Test, LinearAlgebra
using QuAlgorithmZoo

"""
random phase estimation problem setup.
"""
function rand_phaseest_setup(N::Int)
    U = rand_unitary(1<<N)
    b = randn(ComplexF64, 1<<N); b=b/norm(b)
    phases, evec = eigen(U)
    ϕs = @. mod(angle(phases)/2/π, 1)
    return U, b, ϕs, evec
end

@testset "phaseest" begin
    # Generate a random matrix.
    N = 3
    V = rand_unitary(1<<N)

    # Initial Set-up.
    phases = rand(1<<N)
    ϕ = Int(0b111101)/(1<<6)
    phases[3] = ϕ
    signs = exp.(2π*im.*phases)
    U = V*Diagonal(signs)*V'
    b = V[:,3]

    # Define ArrayReg and U operator.
    M = 6
    reg1 = zero_state(M)
    reg2 = ArrayReg(b)
    UG = matblock(U)

    # circuit
    circuit = PEBlock(UG, M, N)

    # run
    reg = apply!(join(reg2, reg1), circuit)

    # measure
    res = breflect(measure(focus!(copy(reg), 1:M); nshots=10)[1]; nbits=M) / (1<<M)

    @test res ≈ ϕ
    @test apply!(reg, circuit |> adjoint) ≈ join(reg2, reg1)
end

@testset "phaseest, non-eigen" begin
    # Generate a random matrix.
    N = 3
    U, b, ϕs, evec = rand_phaseest_setup(N)

    # Define ArrayReg and U operator.
    M = 6
    reg1 = zero_state(M)
    reg2 = ArrayReg(b)
    UG = matblock(U);

    # run circuit
    reg= join(reg2, reg1)
    pe = PEBlock(UG, M, N)
    apply!(reg, pe)

    # measure
    bs, proj, amp_relative = projection_analysis(evec, focus!(reg, M+1:M+N))
    @test isapprox(ϕs, bfloat.(bs, nbits=M), atol=0.05)
end


function KS_solve_Emin_cg( pw::PWGrid, V_ionic, Focc, Nstates::Int;
                           psi0=nothing, Potentials0 = nothing,
                           α_t = 3e-5, NiterMax=1000, verbose=false )

    Ns = pw.Ns
    Npoints = prod(Ns)

    if psi0 == nothing
        psi = rand(ComplexF64,Npoints,Nstates)
        psi = ortho_gram_schmidt(psi)
    else
        psi = copy(psi0)
    end

    if Potentials0 == nothing
        Potentials = PotentialsT( V_ionic, zeros(Npoints), zeros(Npoints) )
        rho = calc_rho( pw, Focc, psi )
        Potentials.Hartree = real( G_to_R( Ns, Poisson_solve(pw, rho) ) )
        Potentials.XC = excVWN( rho ) + rho .* excpVWN( rho )
    else
        Potentials = PotentialsT( Potentials0.Ionic,
                                  Potentials0.Hartree,
                                  Potentials0.XC )
    end

    d      = zeros(ComplexF64, Npoints, Nstates)
    g_old  = zeros(ComplexF64, Npoints, Nstates)
    d_old  = zeros(ComplexF64, Npoints, Nstates)
    Kg     = zeros(ComplexF64, Npoints, Nstates)
    Kg_old = zeros(ComplexF64, Npoints, Nstates)

    yk1 = zeros(ComplexF64, Npoints, Nstates) # for testing Hager-Zhang beta
    v1 = zeros(ComplexF64, Npoints, Nstates)

    β     = 0.0
    Etot_old = 0.0
    Etot     = 0.0
    Energies = EnergiesT( 0.0, 0.0, 0.0, 0.0, 0.0 )

    CONVERGED = 0

    for iter = 1:NiterMax

        g = calc_grad( pw, Potentials, Focc, psi)
        Kg = Kprec(pw,g)
        if iter != 1
            #β = real(sum(conj(g).*Kg))/real(sum(conj(g_old).*Kg_old))
            #β = real(sum(conj(g-g_old).*Kg))/real(sum(conj(g_old).*Kg_old))
            #β = real(sum(conj(g-g_old).*Kg))/real(sum(conj(g-g_old).*d))
            β = real(sum(conj(g).*Kg))/real(sum((g-g_old).*conj(d_old)))

            #yk1 = g - g_old
            #tt = real(tr(d_old'*yk1))
            #v1 = yk1 - 2*d_old*norm(yk1)/tt
            #β = real(tr(v1'*Kg)/tt)

            #@printf("\nβ = %f\n", β)
        end

        if β < 0.0
            println("Encountered β less than 0 setting it to zero")
            β = 0.0
        end
        #
        d = -Kg + β * d_old
        #
        psic = ortho_gram_schmidt(psi + α_t*d)
        rho = calc_rho( pw, Focc, psic )
        Potentials.Hartree = real( G_to_R( Ns, Poisson_solve(pw, rho) ) )
        Potentials.XC = excVWN( rho ) + rho .* excpVWN( rho )
        gt = calc_grad( pw, Potentials, Focc, psic )

        denum = real(sum(conj(g-gt).*d))
        if denum != 0.0
            α = abs( α_t*real(sum(conj(g).*d))/denum )
        else
            α = 0.0
        end

        # Update wavefunction
        psi = psi[:,:] + α*d[:,:]

        # Update potentials
        psi = ortho_gram_schmidt(psi)
        rho = calc_rho( pw, Focc, psi )
        Potentials.Hartree = real( G_to_R( Ns, Poisson_solve(pw, rho) ) )
        Potentials.XC = excVWN( rho ) + rho .* excpVWN( rho )

        Energies = calc_Energies( pw, Potentials, Focc, psi )
        Etot = Energies.Total

        diff = abs(Etot-Etot_old)
        @printf("Emin CG: %8d = %18.10f %18.10f\n", iter, Etot, diff)

        if diff < 1e-6
            CONVERGED = CONVERGED + 1
        else
            CONVERGED = 0
        end

        if CONVERGED >= 2
            @printf("CONVERGENCE ACHIEVED\n")
            break
        end

        g_old = copy(g)
        d_old = copy(d)
        Kg_old = copy(Kg)
        Etot_old = Etot
    end
    return psi, Energies, Potentials
    #
end


# Paralel network simulation

# Add process
using Distributed
nprocs() == Sys.CPU_THREADS || addprocs(Sys.CPU_THREADS - nprocs() + 1)

# Load packages
@everywhere begin
    using Pkg
    Pkg.activate(joinpath(Pkg.envdir(), "dev-env"))
end

@everywhere begin
    import Cairo, Fontconfig
    using GraphPlot
    using Graphs
    using SimpleWeightedGraphs
    using LinearAlgebra
    using DynamicalNetworks
    using DifferentialEquations
end

@everywhere begin
    σr = collect(0.0:0.1:1.0)
    function prob_func(prob, i, repeat)
        @show σr[i]
        prob.p[1].noisemat .*= σr[i]
        prob
    end

    # Settings
    nbits = 10
    tbit = 50.0
    bits = rand(Bool, nbits)
    ti = 0.0
    tf = nbits * tbit
    tswitch = ti:tbit:tf-tbit
    srcnode1, dstnode1 = 5, 8     # Control link nodes
    high = 100.0            # Control coupling strength
    low = eps()             # Control coupling strength
    condition(u, t, integrator) = integrator.t ∈ tswitch
    function action!(integrator)
        net, bits = integrator.sol.prob.p
        if popfirst!(bits)
            add_edge!(net.graph, srcnode1, dstnode1, high)
        else
            add_edge!(net.graph, srcnode1, dstnode1, low)
        end
    end
end

# Topology
n = 8
graph = cycle_graph(n) |> SimpleWeightedGraph
add_edge!(graph, 2, 7)
add_edge!(graph, 3, 6)
ϵ = 20.0

# Network
σ = σr[1]
H = [
    1 0 0 0
    -1 0 0 0
    0 1 0 0
    0 -1 0 0
    0 0 1 0
    0 0 -1 0
    0 0 0 1
    0 0 0 -1
] * σ
P = diagm([1.0, 0.0, 0.0])
net = SDENetwork(Lorenz, ϵ, graph, H, P)
callback = DiscreteCallback(condition, action!)

# Simulation
prob = netproblem(net, (ti, tf), copy(bits), noise_rate_prototype = get_noise_rate_prototype(net), callback = callback, tstops = tswitch)
ensembleprob = EnsembleProblem(prob, prob_func = prob_func)
sim = solve(ensembleprob, SRA(), EnsembleDistributed(), trajectories = length(σr))

# Plots
using PyPlot
pygui(true)
close("all")
getwave(x, i, d) = getindex.(x, (i - 1) * d + 1)
figure()
d = 3
subplot(221)
for (i, sol) in enumerate(sim[[10, 1]])
    plot(sol.t, abs.(getwave(sol.u, 1, d) - getwave(sol.u, 8, d)), label = "$i")
    axvline.(collect(ti:tbit:tf), color = "red", linestyle = "dashed")
    legend()
end


export transform

"""
    transform(problem::InitialValueProblem, options::Options)

Interface function that calls the respective transformation function.

### Input

- `problem` -- discrete or continuous initial-value problem
- `option`  -- problem options

### Output

A tuple containing the transformed problem and the transformed options.

### Notes

The functions that are called in the background should return a the transformed
system components `A`, `X0`, and `U`, and also an inverse transformation matrix `M`.
If the system has an invariant, it is transformed as well.
"""
function transform(problem::InitialValueProblem, options::Options)
    method = options[:coordinate_transformation]
    if method == ""
        nothing # no-op
    elseif method == "schur"
        options[:transformation_matrix] = Tm
        problem = schur_transform(problem)
    else
        error("the transformation method $method is undefined")
    end
    return (problem, options)
end

"""
    schur_transform(problem::InitialValueProblem)

Applies a Schur transformation to a discrete or continuous initial-value problem.

### Input

- `S` -- discrete or continuous initial-value problem

### Output

Transformed problem.

### Algorithm

We use Julia's default `schurfact` function to compute a
[Schur decomposition](https://en.wikipedia.org/wiki/Schur_decomposition)
of the coefficients matrix ``A``.
"""
function schur_transform(problem::InitialValueProblem{PT, ST}
                         ) where {PT <: Union{ConstrainedLinearControlDiscreteSystem, ConstrainedLinearControlContinuousSystem}, ST<:LazySet}

    n = size(problem.s.A, 1)

    # full (dense) matrix is required by schur
    A_new, T_new = schur(Matrix(problem.s.A))

    # recall that for Schur matrices, inv(T) == T'
    Z_inverse = copy(transpose(T_new))

    # apply transformation to the initial states
    X0_new = Z_inverse * problem.x0

    # apply transformation to the inputs
    U_new = Z_inverse * problem.s.U

    # obtain the transformation matrix for reverting the transformation again
    T_inverse = F[:vectors]

    # apply transformation to the invariant
    if hasmethod(stateset, Tuple{typeof(problem.s)})
        invariant_new = T_inverse * problem.s.X
    else
        invariant_new = Universe(n)
    end

    system_new = (A_new, I(n), invariant_new, U_new)
    return InitialValueProblem(PT(system_new), X0_new)
end


#!/usr/local/bin/julia

# Obligatory worldly salutation
println("o hai world")

#=
Use can use block comments
=#

#=
Maths
=#

# Unary operators
1 + 2 - 4 * 3 / 5 ^ 6 % 7

#=
Bitwise operator
=#

# Not
~true
# And
true & false
# Or
false | true
# Xor
true ⊻ false
# Logical shift
true >>> false
# Arithmetic shift
true >> false
# Logical/arithmetic shift
true << true

#=
Every binary arithmetic and bitwise operator also has an
updating version that assigns the result of the operation
back into its left operand.
=#

# Comparative operatations
1_000 == 1000

# Built-in specials
pi
e
π == pi

# Functions that look like maths
f(x) = x^2 + 2x
f(3)

#=
Strings
=#
typeof('x')
typeof("x")
Int('x')
str = "oh hai wurld"
str[1]
str[4:5]
str[end]

#=
Functions
=#
function f(x, y)
  x + y
end
f(3, 4)

f(x, y) = x + y
f(4, 5)

foo(x::Int) = println("$x is an integer")

@time foo(5)
@time foo(π)
foo(x::String) = println("$x is a string")

#=
Arrays
=#
A = [1, 2, 3, 4, 5]
collect(1:5)
[i for i in 1:5]

letters = collect('a':'z')
letters[1]

1:5 .* 2
length(letters)

[1 2 3; 4 5 6]
[[1, 2, 3] [4, 5, 6]]
Array(1:3)
Array([1 3 4])

eye(5)
[(i,j) for i=1:3 for j=1:i]
reshape(1:16, 4, 4)

eye(Int, 10) .* collect(1:10:100)

#=
Control flow
=#

# Compound expressions
z = begin
  x = 1
  y = 2
  x + y
end

z = (x = 1; y = 2; x + y)

# Conditional expressions
if x < y
    println("x is less than y")
elseif x > y
    println("x is greater than y")
else
    println("x is equal to y")
end

x < y ? "less than" : "not less than"

# While loop
i = 1
while true
  println(i)
  if i >= 5
    break
  end
  i += 1
end

# For loop
for i = 1:10
  if i % 3 != 0
    continue
  end
  println(i)
end

# Try-catch
f(x) =
  try
    sqrt(x)
  catch
    sqrt(complex(x, 0))
end

"""
# Tasks aka Coroutines aka Green Threads

Tasks are a control flow feature that allows computations to be suspended and
resumed in a flexible manner.
"""

"""
# Project Euler

## Problem 9: Special Pythagorean triplet

A Pythagorean triplet is a set of three natural numbers, a < b < c, for which,

```math
a^2 + b^2 = c^2
```

For example, 32 + 42 = 9 + 16 = 25 = 5^2.

There exists exactly one Pythagorean triplet for which a + b + c = 1000.
Find the product abc.
"""

"Generates Pythagorean triplets."
function pythagorean_triplet_producer(ch::Channel)
  m = 2
  while true
    for n = 1:(m-1)
      a = 2*m*n
      b = m^2 - n^2
      c = m^2 + n^2
      put!(ch, [a, b, c])
    end
    m += 1
  end
end

"Solves the 9th Projec Euler programming problem."
function pr009()
  chnl = Channel(pythagorean_triplet_producer)
  while true
    a = take!(chnl)
    if sum(a) == 1_000
      return prod(a)
    end
  end
end

@printf "Time elapsed:\t%4.8f\n" @elapsed pr009()

# EOF


Random.seed!(17)
N = 100
net = CompleteBipartiteGraph(N, N, Network)
eprop!(net, "w", e -> rand())
vprop!(net, "b", v -> 1)
res_match = Matching.run_bp(net, maxiters=100, verbose=false)
res_bmatch = BMatching.run_bp(net, maxiters=100, verbose=false)
bmatchmap = [m[1] for m in res_bmatch.match]
@test all(bmatchmap .== res_match.match)

Random.seed!(17)
N = 100
net = CompleteGraph(N, Network)
eprop!(net, "w", e -> rand())
vprop!(net, "b", v -> rand(1:5))
res = BMatching.run_bp(net, maxiters=100, verbose=false)
@test res.num_violations <= 1 # = 0 on Julia 1.6
@test res.iters <= 100


## Hermite
@register0 Hermite AbstractCCOP0
export Hermite

"""
    Hermite

The [Hermite](https://en.wikipedia.org/wiki/Hermite_polynomials) polynomials have two versions the physicists (`Hermite`  or  `H`) and the probablalists (`ChebyshevHermite` or  `Hₑ`). They are  related through  `Hᵢ(x) =  2^(i/2) Hₑᵢ(√2 x)`.

The Hermite   polynomials have weight  function `w(x)=exp(-x^2/2)` and domain the real line.

## Examples
```jldoctest
julia> using Polynomials,  SpecialPolynomials

julia> x = variable(Polynomial{Rational{Int}})
Polynomial(x)

julia> [basis(Hermite, i)(x) for i in 0:5]
6-element Array{Polynomial{Float64},1}:
 Polynomial(1.0)
 Polynomial(2.0*x)
 Polynomial(-2.0 + 4.0*x^2)
 Polynomial(-12.0*x + 8.0*x^3)
 Polynomial(12.0 - 48.0*x^2 + 16.0*x^4)
 Polynomial(120.0*x - 160.0*x^3 + 32.0*x^5)

julia> [basis(ChebyshevHermite, i)(x) for i in 0:5]
6-element Array{Polynomial{Float64},1}:
 Polynomial(1.0)
 Polynomial(1.0*x)
 Polynomial(-1.0 + 1.0*x^2)
 Polynomial(-3.0*x + 1.0*x^3)
 Polynomial(3.0 - 6.0*x^2 + 1.0*x^4)
 Polynomial(15.0*x - 10.0*x^3 + 1.0*x^5)
```

!!! note
    The Hermite family needs help, as the computed values for `Bn`,and,`Cn` are  both 0.
"""
Hermite

basis_symbol(::Type{<:Hermite}) = "H"
Polynomials.domain(::Type{<:Hermite}) = Polynomials.Interval(-Inf, Inf)

abcde(::Type{<:Hermite})  = NamedTuple{(:a,:b,:c,:d,:e)}((1,0,0,-2,0))

k0(P::Type{<:Hermite}) = one(eltype(P))
function k1k0(P::Type{<:Hermite}, k::Int)
    val = 2*one(eltype(P))
    val
end

norm2(::Type{<:Hermite}, n) = sqrt(pi) * 2^n * gamma(n+1)
weight_function(::Type{<: Hermite})  = x -> exp(-x^2)
generating_function(::Type{<:Hermite}) = (t, x)  -> exp(2*t*x - t^2)
function classical_hypergeometric(::Type{<:Hermite}, n, x)
    as = iseven(n) ? (-n ÷ 2, -(n-1)/2) : (-n/2, -(n-1)÷2)
    bs = ()
    (2x)^n * pFq(as, bs, -inv(x)^2)
end

gauss_nodes_weights(p::Type{P}, n) where {P <: Hermite} =
    FastGaussQuadrature.gausshermite(n)

## Overrides
# Use override here, as we get  An,Bn,Cn =  1, 0,0, otherwise
An(P::Type{<:Hermite}, n::Int) = 2*one(eltype(P))
Bn(P::Type{<:Hermite}, n::Int) = zero(eltype(P))
Cn(P::Type{<:Hermite}, n::Int) = 2*n*one(eltype(P))

β̃n(P::Type{<:Hermite},  n::Int) = zero(eltype(P))
γ̃n(P::Type{<:Hermite},  n::Int) = zero(eltype(P))
b̂̃n(P::Type{<:Hermite}, ::Val{N}) where {N} = zero(eltype(P)) #where {M} = error("Don't call me")#zero(S)
ĉ̃n(P::Type{<:Hermite}, ::Val{N}) where {N}  = zero(eltype(P)) #where {M} = error("Don't call me")#zero(S)

## https://arxiv.org/pdf/1901.01648.pdf. Connection formula (14)
##  x^n  = n! sum(H_{n-2j}/ (2^j(n-2j)!j!) j = 0:floor(n/2))

Base.convert(P::Type{<:Hermite}, q::Polynomial) = connection(P,q)
function Base.iterate(o::Connection{P, Q}, state=nothing) where
    {P<:Hermite,
     Q<:Polynomials.StandardBasisPolynomial}

    k, n = o.k, o.n

    if state == nothing
        i = k
        j = 0
        i > n && return nothing
        val = __hermite_lambda(i,k)
    elseif state[1] + 2 > n # terminate
        return nothing
    else
        j,val1 = state
        #val1 *= (i+1)*(i+2)/4/(j+1/4)
        j += 1
        i = k + 2j
        val = __hermite_lambda(i,k)
    end

    return(i, val), (j, val)
end

function __hermite_lambda(n,k)
    val = gamma(1+n)/2^n
    val /= gamma(1+k)
    val /= gamma(1 + (n-k)/2)
    val
end

# 2x more performant
function Polynomials.integrate(p::P, C::Number=0) where {T,P<:Hermite{T}}
    # int H_n = 1/(n+1) H_{n+1}
    R = eltype(one(T)/1)
    d = degree(p)
    qs = zeros(R, d+2)

    for i in 0:d
        qs[i+2] = p[i]/(2(i+1))
    end

    q = ⟒(P)(qs, p.var)
    q = q - q(0) + R(C)

    return q
end

##
## --------------------------------------------------
##

@register0 MonicHermite AbstractCCOP0
export MonicHermite
ϟ(::Type{<:MonicHermite}) = Hermite
ϟ(::Type{<:MonicHermite{T}}) where {T} = Hermite{T}
@register_monic(MonicHermite)

pqr_start(::Type{<:MonicHermite}) = 0
pqr_symmetry(::Type{<:MonicHermite}) = true
pqr_weight(P::Type{<:MonicHermite}, n, x, dπx) = (one(P)*2)*exp(-x^2)/(dπx*dπx)

##
## --------------------------------------------------
##
@register0 ChebyshevHermite AbstractCCOP0
export ChebyshevHermite
"""
    ChebyshevHermite

Type for the Probabalist's  [Hermite](https://en.wikipedia.org/wiki/Hermite_polynomials) polynomials.

"""
ChebyshevHermite

basis_symbol(::Type{<:ChebyshevHermite}) = "Hₑ"
Polynomials.domain(::Type{<:ChebyshevHermite}) = Polynomials.Interval(-Inf, Inf)

# https://arxiv.org/pdf/1901.01648.pdf eqn 17
abcde(::Type{<:ChebyshevHermite})  = NamedTuple{(:a,:b,:c,:d,:e)}((0,0,1,-1,0))

kn(P::Type{<:ChebyshevHermite}, n::Int) =  one(eltype(P))
k1k0(P::Type{<:ChebyshevHermite}, k::Int)  = one(eltype(P))
k1k_1(P::Type{<:ChebyshevHermite}, k::Int) =  one(eltype(P))
ismonic(::Type{<:ChebyshevHermite}) = true

weight_function(P::Type{ChebyshevHermite})  = x -> exp(-(one(eltype(P))*x^2)/2)
generating_function(::Type{<:ChebyshevHermite}) = (t, x)  -> exp(t*x - t^2/2)
function classical_hypergeometric(::Type{<:ChebyshevHermite}, n, x)
    2^(-n/2)*classical_hypergeometric(Hermite, n,  x/sqrt(2))
end

function  gauss_nodes_weights(P::Type{<:ChebyshevHermite}, n)  where {α,β}
    xs, ws = glaser_liu_rokhlin_gauss_nodes(basis(ChebyshevHermite, n))
    xs, ws
end

pqr_start(::Type{<:ChebyshevHermite}) = 0
pqr_symmetry(::Type{<:ChebyshevHermite}) = true
pqr_weight(P::Type{<:ChebyshevHermite}, n, x, dπx) =  sqrt(2)*gamma(1+n)*sqrt(pi)/(dπx)^2

## Overrides
#An(P::Type{<:ChebyshevHermite}, n::Int) = one(eltype(P))
#Bn(P::Type{<:ChebyshevHermite}, n::Int) = zero(eltype(P))
#Cn(P::Type{<:ChebyshevHermite}, n::Int) = -one(eltype(P))*n

##
## --------------------------------------------------
##

##  Multiplication
## TODO: work  on  types
AbstractHermite{T} = Union{Hermite{T}, ChebyshevHermite{T}}

# Default is slow. Instead
# directly use a linearization formula from
# https://arxiv.org/pdf/1901.01648.pdf
# and  for Hermite, Hn(x) =2^(n/2)Hₑ_n(sqrt(2) x)
# so Hm⋅Hn = ∑ C_{m,n,j} 2^j H_{m+n-2j}
function ⊗(p::P, q::Q) where {T,S, P<:AbstractHermite{T}, Q<:AbstractHermite{S}}
    R = eltype(one(promote_type(T,S))/1)
    N,M = degree(p), degree(q)
    N == -1 && return zero(⟒(P){R}, q.var)
    M == -1 && return zero(⟒(P){R}, p.var)
    N == 0  && return q * p[0]
    M == 0  && return p * q[0]
    q.var != p.var && throw(ArgumentError("Variable names must   match"))

    as =  zeros(R, 1+N+M)
    for i  in eachindex(p)
        pᵢ = p[i]
        iszero(pᵢ) && continue
        for  j in eachindex(q)
            qⱼ = q[j]
            pᵢqⱼ = pᵢ*qⱼ
            iszero(qⱼ) &&  continue
            λᵤ = one(R)
            for  k in 0:min(i,j)
                as[1 + i  +  j - 2k] += pᵢqⱼ * λᵤ
                λᵤ *= (i-k) * (j-k) / (k+1) * hermite_α(P)
            end
        end
    end
    ⟒(P){R}(as, p.var)
end

hermite_α(P::Type{<:ChebyshevHermite}) = 1
hermite_α(P::Type{<:Hermite}) = 2

#⊗(p::Hermite, q::Hermite) = linearization_product(p, q)
#⊗(p::ChebyshevHermite, q::ChebyshevHermite) = linearization_product(p, q)

# function Base.iterate(o::Linearization{P,R}, state =  nothing) where {P <: AbstractHermite,R}

#     l, m, n = o.l, o.m, o.n
#     l  > m + n && return nothing

#     if state == nothing

#         #k =  0
#         j = 0
#         p = min(l, n)
#         q = l - p
#         val = linearization_α(P, R, j, l, p, q)

#     else
#         # we work with l + 2j as there is a parity consideration
#         j, p, q, val  = state
#         s = l + j # l +2j + p + q = l +2j + l = 2l + 2j, so s=l+j
#         if p == 0 ||  q == m || q  > s
#             j += 1
#             l + 2j > n+m && return nothing #  l + 2j  is too  big now
#             p = min(l + j, n)  # p <= s
#             q = l + 2j - p
#             q > s + 1 && return nothing
#             val = linearization_α(P, R, j, l, p, q)
#         else
#             p -= 1
#             q += 1

#             λ = q/(p+1)*(s-(q-1))/(s-p)
#             val *= λ
#         end

#     end

#     return (p,q,val), (j, p, q, val)
# end

# #  https://arxiv.org/pdf/1901.01648.pdf equation (74)
# function linearization_α(P::Type{<:AbstractHermite}, R, j, l, n, m)
#     s = l + j # pass in j to avoid s = divrem(l + m + n,  2) call
#     val = one(R)
#     val *= gamma(1+m)*gamma(1+n)/gamma(1+s-m)/gamma(1+s-n)/gamma(1+s-l)
#     val *= linearization_λ(P, l, m,n)
# end
# linearization_λ(::Type{<:Hermite}, l, m, n) = 2.0^((m+n-l)/2)
# linearization_λ(::Type{<:ChebyshevHermite}, l, m, n) = 1


module TimeModels

using Base.Dates
using Distributions
using StatsBase
using TimeSeries
using Optim
using NLopt
using Compat

import Base: show

export
  # General state space model
  StateSpaceModel, simulate,

  #Kalman
  loglikelihood, kalman_filter, KalmanFiltered,
  kalman_smooth, KalmanSmoothed, KalmanSmoothedMinimal,

  # Parameter fitting
  ParametrizedMatrix,
  parametrize_full, parametrize_diag, parametrize_none,
  ParametrizedSSM, SSMParameters, fit,

  # ARIMA exports
  arima_statespace,
  arima,

  # GARCH exports
  garchFit,
  predict,

  # diagnostic tests exports
  jbtest,

  # utilities
  em_checkmodel

# Core functionality
include("statespacemodel.jl")
include("kalman_filter.jl")
include("kalman_smooth.jl")
include("parameter_estimation.jl")

# Model specifications
include("ARIMA.jl")
include("GARCH.jl")

# Tests
include("diagnostic_tests.jl")

include("utilities.jl")

end


using Interact
using Blink
using Plots

# specify backend for Plots.jl
pyplot()

plotsui=@manipulate for a in -4:0.05:4
    x= -π:0.01:π
    y=sin.(x.-a)
    Plots.plot(x, y, xlim=(-π, π), ylim=(-1,1))
end

w=Window()
body!(w, plotsui)


# This file was generated, do not modify it. # hide
PCA = @load PCA
pca_pipe = Standardizer() |> PCA(maxoutdim=2)
pca = machine(pca_pipe, Xtrain)
fit!(pca)
W = transform(pca, Xtrain)


using DataFrames
using LinearAlgebra
using MixedModels
using Test

@testset "contra" begin
    contra = MixedModels.dataset(:contra)
    contraform = @formula(use ~ 1+age+abs2(age)+urban+livch+(1|urbdist))
    gm0 = fit(MixedModel, contraform, contra, Bernoulli(), fast=true);
    @test gm0.lowerbd == zeros(1)
    @test isapprox(gm0.θ, [0.5720734451352923], atol=0.001)
    @test isapprox(deviance(gm0,true), 2361.657188518064, atol=0.001)
    gm1 = fit(MixedModel, contraform, contra, Bernoulli());
    @test isapprox(gm1.θ, [0.573054], atol=0.005)
    @test lowerbd(gm1) == vcat(fill(-Inf, 7), 0.)
    @test isapprox(deviance(gm1,true), 2361.54575, rtol=0.00001)
    @test isapprox(loglikelihood(gm1), -1180.77288, rtol=0.00001)
    @test dof(gm0) == length(gm0.β) + length(gm0.θ)
    @test nobs(gm0) == 1934
    fit!(gm0, fast=true, nAGQ=7)
    @test isapprox(deviance(gm0), 2360.9838, atol=0.001)
    gm1 = fit(MixedModel, contraform, contra, Bernoulli(), nAGQ=7)
    @test isapprox(deviance(gm1), 2360.8760, atol=0.001)
    @test gm1.β == gm1.beta
    @test gm1.θ == gm1.theta
    @test isnan(gm1.σ)
    @test length(gm1.y) == size(gm1.X, 1)
    @test :θ in propertynames(gm0)
    @testset "GLMM rePCA" begin
        @test length(MixedModels.PCA(gm0)) == 1
        @test length(MixedModels.rePCA(gm0)) == 1
        @test length(gm0.rePCA) == 1
    end
    # gm0.βθ = vcat(gm0.β, gm0.theta)
    # the next three values are not well defined in the optimization
    #@test isapprox(logdet(gm1), 75.7217, atol=0.1)
    #@test isapprox(sum(abs2, gm1.u[1]), 48.4747, atol=0.1)
    #@test isapprox(sum(gm1.resp.devresid), 2237.349, atol=0.1)
    show(IOBuffer(), gm1)
    show(IOBuffer(), BlockDescription(gm0))
end

@testset "cbpp" begin
    cbpp = MixedModels.dataset(:cbpp)
    gm2 = fit(MixedModel, @formula((incid/hsz) ~ 1 + period + (1|herd)), cbpp, Binomial(), wts=float(cbpp.hsz))
    @test deviance(gm2,true) ≈ 100.09585619892968 atol=0.0001
    @test sum(abs2, gm2.u[1]) ≈ 9.723054788538546 atol=0.0001
    @test logdet(gm2) ≈ 16.90105378801136 atol=0.0001
    @test isapprox(sum(gm2.resp.devresid), 73.47174762237978, atol=0.001)
    @test isapprox(loglikelihood(gm2), -92.02628186840045, atol=0.001)
    @test isnan(sdest(gm2))
    @test varest(gm2) == 1
end

@testset "verbagg" begin
    gm3 = fit(MixedModel, @formula(r2 ~ 1+anger+gender+btype+situ+(1|subj)+(1|item)),
        MixedModels.dataset(:verbagg), Bernoulli())
    @test deviance(gm3) ≈ 8151.40 rtol=1e-5
    @test lowerbd(gm3) == vcat(fill(-Inf, 6), zeros(2))
    @test fitted(gm3) == predict(gm3)
    # these two values are not well defined at the optimum
    @test isapprox(sum(x -> sum(abs2, x), gm3.u), 273.29266717430795, rtol=1e-3)
    @test sum(gm3.resp.devresid) ≈ 7156.547357801238 rtol=1e-4
end

@testset "grouseticks" begin
    center(v::AbstractVector) = v .- (sum(v) / length(v))
    grouseticks = MixedModels.dataset(:grouseticks)
    grouseticks.ch = center(grouseticks.height)
    gm4 = fit(MixedModel, @formula(ticks ~ 1+year+ch+ (1|index) + (1|brood) + (1|location)),
        grouseticks, Poisson(), fast=true)  # fails in pirls! with fast=false
    @test isapprox(deviance(gm4), 851.4046, atol=0.001)
    # these two values are not well defined at the optimum
    #@test isapprox(sum(x -> sum(abs2, x), gm4.u), 196.8695297987013, atol=0.1)
    #@test isapprox(sum(gm4.resp.devresid), 220.92685781326136, atol=0.1)
end


function river_length(River)
    x, z, dx = River.x, River.z, River.Δx^2
    L = zero(x)
    @tturbo for i in 2:length(x)
        L[i] = √(dx  + (z[i]-z[i-1])^2)
    end
    cumsum(L)
end

function lazycumsum(A::Vector{T}, idx) where T
    s = zero(T)
    @turbo for i in 1:idx
        s += A[i]
    end
    s
end


# Unconstrained problem
#
export LevenbergMarquardt_Conf
export set_max_iteration!, set_ε_grad_Inf_norm!, set_ε_step_Inf_norm!

using LinearAlgebra: norm, I

#
# Levenberg-Marquardt implementation
#
function LevenbergMarquardt(nls::AbstractNLS,
                            θ_init::AbstractVector;
                            # parameters
                            max_iter::Int=50,
                            ε_grad_Inf_norm::Float64=1e-8,
                            ε_step_Inf_norm::Float64=1e-8,
                            # initial regularization
                            τ::Float64=1.0e-3)
    # Sanity check
    #
    @assert parameter_size(nls) == length(θ_init)

    @assert max_iter > 0
    @assert ε_grad_Inf_norm ≥ 0
    @assert ε_step_Inf_norm ≥ 0
    @assert τ > 0

    # Compute, r,J, ∇fobj=J'r
    #
    θ=copy(θ_init)

    (r,J) = eval_r_J(nls,θ)
    ∇fobj = eval_nls_∇fobj(r,J)

    # Check CV: |∇fobj| ≤ ϵ ?
    #
    inf_norm_∇fobj = norm(∇fobj,Inf)
    if  inf_norm_∇fobj ≤ ε_grad_Inf_norm

        result = LevenbergMarquardt_Result(_converged=true,
                                         _iter_count=0,
                                         _fobj=eval_nls_fobj(r),
                                         _solution=θ)

        # @info "Initial point was already a soluion" result = result

        return result
    end

    # Compute H=J'J
    #
    H = eval_nls_∇∇fobj(J)

    # Initial μ
    #
    # (maybe add (Abstract)Damping type into conf)
    #
    damping = LM_Damping(τ * norm(H,Inf))

    for iter ∈ 1:max_iter
        # regularize Hessian
        #
        H_μD = H + get_μ(damping)*I

        # Newton step = -inv(H).∇f
        #
        local step
        try
            step = -H_μD\∇fobj
        catch
            result = LevenbergMarquardt_Result(_converged=false,
                                               _iter_count=iter,
                                               _fobj=eval_nls_fobj(r),
                                               _solution=θ
                                               )

            @warn "Leaving LM (singular system)" result = result

            return result
        end

        # Check if step not too small -> CV
        #
        norm_Inf_step = norm(step,Inf)

        if norm_Inf_step ≤ ε_step_Inf_norm

            result = LevenbergMarquardt_Result(_converged=true,
                                             _iter_count=iter,
                                             _fobj=eval_nls_fobj(r),
                                             _solution=θ,
                                               )

            # @info "Small step" result = result

            return result
        end

        # Compute  δL = L(0)-L(step)
        #
        δL = compute_δL_unconstrained(∇fobj,get_μ(damping),step)
        @assert δL > 0

        # Compute new θ & residue
        #
        θ_new = θ + step
        r_new = eval_r(nls,θ_new)

        # Compute δfobj = fobj(θ) - fobj(θ_new)
        #
        δfobj = compute_δf(r,r_new)

        # compute ρ = δf/δL
        #
        ρ = δfobj/δL

        # @info "LM: iter=$(_fmt(iter)), |step|=$(_fmt(norm_Inf_step)), |∇f|=$(_fmt(inf_norm_∇fobj)), μ=$(_fmt(get_μ(damping)))"

        # Accept new point?
        #
        # -> update position and check for CV
        #
        if ρ>0
            θ = θ_new
            r, J = eval_r_J(nls,θ) # r_new was already know, but not J
            ∇fobj = eval_nls_∇fobj(r,J)
            H = eval_nls_∇∇fobj(J)

            inf_norm_∇fobj = norm(∇fobj,Inf)
            if  inf_norm_∇fobj ≤ ε_grad_Inf_norm

                result = LevenbergMarquardt_Result(_converged=true,
                                                 _iter_count=iter,
                                                 _fobj=eval_nls_fobj(r),
                                                 _solution=θ_new)

                # @info "Found a critical point" result = result

                return result
            end
        end

        # In all cases (accepted or not) update damping factor μ
        #
        damping = update_μ(damping,ρ)
    end

    # end of loop... not convergence
    return LevenbergMarquardt_Result(_converged=false,
                                     _iter_count=max_iter,
                                     _fobj=eval_nls_fobj(r),
                                     _solution=θ
                                     )
end

# ================================================================
# Use the "Solver Conf + Solve method" framework:
# 1. define "LevenbergMarquardt_Conf"
# 2. overwrite the "solve()" function
# ================================================================

# ----------------------------------------------------------------
# 1. define "LevenbergMarquardt_Conf"
# ----------------------------------------------------------------
#
@doc raw"""
```julia
mutable struct LevenbergMarquardt_Conf <: Abstract_Solver_Conf
    ...
end
```

Use this constructor
```julia
LevenbergMarquardt_Conf()
```
to initialize the Levenberg-Marquardt solver default configuration
parameters.

To solve a problem with this method, you must then call
[`solve(nls::AbstractNLS, θ_init::AbstractVector, conf::Abstract_Solver_Conf)`](@ref)

See:
- [`set_max_iteration!(conf::LevenbergMarquardt_Conf,max_iter::Int)`](@ref)
- [`set_ε_grad_Inf_norm!(conf::LevenbergMarquardt_Conf,ε_grad_Inf_norm::Float64)`](@ref)
- [`set_ε_step_Inf_norm!(conf::LevenbergMarquardt_Conf,ε_step_Inf_norm::Float64)`](@ref)

"""
mutable struct LevenbergMarquardt_Conf <: Abstract_Solver_Conf
    # The structure is mutable as we will add methods such as:
    # set_max_iter().
    #

    # Related to CV test
    #
    _max_iter::Int
    _ε_grad_Inf_norm::Float64
    _ε_step_Inf_norm::Float64

    # initial regularization μ = τ max_ij(|H_ij|)
    _τ::Float64

    # default values
    function LevenbergMarquardt_Conf(;
                     max_iter::Int=1000,
                     ε_grad_Inf_norm::Float64=1e-8,
                     ε_step_Inf_norm::Float64=1e-8,

                     τ::Float64=1.0e-3)

        # note: parameters values are controlled directly by the
        # LevenbergMarquardt() function

        new(max_iter,
            ε_grad_Inf_norm,
            ε_step_Inf_norm,

            τ)
    end
end

@doc raw"""
```julia
set_max_iteration!(conf::LevenbergMarquardt_Conf,
                   max_iter::Int)
```

Modify the maximum number of iterations

See: [`LevenbergMarquardt_Conf`](@ref)
"""
function set_max_iteration!(conf::LevenbergMarquardt_Conf,max_iter::Int)
    @assert max_iter>0
    conf._max_iter=max_iter
    conf
end

@doc raw"""
```julia
set_ε_grad_Inf_norm!(conf::LevenbergMarquardt_Conf,
                     ε_grad_Inf_norm::Float64)
```

Modify the stopping criterion ``|\nabla f|_\infty\le\epsilon``

See: [`LevenbergMarquardt_Conf`](@ref)
"""
function set_ε_grad_Inf_norm!(conf::LevenbergMarquardt_Conf,ε_grad_Inf_norm::Float64)
    @assert ε_grad_Inf_norm>0
    conf._ε_grad_Inf_norm = ε_grad_Inf_norm
    conf
end

@doc raw"""
```julia
set_ε_step_Inf_norm!(conf::LevenbergMarquardt_Conf,
                     ε_step_Inf_norm::Float64)
```

Modify the stopping criterion ``|\delta x|_\infty\le\epsilon``

See: [`LevenbergMarquardt_Conf`](@ref)
"""
function set_ε_step_Inf_norm!(conf::LevenbergMarquardt_Conf,ε_step_Inf_norm::Float64)
    @assert ε_step_Inf_norm>0
    conf._ε_step_Inf_norm = ε_step_Inf_norm
    conf
end

# ----------------------------------------------------------------
# 2. overwrite the "solve()" function
# ----------------------------------------------------------------
#
function solve(nls::AbstractNLS,
               θ_init::AbstractVector,
               conf::LevenbergMarquardt_Conf)

    LevenbergMarquardt(nls,θ_init,

                        max_iter=conf._max_iter,
                        ε_grad_Inf_norm=conf._ε_grad_Inf_norm,
                        ε_step_Inf_norm= conf._ε_step_Inf_norm,

                        τ=conf._τ)
end


#=
Copyright 2021 BlackRock, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
=#

"""
    BoundedQuadratic(lb::Real, ub::Real, p::Real, q::Real, r::Real)

Represents bounded quadratic function
```math
f(x) = px^2 + qx + r, ∀ x ∈ [lb, ub]
```

The fields represent:
- `lb`: Lower bound of the function
- `ub`: Upper bound of the function
- `p`: Coefficient of the quadratic term
- `q`: Coefficient of the linear term
- `r`: Constant

# Example
```jldoctest
julia> bq = BoundedQuadratic(0., 5., 3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ [0.00000, 5.00000]

```
"""
mutable struct BoundedQuadratic
    lb::Float64
    ub::Float64
    p::Float64
    q::Float64
    r::Float64

    function BoundedQuadratic(lb::Real, ub::Real, p::Real, q::Real, r::Real)
        @assert !isnan(lb)
        @assert !isnan(ub)
        @assert isfinite(p)
        @assert isfinite(q)
        @assert isfinite(r)
        return new(lb, ub, p, q, r)
    end
end

"""
    BoundedQuadratic()

Constructs an unbounded BoundedQuadratic `f(x) = 0, ∀ x`.
"""
BoundedQuadratic() = BoundedQuadratic(0.0, 0.0, 0.0)

"""
    BoundedQuadratic(p::Real, q::Real, r::Real)

Constructs a unbounded `BoundedQuadratic` function `f(x) = px^2 + qx + r, ∀ x ∈ ℝ`.
"""
BoundedQuadratic(p::Real, q::Real, r::Real) = BoundedQuadratic(Interval(), p, q, r)

"""
    BoundedQuadratic(dom::Interval, p::Real, q::Real, r::Real

Constructs a `BoundedQuadratic` function on an Interval `dom`.
"""
function BoundedQuadratic(dom::Interval, p::Real, q::Real, r::Real)
    return BoundedQuadratic(dom.lb, dom.ub, p, q, r)
end

function Base.show(io::IO, f::BoundedQuadratic)
    @printf(io, "BoundedQuadratic: f(x) = ")
    if f.p > 0
        @printf(io, "%.5f x² ", f.p)
    elseif f.p < 0
        @printf(io, "-%.5f x² ", -f.p)
    end
    if f.q > 0
        @printf(io, "+ %.5f x ", f.q)
    elseif f.q < 0
        @printf(io, "- %.5f x ", -f.q)
    end
    if f.r > 0
        @printf(io, "+ %.5f", f.r)
    elseif f.r < 0
        @printf(io, "- %.5f", -f.r)
    end
    if (f.p == 0) & (f.q == 0)
        @printf(io, "0")
    end
    @printf(io, ", ∀x ∈ ")
    print(io, domain(f))
    return
end

"""
    zero(::Type{BoundedQuadratic})

Construct an empty BoundedQuadratic with no constraints.

# Example
```jldoctest
julia> zero(BoundedQuadratic)
BoundedQuadratic: f(x) = 0, ∀x ∈ ℝ

```
"""
zero(::Type{BoundedQuadratic}) = BoundedQuadratic()
copy(f::BoundedQuadratic) = BoundedQuadratic(f.lb, f.ub, f.p, f.q, f.r)
function copy!(out::BoundedQuadratic, f::BoundedQuadratic)
    out.lb = f.lb
    out.ub = f.ub
    out.p = f.p
    out.q = f.q
    out.r = f.r
    return out
end

"""
    get_tangent(f::BoundedQuadratic, x::Real)

Construct a new unbounded `BoundedQuadratic` representing the (unbounded) tangent line to `f` at the point `x`.

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 1., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-1.00000, 1.00000]

julia> tangent = get_tangent(bq, 0.5)
BoundedQuadratic: f(x) = + 2.00000 x + 0.75000, ∀x ∈ ℝ

```
"""
function get_tangent(f::BoundedQuadratic, x::Real)
    q_tan = 2 * f.p * x + f.q
    r_tan = f.p * x^2 + f.q * x + f.r - q_tan * x
    return BoundedQuadratic(Interval(), 0.0, q_tan, r_tan)
end

"""
    get_line(x1::Real, y1::Real, x2::Real, y2::Real)

Construct a new unbounded `BoundedQuadratic` representing a line passing through `(x1, y1)` and `(x2, y2)`.

# Example
```jldoctest
julia> line = get_line(1., 2., 3., 4.)
BoundedQuadratic: f(x) = + 1.00000 x + 1.00000, ∀x ∈ ℝ

```
"""
function get_line(x1::Real, y1::Real, x2::Real, y2::Real)
    @assert x1 != x2
    q = (y2 - y1) / (x2 - x1)
    return BoundedQuadratic(0.0, q, y1 - q * x1)
end

"""
    domain(f::BoundedQuadratic)

Return the Interval on which the BoundedQuadratic is defined.

# Example
```jldoctest
julia> bq = BoundedQuadratic(0., 5., 3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ [0.00000, 5.00000]

julia> dom = domain(bq)
[0.00000, 5.00000]

```
"""
domain(f::BoundedQuadratic) = Interval(f.lb, f.ub)

#####
##### Boolean
#####

"""
    isempty(f::BoundedQuadratic)

Return `true` if the domain of the BoundedQuadratic `f` is empty.
"""
isempty(f::BoundedQuadratic) = isempty(domain(f))

function ≤(g::BoundedQuadratic, f::BoundedQuadratic)
    @assert g.p == 0
    return (g.lb ≤ f.lb) & (g.ub ≥ f.lb) & (minimize(f - g)[2] ≥ 0)
end

≥(g::BoundedQuadratic, f::BoundedQuadratic) = f ≤ g

function ≲(g::BoundedQuadratic, f::BoundedQuadratic)
    @assert g.p == 0
    return (g.lb ≲ f.lb) & (g.ub ≳ f.ub) & (minimize(f - g)[2] ≳ 0.0)
end

≳(g::BoundedQuadratic, f::BoundedQuadratic) = f ≲ g

function ≈(f::BoundedQuadratic, g::BoundedQuadratic)
    return ((abs(f.lb - g.lb) ≤ ϵ) | (f.lb == g.lb)) &
           ((abs(f.ub - g.ub) ≤ ϵ) | (f.ub == g.ub)) &
           (abs(f.p - g.p) ≤ ϵ) &
           (abs(f.q - g.q) ≤ ϵ) &
           (abs(f.r - g.r) ≤ ϵ)
end

"""
    is_point(f::BoundedQuadratic)

Return `true` if the BoundedQuadratic is defined only on a single point (`lb == ub`).

See also: [`is_almost_point`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(0., 5., 3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ [0.00000, 5.00000]

julia> is_point(bq)
false

julia> bq = BoundedQuadratic(5., 5., 3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ [5.00000, 5.00000]

julia> is_point(bq)
true

```
"""
is_point(f::BoundedQuadratic) = f.lb == f.ub

"""
    is_almost_point(f::BoundedQuadratic)

Return `true` if the BoundedQuadratic is approximately defined only on a single point (`lb ≈ ub`).

See also: [`is_point`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(5., 5. + 1e-14, 3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ [5.00000, 5.00000]

julia> is_point(bq)
false

julia> is_almost_point(bq)
true

```
"""
function is_almost_point(f::BoundedQuadratic)
    return (is_point(f)) || (f.lb ≈ f.ub)
end

"""
    continuous_and_overlapping(f::BoundedQuadratic, g::BoundedQuadratic)

Return `true` if `f`'s right endpoint corresponds with `g`'s left endpoint.
"""
function continuous_and_overlapping(f::BoundedQuadratic, g::BoundedQuadratic)
    return (f.ub ≈ g.lb) && (f(f.ub) ≈ g(g.lb))
end

is_convex(f::BoundedQuadratic) = f.p >= 0.0

#####
##### Operations
#####

"""
    (f::BoundedQuadratic)(x::Real)

Evaluate `f(x)` if `x` is in the domain of `f`, else return `Inf`.

# Example
```jldoctest
julia> bq = BoundedQuadratic(3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ ℝ

julia> bq(1.)
6.0

```
"""
function (f::BoundedQuadratic)(x::Real)
    if x in domain(f)
        return f.p * x^2 + f.q * x + f.r
    else
        return Inf
    end
end

function +(f::BoundedQuadratic, a::Real)
    return BoundedQuadratic(f.lb, f.ub, f.p, f.q, f.r + a)
end

function -(f::BoundedQuadratic)
    @assert f.p == 0.0
    return BoundedQuadratic(f.lb, f.ub, -f.p, -f.q, -f.r)
end

function +(f::BoundedQuadratic, g::BoundedQuadratic)
    dom = domain(f) ∩ domain(g)
    return BoundedQuadratic(dom, f.p + g.p, f.q + g.q, f.r + g.r)
end

function -(f::BoundedQuadratic, g::BoundedQuadratic)
    return f + (-g)
end

*(f::BoundedQuadratic, α::Real) = BoundedQuadratic(f.lb, f.ub, α * f.p, α * f.q, α * f.r)
*(α::Real, f::BoundedQuadratic) = f * α

"""
    scale(f::BoundedQuadratic, α::Real)

Return a new `BoundedQuadratic` that has been scaled by `α`. That is, given `f(x)` and `α`, returns `f(αx)`.

Note: this operation requires scaling the domain.

See also: [`scale!`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 1., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-1.00000, 1.00000]

julia> scale(bq, 2.)
BoundedQuadratic: f(x) = 4.00000 x² + 2.00000 x + 1.00000, ∀x ∈ [-0.50000, 0.50000]

```
"""
function scale(f::BoundedQuadratic, α::Real)
    return BoundedQuadratic(f.lb / α, f.ub / α, α^2 * f.p, α * f.q, f.r)
end

"""
     scale!(f::BoundedQuadratic, α::Real)
     scale!(f::BoundedQuadratic, α::Real, out::BoundedQuadratic)

Scale `f` inplace.

See also: [`scale`](@ref)
"""
scale!(f::BoundedQuadratic, α::Real) = scale!(f, α, f)
scale!(f::BoundedQuadratic, α::Real, out::BoundedQuadratic) = copy!(out, scale(f, α))

"""
    perspective(f::BoundedQuadratic, α::Real)

Return the perspective function of `f`. That is, given `f(x)` and `α`, return `α * f(x / α)`.

Note: that this operation requires scaling of the domain.

See also: [`perspective!`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 1., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-1.00000, 1.00000]

julia> perspective(bq, 2.)
BoundedQuadratic: f(x) = 0.50000 x² + 1.00000 x + 2.00000, ∀x ∈ [-2.00000, 2.00000]

```
"""
function perspective(f::BoundedQuadratic, α::Real)
    return BoundedQuadratic(α * f.lb, α * f.ub, f.p / α, f.q, f.r * α)
end

"""
    perspective!(f::BoundedQuadratic, α::Real)
    perspective!(f::BoundedQuadratic, α::Real, out::BoundedQuadratic)

Shift perspective of `f` inplace.

See also: [`perspective`](@ref)
"""
perspective!(f::BoundedQuadratic, α::Real) = perspective!(f, α, f)
function perspective!(f::BoundedQuadratic, α::Real, out::BoundedQuadratic)
    return copy!(out, perspective(f, α))
end

"""
    shift(f::BoundedQuadratic, δ::Real)

Return `f` shifted along the `x`-axis by `δ`.

Note: for `δ > 0`, this is a right shift. For `δ < 0`, this is a left shift.

See also: [`shift!`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 1., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-1.00000, 1.00000]

julia> shift(bq, 2.)
BoundedQuadratic: f(x) = 1.00000 x² - 3.00000 x + 3.00000, ∀x ∈ [1.00000, 3.00000]

```
"""
function shift(f::BoundedQuadratic, δ::Real)
    return BoundedQuadratic(f.lb + δ, f.ub + δ, f.p, f.q - 2 * f.p * δ,
                            f.p * δ^2 - f.q * δ + f.r)
end

"""
    shift!(f::BoundedQuadratic, δ::Real)
    shift!(f::BoundedQuadratic, δ::Real, out::BoundedQuadratic)

Shift `f` inplace along the `x`-axis by `δ`.

See also: [`shift`](@ref)
"""
shift!(f::BoundedQuadratic, δ::Real) = shift!(f, δ, f)
function shift!(f::BoundedQuadratic, δ::Real, out::BoundedQuadratic)
    return copy!(out, shift(f, δ))
end

"""
    tilt(f::BoundedQuadratic, α::Real)

Return `f` tilted by `α`. This shifts linear coefficient `q` by `α`.

See also: [`tilt!`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 1., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-1.00000, 1.00000]

julia> tilt(bq, 2.)
BoundedQuadratic: f(x) = 1.00000 x² + 3.00000 x + 1.00000, ∀x ∈ [-1.00000, 1.00000]

```
"""
function tilt(f::BoundedQuadratic, α::Real)
    return BoundedQuadratic(f.lb, f.ub, f.p, f.q + α, f.r)
end

"""
    tilt!(f::BoundedQuadratic, α::Real)
    tilt!(f::BoundedQuadratic, α::Real, out::BoundedQuadratic)

Tilt `f` inplace.

See also: [`tilt`](@ref)
"""
tilt!(f::BoundedQuadratic, α::Real) = tilt!(f, α, f)
function tilt!(f::BoundedQuadratic, α::Real, out::BoundedQuadratic)
    return copy!(out, tilt(f, α))
end

"""
    restrict_dom(f::BoundedQuadratic, dom::Interval)
    restrict_dom(f::BoundedQuadratic, lb::Real, ub::Real)

Return a new BoundedQuadratic with domain restricted to the intersect of the passed domain.

See also: [`restrict_dom!`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(-10., 10., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-10.00000, 10.00000]

julia> restrict_dom(bq, 2., 3.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [2.00000, 3.00000]

```
"""
function restrict_dom(f::BoundedQuadratic, dom::Interval)
    restricted_dom = domain(f) ∩ dom
    @assert restricted_dom.ub ≥ restricted_dom.lb
    return BoundedQuadratic(restricted_dom, f.p, f.q, f.r)
end
restrict_dom(f::BoundedQuadratic, lb::Real, ub::Real) = restrict_dom(f, Interval(lb, ub))

"""
    restrict_dom!(f::BoundedQuadratic, dom::Interval)
    restrict_dom!(f::BoundedQuadratic, lb::Real, ub::Real)
    restrict_dom!(f::BoundedQuadratic, lb::Real, ub::Real, out::BoundedQuadratic)

Restrict domain of `f` inplace.

See also: [`restrict_dom`](@ref)
"""
restrict_dom!(f::BoundedQuadratic, dom::Interval) = restrict_dom!(f, dom, f)
function restrict_dom!(f::BoundedQuadratic, dom::Interval, out::BoundedQuadratic)
    return copy!(out, restrict_dom(f, dom))
end
restrict_dom!(f::BoundedQuadratic, lb::Real, ub::Real) = restrict_dom!(f, lb, ub, f)
function restrict_dom!(f::BoundedQuadratic, lb::Real, ub::Real, out::BoundedQuadratic)
    return restrict_dom!(f, Interval(lb, ub), f)
end

"""
    extend_dom(f::BoundedQuadratic)

Return an unbounded version of `f`.

See also: [`extend_dom!`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 1., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-1.00000, 1.00000]

julia> extend_dom(bq)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ ℝ

```
"""
extend_dom(f::BoundedQuadratic) = BoundedQuadratic(f.p, f.q, f.r)

"""
    extend_dom!(f::BoundedQuadratic)
    extend_dom!(f::BoundedQuadratic, out::BoundedQuadratic)

Remove the bounds of `f` inplace.

See also: [`extend_dom`](@ref)
"""
extend_dom!(f::BoundedQuadratic) = extend_dom!(f, f)
extend_dom!(f::BoundedQuadratic, out::BoundedQuadratic) = copy!(out, extend_dom(f))

"""
    reverse(f::BoundedQuadratic)

Return `f` reversed over the `y` axis. That is, given `f(x)`, return `f(-x)`.

See also: [`reverse!`](@ref)

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 2., 1., 1., 1.)
BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-1.00000, 2.00000]

julia> reverse(bq)
BoundedQuadratic: f(x) = 1.00000 x² - 1.00000 x + 1.00000, ∀x ∈ [-2.00000, 1.00000]

```
"""
reverse(f::BoundedQuadratic) = BoundedQuadratic(-f.ub, -f.lb, f.p, -f.q, f.r)

"""
    reverse!(f::BoundedQuadratic)
    reverse!(f::BoundedQuadratic, out::BoundedQuadratic)

Reverse `f` inplace.

See also: [`reverse`](@ref)
"""
reverse!(f::BoundedQuadratic) = reverse!(f, f)
function reverse!(f::BoundedQuadratic, out::BoundedQuadratic)
    return copy!(out, reverse(f))
end

"""
    derivative(f::BoundedQuadratic)

Return the derivative of `f`, `f'`

# Example
```jldoctest
julia> bq = BoundedQuadratic(3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ ℝ

julia> derivative(bq)
BoundedQuadratic: f(x) = + 6.00000 x + 2.00000, ∀x ∈ ℝ

```
"""
function derivative(f::BoundedQuadratic)
    return BoundedQuadratic(f.lb, f.ub, 0.0, 2.0 * f.p, f.q)
end

"""
    derivative(f::BoundedQuadratic, x::Real)

Evaluate the derivative of `f` at `x`, `f'(x)`.

# Example
```jldoctest
julia> bq = BoundedQuadratic(3., 2., 1.)
BoundedQuadratic: f(x) = 3.00000 x² + 2.00000 x + 1.00000, ∀x ∈ ℝ

julia> derivative(bq, 2.)
14.0

```
"""
derivative(f::BoundedQuadratic, x::Real) = derivative(f)(x)

function _sum(f_list::Vector{BoundedQuadratic})
    dom, is_valid = _intersect(f_list)

    p = 0.0
    q = 0.0
    r = 0.0
    if is_valid
        for fi in f_list
            p += fi.p
            q += fi.q
            r += fi.r
        end
        return BoundedQuadratic(dom, p, q, r), is_valid
    else
        return BoundedQuadratic(dom, 0.0, 0.0, 0.0), is_valid
    end
end

"""
    sum(f_list::Vector{BoundedQuadratic})

Return the BoundedQuadratic sum of a list of BoundedQuadratics if it is valid, else `nothing`.

# Example
```jldoctest
julia> bq = BoundedQuadratic(-1., 5., 1., 2., 3.)
BoundedQuadratic: f(x) = 1.00000 x² + 2.00000 x + 3.00000, ∀x ∈ [-1.00000, 5.00000]

julia> sum([bq, bq])
BoundedQuadratic: f(x) = 2.00000 x² + 4.00000 x + 6.00000, ∀x ∈ [-1.00000, 5.00000]

```
"""
function sum(f_list::Vector{BoundedQuadratic})
    g, is_valid = _sum(f_list)
    if is_valid
        return g
    else
        return nothing
    end
end

function _intersect(f_list::Vector{BoundedQuadratic})
    k = length(f_list)
    doms = [Interval(), map(domain, f_list)...]
    dom = reduce(∩, doms)
    is_valid = !isempty(dom)
    return dom, is_valid
end

# TODO: the following example code fails as a doctest with Julia 1.0...
# it appears that the output of `Vector{T}` displays as `Array{T,1}` instead
"""
    intersect(f_list::Vector{BoundedQuadratic})

Intersect the domains of a list of BoundedQuadratics if possible.

# Example
```julia
julia> f_list = [BoundedQuadratic(-100, 100, 1, 1, 1),
          BoundedQuadratic(-50, 25, 1, 1, 1),
          BoundedQuadratic(0, 50, 1, 1, 1)]
3-element Vector{BoundedQuadratic}:
 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-100.00000, 100.00000]

 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-50.00000, 25.00000]

 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [0.00000, 50.00000]

julia> out, valid = intersect(f_list);

julia> valid
true

julia> out
3-element Vector{BoundedQuadratic}:
 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [0.00000, 25.00000]

 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [0.00000, 25.00000]

 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [0.00000, 25.00000]

julia> f_list = [BoundedQuadratic(-100, 0, 1,1,1)
            BoundedQuadratic(25, 50, 1,1,1)]  # non-overlapping
2-element Vector{BoundedQuadratic}:
 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [-100.00000, 0.00000]

 BoundedQuadratic: f(x) = 1.00000 x² + 1.00000 x + 1.00000, ∀x ∈ [25.00000, 50.00000]

julia> out, valid = intersect(f_list);

julia> valid
false

julia> out
2-element Vector{BoundedQuadratic}:
 #undef
 #undef

```
"""
function intersect(f_list::Vector{BoundedQuadratic})
    k = length(f_list)
    dom, is_valid = _intersect(f_list)

    g_list = Vector{BoundedQuadratic}(undef, k)
    if is_valid
        for (i, f) in enumerate(f_list)
            g_list[i] = restrict_dom(f, dom)
        end
    end
    return g_list, is_valid
end

"""
    minimize(f::BoundedQuadratic)

Return the minimum `x` and `f(x)` of `f` over its domain.
"""
function minimize(f::BoundedQuadratic)
    if f.lb > f.ub  # Infeasible.
        xstar = NaN
        min_val = Inf
    else  # Feasible, but possibly unbounded.
        if f.p > 0
            xstar = -f.q / (2 * f.p)
            if xstar < f.lb
                xstar = f.lb
            elseif xstar > f.ub
                xstar = f.ub
            end
        elseif f.q > 0
            xstar = (f.lb == -Inf) ? NaN : f.lb
        elseif f.q < 0
            xstar = (f.ub == Inf) ? NaN : f.ub
        else  # Case of p = 0, q = 0.
            if f.lb > -Inf
                xstar = f.lb
            elseif f.ub < Inf
                xstar = f.ub
            else
                xstar = 0.0
            end
        end
        if isfinite(xstar)
            min_val = f.p * xstar^2 + f.q * xstar + f.r
        else
            min_val = -Inf
        end
    end
    return xstar, min_val
end