# store psis in vector and iterate to do derivatives

function solovev_psi_functions()
    # up down symmetric homogeneous solutions
    psi1(x,y) = 1
    psi2(x,y) = x^2
    psi3(x,y) = y^2 - x^2*log(x)
    psi4(x,y) = x^4 - 4*x^2*y^2
    psi5(x,y) = 2*y^4 - 9*y^2*x^2 + 3*x^4*log(x) - 12*x^2*y^2*log(x)
    psi6(x,y) = x^6 - 12*x^4*y^2 + 8*x^2*y^4
    psi7(x,y) = 8*y^6 - 140*y^4*x^2 + 75*y^2*x^4 - 15*x^6*log(x) + 180*x^4*y^2*log(x) - 120*x^2*y^4*log(x)

    # up down asymmetric homogenous solutions
    psi8(x,y) = y
    psi9(x,y) = y*x^2
    psi10(x,y) = y^3 - 3*y*x^2*log(x)
    psi11(x,y) = 3*y*x^4 - 4*y^3*x^2
    psi12(x,y) = 8*y^5 - 45*y*x^4 - 80*y^3*x^2*log(x) + 60*y*x^4*log(x)

    psibasis = [psi1 psi2 psi3 psi4 psi5 psi6 psi7 psi8 psi9 psi10 psi11 psi12]

    return psibasis
end

function solve_solovev_psi_functions(x, y) #XXX might not want to call this function the same as the function above?
    # up down symmetric homogeneous solutions
    psi1 = 1
    psi2 = x^2
    psi3 = y^2 - x^2*log(x)
    psi4 = x^4 - 4*x^2*y^2
    psi5 = 2*y^4 - 9*y^2*x^2 + 3*x^4*log(x) - 12*x^2*y^2*log(x)
    psi6 = x^6 - 12*x^4*y^2 + 8*x^2*y^4
    psi7 = 8*y^6 - 140*y^4*x^2 + 75*y^2*x^4 - 15*x^6*log(x) + 180*x^4*y^2*log(x) - 120*x^2*y^4*log(x)

    # up down asymmetric homogenous solutions
    psi8 = y
    psi9 = y*x^2
    psi10 = y^3 - 3*y*x^2*log(x)
    psi11 = 3*y*x^4 - 4*y^3*x^2
    psi12 = 8*y^5 - 45*y*x^4 - 80*y^3*x^2*log(x) + 60*y*x^4*log(x)

    psivals = [psi1 psi2 psi3 psi4 psi5 psi6 psi7 psi8 psi9 psi10 psi11 psi12]

    return psivals
end

function solovev_parameterize(n, delta, epsilon, kappa)
    # epsilon = reverse aspect ratio (epsilon = a/R0 for R0 the major radius (center of machine to
    # center of region of plasma) and a the minor radius (center of plasma to wall))
    # kappa = elongation
    # sin(alpha) = delta = triangularity
    # n = resolution of points in x/y

    alpha = asin(delta)

    # calculating x and y coordinates of flux surface cross section
    tau = LinRange(0, 2*pi, n)
    x = zeros(n)
    y = zeros(n)
    for i=1:n
        x[i] = 1 + epsilon*cos(tau[i] + alpha*sin(tau[i]))
        y[i] = epsilon*kappa*sin(tau[i])
    end

    return tau, x, y
end

function solovev_particular_psi(A, x, y)
    psi = x^4/8 + A * (1/2*x^2*log(x) - x^4/8)
    return psi
end

function solovev_homo_partial_x(functs, xval, yval) #XXX could be confusing that the input variable psis is also called the same as two seperate functions defined above.
    partial_x = zeros(12)
    for i=1:12
        # partial_x[i] = ForwardDiff.derivative(x -> psis[i](x,y), x)

        # XXX: ForwardDiff.derivative returns a number, not a function. The input looks like (::Function, ::Real), and the function has to be single variable
        #      so it doesn't know what to do with psis[i](x,y), it has to be psis[i](x,yval)
        partial_x[i] = ForwardDiff.derivative(x -> functs[i](x,yval), xval) #XXX <-- applying this correction to the other instances should fix the issue.
    end
    # for j=1:12
    #     partial_x[j] = partial_x[j][xval, yval]
    # end

    return partial_x
end

function solovev_homo_partial_xx(functs, xval, yval)
    # px = []
    pxx = zeros(12)
    for i=1:12
        # XXX Since ForwardDiff isn't returning functions, this is the way to take a second derivative:
        pxx[i] = ForwardDiff.derivative(x -> ForwardDiff.derivative(x -> functs[i](x,yval), x), xval)
    end
    return pxx
end

function solovev_homo_partial_y(functs, xval, yval) #XXX Need to just apply the changes above to the y versions.
    partial_y = zeros(12)
    for i=1:12
        partial_y[i] = ForwardDiff.derivative(y -> functs[i](xval,y), yval)
    end
    return partial_y
end

function solovev_homo_partial_yy(functs, xval, yval)
    # py = zeros(12)
    pyy = zeros(12)
    for i=1:12
        # py[i] = ForwardDiff.derivative(y -> psis[i](x,y), y)
        pyy[i] = ForwardDiff.derivative(y -> ForwardDiff.derivative(y -> functs[i](xval,y), y), yval)
    end
    return pyy
end

function solovev_particular_partial_x(A, xval, yval)
    partial_x = ForwardDiff.derivative(x -> solovev_particular_psi(A, x, yval), xval)
    return partial_x
end

function solovev_particular_partial_xx(A, xval, yval)
    # partial_x = ForwardDiff.derivative(x -> solovev_particular_psi(A, x, y), x)
    partial_xx = ForwardDiff.derivative(x -> ForwardDiff.derivative(x -> solovev_particular_psi(A, x, yval), x), xval)
    return partial_xx
end

function solovev_particular_partial_y(A, xval, yval)
    partial_y = ForwardDiff.derivative(y -> solovev_particular_psi(A, xval, y), yval)
    return partial_y
end

function solovev_particular_partial_yy(A, xval, yval)
    # partial_y = ForwardDiff.derivative(y -> solovev_particular_psi(A)(y), yval)
    partial_yy = ForwardDiff.derivative(y -> ForwardDiff.derivative(y -> solovev_particular_psi(A, xval, y), y), yval)
    return partial_yy
end

function solovev_flux_function(A, delta, epsilon, kappa, B0; upsep=[0,0], downsep=[0,0])
    alpha = asin(delta)

    # last terms are xsep/ysep - can be changed to be inputs if need be
    N1 = -(1 + alpha)^2/(epsilon*kappa^2)
    N2 = (1 - alpha)^2/(epsilon*kappa^2)
    N3 = -kappa/(epsilon*(cos(alpha)^2))

    x = [1 + epsilon, 1 - epsilon, 1 - delta*epsilon, upsep[1], downsep[1]]
    y = [0, 0, kappa*epsilon, upsep[2], downsep[2]]

    psisbasis = solovev_psi_functions()

    rhs1 = solovev_particular_psi(A, x[1], y[1])
    rhsy2 = solovev_particular_partial_y(A, x[1], y[1])
    rhsyy3 = solovev_particular_partial_yy(A, x[1], y[1])
    rhsx3 = solovev_particular_partial_x(A, x[1], y[1])

    rhs4 = solovev_particular_psi(A, x[2], y[2])
    rhsy5 = solovev_particular_partial_y(A, x[2], y[2])
    rhsyy6 = solovev_particular_partial_yy(A, x[2], y[2])
    rhsx6 = solovev_particular_partial_x(A, x[2], y[2])

    psis1 = solve_solovev_psi_functions(x[1], y[1])
    psisy2 = solovev_homo_partial_y(psisbasis, x[1], y[1])
    psisyy3 = solovev_homo_partial_yy(psisbasis, x[1], y[1])
    psisx3 = solovev_homo_partial_x(psisbasis, x[1], y[1])

    psis4 = solve_solovev_psi_functions(x[2], y[2])
    psisy5 = solovev_homo_partial_y(psisbasis, x[2], y[2])
    psisyy6 = solovev_homo_partial_yy(psisbasis, x[2], y[2])
    psisx6 = solovev_homo_partial_x(psisbasis, x[2], y[2])

    Amat = zeros(12, 12)
    Amat[1,:] = psis1
    Amat[2,:] = psisy2
    Amat[3,:] = psisyy3 + N1*psisx3
    Amat[4,:] = psis4
    Amat[5,:] = psisy5
    Amat[6,:] = psisyy6 + N2*psisx6

    if upsep==[0,0] && downsep==[0,0]
        # calculating values of flux function and derivatives at boundaries
        rhs7 = solovev_particular_psi(A, x[3], y[3])
        rhsx8 = solovev_particular_partial_x(A, x[3], y[3])
        rhsxx9 = solovev_particular_partial_xx(A, x[3], y[3])
        rhsy9 = solovev_particular_partial_y(A, x[3], y[3])
        rhs10 = solovev_particular_psi(A, x[3], -y[3])
        rhsx11 = solovev_particular_partial_x(A, x[3], -y[3])
        rhsxx12 = solovev_particular_partial_xx(A, x[3], -y[3])
        rhsy12 = solovev_particular_partial_y(A, x[3], -y[3])

        psis7 = solve_solovev_psi_functions(x[3], y[3])
        psisx8 = solovev_homo_partial_x(psisbasis, x[3], y[3])
        psisxx9 = solovev_homo_partial_xx(psisbasis, x[3], y[3])
        psisy9 = solovev_homo_partial_y(psisbasis, x[3], y[3])
        psis10 = solve_solovev_psi_functions(x[3], -y[3])
        psisx11 = solovev_homo_partial_x(psisbasis, x[3], -y[3])
        psisxx12 = solovev_homo_partial_xx(psisbasis, x[3], -y[3])
        psisy12 = solovev_homo_partial_y(psisbasis, x[3], -y[3])

        Amat[7,:] = psis7
        Amat[8,:] = psisx8
        Amat[9,:] = psisxx9 + N3*psisy9
        Amat[10,:] = psis10
        Amat[11,:] = psisx11
        Amat[12,:] = psisxx12 - N3*psisy12

        b = [rhs1, rhsy2, rhsyy3+N1*rhsx3, rhs4, rhsy5, rhsyy6+N2*rhsx6,
        rhs7, rhsx8, rhsxx9+N3*rhsy9, rhs10, rhsx11, rhsxx12-N3*rhsy12]
    elseif downsep==[0,0]
        rhs7 = solovev_particular_psi(A, x[4], y[4])
        rhsx8 = solovev_particular_partial_x(A, x[4], y[4])
        rhsy9 = solovev_particular_partial_y(A, x[4], y[4])
        rhs10 = solovev_particular_psi(A, x[3], -y[3])
        rhsx11 = solovev_particular_partial_x(A, x[3], -y[3])
        rhsxx12 = solovev_particular_partial_xx(A, x[3], -y[3])
        rhsy12 = solovev_particular_partial_y(A, x[3], -y[3])

        psis7 = solve_solovev_psi_functions(x[4], y[4])
        psisx8 = solovev_homo_partial_x(psisbasis, x[4], y[4])
        psisy9 = solovev_homo_partial_y(psisbasis, x[4], y[4])
        psis10 = solve_solovev_psi_functions(x[3], -y[3])
        psisx11 = solovev_homo_partial_x(psisbasis, x[3], -y[3])
        psisxx12 = solovev_homo_partial_xx(psisbasis, x[3], -y[3])
        psisy12 = solovev_homo_partial_y(psisbasis, x[3], -y[3])

        Amat[7,:] = psis7
        Amat[8,:] = psisx8
        Amat[9,:] = psisy9
        Amat[10,:] = psis10
        Amat[11,:] = psisx11
        Amat[12,:] = psisxx12 - N3*psisy12

        b = [rhs1, rhsy2, rhsyy3+N1*rhsx3, rhs4, rhsy5, rhsyy6+N2*rhsx6,
        rhs7, rhsx8, rhsy9, rhs10, rhsx11, rhsxx12+N3*rhsy12]
    elseif upsep==[0,0]
        rhs7 = solovev_particular_psi(A, x[3], y[3])
        rhsx8 = solovev_particular_partial_x(A, x[3], y[3])
        rhsxx9 = solovev_particular_partial_xx(A, x[3], y[3])
        rhsy9 = solovev_particular_partial_y(A, x[3], y[3])
        rhs10 = solovev_particular_psi(A, x[5], y[5])
        rhsx11 = solovev_particular_partial_x(A, x[5], y[5])
        rhsy12 = solovev_particular_partial_y(A, x[5], y[5])

        psis7 = solve_solovev_psi_functions(x[3], y[3])
        psisx8 = solovev_homo_partial_x(psisbasis, x[3], y[3])
        psisxx9 = solovev_homo_partial_xx(psisbasis, x[3], y[3])
        psisy9 = solovev_homo_partial_y(psisbasis, x[3], y[3])
        psis10 = solve_solovev_psi_functions(x[5], y[5])
        psisx11 = solovev_homo_partial_x(psisbasis, x[5], y[5])
        psisy12 = solovev_homo_partial_y(psisbasis, x[5], y[5])

        Amat[7,:] = psis7
        Amat[8,:] = psisx8
        Amat[9,:] = psisxx9 + N3*psisy9
        Amat[10,:] = psis10
        Amat[11,:] = psisx11
        Amat[12,:] = psisy12

        b = [rhs1, rhsy2, rhsyy3+N1*rhsx3, rhs4, rhsy5, rhsyy6+N2*rhsx6,
        rhs7, rhsx8, rhsxx9+N3*rhsy9, rhs10, rhsx11, rhsy12]
    else
        rhs7 = solovev_particular_psi(A, x[4], y[4])
        rhsx8 = solovev_particular_partial_x(A, x[4], y[4])
        rhsy9 = solovev_particular_partial_y(A, x[4], y[4])
        rhs10 = solovev_particular_psi(A, x[5], y[5])
        rhsx11 = solovev_particular_partial_x(A, x[5], y[5])
        rhsy12 = solovev_particular_partial_y(A, x[5], y[5])

        psis7 = solve_solovev_psi_functions(x[4], y[4])
        psisx8 = solovev_homo_partial_x(psisbasis, x[4], y[4])
        psisy9 = solovev_homo_partial_y(psisbasis, x[4], y[4])
        psis10 = solve_solovev_psi_functions(x[5], y[5])
        psisx11 = solovev_homo_partial_x(psisbasis, x[5], y[5])
        psisy12 = solovev_homo_partial_y(psisbasis, x[5], y[5])

        Amat[7,:] = psis7
        Amat[8,:] = psisx8
        Amat[9,:] = psisy9
        Amat[10,:] = psis10
        Amat[11,:] = psisx11
        Amat[12,:] = psisy12

        b = [rhs1, rhsy2, rhsyy3+N1*rhsx3, rhs4, rhsy5, rhsyy6+N2*rhsx6,
        rhs7, rhsx8, rhsy9, rhs10, rhsx11, rhsy12]
    end

    c = Amat \ -b
    psifunctions = solovev_psi_functions()
    psi(x,y) = solovev_particular_psi(A,x,y) .+ dot(c, [p(x,y) for p in psifunctions])

    Bx(x, y) = ForwardDiff.derivative(u -> -psi(x,u), y)
    By(x, y) = ForwardDiff.derivative(u -> psi(u,y), x) #XXX changed the sign of this to make consistent with trace.jl, should check later which way is actually correct.
    # B0 = R0^4*B0^2/psi0^2
    Bphi(x,y) = (B0^2 - 2*A*psi(x,y))^(1/2)
    B(x,y) = norm(Float64[Bx(x,y), By(x,y), Bphi(x,y)])

    bx(x,y) = Bx(x,y) / B(x,y)
    by(x,y) = By(x,y) / B(x,y)
    bz(x,y) = Bphi(x,y) / B(x,y)

    return psi, bx, by, bz
end


"""
    nx = dsmap(u, ABCD, nlev, x, e, v)

For a DSM with input `u`, a structure `ABCD` and an `nlev`-level quantizer,
compute the (potential) vertices of the image of a convex object described
in terms of its vertices `x` and edges `e`. `v` is the assumed quantizer
output; it is computed if it is not supplied.

Basic Algorithm:
1) Compute the images of the vertices.
2) For those edges which cross a splitting hyperplane,
   compute the images of each split point.
"""
function dsmap(u, ABCD, nlev, x, e, v=nothing)
    n = size(ABCD, 1) - 1
    A = ABCD[1:n, 1:n]
    B = ABCD[1:n, n+1:n+2]
    C = ABCD[n+1, 1:n]
    D1 = ABCD[n+1, n+1] # For any DSM, D2=ABCD(n+1,n+2) must be zero.

    N = size(x, 2)

    # Compute v. The assumption that D1=0 for u-ranges is implicit in this step.
    if isnothing(v)
        y = x'*C .+ D1*u
        v = ds_quantize.(y, nlev)
    elseif size(v) == ()
        v = v * ones(Int, N)
    elseif size(v, 2) != N
        error("the supplied v argument is the wrong size.")
    end

    # 1) Compute the images of the vertices.
    B1u = B[:,1] * u
    nx = A*x + B1u*ones(1, N) + B[:,2]*v'

    # 2) For those edges which cross a (or several) splitting hyperplanes,
    #    compute the two images of each split point.
    diff = abs.(v[e[1,:]] - v[e[2,:]])
    split1 = diff .== 2	# edges split in one place only

    # Handle the split1 edges en masse.
    if any(split1)
        i1 = e[1,split1]
        i2 = e[2,split1]
        y0 = 0.5 * (v[i1] + v[i2]) # the appropriate quantizer thresholds
        k1 = (y[i2] - y0) ./ (y[i2] - y[i1])
        k2 = 1 .- k1
        psplit = ones(n)*k1' .* x[:,i1] + ones(n)*k2' .* x[:,i2]
        N = length(k1)
        images1 = A*psplit + B1u*ones(1,N) + B[:,2]*v[i1]'
        images2 = images1 + B[:,2]*(v[i2] - v[i1])'
        nx = [nx images1 images2]
    end

    return nx
end


using AugmentedGaussianProcesses
using Distributions
using Distances
using Random
ν=3.0

testfunction(x) = (1+x/ν)^(-0.5*(ν+1))
testfunction(x) = sech(sqrt(x/2))
# testfunction(x) = (1+sqrt(3*x))*exp(-sqrt(3*x))
# testfunction(x) = (1+sqrt(5*x)+5/3*x)*exp(-sqrt(5*x))
testdist = AGP.LaplaceTransformDistribution(testfunction,0.0,AGP.BromwichInverseLaplace(19,1,10,5))
@btime rand($testdist)
@btime rand(Gamma((ν+1)/2,1/ν))
rand(testdist)
# @btime AGP.draw(AugmentedGaussianProcesses.PolyaGammaDist(),1,0)
evaluate(KLDivergence(),rand(testdist,10000),rand(Gamma((ν+1)/2,1/ν),10000))
# laptrans(testdist,n=1)[1]
p = histogram(rand(testdist,10000),bins=range(0,3,length=100),normalize=true)|>display
histogram!(rand(Gamma((ν+1)/2,1/ν),10000),alpha=0.5,bins=range(0,10,length=100),normalize=true)
# histogram!([AGP.draw(AugmentedGaussianProcesses.PolyaGammaDist(),1,0) for i in 1:10000],alpha=0.5,bins=range(0,10,length=100),normalize=true)


using ChaosTools
using DynamicalSystemsBase
using Test

@testset "Tipping points" begin
    xg = yg = range(-4, 4, length = 100)
    ds = Systems.magnetic_pendulum(d=0.2, α=0.2, ω=0.8, N=3)
    basins_before, = basins_of_attraction((xg, yg), ds; idxs=1:2,
    diffeq=(;:reltol => 1e-9), show_progress = false)
    ds = Systems.magnetic_pendulum(d=0.2, α=0.2, ω=0.8, N=3, γs = [1.0, 1.0, 0.05])
    basins_after, = basins_of_attraction((xg, yg), ds; idxs = 1:2,
    diffeq=(;:reltol => 1e-9), show_progress = false)
    @testset "Basin fractions" begin
        fs = basin_fractions(basins_before)
        @test sum(values(fs)) ≈ 1
        @test all(0.32 .≤ values(fs) .≤ 0.34)
        fs_after = basin_fractions(basins_after)
        @test length(fs_after) == 2
    end
    @testset "tipping probabilities" begin
        P = tipping_probabilities(basins_before, basins_after)
        @test size(P) == (3,2)
        @test all(0.495 .≤ P[1, :] .≤ 0.505) # must be ≈ 50%
        for i in 1:3; @test sum(P[i, :]) ≈ 1; end
    end
end


# # this might be used for Nesterov lookahead, or similar
# before_grad_calc(θ::AbstractVector, updater::ParamUpdater, ∇::AbstractVector) = return

# TODO: add citations/links for each method
# note: good ref: http://sebastianruder.com/optimizing-gradient-descent/
# note: good ref: https://www.quora.com/What-are-differences-between-update-rules-like-AdaDelta-RMSProp-AdaGrad-and-AdaM

init(updater::ParamUpdater, model::Learnable) = init(updater, params(model))

"Stochastic Gradient Descent with Momentum"
type SGD{T<:Number} <: ParamUpdater
    mom::T              # momentum
    lastchg::Vector{T}   # most recent grad change
    (::Type{SGD{T}}){T <: Number}(mom::T) = new{T}(mom)
end
SGD{T}(::Type{T}, mom = T(0.0)) = SGD{T}(T(mom))
SGD{T}(mom::T = 0.0) = SGD{T}(mom)

function init(updater::SGD, θ::AbstractVector)
    updater.lastchg = zeros(θ)
    return
end

# # add momentum into θ, and reset lastchg
# function before_grad_calc(θ::AbstractVector, updater::SGD, ∇::AbstractVector)
#     @inbounds for (i,k) in zip(eachindex(θ), eachindex(updater.lastchg))
#         momchg = updater.mom * updater.lastchg[k]
#         updater.lastchg[k] = momchg
#         θ[i] += momchg
#     end
# end

#
function update!(θ::AbstractVector, updater::SGD, ∇::AbstractVector, lr::Number)
    @inbounds for (i,j,k) in zip(eachindex(θ), eachindex(∇), eachindex(updater.lastchg))
        chg = -lr * ∇[j] + updater.mom * updater.lastchg[k]
        θ[i] += chg
        updater.lastchg[k] = chg
    end
end

# -----------------------------------------------------------------------

"Adaptive Gradient"
type Adagrad{T<:Number} <: ParamUpdater
    ϵ::T
    G::Vector{T}  # sum of squared gradients
    (::Type{Adagrad{T}}){T <: Number}(ϵ::T) = new{T}(ϵ)
end
Adagrad{T}(::Type{T}, ϵ = 1e-2) = Adagrad{T}(T(ϵ))
Adagrad() = Adagrad(Float64)

function init(updater::Adagrad, θ::AbstractVector)
    updater.G = zeros(θ)
    return
end

function update!(θ::AbstractVector, updater::Adagrad, ∇::AbstractVector, lr::Number)
    @inbounds for (i,j,k) in zip(eachindex(θ), eachindex(∇), eachindex(updater.G))
        updater.G[k] += ∇[j]^2
        θ[i] -= lr * ∇[j] / sqrt(updater.ϵ + updater.G[k])
    end
end

# -----------------------------------------------------------------------

"""
See: ADADELTA: An Adaptive Learning Rate Method (Zeiler 2012)

Relatively parameter-free... can probably avoid changing ε and ρ
"""
type Adadelta{T<:Number} <: ParamUpdater
    ϵ::T
    ρ::T
    dmean::Vector{T} # exponential average of squared param changes
    Gmean::Vector{T} # exponential average of squared gradients
    (::Type{Adadelta{T}}){T<:Number}(ϵ::T, ρ::T) = new{T}(ϵ, ρ)
end
Adadelta{T}(::Type{T}, ϵ = 1e-2, ρ = 0.97) = Adadelta{T}(T(ϵ), T(ρ))
Adadelta{T}(ϵ::T = 1e-2, ρ::T = 0.97) = Adadelta{T}(ϵ, ρ)

function init(updater::Adadelta, θ::AbstractVector)
    updater.dmean = zeros(θ)
    updater.Gmean = zeros(θ)
    return
end

function update!{T}(θ::AbstractVector, updater::Adadelta{T}, ∇::AbstractVector, lr::Number)
    ϵ = updater.ϵ
    ρ = updater.ρ
    dm = updater.dmean
    Gm = updater.Gmean
    @inbounds for i=1:length(θ)
        # average g²
        Gm[i] = ρ * Gm[i] + (one(T) - ρ) * ∇[i]^2

        # learning rate is function of prev avg dw² and current avg g²
        δ = lr * sqrt((dm[i] + ϵ) / (Gm[i] + ϵ)) * ∇[i]
        θ[i] -= δ

        # update avg dw²
        dm[i] = ρ * dm[i] + (one(T) - ρ) * δ^2
    end
end

# -----------------------------------------------------------------------

"""
see: ADAM: A method for Stochastic Optimization (Kingma and Ba 2015)

Tracks an exponential moving average of the first and second moments of the gradient,
adjusting for zero-bias.  The defaults are those suggested in the paper.

TODO: AdaMax is similar, using the p-norm as p -> ∞
"""
type Adam{T<:Number} <: ParamUpdater
    ϵ::T  # small number so we don't divide by 0
    ρ₁::T # decay for first moment (β₁ in the paper)
    ρ₂::T # decay for second moment (β₂ in the paper)
    m::Vector{T} # average first moment
    v::Vector{T} # average second moment
    ρ₁ᵗ::Vector{T} # β₁ᵗ from the paper... t-th power of β₁
    ρ₂ᵗ::Vector{T} # β₂ᵗ from the paper... t-th power of β₂
    (::Type{Adam{T}}){T<:Number}(ϵ::T, ρ₁::T, ρ₂::T) = new{T}(ϵ, ρ₁, ρ₂)
end
Adam{T}(::Type{T}, ϵ = T(1e-8), ρ₁ = T(0.9), ρ₂ = T(0.999)) = Adam{T}(T(ϵ), T(ρ₁), T(ρ₂))
Adam{T}(ϵ::T = 1e-8, ρ₁::T = 0.9, ρ₂::T = 0.999) = Adam{T}(ϵ, ρ₁, ρ₂)

function init(updater::Adam, θ::AbstractVector)
    updater.m = zeros(θ)
    updater.v = zeros(θ)
    updater.ρ₁ᵗ = ones(θ)
    updater.ρ₂ᵗ = ones(θ)
    return
end

function update!{T}(θ::AbstractVector, updater::Adam{T}, ∇::AbstractVector, lr::Number)
    ϵ = updater.ϵ
    ρ₁ = updater.ρ₁
    ρ₂ = updater.ρ₂
    m = updater.m
    v = updater.v
    ρ₁ᵗ = updater.ρ₁ᵗ
    ρ₂ᵗ = updater.ρ₂ᵗ
    @inbounds for i=1:length(θ)
        m[i] = ρ₁ * m[i] + (one(T) - ρ₁) * ∇[i]
        v[i] = ρ₂ * v[i] + (one(T) - ρ₂) * ∇[i]^2
        ρ₁ᵗ[i] *= ρ₁
        ρ₂ᵗ[i] *= ρ₂
        θ[i] -= lr * m[i] * sqrt((one(T) - ρ₂ᵗ[i]) / (v[i] + ϵ)) / (one(T) - ρ₁ᵗ[i])
    end
end

# -----------------------------------------------------------------------

"""
see: ADAM: A method for Stochastic Optimization (Kingma and Ba 2015)

AdaMax is similar to Adam, using the p-norm as p -> ∞
"""
type Adamax{T<:Number} <: ParamUpdater
    ρ₁::T # decay for first moment (β₁ in the paper)
    ρ₂::T # decay for second moment (β₂ in the paper)
    m::Vector{T} # average first moment
    u::Vector{T} # average second moment
    ρ₁ᵗ::Vector{T} # β₁ᵗ from the paper... t-th power of β₁
    (::Type{Adamax{T}}){T<:Number}(ρ₁::T, ρ₂::T) = new{T}(ρ₁, ρ₂)
end
Adamax{T}(::Type{T}, ρ₁ = T(0.9), ρ₂ = T(0.999)) = Adamax{T}(T(ρ₁), T(ρ₂))
Adamax{T}(ρ₁::T = 0.9, ρ₂::T = 0.999) = Adamax{T}(ρ₁, ρ₂)

function init(updater::Adamax, θ::AbstractVector)
    updater.m = zeros(θ)
    updater.u = zeros(θ)
    updater.ρ₁ᵗ = ones(θ)
    return
end

function update!{T}(θ::AbstractVector, updater::Adamax{T}, ∇::AbstractVector, lr::Number)
    ρ₁ = updater.ρ₁
    ρ₂ = updater.ρ₂
    m = updater.m
    u = updater.u
    ρ₁ᵗ = updater.ρ₁ᵗ
    @inbounds for i=1:length(θ)
        m[i] = ρ₁ * m[i] + (one(T) - ρ₁) * ∇[i]
        u[i] = max(ρ₂ * u[i], abs(∇[i]))
        ρ₁ᵗ[i] *= ρ₁
        θ[i] -= lr * m[i] / ((u[i] + T(1e-10)) * (one(T) - ρ₁ᵗ[i]))
    end
end

# -----------------------------------------------------------------------

# TODO: RMSProp: http://climin.readthedocs.io/en/latest/rmsprop.html#tieleman2012rmsprop

type RMSProp{T<:Number} <: ParamUpdater
    γ::T # exponential weight of ∇² avg
    g::Vector{T}  # the exponential mean
    (::Type{RMSProp{T}}){T<:Number}(γ::T) = new{T}(γ)
end
RMSProp{T}(::Type{T}, γ = T(0.95)) = RMSProp{T}(T(γ))
RMSProp{T}(γ::T) = RMSProp{T}(γ)
RMSProp() = RMSProp{Float64}(0.95)

function init(updater::RMSProp, θ::AbstractVector)
    updater.g = ones(θ)
    return
end

function update!{T}(θ::AbstractVector, updater::RMSProp{T}, ∇::AbstractVector, lr::Number)
    γ = updater.γ
    g = updater.g
    @inbounds for i=1:length(θ)
        g[i] = γ * g[i] + (one(T) - γ) * ∇[i]^2
        θ[i] -= lr * ∇[i] / sqrt(g[i] + 1e-10)
    end
end


# unit test the bGrad

include("../src/DMD_Util.jl");
include("../src/DMD_Type.jl");
include("../src/DMD_Func.jl");

# Generate DMD Synthetic Data
#------------------------------------------------------------------------------
# dimensions
m = 100;    # temporal dimension
n = 100;    # spatial dimension
k = 3;      # number of modes
T = Float64;
# generate data
X, t, at, Bt = simDMD(m, n, k, T; seed=123456);
# specify the loss
function lossFunc(r)
	return sum(abs2, r)
end
#
function lossGrad(r)
	conj!(r)
end
#
# Create object
#------------------------------------------------------------------------------
params = DMDParams(k, X, t, lossFunc, lossGrad);
#
# Initialize with true parameter
#------------------------------------------------------------------------------
copyto!(params.a, at);
copyto!(params.B, Bt);
#
# Evaluate bGrad
#------------------------------------------------------------------------------
update_P!(params);
id = rand(1:n);
function func(br, params, id)
	copyto!(params.br[id], br);
	return bFunc(params, id)
end
func(br) = func(br, params, id);
#
function grad(gbr, br, params, id)
	copyto!(params.br[id], br);
	bGrad(gbr, params, id);
end
grad(gbr, br) = grad(gbr, br, params, id);
#
br = randn(k<<1);
gbr = zeros(k<<1);
grad(gbr, br);
#
# finite difference method verification
ϵ = 1e-8;
br_fd = zeros(k<<1);
gbr_fd = zeros(k<<1);
#
for i in eachindex(gbr_fd)
	copyto!(br_fd, br);
	br_fd[i] += ϵ;
	gbr_fd[i] = (func(br_fd) - func(br))/ϵ;
end
#
err = sum(abs2, gbr_fd - gbr);
if err < 1e-6
	println("bGrad: OK");
else
	println("bGrad: Wrong, err: $err");
end


using Base.Test

using MarketData
using TimeSeries

using MarketTechnicals

@testset "Moving Averages" begin

@testset "Moving Averages on TimeArrays" begin
    @testset "sma" begin
        ta = sma(cl, 10)
        @test isapprox(ta.values[1]  , 98.782, atol=.01)  # TTR value 98.782
        @test isapprox(ta.values[2]  , 97.982, atol=.01)  # TTR value 97.982
        @test isapprox(ta.values[3]  , 98.388, atol=.01)  # TTR value 98.388
        @test isapprox(ta.values[490], 21.266, atol=.01)  # TTR value 21.266
        @test isapprox(ta.values[491], 21.417, atol=.01)  # TTR value 21.417
        @test ta.timestamp[491] == Date(2001,12,31)

        ta = sma(ohlc, 10)
        @test isapprox(ta.values[1,:]  , [100.692, 103.981, 96.001, 98.782] , atol=.01)
        @test isapprox(ta.values[2,:]  , [100.304, 103.331, 95.876, 97.982], atol=.01)
        @test isapprox(ta.values[3,:]  , [100.041, 103.144, 96.095,98.388], atol=.01)
        @test isapprox(ta.values[490,:], [21.081, 21.685, 20.797, 21.266], atol=.01)
        @test isapprox(ta.values[491,:], [21.259, 21.868, 20.971, 21.417], atol=.01)
        @test ta.timestamp[491] == Date(2001,12,31)
    end

    @testset "ema" begin
        ta = ema(cl, 10)
        @test isapprox(ta.values[1]  , 98.782, atol=.01)   # TTR value 98.78200
        @test isapprox(ta.values[2]  , 99.719, atol=.01)   # TTR value 99.71982
        @test isapprox(ta.values[3]  , 100.963, atol=.01)  # TTR value 100.96349
        @test isapprox(ta.values[490], 21.585, atol=.01)   # TTR value 21.58580
        @test isapprox(ta.values[491], 21.642, atol=.01)   # TTR value 21.64293
        @test ta.timestamp[491] == Date(2001,12,31)

        ta = ema(ohlc, 10)
        @test isapprox(ta.values[1,:]  , [100.692, 103.981, 96.001, 98.782], atol=.01)
        @test isapprox(ta.values[2,:]  , [100.748, 104.348, 96.808, 99.719], atol=.01)
        @test isapprox(ta.values[3,:]  , [101.634, 105.148, 98.003, 100.963], atol=.01)
        @test isapprox(ta.values[490,:], [21.369, 22.030, 21.125, 21.585], atol=.01)
        @test isapprox(ta.values[491,:], [21.576, 22.145, 21.253, 21.642], atol=.01)
        @test ta.timestamp[491] == Date(2001,12,31)
    end

    @testset "ema wilder true" begin
        ta = ema(cl, 10, wilder=true)
        @test isapprox(ta.values[1]  , 98.782, atol=.01)  # TTR value 98.7820
        @test isapprox(ta.values[2]  , 99.297, atol=.01)  # TTR value 99.2978
        @test isapprox(ta.values[3]  , 100.024, atol=.01)  # TTR value 100.0240
        @test isapprox(ta.values[490], 21.345, atol=.01)  # TTR value 21.34556
        @test isapprox(ta.values[491], 21.401, atol=.01)  # TTR value 21.40100
        @test ta.timestamp[491] == Date(2001,12,31)

        ta = ema(ohlc, 10, wilder=true)
        @test isapprox(ta.values[1,:]  , [100.692, 103.981, 96.001, 98.782], atol=.01)
        @test isapprox(ta.values[2,:]  , [100.723, 104.183, 96.444, 99.297], atol=.01)
        @test isapprox(ta.values[3,:]  , [101.213, 104.64, 97.138, 100.024], atol=.01)
        @test isapprox(ta.values[490,:], [21.184, 21.776, 20.847, 21.345], atol=.01)
        @test isapprox(ta.values[491,:], [21.317, 21.865, 20.945, 21.401], atol=.01)
        @test ta.timestamp[491] == Date(2001,12,31)
    end

    @testset "kama" begin
        ta = kama(cl)
        @test isapprox(ta.values[1], 98.9052, atol=.01)
        @test isapprox(ta.values[2], 99.0098, atol=.01)
        @test isapprox(ta.values[3], 99.4499, atol=.01)
        @test ta.timestamp[1] == Date(2000, 1, 18)
        @test ta.colnames     == ["kama"]

        ta = kama(ohlc)
        @test length(ta.colnames) == 4
        @test ta.timestamp[1]     == Date(2000, 1, 18)

        ta = kama(TimeArray(collect(Date(2011, 1, 1):Date(2011, 1, 20)), 1:20))
        @test ta.timestamp[end] == Date(2011, 1, 20)

        ta = kama(TimeArray(collect(Date(2011, 1, 1):Date(2011, 1, 20)), fill(42, 20)))
        @test ta.values == fill(42, length(ta))
    end

    @testset "env" begin
        ta = env(cl, 10)
        @test isapprox(ta.values[1,2]  , 108.66, atol=.01)
        @test isapprox(ta.values[2,2]  , 107.78, atol=.01)
        @test isapprox(ta.values[490,2]  , 23.3926, atol=.01)
        @test isapprox(ta.values[491,2]  , 23.5587, atol=.01)
        @test isapprox(ta.values[1,1]  , 88.9038, atol=.01)
        @test isapprox(ta.values[2,1]  , 88.1838, atol=.01)
        @test isapprox(ta.values[490,1]  , 19.1394, atol=.01)
        @test isapprox(ta.values[491,1]  , 19.2753, atol=.01)

        ta = env(ohlc, 10)
        @test isapprox(ta.values[1,5:end]  , [110.761, 114.379, 105.601, 108.66] , atol=.01)
        @test isapprox(ta.values[2,5:end]  , [110.334, 113.664, 105.464, 107.78], atol=.01)
        @test isapprox(ta.values[490,5:end]  , [23.1891, 23.8535, 22.8767, 23.3926], atol=.01)
        @test isapprox(ta.values[491,5:end]  , [23.3849, 24.0548, 23.0681, 23.5587], atol=.01)
        @test isapprox(ta.values[1,1:4]  , [90.6228, 93.5829, 86.4009, 88.9038] , atol=.01)
        @test isapprox(ta.values[2,1:4]  , [90.2736, 92.9979, 86.2884, 88.1838], atol=.01)
        @test isapprox(ta.values[490,1:4]  , [18.9729, 19.5165, 18.7173, 19.1394], atol=.01)
        @test isapprox(ta.values[491,1:4]  , [19.1331, 19.6812, 18.8739, 19.2753], atol=.01)
    end
end

@testset "Moving Averages on arrays" begin
    @testset "Array dispatch on sma" begin
        arr = sma(cl.values, 10)
        # same values as above TimeArray dispatch
        @test isapprox(arr[1]  , 98.782, atol=.01)
        @test isapprox(arr[2]  , 97.982, atol=.01)
        @test isapprox(arr[3]  , 98.388, atol=.01)
        @test isapprox(arr[490], 21.266, atol=.01)
        @test isapprox(arr[491], 21.417, atol=.01)

        arr = sma(ohlc.values, 10)
        @test isapprox(arr[1,:]  , [100.692, 103.981, 96.001, 98.782] , atol=.01)
        @test isapprox(arr[2,:]  , [100.304, 103.331, 95.876, 97.982], atol=.01)
        @test isapprox(arr[3,:]  , [100.041, 103.144, 96.095,98.388], atol=.01)
        @test isapprox(arr[490,:], [21.081, 21.685, 20.797, 21.266], atol=.01)
        @test isapprox(arr[491,:], [21.259, 21.868, 20.971, 21.417], atol=.01)
    end

    @testset "Array dispatch on ema" begin
        arr = ema(cl.values, 10)
        @test isapprox(arr[1]  , 98.782, atol=.01)
        @test isapprox(arr[2]  , 99.719, atol=.01)
        @test isapprox(arr[3]  , 100.963, atol=.01)
        @test isapprox(arr[490], 21.585, atol=.01)
        @test isapprox(arr[491], 21.642, atol=.01)

        arr = ema(ohlc.values, 10)
        @test isapprox(arr[1,:]  , [100.692, 103.981, 96.001, 98.782], atol=.01)
        @test isapprox(arr[2,:]  , [100.748, 104.348, 96.808, 99.719], atol=.01)
        @test isapprox(arr[3,:]  , [101.634, 105.148, 98.003, 100.963], atol=.01)
        @test isapprox(arr[490,:], [21.369, 22.030, 21.125, 21.585], atol=.01)
        @test isapprox(arr[491,:], [21.576, 22.145, 21.253, 21.642], atol=.01)
        @test ema(ohlc, 10).timestamp[491] == Date(2001,12,31)
    end

    @testset "Array dispatch on ema wilder true" begin
        arr = ema(cl.values, 10, wilder=true)
        @test isapprox(arr[1]  , 98.782, atol=.01)
        @test isapprox(arr[2]  , 99.297, atol=.01)
        @test isapprox(arr[3]  , 100.024, atol=.01)
        @test isapprox(arr[490], 21.345, atol=.01)
        @test isapprox(arr[491], 21.401, atol=.01)

        arr = ema(ohlc.values, 10, wilder=true)
        @test isapprox(arr[1,:]  , [100.692, 103.981, 96.001, 98.782], atol=.01)
        @test isapprox(arr[2,:]  , [100.723, 104.183, 96.444, 99.297], atol=.01)
        @test isapprox(arr[3,:]  , [101.213, 104.64, 97.138, 100.024], atol=.01)
        @test isapprox(arr[490,:], [21.184, 21.776, 20.847, 21.345], atol=.01)
        @test isapprox(arr[491,:], [21.317, 21.865, 20.945, 21.401], atol=.01)
    end

    @testset "Array dispatch on env" begin
        ta = env(cl.values, 10)
        @test isapprox(ta[1,:]  , [88.9038, 108.66], atol=.01)
        @test isapprox(ta[2,:]  , [88.1838, 107.78], atol=.01)
        @test isapprox(ta[490,:]  , [19.1394, 23.3926], atol=.01)
        @test isapprox(ta[491,:]  , [19.2753, 23.5587], atol=.01)

        ta = env(ohlc.values, 10)
        @test isapprox(ta[1,:]  , [90.6228, 93.5829, 86.4009, 88.9038, 110.761, 114.379, 105.601, 108.66] , atol=.01)
        @test isapprox(ta[2,:]  , [90.2736, 92.9979, 86.2884, 88.1838, 110.334, 113.664, 105.464, 107.78], atol=.01)
        @test isapprox(ta[490,:]  , [18.9729, 19.5165, 18.7173, 19.1394, 23.1891, 23.8535, 22.8767, 23.3926], atol=.01)
        @test isapprox(ta[491,:]  , [19.1331, 19.6812, 18.8739, 19.2753, 23.3849, 24.0548, 23.0681, 23.5587], atol=.01)
    end
end

end  # @testset "Moving Averages"


module DynamicalSystems

# Creates the functions for the harmonic oscillator with a sinusoidal driving
# force.
#
# x'' + beta x' + omega^2 x = gamma sin(omega t)
#
# Args:
#   omega0: The frequency term.
#   beta: The damping term (default: 0.0).
#   gamma: The driving force amplitude (default: 0.0).
#   omega: The driving force frequency (default: 1.0).
#
# Returns: An array of two functions defining the harmonic oscillator system.
function harmonicOscillator(omega0::AbstractFloat;
    beta::AbstractFloat=0.0, gamma::AbstractFloat=0.0,
    omega::AbstractFloat=1.0)

    return [(t,x) -> x[2],
            (t,x) -> (gamma*sin(omega*t) -
                 beta*x[2] -
                 (omega0^2)*x[1])];

end #Close simpleHarmonicOscillator.

# Creates the functions for a Duffing oscillator.
#
# x'' + delta x' + alpha x + beta x^3 = gamma sin(omega t)
#
# Args:
#   delta: The damping term.
#   gamma: The driving amplitude.
#   alpha: The stiffness (default: 1.0).
#   beta: The restoring force non-linearity (default: -1.0).
#   omega: The driving force frequency (default: 1.0).
#
# Returns: An array of two functions defining the Duffing oscillator system.
function duffingOscillator(delta::AbstractFloat, gamma::AbstractFloat;
    alpha::AbstractFloat=-1.0, beta::AbstractFloat=1.0,
    omega::AbstractFloat=1.0)

    return [(t,x) -> x[2],
     (t,x) -> (gamma*sin(omega*t) -
              delta*x[2] -
              alpha*x[1] -
              beta*(x[1]^3))];

end #Close duffingOscillator.

# Creates the functions for a Lorenz attractor.
#
# x' = sigma (y - x)
# y' = x (rho - z) - y
# z' = x y - beta z
#
# Args:
#   sigma: The sigma parameter of the system.
#   rho: The rho parameter of the system.
#   beta: The beta parameter of the system.
#
# Returns: An array of three function defining the Lorenz attractor system.
function lorenzAttractor(sigma::AbstractFloat, rho::AbstractFloat,
    beta::AbstractFloat)

    return [(t,x) -> sigma*(x[2] - x[1]),
            (t,x) -> x[1]*(rho - x[3]) - x[2],
            (t,x) -> x[1]*x[2] - beta*x[3]];
end

export harmonicOscillator, duffingOscillator, lorenzAttractor

end # module


# https://adventofcode.com/2020/day/18

function parse_expression(chars::Vector{Char})
    expression = []
    while !isempty(chars)
        ch = popfirst!(chars)

        if ch == ' '
            continue
        elseif isdigit(ch)
            digit = parse(Int, ch)
            if !isempty(expression) && isa(expression[end], Number)
                expression[end] = 10 * expression[end] + digit
            else
                push!(expression, digit)
            end
        elseif ch == '*'
            push!(expression, :*)
        elseif ch == '+'
            push!(expression, :+)
        elseif ch == '('
            partial, chars = parse_expression(chars)
            push!(expression, partial)
        elseif ch == ')'
            return expression, chars
        else
            error("invalid symbol: '$(ch)'")
        end
    end
    return expression, chars
end

function parse_expression(string::AbstractString)
    expression, _ = parse_expression(collect(string))
    return expression
end

@assert parse_expression("1 + 2") == [1, :+, 2]
@assert parse_expression("1 + (2 * 3)") == [1, :+, [2, :*, 3]]

function evaluate(expression)
    result = 0
    op = (+)
    for x in expression
        if isa(x, Symbol)
            op = (x == :+) ? (+) : (*)
            continue
        elseif isa(x, Vector)
            x = evaluate(x)
        end
        result = op(result, x)
    end
    return result
end

parse_eval(expr) = evaluate(parse_expression(expr))

@assert parse_eval("1 + (2 * 3) + (4 * (5 + 6))") == 51
@assert parse_eval("2 * 3 + (4 * 5)") == 26
@assert parse_eval("5 + (8 * 3 + 9 + 3 * 4 * 3)") == 437
@assert parse_eval("5 * 9 * (7 * 3 * 3 + 9 * 3 + (8 + 6 * 4))") == 12240
@assert parse_eval("((2 + 4 * 9) * (6 + 9 * 8 + 6) + 6) + 2 + 4 * 2") == 13632

function part1(input)
    return reduce(+, map(parse_eval, split(input, '\n', keepempty=false)))
end

function bracket(expr)
    new_expr = []
    i = 1
    while i <= length(expr)
        x = expr[i]
        if isa(x, Vector)
            x = bracket(x)
        end

        if x == :+
            y = expr[i + 1]
            if isa(y, Vector)
                y = bracket(y)
            end

            new_expr[end] = [new_expr[end], :+, y]
            i += 2
        else
            push!(new_expr, x)
            i += 1
        end
    end
    return length(new_expr) == 1 ? first(new_expr) : new_expr
end

@assert bracket([2, :*, 3, :+, 4]) == [2, :*, [3, :+, 4]]
@assert bracket([2, :*, [3, :*, 4], :+, 5]) == [2, :*, [[3, :*, 4], :+, 5]]
@assert bracket([2, :+, 3, :+, 4, :*, 5]) == [[[2, :+, 3], :+, 4], :*, 5]
@assert bracket([5, :+, [8, :*, 3, :+, 9, :+, 3, :*, 4, :*, 3]]) == [5, :+, [8, :*, [[3, :+, 9], :+, 3], :*, 4, :*, 3]]

parse_bracket_eval(expr) = evaluate(bracket(parse_expression(expr)))

@assert parse_bracket_eval("1 + (2 * 3) + (4 * (5 + 6))") == 51
@assert parse_bracket_eval("2 * 3 + (4 * 5)") == 46
@assert parse_bracket_eval("5 + (8 * 3 + 9 + 3 * 4 * 3)") == 1445
@assert parse_bracket_eval("5 * 9 * (7 * 3 * 3 + 9 * 3 + (8 + 6 * 4))") == 669060
@assert parse_bracket_eval("((2 + 4 * 9) * (6 + 9 * 8 + 6) + 6) + 2 + 4 * 2") == 23340

function part2(input)
    return reduce(+, map(parse_bracket_eval, split(input, '\n', keepempty=false)))
end

test = read("data/day-18.txt", String)
println("Part 1: $(result1 = part1(test))")
println("Part 2: $(result2 = part2(test))")

@assert result1 == 1408133923393
@assert result2 == 314455761823725


using Test
using Petri
import Petri: fluxes, odefunc
using ModelingToolkit
using LabelledArrays
using OrdinaryDiffEq
import OrdinaryDiffEq: solve

@testset "Generation of ODE formulas" begin
    @variables S,E,I,R, β,γ,μ
    sir = Petri.Model([S,I,R],[(S+I, 2I), (I,R)])
    seir = Petri.Model([S,E,I,R],[(S+I, E+I), (E,I), (I,R)])
    seirs = Petri.Model([S,E,I,R],[(S+I, E+I), (E,I), (I,R), (R,S)])

    sirf = fluxes(sir)
    @test length(sirf) == 3

    seirf = fluxes(seir)
    @test length(seirf) == 4

    seirsf = fluxes(seirs)
    @test length(seirsf) == 4
end

N(x) = sum(x)

@testset "Generation of ODE solutions" begin
    @variables S,E,I,R, β,γ,μ
    sir = Petri.Model([S,I,R],[(S+I, 2I), (I,R)])
    seir = Petri.Model([S,E,I,R],[(S+I, E+I), (E,I), (I,R)])
    seirs = Petri.Model([S,E,I,R],[(S+I, E+I), (E,I), (I,R), (R,S)])
    fex = odefunc(sir,:sir)
    f = eval(fex)
    u0 = @LArray [100.0, 1, 0] (:S, :I, :R)
    dusir = @LArray [0.0, 0.0, 0.0] (:S, :I, :R)
    p = @LArray [0.35, 0.05] (:μ, :β)
    f(dusir, u0, p, 0.0)
    @show dusir

    @testset "SIR Solving" begin
        u0 = @LArray [100.0, 1, 0] (:S, :I, :R)
        p = @LArray [0.35, 0.05] (:μ, :β)
        fex = odefunc(sir, :sir)
        f = eval(fex)
        prob = ODEProblem(f,u0,(0.0,365.0),p)
        sol = solve(prob,Tsit5())
        @test sol[end].S < 1
        @test sol[end].I < 1
        @test sol[end].R > 99
    end

    @testset "SEIR Solving" begin
        u0 = @LArray [100.0, 1, 0, 0] (:S, :E, :I, :R)
        p = @LArray [0.35, 0.05, 0.05] (:μ, :β, :γ)
        fex = odefunc(seir, :seir)
        f = eval(fex)
        prob = ODEProblem(f,u0,(0.0,365.0),p)
        sol = solve(prob,Tsit5())
        @test sol[end].S < 1
        @test sol[end].E < 1
        @test sol[end].I < 1
        @test sol[end].R > 99
    end

    @testset "SEIRS Solving" begin
        u0 = @LArray [100.0, 1, 0, 0] (:S, :E, :I, :R)
        p = @LArray [0.35, 0.05, 0.07, 0.3] (:μ, :β, :γ, :η)
        fex = odefunc(seirs, :seirs)
        f = eval(fex)
        prob = ODEProblem(f,u0,(0.0,365.0),p)
        sol = solve(prob,Tsit5())
        @test sol[end].S > 5
        @test sol[end].E > 5
        @test sol[end].I > 5
        @test sol[end].R > 5
        @test sol[end].S < 95
        @test sol[end].E < 95
        @test sol[end].I < 95
        @test sol[end].R < 95
    end
end


using Empirikos
using Test
using Hypatia
using Random
Zs_raw = NormalSample.(randn(10), rand(10))
Zs_compound = compound(Zs_raw)

Zs_standard = StandardNormalSample.(randn(100))
Zs_standard_compound = compound(Zs_standard)

@test Zs_standard_compound[1].vec == [StandardNormalSample()]

@test cdf.(Normal(0,2), Zs_standard) == cdf.(Normal(0,2), Zs_standard)
@test pdf.(Normal(0,2), Zs_standard) == pdf.(Normal(0,2), Zs_standard)

dkw_loc_standard_compound = fit(DvoretzkyKieferWolfowitz(α=0.05*exp(1)), Zs_standard_compound)
dkw_loc_standard = fit(DvoretzkyKieferWolfowitz(α=0.05), Zs_standard)

@test dkw_loc_standard.homoskedastic
@test !dkw_loc_standard_compound.homoskedastic

@test dkw_loc_standard.band ≈ dkw_loc_standard_compound.band

target = PosteriorMean(NormalSample(2.0, 2.0))

gcal = DiscretePriorClass(-2:0.2:2)
floc_method_standard = FLocalizationInterval(flocalization=DvoretzkyKieferWolfowitz(0.05),
                             convexclass=gcal,
                             solver=Hypatia.Optimizer)
floc_method_compound = FLocalizationInterval(flocalization=DvoretzkyKieferWolfowitz(0.05*exp(1)),
                             convexclass=gcal,
                             solver=Hypatia.Optimizer)

ci1 = confint(floc_method_standard, target, Zs_standard)
ci2 = confint(floc_method_compound, target, Zs_standard_compound)
@test ci1.lower ≈ ci2.lower
@test ci1.upper ≈ ci2.upper
@test ci1.α ≈ 0.05
@test ci2.α ≈ 0.05*exp(1)

# Same with actually heteroskedastic samples

Random.seed!(1)
σs = rand(1:10,1000)
μs = randn(1000)
Zs = NormalSample.(μs, σs)
Zs_compound = compound(Zs)
@test length(Zs_compound[1].vec) == 10

dkw_loc_not_compound = fit(DvoretzkyKieferWolfowitz(α=0.05), Zs)
dkw_loc_compound = fit(DvoretzkyKieferWolfowitz(α=0.05), Zs_compound)

@test !dkw_loc_not_compound.homoskedastic
@test !dkw_loc_compound.homoskedastic

@test dkw_loc_not_compound.band ≈ dkw_loc_compound.band
@test length(dkw_loc_not_compound.summary) == length(dkw_loc_compound.summary)

Random.seed!(1)
Zs_binom_heterosk = BinomialSample.( sample(1:4, 100), sample(20:22,100))
Zs_binom_summary = summarize(Zs_binom_heterosk)
@test nobs(Zs_binom_summary) == 100
@test length(keys(Zs_binom_summary)) == 3*4
comp_binom = compound(Zs_binom_heterosk)
Zs_binom_comp_summary = summarize(comp_binom)
@test nobs(Zs_binom_comp_summary) == 100
@test length(keys(Zs_binom_comp_summary)) == 4

Zs_binom_summary_comp = compound(Zs_binom_summary)
@test nobs(Zs_binom_summary_comp) == 100

Empirikos.multiplicity(Zs_binom_comp_summary) == Empirikos.multiplicity(Zs_binom_summary_comp)


function bool3(model::CAGD.Model)
    arranged_model = CAGD.spatial_arrangement(model, outerCell = true)
    innerpoints, intersectedfaces = CAGD.internalpoints(arranged_model)
    # number of input complexes
    nInC = size(model, 3, 1)
    infaces = [CAGD.getModelLoCell(model, 3, i) for i = 1 : nInC]
    # Determine in which initial element was h
    span(h) = [j for j = 1 : nInC if h in infaces[j]]
	containmenttest = CAGD.testinternalpoint(model)
	# currently copCF contains the outercell in first column ...
	# TODO remove first row and column, in case (look at file src/)
	boolmatrix = BitArray(undef, length(innerpoints)+1, nInC+1)
	boolmatrix[1,1] = 1
	for (k, point) in enumerate(innerpoints) # k runs on columns
		cells = containmenttest(point) # contents of columns
		#println(k," ",faces)
		rows = [span(h) for h in cells]
		for l in cat(rows)
			boolmatrix[k+1, l+1] = 1
		end
	end
	return arranged_model, boolmatrix
end

function internalpoints(model::CAGD.Model)
    return Lar.internalpoints(model.G, model.T[1], model.T[2], model.T[3][2:end, :])
end

"""
	testinternalpoint(V::Lar.Points, EV::Lar.Cells, FV::Lar.Cells)

"""
function testinternalpoint(model)
	copEV = abs.(model.T[1])
    copFE = abs.(model.T[2])
    V = model.G
    EV = Lar.cop2lar(model.T[1])
    FV = Lar.cop2lar(map(x -> floor(Int8, x / 2), copFE * copEV))

	I,J,Val = findnz(copFE)
	Val = convert(Array{Int8,1},Val)
	copFE = sparse(I,J,Val)
	function testinternalpoint0(testpoint)
		intersectedfaces = Int64[]
		# spatial index for possible intersections with ray
		faces = Lar.spaceindex(testpoint)((V, FV))
		depot = []
		# face in faces :  indices of faces of possible intersection with ray
		for face in faces
			value = Lar.rayintersection(testpoint)(V, FV, face)
			if typeof(value) == Array{Float64,1} push!(depot, (face,value)) end
		end
		# actual containment test of ray point in faces within depot
		for (face,point3d) in depot
			vs, edges, point2d = Lar.planemap(V, copEV, copFE, face)(point3d)
			classify = Lar.pointInPolygonClassification(vs,edges)
			inOut = classify(point2d)
			if inOut != "p_out"
				push!(intersectedfaces,face)
			end
		end
		return intersectedfaces
	end
	return testinternalpoint0
end


# using Base.Test
# using MIToS.Information
# using MIToS.Utils
# using MIToS.MSA
# using PairwiseListMatrices

const Gaoetal2011 = joinpath(pwd(), "data", "Gaoetal2011.fasta")

gao11_buslje09(measure) = joinpath(pwd(), "data", string("data_Gaoetal2011_soft_Busljeetal2009_measure_", measure, ".txt"))

## Column numbers for the output of Buslje et. al. 2009
const SCORE = 9
const ZSCORE = 12

const MIToS_SCORE = 2
const MIToS_ZSCORE = 1

print("""

Test APC!
=========
""")

let MI = [ 0 2 4
           2 0 6
           4 6 0 ]

  mean_col = Information._mean_column(MI)
  mean_tot = Information._mean_total(MI)
  @test mean_col == [3., 4., 5.]
  @test mean_tot == 4.

  MIp = APC!(convert(Matrix{Float64}, MI))
  @test_approx_eq MIp [  NaN -1.0  0.25
                        -1.0  NaN  1.00
                        0.25 1.00   NaN ]
end

let MI = PairwiseListMatrix([2, 4, 6])

  mean_col = mean_nodiag(MI, 1)
  mean_tot = mean_nodiag(MI)
  @test mean_col == [3. 4. 5.]
  @test mean_tot == 4.

  MIp = APC!(convert(PairwiseListMatrix{Float64, false}, MI))
  @test_approx_eq MIp [  NaN -1.0  0.25
                        -1.0  NaN  1.00
                        0.25 1.00   NaN ]
end

let MI = PairwiseListMatrix([0, 2, 4, 0, 6, 0], true)

  mean_col = mean_nodiag(MI, 1)
  mean_tot = mean_nodiag(MI)
  @test mean_col == [3. 4. 5.]
  @test mean_tot == 4.

  MIp = APC!(convert(PairwiseListMatrix{Float64, true}, MI))
  @test_approx_eq MIp [  NaN -1.0  0.25
                        -1.0  NaN  1.00
                        0.25 1.00   NaN ]
end

print("""

Simple examples
===============
""")

let aln = Residue[ 'A' 'A'
                   'A' 'R' ]

  print("""
  MI
  """)
  # Fill the Pij matrix of the example
  Pij = zeros(Float64, 20, 20);
  N = 2 # There are 2 sequences
  Pij[1,1] = (1/N) # A A
  Pij[1,2] = (1/N) # A R
  @test_approx_eq sum(Pij) 1.0 # Is the matrix correct?
  # Fill the marginals
  Pi = squeeze(Base.sum(Pij,2),2)
  @test Pi[1] == 1.0 # Is always A
  Pj = squeeze(Base.sum(Pij,1),1)
  @test Pj[1] == 0.5 # A
  @test Pj[2] == 0.5 # R
  # Start to sum with 0.0
  total = 0.0
  for i in 1:20, j in 1:20
    if Pij[i,j] != 0.0 && Pi[i] != 0.0 && Pj[j] != 0.0
      total += (Pij[i,j] * log(Pij[i,j]/(Pi[i]*Pj[j]))) # 0.5 * log(0.5/(0.5 * 1.0)) == 0.0
    end
  end
  # Compare with MIToS result
  mi = estimateincolumns(aln, Float64, ResidueProbability{Float64, 2, false}, MutualInformation{Float64}())
  @test mi[1,2] == total

  print("""

  MI: Using pseudocount (0.05)
  ----------------------------
  """)
  Pij = zeros(Float64, 20, 20);
  N = (400 * 0.05) + 2 # 0.05 is the pseudocount and there are 2 sequences
  fill!(Pij, 0.05/N);
  Pij[1,1] =(1.05/N) # A A
  Pij[1,2] =(1.05/N) # A R
  @test_approx_eq sum(Pij) 1.0
  Pi = squeeze(Base.sum(Pij,2),2)
  Pj = squeeze(Base.sum(Pij,1),1)
  total = 0.0
  for i in 1:20, j in 1:20
    total += (Pij[i,j] * log(Pij[i,j]/(Pi[i]*Pj[j])))
  end
  mi = estimateincolumns(aln, Float64, ResidueProbability{Float64, 2, false}, MutualInformation{Float64}(), AdditiveSmoothing(0.05))
  @test_approx_eq mi[1,2] total

  print("""

  Test APC!
  """)
  let zerodiagonal = Float64[ 0.0 total
                             total 0.0 ]

    # MI.. == MI.j == MIi. == total
    # APC = (total * total) / total == total
    # MI - APC == total - total == 0.0
    APC!(zerodiagonal)
    @test_approx_eq zerodiagonal[1,2] 0.0
  end

  APC!(mi)
  @test isnan(mi[1,1])
  @test_approx_eq_eps mi[1,2] 0.0 1e-16

  print("""

  Z-score
  """)
  # 2 Possibilities:  A A   A A
  #                   A R   R A
  # Is almost the same MI, Z score should be 0.0
  results = buslje09(aln, lambda=0.05, clustering=false, apc=false)
  @test_approx_eq results[MIToS_ZSCORE][1,2] 0.0

  @test aln == Residue[ 'A' 'A'
                        'A' 'R' ]

end

let aln = Residue[ 'R' 'A'
                   'A' 'R' ]

  print("""
  MI
  """)
  Pij = zeros(Float64, 20, 20);
  N = 2 # There are 2 sequences
  Pij[2,1] = (1/N) # R A
  Pij[1,2] = (1/N) # A R
  @test_approx_eq sum(Pij) 1.0
  Pi = squeeze(Base.sum(Pij,2),2)
  @test Pi[2] == 0.5 # R
  @test Pi[1] == 0.5 # A
  Pj = squeeze(Base.sum(Pij,1),1)
  @test Pj[1] == 0.5 # A
  @test Pj[2] == 0.5 # R
  total = 0.0
  for i in 1:20, j in 1:20
    if Pij[i,j] != 0.0 && Pi[i] != 0.0 && Pj[j] != 0.0
      total += (Pij[i,j] * log(Pij[i,j]/(Pi[i]*Pj[j])))
    end
  end
  @test total == 0.5 * log(2) + 0.5 * log(2) # 0.5/(0.5*0.5) == 2
  mi = estimateincolumns(aln, Float64, ResidueProbability{Float64, 2, false}, MutualInformation{Float64}())
  @test mi[1,2] == total

  print("""

  MI: Using pseudocount (0.05)
  ----------------------------
  """)
  Pij = zeros(Float64, 20, 20);
  N = (400 * 0.05) + 2 # 0.05 is the pseudocount and there are 2 sequences
  fill!(Pij, 0.05/N);
  Pij[2,1] =(1.05/N) # R A
  Pij[1,2] =(1.05/N) # A R
  @test_approx_eq sum(Pij) 1.0
  Pi = squeeze(Base.sum(Pij,2),2)
  Pj = squeeze(Base.sum(Pij,1),1)
  total = 0.0
  for i in 1:20, j in 1:20
    total += (Pij[i,j] * log(Pij[i,j]/(Pi[i]*Pj[j])))
  end
  mi = estimateincolumns(aln, Float64, ResidueProbability{Float64, 2, false}, MutualInformation{Float64}(), AdditiveSmoothing(0.05))
  @test_approx_eq mi[1,2] total

  print("""

  Test APC!
  """)
  let zerodiagonal = Float64[ 0.0 total
                             total 0.0 ]

    # MI.. == MI.j == MIi. == total
    # APC = (total * total) / total == total
    # MI - APC == total - total == 0.0
    APC!(zerodiagonal)
    @test_approx_eq zerodiagonal[1,2] 0.0
  end

  APC!(mi)
  @test isnan(mi[1,1])
  @test_approx_eq mi[1,2] 0.0

  print("""

  Z-score
  """)
  # 4 Possibilities:  R A   R A   A R   A R
  #                   A R   R A   A R   R A
  mi = estimateincolumns(aln, Float64, ResidueProbability{Float64, 2, false}, MutualInformation{Float64}(), AdditiveSmoothing(0.05))
  other_posib = Residue[ 'A' 'R'
                         'A' 'R' ]
  other_mi = estimateincolumns(other_posib, Float64, ResidueProbability{Float64, 2, false}, MutualInformation{Float64}(), AdditiveSmoothing(0.05))
  # The mean should be:
  r_mean = 0.5 * ( mi[1,2] + other_mi[1,2] )
  # And the std should be similar to:
  r_std = 0.5 * ( sqrt((mi[1,2]-r_mean)^2) + sqrt((other_mi[1,2]-r_mean)^2) )

  results = buslje09(aln, lambda=0.05, clustering=false, apc=false, samples=100)
  @test_approx_eq_eps results[MIToS_ZSCORE][1,2] ((mi[1,2] - r_mean) / r_std) 0.55

  results = buslje09(aln, lambda=0.05, clustering=false, apc=false, samples=1000)
  @test_approx_eq_eps results[MIToS_ZSCORE][1,2] ((mi[1,2] - r_mean) / r_std) 0.15

  results = buslje09(aln, lambda=0.05, clustering=false, apc=false, samples=10000)
  @test_approx_eq_eps results[MIToS_ZSCORE][1,2] ((mi[1,2] - r_mean) / r_std) 0.055

  @test aln == Residue[ 'R' 'A'
                        'A' 'R' ]

end

print("""

Results from Buslje et. al 2009
===============================
""")

println("""

MI
""")

let data = readdlm(joinpath(pwd(), "data", "data_simple_soft_Busljeetal2009_measure_MI.txt")),
    results = buslje09(joinpath(pwd(), "data", "simple.fasta"), FASTA, lambda=0.0, clustering=false, apc=false)

  @test_approx_eq_eps Float64(data[1, SCORE]) results[MIToS_SCORE][1,2] 1e-6
  @test_approx_eq_eps Float64(data[1, ZSCORE]) results[MIToS_ZSCORE][1,2]  1.5
end

let data = readdlm(gao11_buslje09("MI")),
    results = buslje09(Gaoetal2011, FASTA, lambda=0.0, clustering=false, apc=false)

  @test_approx_eq_eps convert(Vector{Float64}, data[:, SCORE]) matrix2list(results[MIToS_SCORE]) 1e-6
  @test_approx_eq_eps convert(Vector{Float64}, data[:, ZSCORE]) matrix2list(results[MIToS_ZSCORE]) 1.5

  println("Pearson for Z-score: ", cor(convert(Vector{Float64}, data[:, ZSCORE]), matrix2list(results[MIToS_ZSCORE])))

  print("""

  cMI
  """)
  @test cumulative(results[1], -Inf) == Base.vec(sum(results[1], 1))
  #                                                      1.38629 = 0.693147 + 0.693147
  @test all( ( cumulative(results[2], 0.5) .- [0.0, 0.0, 1.38629, 1.38629, 1.38629, 0.0] ) .< 0.00001 )
end

@test_approx_eq_eps buslje09(Gaoetal2011, FASTA, lambda=0.05, clustering=false, apc=false)[MIToS_SCORE][1,2] 0.33051006116310444 1e-14

println("""

MI + clustering
""")

let data = readdlm(gao11_buslje09("MI_clustering")),
    results = buslje09(Gaoetal2011, FASTA, lambda=0.0, clustering=true, apc=false)

  @test_approx_eq_eps convert(Vector{Float64}, data[:, SCORE]) matrix2list(results[MIToS_SCORE]) 1e-6
  @test_approx_eq_eps convert(Vector{Float64}, data[:, ZSCORE]) matrix2list(results[MIToS_ZSCORE]) 1.5

  println("Pearson for Z-score: ", cor(convert(Vector{Float64}, data[:, ZSCORE]), matrix2list(results[MIToS_ZSCORE])))
end

println("""

MIp
""")

let data = readdlm(gao11_buslje09("MI_APC")); results = buslje09(Gaoetal2011, FASTA, lambda=0.0, clustering=false, apc=true)

  @test_approx_eq_eps convert(Vector{Float64}, data[:, SCORE]) matrix2list(results[MIToS_SCORE]) 1e-6
  @test_approx_eq_eps convert(Vector{Float64}, data[:, ZSCORE]) matrix2list(results[MIToS_ZSCORE]) 1.5

  @test_approx_eq_eps results[MIToS_SCORE][5,6] 0.018484 0.000001

  println("Pearson for MIp: ", cor(convert(Vector{Float64}, data[:, SCORE]), matrix2list(results[MIToS_SCORE])))
  println("Pearson for Z-score: ", cor(convert(Vector{Float64}, data[:, ZSCORE]), matrix2list(results[MIToS_ZSCORE])))
end

println("""

MIp + clustering
""")

let data = readdlm(gao11_buslje09("MI_APC_clustering")); results = buslje09(Gaoetal2011, FASTA, lambda=0.0, clustering=true, apc=true)

  @test_approx_eq_eps convert(Vector{Float64}, data[:, SCORE]) matrix2list(results[MIToS_SCORE]) 1e-6
  @test_approx_eq_eps convert(Vector{Float64}, data[:, ZSCORE]) matrix2list(results[MIToS_ZSCORE]) 1.5

  @test_approx_eq_eps results[MIToS_SCORE][5,6] 0.018484 0.000001

  println("Pearson for MIp: ", cor(convert(Vector{Float64}, data[:, SCORE]), matrix2list(results[MIToS_SCORE])))
  println("Pearson for Z-score: ", cor(convert(Vector{Float64}, data[:, ZSCORE]), matrix2list(results[MIToS_ZSCORE])))
end

# TO DO: Test labels!

print("""

Test for BLMI
=============
""")

let file = joinpath(pwd(), "data", "simple.fasta"),
    busl = buslje09(file, FASTA),
    blmi = BLMI(file, FASTA)

  @test_approx_eq busl[1] blmi[1]
  @test_approx_eq busl[2] blmi[2]

end

let msa = read(Gaoetal2011, FASTA),
    busl = buslje09(Gaoetal2011, FASTA, lambda=0.0, samples=0),
    blmi = BLMI(msa, lambda=0.0, beta=0.0, samples=5) # BLMI should be equal to Buslje09 if beta is zero

  @test_approx_eq busl[2] blmi[2] # MIapc

  @test msa == read(Gaoetal2011, FASTA)

end

let busl = buslje09(Gaoetal2011, FASTA, lambda=0.5, samples=0),
    blmi = BLMI(Gaoetal2011, FASTA, lambda=0.5, beta=0.0, samples=0) # BLMI should be equal to Buslje09 if beta is zero

  @test_approx_eq busl[2] blmi[2] # MIapc
end

print("""

Test for Pairwise Gap Percentage
================================
""")

let file = joinpath(pwd(), "data", "simple.fasta"),
    mat = [ 0. 0.
            0. 0. ]

  (gu, gi) = pairwisegapfraction(file, FASTA)

  @test gu == mat
  @test gi == mat
end

let file = joinpath(pwd(), "data", "gaps.txt")

  gu, gi = pairwisegapfraction(file, Raw)
  cl = hobohmI(read(file, Raw), 62)
  ncl = nclusters(cl)

  @test_approx_eq gu[1, 1] 0.0
  @test_approx_eq gi[1, 1] 0.0

  @test_approx_eq gu[1, 2] 100.0 * getweight(cl, 10)/ncl
  @test_approx_eq gi[1, 2] 0.0

  @test_approx_eq gu[10, 9] 100.0 * (ncl - getweight(cl, 1))/ncl
  @test_approx_eq gi[10, 9] 100.0 * (ncl - getweight(cl, 1) - getweight(cl, 2))/ncl

  @test_approx_eq gu[10, 10] 100.0 * (ncl - getweight(cl, 1))/ncl
  @test_approx_eq gu[10, 10] 100.0 * (ncl - getweight(cl, 1))/ncl
end


export Curve

"""
`Curve` Represents a domain defined by the image of a Fun
"""

immutable IntervalCurve{S<:Space,T} <: IntervalDomain{T}
    curve::Fun{S,T}
end

immutable PeriodicCurve{S<:Space,T} <: PeriodicDomain{T}
    curve::Fun{S,T}
end

typealias Curve{S,T} Union{IntervalCurve{S,T},PeriodicCurve{S,T}}

==(a::Curve,b::Curve)=a.curve==b.curve

for TYP in (:IntervalCurve,:PeriodicCurve)
    @eval points(c::$TYP,n::Integer)=c.curve(points(domain(c.curve),n))
end

checkpoints(d::Curve) = fromcanonical(d,checkpoints(domain(d.curve)))

for op in (:(Base.first),:(Base.last),:(Base.rand))
    @eval $op(c::Curve)=c.curve($op(domain(c.curve)))
end

canonicaldomain(c::Curve)=domain(c.curve)

fromcanonical{S<:Space,T<:Number}(d::Curve{S,T},v::AbstractArray)=eltype(d)[fromcanonical(d,vk) for vk in v]
fromcanonical{S<:Space,T<:Number}(c::Curve{S,T},x) = c.curve(x)
function tocanonical(c::Curve,x)
    rts=roots(c.curve-x)
    @assert length(rts)==1
    first(rts)
end

fromcanonicalD(c::Curve,x)=differentiate(c.curve)(x)

Base.in(x,d::Curve)=in(tocanonical(d,x),canonicaldomain(d))

Base.reverse(d::Curve)=Curve(reverseorientation(d.curve))

isambiguous(d::Curve)=ncoefficients(d.curve)==0 && isambiguous(domain(d.curve))
Base.convert{S,T}(::Type{IntervalCurve{S,T}},::AnyDomain)=Fun([NaN],S(AnyDomain()))
Base.convert{S,T}(::Type{PeriodicCurve{S,T}},::AnyDomain)=Fun([NaN],S(AnyDomain()))


# A./*B

FAhadamarddivB(A,B)=(A./B,nothing)

function FAhadamarddivB_inplace(value,aux,A,B)
    copy!(value,A./B)
end

function DAhadamarddivB(derivativeIDX,f_c,faux_c,grad_c,grad_n,A,B)
    if derivativeIDX==1
        axpy!(1.0,grad_c./B,grad_n) # should really use BLAS here
    elseif derivativeIDX==2
        axpy!(-1.0,grad_c.*A./(B.*B),grad_n)
    end
end

if PROC=="GPU"
    function FAhadamarddivB(A::CudaArray,B::CudaArray)
        tmp=CudaArray(Float64,size(A))
        vdiv!(1.0,A,B,tmp)
        return (tmp,nothing)
    end

    function FAhadamarddivB_inplace(value::CudaArray,aux,A::CudaArray,B::CudaArray)
        vdiv!(1.0,A,B,value)
    end

    function DAhadamarddivB(derivativeIDX,f_c,faux_c,grad_c,grad_n,A::CudaArray,B::CudaArray)
    if derivativeIDX==1
        vdivupdate!(1.0,grad_c,B,grad_n)
    elseif derivativeIDX==2
        vAoverBupdate!(1.0,grad_c,A,B,grad_n)
    end
end

end

Derivative[FAhadamarddivB]=DAhadamarddivB
Inplace[FAhadamarddivB]=FAhadamarddivB_inplace

import Base. ./
./(A::ADnode,B::ADnode)=ADnode(FAhadamarddivB,[A B])
export ./


using LinearAlgebra
using JuMP

function test(n::Int, verbose::Bool)
    model = Model()
    a = rand(n, n)
    @variable(model, x[1:n, 1:n])

    # Compile it
    x * a
    a * x * a

    elapsed = @elapsed x * a
    if verbose
        println("2D Matrix product `x * a`: $(elapsed)")
    end

    elapsed = @elapsed a * x * a
    if verbose
        println("2D Matrix product `a * x * a`: $(elapsed)")
    end

    return
end

# Compile the methods needed.
test(2, false)

for n in [10, 20, 50, 100]
    println("n = $n")
    test(n, true)
end


function gen_carrier_replica!(
    carrier_replica::StructArray{Complex{T}},
    carrier_frequency,
    sampling_frequency,
    start_phase,
    start_sample,
    num_samples
) where T
    c_re = carrier_replica.re; c_im = carrier_replica.im
    carr_freq = T(upreferred(carrier_frequency / Hz))
    sample_freq = T(upreferred(sampling_frequency / Hz))
    twopi = T(2π)
    phase = T(start_phase)
    @avx for i in 0:num_samples - 1
        c_im[i + start_sample], c_re[i + start_sample] =
            sincos(twopi * (i * carr_freq / sample_freq + phase))
    end
    carrier_replica
end

"""
$(SIGNATURES)

Updates the carrier phase.
"""
function update_carrier_phase(
    num_samples,
    carrier_frequency,
    sampling_frequency,
    start_phase,
)
    phase = num_samples * carrier_frequency / sampling_frequency + start_phase
    mod(phase + 0.5, 1) - 0.5
end

"""
$(SIGNATURES)

Calculates the current carrier frequency.
"""
function get_current_carrier_frequency(intermediate_frequency, carrier_doppler)
    carrier_doppler + intermediate_frequency
end


# FunctionGraph

@testset "ex-fg" begin
    b = board("brd", shownavigation=true, showscreenshot=true)
    b ++ slider("a", [[1,-1],[5,-1],[0,1.5,3]])
    b ++ @jsf foo(x) = val(a)*x^2
    b ++ functiongraph(foo, dash=2)
    s = str(b, preamble=false)
    @test isapproxstr(s, """(function(){
        function foo(x){return (val(a)*Math.pow(x,2))};
        var brd = JXG.JSXGraph.initBoard("brd",
                {"boundingbox":[-10,10,10,-10],
                 "axis":false,
                 "showcopyright":false,
                 "shownavigation":true,
                 "showscreenshot":true});
        var a = brd.create(
                'slider',
                [[1.0,-1.0],[5.0,-1.0],[0.0,1.5,3.0]],
                {});
        brd.create(
                'functiongraph',
                [function(t){return foo(t);}, -10, 10],
                {"dash":2});})();
        """)
end

# ParametricCurve
@testset "ex-pc" begin
    b = board("brd", xlim=[-1, 15], ylim=[-0.5, 2.5])
    @jsf f1(t) = t - sin(t)
    @jsf f2(t) = 1 - cos(t)
    sT = slider("T", [[0,2.1],[3,2.1],[0,π,5π]])
    @jsf fb() = val(T)
    @jsf pa() = f1(val(T))
    @jsf pb() = f2(val(T))
    b ++ (f1, f2, sT, fb, pa, pb)
    b ++ plot(f1, f2; a=0, b=fb, dash=2)
    b ++ point(pa, pb, withlabel=false)
    s = str(b, preamble=false)
    @test isapproxstr(s, """(function(){
        function f1(t){return (t-sin(t))};
        function f2(t){return (1-cos(t))};
        function fb(){return val(T)};
        function pa(){return f1(val(T))};
        function pb(){return f2(val(T))};
        var brd=JXG.JSXGraph.initBoard("brd",
                {"boundingbox":[-1.0,2.5,15.0,-0.5],
                "axis":false,
                "showcopyright":false,
                "shownavigation":false});
        var T=brd.create(
                'slider',
                [[0.0,2.1],[3.0,2.1],[0.0,3.141592653589793,15.707963267948966]], {});
        brd.create(
                'curve',
                [function(t){return f1(t);}, function(t){return f2(t);},
                  0, function(t){return fb(t);}],
                {"dash":2});
        brd.create(
                'point',
                [function(t){return pa(t);}, function(t){return pb(t);}],
                {"withlabel":false});})();
        """)
end

# DataPlot
@testset "ex-dp" begin
    b = board("b", xlim=[0, 1], ylim=[0, 1])
    x = [1,2,3,4]
    y = [1,2,3,4]
    b ++ plot(x, y)
    s = str(b, preamble=false)
    @test isapproxstr(s, """(function(){
        var b=JXG.JSXGraph.initBoard("b"
                ,
                {"boundingbox":[0,1,1,0],
                "axis":false,
                "showcopyright":false,
                "shownavigation":false});
        b.create('curve', [[1,2,3,4], [1,2,3,4]], {});})();
        """)
end

# Lissajou
@testset "ex-li" begin
    b = board("brd", xlim=[-12, 12], ylim=[-10,10])
    b ++ (
        slider("a", [[2,8],[6,8],[0,3,6]]),
        slider("b", [[2,7],[6,7],[0,2,6]]),
        slider("A", [[2,6],[6,6],[0,3,6]]),
        slider("B", [[2,5],[6,5],[0,3,6]]),
        slider("delta", [[2,4],[6,4],[0,0,π]], name="δ")
        )
    @jsf f1(t) = val(A)*sin(val(a)*t+val(delta))
    @jsf f2(t) = val(B)*sin(val(b)*t)
    b ++ plot(f1, f2, a=0, b=2π, strokecolor="#aa2233", strokewidth=3)
    s = str(b, preamble=false)
    @test isapproxstr(s, """(function(){
        var brd=JXG.JSXGraph.initBoard("brd"
                ,
                {"boundingbox":[-12,10,12,-10],
                "axis":false,
                "showcopyright":false,
                "shownavigation":false});
        var a=brd.create('slider', [[2,8],[6,8],[0,3,6]], {});
        var b=brd.create('slider', [[2,7],[6,7],[0,2,6]], {});
        var A=brd.create('slider', [[2,6],[6,6],[0,3,6]], {});
        var B=brd.create('slider', [[2,5],[6,5],[0,3,6]], {});
        var delta=brd.create('slider', [[2.0,4.0],[6.0,4.0],[0.0,0.0,3.141592653589793]], {"name":"δ"});
        function f1(t){return (val(A)*sin(((val(a)*t)+val(delta))))};
        function f2(t){return (val(B)*sin((val(b)*t)))};
        brd.create(
                'curve',
                [function(t){return f1(t);}, function(t){return f2(t);},
                 0, 6.283185307179586],
                {"strokecolor":"#aa2233",
                "strokewidth":3});})();
        """)
end


using InfiniteOpt, Ipopt, Plots

# Parmeters
β = 0.3
γ = 0.1

i_max = 1000
# Time span
t0 = 0
tf = 50
# Initial values
i0 = 1

# Model
model = InfiniteModel(Ipopt.Optimizer)

# infinite_parameter
@infinite_parameter(model, t ∈ [t0, tf], num_supports = 101,
    derivative_method = OrthogonalCollocation(2))

# variable
@variable(model, i ≥ 0, Infinite(t))

@variable(model, 0 ≤ u ≤ 1, Infinite(t), start = 0)

# objective
@objective(model, Min, ∫(u, t))

# constraint
@constraint(model, i(0) == i0)
@constraint(model, i_constr, ∂(i, t) == (1 - u) * β * i - γ * i )
@constraint(model, imax_constr, i ≤ i_max)

# Optimization

print(model)

optimize!(model)

# Visulization
ts = value(t)
u = value(u)
I = value(i)

plot(ts,u)
plot(ts,I)


function init_pot_harm_3d( LF, ω, center )
  Npoints = LF.Nx * LF.Ny * LF.Nz
  Vpot = zeros( Float64,Npoints )
  for ip = 1:Npoints
    Vpot[ip] = 0.5 * ω^2 * norm(LF.lingrid[:,ip] - center[:])^2
  end
  return Vpot
end


function main()

    io = open("krypton.txt", "w");
    B1 = 26102.88e-8
    B2 = 56946.82e-8
    λ1 = 14.36*10^(-9)
    λ2 = 89.81*10^(-9)
    C1 = 2.01e-6*(1e-6)^2
    C2 = 10.043e-3*(1e-6)^2

    p0 = 1
    k0(λ) = 2π/λ
    a = 14.9e-6
    T = 273+20
    T0 = 273
    λ0 = 800e-9
    δ(λ) = T0 / T * (B1 * λ^2/(λ^2 - C1) + B2 * λ^2 / (λ^2 - C2))
    n(λ, p, u) = sqrt(1 + δ(λ)*p/p0 - (u / k0(λ) / a)^2 )

    e11 = 2.405
    e12 = 5.52
    e13 = 8.6573
    e23 = 10.1735
    e33 = 11.6198
    e43 = 13.0152

    ps = range(0, 12, length=1000)

    for p in ps
        res13 = (n(λ0, p, e11) - n(λ0/3, p, e13)) * 10^4
        res23 = (n(λ0, p, e11) - n(λ0/3, p, e23)) * 10^4
        res33 = (n(λ0, p, e11) - n(λ0/3, p, e33)) * 10^4
        write(io, "$p \t $res13 \t $res23 \t $res33 \n")
    end
    close(io)
end
main()


"""
    b = int_F²(f)

Compute the time profile contribution to the b-value. To obtain the b-value,
multiply the result by `q^2 = (γg)^2`, where `γ` is the gyromagnetic ratio of the
water proton, `g` is the gradient amplitude, and `b = q^2 * int_F²(f)`.
"""
int_F²(f::TimeProfile{T}) where {T} = quadgk(s -> integral(f, s)^2, zero(T), echotime(f))
int_F²(f::PGSE) = f.δ^2 * (f.Δ - f.δ / 3)
int_F²(f::DoublePGSE) = 2f.δ^2 * (f.Δ - f.δ / 3)
int_F²(f::CosOGSE) = f.δ^3 / (2π * f.nperiod)^2
int_F²(f::SinOGSE) = 3f.δ^3 / (2π * f.nperiod)^2


"""
    coupled_states(E[; i₀=1])

Find all states coupled by the energy expression `E`, starting from
the state with index `i₀`. This can be useful to reduce the necessary
basis or to generate invariant sets for split-operator propagation.
"""
function coupled_states(E::AbstractSparseMatrix; i₀=1)
    m = size(E,1)
    visited = falses(m)
    visited[i₀] = true

    rows = rowvals(E)

    istack = [i₀]
    while !isempty(istack)
        icur = pop!(istack)
        neighbours = rows[nzrange(E, icur)]
        for n in neighbours
            if !visited[n]
                push!(istack, n)
                visited[n] = true
            end
        end
    end

    visited
end

function invariant_sets(E::AbstractSparseMatrix)
    m = size(E,1)
    visited = falses(m)

    sets = Vector{Vector{Int}}()

    while !all(visited)
        icur = findfirst(.!visited)
        set = coupled_states(E; i₀=icur)
        visited[:] .|= set
        j = findall(set)
        # If the state icur is the only one in its set, it may be that
        # it is actually not coupled to anything.
        length(j) == 1 && iszero(E[icur,j]) && continue
        push!(sets, j)
    end
    sets
end

export coupled_states, invariant_sets


# evaluate and understand the 4 types of linear phase FIR filter responses.

# from type 1, degree L to type 2, degree L.
# 𝑎 here is b_tilde from Flilip's thesis Section 2.2.3.
# a here is b from thesis.
function converttotype2(𝑎::Vector{T}) where T
    L = length(𝑎)

    a = Vector{T}(undef,L)

    a[1] = 𝑎[1] + 𝑎[2]/2
    for k = 2:L-1
        a[k] = (𝑎[k-1+1]+𝑎[k+1])/2
    end

    a[L] = 𝑎[L]/2

    return a
end

# 𝑎 is c_tilde from Flilip's thesis Section 2.2.3.
# a here is c from thesis.
function converttotype3(𝑎::Vector{T}) where T
    L = length(𝑎)
    @assert L >= 3

    a = Vector{T}(undef,L)

    a[1] = 𝑎[1] - 𝑎[3]/2
    for k = 2:L-2
        a[k] = (𝑎[k-1+1]-𝑎[k+1+1])/2
    end

    a[L-1] = 𝑎[L-1]/2
    a[L] = 𝑎[L]/2

    return a
end

# 𝑎 is d_tilde from Flilip's thesis Section 2.2.3.
# a here is d from thesis.
function converttotype4(𝑎::Vector{T}) where T
    L = length(𝑎)

    a = Vector{T}(undef,L)

    a[1] = 𝑎[1] - 𝑎[2]/2
    for k = 2:L-1
        a[k] = (𝑎[k-1+1]-𝑎[k+1])/2
    end

    a[L] = 𝑎[L]/2

    return a
end

function getfilterfuncs(type_num::Int)
    @assert 1 <= type_num <= 4

    getafunc = getatype1
    gethfunc = gethtype1

    if type_num == 2
        getafunc = getatype2
        gethfunc = gethtype2

    elseif type_num == 3
        getafunc = getatype3
        gethfunc = gethtype3

    elseif type_num == 4
        getafunc = getatype4
        gethfunc = gethtype4
    end

    return getafunc, gethfunc
end

#### type 1
function gethtype1(a::Vector{T}) where T
    L = length(a) -1
    N = 2*L+1

    h = Vector{T}(undef, N)
    fill!(h,-1.23)

    #tmp = Vector{T}(undef, L+1)
    h[L+1] = a[1]
    for k = 1:L
        h[L-k+1] = a[k+1]/2
    end

    h[L+2:end] = reverse(h[1:L])

    return h
end

function getatype1(h::Vector{T}) where T
    N = length(h)
    @assert isodd(length(h))

    L = div(N-1,2)

    a = Vector{T}(undef,L+1)
    fill!(a,-1.23)

    a[1] = h[L+1]
    for k = 1:L
        a[k+1] = 2*h[L-k+1]
    end

    return a
end

function evalHeqn211(h::Vector{T}, ω ) where T

    a = getatype1(h)
    L = length(a)-1

    return sum( a[k+1]*cos(k*ω) for k = 0:L)
end

#### type 2

function evalHeqn212(h::Vector{T}, ω ) where T

    a = getatype2(h)
    L = length(a)

    return sum( a[k]*cos((k-0.5)*ω) for k = 1:L)
end

function getatype2(h::Vector{T}) where T

    N = length(h)
    @assert iseven(N)

    L = div(N,2)

    a = Vector{T}(undef,L)
    fill!(a,-1.23)

    for k = 1:L
        a[k] = 2*h[L-k+1]
    end

    return a
end

function gethtype2(a::Vector{T}) where T
    L = length(a)
    N = 2*L

    h = Vector{T}(undef, N)
    fill!(h,-1.23)

    for k = 1:L
        h[L-k+1] = a[k]/2
    end

    h[L+1:end] = reverse(h[1:L])

    return h
end

#### type 3

function evalHeqn213(h::Vector{T}, ω ) where T

    a = getatype3(h)
    L = length(a)

    return sum( a[k]*sin(k*ω) for k = 1:L)
end

function gethtype3(a::Vector{T}) where T
    L = length(a)
    N = 2*L+1

    h = Vector{T}(undef, N)
    fill!(h,-1.23)

    #tmp = Vector{T}(undef, L+1)
    h[L+1] = zero(T)
    for k = 1:L
        h[L-k+1] = a[k]/2
    end

    h[L+2:end] = reverse(-h[1:L])

    return h
end

function getatype3(h::Vector{T}) where T
    N = length(h)
    @assert isodd(length(h))

    L = div(N-1,2)
    @assert isnumericallyclose(h[L+1],zero(T))

    a = Vector{T}(undef,L)
    fill!(a,-1.23)

    a[1] = zero(T)
    for k = 1:L
        a[k] = 2*h[L-k+1]
    end

    return a
end

#### type 4

function evalHeqn214(h::Vector{T}, ω ) where T

    a = getatype4(h)
    L = length(a)

    return sum( a[k]*sin((k-0.5)*ω) for k = 1:L )
end

function getatype4(h::Vector{T}) where T

    N = length(h)
    @assert iseven(N)

    L = div(N,2)

    a = Vector{T}(undef,L)
    fill!(a,-1.23)

    for k = 1:L
        a[k] = 2*h[L-k+1]
    end

    return a
end

function gethtype4(a::Vector{T}) where T
    L = length(a)
    N = 2*L

    h = Vector{T}(undef, N)
    fill!(h,-1.25)

    for k = 1:L
        h[L-k+1] = a[k]/2
    end

    h[L+1:end] = reverse(-h[1:L])

    return h
end

#### generic for all 4 types.

function evalHeqn210(h,ω,type_num)
    L = 0
    if type_num == 3 || type_num == 4
        # case negative symmetric.
        L = 1
    end

    N = length(h)
    G = naiveimpulsersp(h,ω)

    return G*exp( -im*(L*π - (N-1)*ω)/2 )
end

function naiveimpulsersp(h,ω)
    N = length(h)
    return sum( h[n+1]*exp(-im*ω*n) for n = 0:N-1 )
end


# ---
# title: 238. Product of Array Except Self
# id: problem238
# author: Indigo
# date: 2021-06-03
# difficulty: Medium
# categories: Array
# link: <https://leetcode.com/problems/product-of-array-except-self/description/>
# hidden: true
# ---
#
# Given an array `nums` of _n_ integers where _n_ > 1,  return an array `output`
# such that `output[i]` is equal to the product of all the elements of `nums`
# except `nums[i]`.
#
# **Example:**
#
#
#
#     Input:  [1,2,3,4]
#     Output: [24,12,8,6]
#
#
# **Constraint:**  It's guaranteed that the product of the elements of any
# prefix or suffix of the array (including the whole array) fits in a 32 bit
# integer.
#
# **Note:** Please solve it **without division** and in O( _n_ ).
#
# **Follow up:**
# Could you solve it with constant space complexity? (The output array **does
# not** count as extra space for the purpose of space complexity analysis.)
#
#
## @lc code=start
using LeetCode

function product_except_self(nums::Vector{Int})
    n = length(nums)
    left = right = 1
    res = fill(1, n)
    @inbounds for i in 1:n
        res[i] *= left
        left *= nums[i]
        res[n - i + 1] *= right
        right *= nums[n - i + 1]
    end
    return res
end
## @lc code=end


cd(raw"C:\Users\francis.smart.ctr\RandomnessAndTalent.jl")
Pkg.activate(".")

module RandomnessAndTalent

using DataFrames, Gadfly

xgrid = 200
ygrid = 200

ni = 1000
ix = rand(1:xgrid, ni)
iy = rand(1:ygrid, ni)

talentave = .6
talentsd  = .1

italent = max.(0, min.(1, talentave .+ talentsd.*rand(Normal(), ni)))

maxt = round(maximum(italent), digits=2)
mint = round(minimum(italent), digits=2)

print("Maximum Talent:$maxt \nMinimum Talent:$mint")

ngood = 250

goodx = rand(1:xgrid, ngood)
goody = rand(1:ygrid, ngood)

nbad  = 250
badx = rand(1:xgrid, ngood)
bady = rand(1:ygrid, ngood)

timeperiods    = 80
initialcapital = 100.0

people = DataFrame(x = ix,    y = iy,    type = "individual")
good   = DataFrame(x = goodx, y = goody, type = "good")
bad    = DataFrame(x = badx,  y = bady,  type = "bad")

gridset = vcat(people, good, bad)

set_default_plot_size(20cm, 16cm)  # hide

plot(gridset, x = "x", y = "y", color = "type", Geom.point, Theme(point_size = 2pt),
     Scale.x_continuous(minvalue=0, maxvalue=xgrid),
     Scale.y_continuous(minvalue=0, maxvalue=ygrid))

function randomwalk!(dataframe, rstep=1, xborder = xgrid, yborder = ygrid)
  roundboundary(xmax, x) = round.(max.(0, min.(x, xmax)))

  dataframe.x =  roundboundary(xborder, dataframe.x + rstep.*rand(Normal(), size(dataframe, 1)))
  dataframe.y =  roundboundary(yborder, dataframe.y + rstep.*rand(Normal(), size(dataframe, 1)))

end

function nextphase(dataframe, good = dataframe(x=-1, y=-1), bad = dataframe(x=-1, y=-1), verbose = false)
  t = maximum(dataframe.t)
  recentdata = dataframe[dataframe.t .== t, :]
  recentdata.t .+= 1
  recentdata.good .= 0
  recentdata.realised .= 0
  recentdata.bad .= 0

  #Loop through each individual and check if a good event happened
  for i in 1:size(recentdata, 1)
    # Check if good event could happen
    if sum((good.x .== recentdata.x[i]) .& (good.y .== recentdata.y[i])) > 0
        verbose && println("$i possibly get fortuitous event")
        recentdata.good[i] = 1
        if rand(1,1)[1] < recentdata.talent[i]
            recentdata.capital[i] *= 2
            recentdata.realised[i] = 1
            verbose && println("$i get fortuitous event")

        end
    end

    # Check if bad even happens
    if sum((bad.x .== recentdata.x[i]) .& (bad.y .== recentdata.y[i])) > 0
        verbose && println("$i is unlucky")
        recentdata.capital[i] *= .5
        recentdata.bad[i] = 1
    end
  end

  dataframe = vcat(dataframe, recentdata)
end # module

peopleset = DataFrame(i = 1:ni, t = 0, capital = initialcapital, talent = italent,
  x = ix, y = iy, good=0, realised=0, bad=0)

for i in 1:timeperiods;
    global peopleset
    peopleset = nextphase(peopleset, good, bad);
    randomwalk!(good)
    randomwalk!(bad)
end

retirement = peopleset[peopleset.t .== timeperiods,:]

top = retirement[retirement.capital .> 1000, :].i

plot(peopleset[[i in top for i in peopleset.i],:], x = "t", y = "capital", color = "i",
  Geom.point, Geom.line, Scale.y_log)

plot(peopleset[[i in top for i in peopleset.i],:], x = "t", y = "capital", color = "i",
  Geom.point, Geom.line)

peopleset[[i in top for i in peopleset.i] .& (peopleset.t .== 0), :talent]

plot(peopleset[peopleset.t .== 80,:], x = "talent", y = "capital", Geom.point, Scale.y_log)

eventcounts = by(peopleset, :i,
  (:good, :bad, :realised) => x -> (good = sum(x[1]), bad = sum(x[2]), realised = sum(x[3])))

peopleset[peopleset.good .== 1, :]

mean(eventcounts.good)
mean(eventcounts.bad)
mean(eventcounts.realised)

end #module


@testset "Manopt Cyclic Proximal Point" begin
    using Dates
    n = 100
    N = PowerManifold(Circle(),n)
    f = artificial_S1_signal(n)
    F = x -> costL2TV(N,f,0.5,x)
    proxes = [ (λ,x) -> prox_distance(N,λ,f,x), (λ,x) -> prox_TV(N,0.5*λ,x) ]
    o = cyclic_proximal_point(
        N,
        F,
        proxes,
        f;
        λ = i -> π/(2*i),
        stopping_criterion = StopAfterIteration(100),
        debug = [DebugIterate()," ",DebugCost()," ",DebugProximalParameter(),"\n",10000],
        record = [RecordProximalParameter(), RecordIterate(f), RecordCost()],
        return_options=true
        )
    fR = get_solver_result(o)
    fR2 = cyclic_proximal_point(
        N,
        F,
        proxes,
        f;
        λ = i -> π/(2*i),
        stopping_criterion = StopAfterIteration(100),
    )
    @test fR == fR2
    rec = get_record(o)
    @test F(f) > F(fR)
    #
    o = CyclicProximalPointOptions(f, StopAfterIteration(1), i -> π/(2*i))
    p = ProximalProblem(N,F,proxes,[1,2])
    @test_throws ErrorException getProximalMap(p,1.,f,3)

    order1 = Manopt.update_cpp_order(3,0,1:3,RandomEvalOrder())
    @test all(isinteger.(order1))
    @test minimum(order1) == 1
    @test maximum(order1) == 3

    # i=0 introduce new fixed ranom order
    order2 = Manopt.update_cpp_order(3,0,1:3,FixedRandomEvalOrder())
    @test all(isinteger.(order2))
    @test minimum(order2) == 1
    @test maximum(order2) == 3
    order3 = Manopt.update_cpp_order(3,1,order2,FixedRandomEvalOrder())
    @test order3 == order2 # only update on i=0

    @test Manopt.update_cpp_order(3,0,1:3,LinearEvalOrder()) == 1:3
    @test Manopt.update_cpp_order(3,1,1:3,LinearEvalOrder()) == 1:3
end


# ---
# title: 1223. Dice Roll Simulation
# id: problem1223
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Dynamic Programming
# link: <https://leetcode.com/problems/dice-roll-simulation/description/>
# hidden: true
# ---
#
# A die simulator generates a random number from 1 to 6 for each roll. You
# introduced a constraint to the generator such that it cannot roll the number
# `i` more than `rollMax[i]` (1-indexed) **consecutive** times.
#
# Given an array of integers `rollMax` and an integer `n`, return the number of
# distinct sequences that can be obtained with exact `n` rolls.
#
# Two sequences are considered different if at least one element differs from
# each other. Since the answer may be too large, return it modulo `10^9 + 7`.
#
#
#
# **Example 1:**
#
#
#
#     Input: n = 2, rollMax = [1,1,2,2,2,3]
#     Output: 34
#     Explanation: There will be 2 rolls of die, if there are no constraints on the die, there are 6 * 6 = 36 possible combinations. In this case, looking at rollMax array, the numbers 1 and 2 appear at most once consecutively, therefore sequences (1,1) and (2,2) cannot occur, so the final answer is 36-2 = 34.
#
#
# **Example 2:**
#
#
#
#     Input: n = 2, rollMax = [1,1,1,1,1,1]
#     Output: 30
#
#
# **Example 3:**
#
#
#
#     Input: n = 3, rollMax = [1,1,1,2,2,3]
#     Output: 181
#
#
#
#
# **Constraints:**
#
#   * `1 <= n <= 5000`
#   * `rollMax.length == 6`
#   * `1 <= rollMax[i] <= 15`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


"""
  daxpy(alpha, x, y)

BLAS daxpy: y = alpha*x + y
"""
function daxpy(a::Float64, x::Vector{Float64}, y::Vector{Float64})
    @inbounds for i in 1:length(x)
        y[i] = a*x[i] + y[i]
    end
end

"""
    axpy(alpha, x, y)

Template version of BLAS axpy: y = alpha*x + y
"""
function axpy(a::T, x::Vector{T}, y::Vector{T}) where
    {T <: Real}
    # - This would do but slower - #
    # y = a*x + y

    # - This is equivalent to the for-loop below
    @. y = a*x + y # This is eaiser to read
    # or
    #y .= a.*x .+ y

    # - Keep this for doc reason - #
    # @inbounds for i in 1:length(x)
    #     y[i] = a*x[i] + y[i]
    # end
end


"""
    add_reciprocal_links!(g, p, w)

Add reciprocal links to a weighted digraph, with probability `p` and weights sampled from `w`.

If `w` is a number, then it is assigned as the weight for every created link.
"""
function add_reciprocal_links!(g::MetaDiGraph, p::Real, w::Distribution = Uniform(0.0,1.0))
    for e in edges(g)
        (u, v) = src(e), dst(e)
        if !has_edge(g, v, u) && rand() < p
            add_edge!(g, v, u)
            set_prop!(g, v, u, :weight, rand(w))
        end
    end
    return g
end

function add_reciprocal_links!(g::MetaDiGraph, p::Real, w::Real)
    for e in edges(g)
        (u,v) = src(e), dst(e)
        if !has_edge(g, v, u) && rand() < p
            add_edge!(g, v, u)
            set_prop!(g, v, u, :weight, w)
        end
    end
    return g
end

"""
    add_reciprocal_links(g, p, w)

Add reciprocal links to a weighted digraph, with probability `p` and weights sampled from `w`.

This function creates a copy of `g`. See [add_reciprocal_links!](@ref) for the in-place version.
"""
add_reciprocal_links(g, p, w=Uniform(0.0,1.0)) = add_reciprocal_links!(copy(g), p, w)

add_reciprocal_links!(p::Real, w=Uniform(0.0,1.0)) = g -> add_reciprocal_links!(g, p, w)
add_reciprocal_links(p::Real, w=Uniform(0.0,1.0)) = g -> add_reciprocal_links(g, p, w)


using EconometricsLH
using Random, Test

function regression_test_test()
    @testset "Regression test" begin
        rng = MersenneTwister(44);
        n = Int(1e4);
        trueM = randn(rng, n);
        simM  = trueM .+ 0.3 .* randn(rng, n);
        wtM = 1.0 .+ rand(rng, n);

        for useWeights in [false, true]
            if useWeights
                dev, betaV, seV = regression_test(trueM, simM; weights = wtM);
            else
                dev, betaV, seV = regression_test(trueM, simM);
            end

            @test size(betaV) == (2,)
            @test size(seV) == (2,)
            @test all(seV .> 0.0)
            @test abs(betaV[1]) < 2.2 * seV[1]
            @test abs(betaV[2] - 1.0) < 2.2 * seV[2]
            @test (dev < 2.2);
        end
    end
end

@testset "Regression test" begin
    regression_test_test();
end

# ---------------


"""
    MaxSens(resolution::Float64, tight::Bool)

MaxSens performs over-approximated reachability analysis to compute the over-approximated output reachable set for a network.

# Problem requirement
1. Network: any depth, any activation that is monotone
2. Input: `Hyperrectangle` or `HPolytope`
3. Output: `AbstractPolytope`

# Return
`ReachabilityResult`

# Method
First partition the input space into small grid cells according to `resolution`.
Then use interval arithmetic to compute the reachable set for each cell.
Two versions of interval arithmetic is implemented with indicator `tight`.
Default `resolution` is `1.0`. Default `tight = false`.

# Property
Sound but not complete.

# Reference
[W. Xiang, H.-D. Tran, and T. T. Johnson,
"Output Reachable Set Estimation and Verification for Multi-Layer Neural Networks,"
*ArXiv Preprint ArXiv:1708.03322*, 2017.](https://arxiv.org/abs/1708.03322)
"""
@with_kw struct MaxSens <: Solver
    resolution::Float64 = 1.0
    tight::Bool         = false
end

# This is the main function
function solve(solver::MaxSens, problem::Problem)
    inputs = partition(problem.input, solver.resolution)
    f_n(x) = forward_network(solver, problem.network, x)
    outputs = map(f_n, inputs)
    return check_inclusion(outputs, problem.output)
end

# This function is called by forward_network
function forward_layer(solver::MaxSens, L::Layer, input::Hyperrectangle)
    output = approximate_affine_map(L,  input)
    β    = L.activation.(output.center)
    βmax = L.activation.(high(output))
    βmin = L.activation.(low(output))

    if solver.tight
        center = (βmax + βmin)/2
        rad =  (βmax - βmin)/2
    else
        center = β
        rad = @. max(abs(βmax - β), abs(βmin - β))
    end
    return Hyperrectangle(center, rad)
end

function partition(input::Hyperrectangle, Δ)
    # treat radius = 0 as a flag not to partition the input set at all.
    Δ == 0 && return [input]

    lower, upper = low(input), high(input)

    # The number of sub-hyperrectangles that fit in each dimension, rounding up
    n_hypers_per_dim = max.(ceil.(Int, (upper-lower) / Δ), 1)
    N = prod(big, n_hypers_per_dim)

    N > typemax(Int64) && throw(ArgumentError("Cannot partition the given input. The operation would require $N sets."))
    N > 10^5 && @warn "Propagating $N partitions may be time consuming or impossible."

    # preallocate
    hypers = Vector{typeof(input)}(undef, N)
    local_lower, local_upper = similar(lower), similar(upper)
    CI = CartesianIndices(Tuple(n_hypers_per_dim))

    # iterate from the lower corner to the upper corner
    for i in 1:length(CI)
        I = collect(Tuple(CI[i])) .- 1
        @. local_lower = min(lower + Δ*I,     upper)
        @. local_upper = min(local_lower + Δ, upper)

        hypers[i] = Hyperrectangle(low = local_lower, high = local_upper)
    end
    return hypers
end

function partition(input::HPolytope, delta::Float64)
    @info "MaxSens overapproximates HPolytope input sets as Hyperrectangles."
    partition(overapproximate(input), delta)
end


using ParamLevelSet
using jInv.Mesh
using LinearAlgebra
using SparseArrays
using Test
plotting = false;
if plotting
	using jInvVis
	using PyPlot
end

println("~~~~~~~~~~~~~~~~Testing extended RBFs.~~~~~~~~~~~~~~~~~~")

n = [32,32,32];

Mesh = getRegularMesh([0.0;3.0;0.0;3.0;0.0;3.0],n);
midmesh = [(Mesh.domain[2]+Mesh.domain[1]/2) ; (Mesh.domain[4]+Mesh.domain[3]/2); (Mesh.domain[5]+Mesh.domain[6]/2)];
lenmesh = [(Mesh.domain[2]-Mesh.domain[1]) ; (Mesh.domain[4]-Mesh.domain[3]); (Mesh.domain[6]-Mesh.domain[5])];
numParamOfRBF = 10;
nRBF = 5
m5 = zeros(nRBF*5);
m10 = zeros(nRBF*numParamOfRBF);

for k=1:nRBF
	offset10 = (k-1)*numParamOfRBF + 1;
	m10[offset10] = randn();
	beta = max(abs(randn()),0.25);
	# println(beta)
	# m10[offset10+1] = beta; m10[offset10+4] = beta; m10[offset10+6] = beta; ## for the L version.
	m10[offset10+1] = beta^2; m10[offset10+4] = beta^2; m10[offset10+6] = beta^2; ## for the A version.
	m10[(offset10+7):(offset10+9)] = midmesh + 0.1*lenmesh.*randn(size(lenmesh));
	offset5 = (k-1)*5 + 1;
	m5[offset5] = m10[offset10];
	m5[offset5+1] = beta;
	m5[(offset5+2):(offset5+4)] = m10[(offset10+7):(offset10+9)];
end

u5, = ParamLevelSetModelFunc(Mesh,m5;computeJacobian=0,numParamOfRBF = 5);
u10, = ParamLevelSetModelFunc(Mesh,m10;computeJacobian=0,numParamOfRBF = 10);

@test norm(u5-u10) < 1e-5

n = [32,32,32];
Mesh = getRegularMesh([0.0;5.0;0.0;5.0;0.0;5.0],n);
midmesh = [(Mesh.domain[2]+Mesh.domain[1])/2 ; (Mesh.domain[4]+Mesh.domain[3])/2; (Mesh.domain[5]+Mesh.domain[6])/2];
lenmesh = [(Mesh.domain[2]-Mesh.domain[1]) ; (Mesh.domain[4]-Mesh.domain[3]); (Mesh.domain[6]-Mesh.domain[5])];
numParamOfRBF = 10;
nRBF = 5
m = zeros(nRBF*numParamOfRBF);

for k=1:nRBF
	offset = (k-1)*numParamOfRBF + 1;
	m[offset] = randn();
	A = randn(3,3); A = 0.3*A'*A + 0.5*Matrix(1.0I,3,3); #A = eye(3) + 1e-5;
	# L = chol(A)';
	# m[(offset+1):(offset+6)] = L[L.!=0.0];
	m[(offset+1):(offset+6)] = A[tril(A).!=0.0];

	# m[offset+1] = 1.0; m[offset+4] = 1.0; m[offset+6] = 1.0;
	m[(offset+7):(offset+9)] = midmesh + 0.05*lenmesh.*randn(size(lenmesh));
	#println(midmesh + 0.1*lenmesh.*randn(size(lenmesh)))
end
u, = ParamLevelSetModelFunc(Mesh,m;computeJacobian=0,numParamOfRBF = 10);
if plotting
	figure()
	u = reshape(u,tuple(Mesh.n...));
	plotModel(u)
end

sigmaH = getDefaultHeaviside();
us, = ParamLevelSetModelFunc(Mesh,m;computeJacobian=0,sigma = sigmaH,numParamOfRBF = 10);
if plotting
	figure()
	plotModel(us)
end

bf = 1;
dm = 0.01*zeros(length(m));

for k=1:nRBF
	offset = (k-1)*numParamOfRBF + 1;
	dm[offset] = 0.01*randn();
	dm[(offset+1):(offset+6)] = 0.01*randn(6);# 0.01*[randn(); randn(); 0.0; randn(); 0.0; randn()];#
	dm[(offset+7):(offset+9)] = 0.01*randn(3)
end

hh = 1.0;
# u0,II,JJ,VV = ParamLevelSetModelFunc_old(Mesh,m;computeJacobian=1,bf = bf,numParamOfRBF = 10);
# J0 = sparse(II,JJ,VV,prod(Mesh.n),length(m))

u0,JBuilder = ParamLevelSetModelFunc(Mesh,m;computeJacobian=1,bf = bf,numParamOfRBF = 10);
J0 = getSparseMatrix(JBuilder);

for k=0:8
	hhh = (0.5^k)*hh;
	ut = ParamLevelSetModelFunc(Mesh,m+ hhh*dm;computeJacobian=0,bf = bf,numParamOfRBF = 10)[1];
	println("norm(ut-u0): ",norm(ut[:]-u0[:]),", norm(ut - u0 - J0*dm): ",norm(ut[:] - u0[:] - J0*hhh*dm));
end

println("With Heaviside func");

hh = 1.0;

# u0,II,JJ,VV = ParamLevelSetModelFunc_old(Mesh,m;computeJacobian=1,sigma=sigmaH,bf = bf,numParamOfRBF = 10);
# J0 = sparse(II,JJ,VV,prod(Mesh.n),length(m));

u0,JBuilder = ParamLevelSetModelFunc(Mesh,m;computeJacobian=1,sigma=sigmaH,bf = bf,numParamOfRBF = 10);
J0 = getSparseMatrix(JBuilder);
for k=0:8
	hhh = (0.5^k)*hh;
    ut = ParamLevelSetModelFunc(Mesh,m + hhh*dm;computeJacobian=0,sigma=sigmaH,bf = bf,numParamOfRBF = 10)[1];
	println("norm(ut-u0): ",norm(ut[:]-u0[:]),", norm(ut - u0 - J0*dm): ",norm(ut[:] - u0[:] - hhh*(J0*dm)));
end


module WaveletRegression

using Wavelets
using Statistics
using LinearAlgebra

export
    wavelm,
    AbstractWaveletTransform,
    MODWT,
    DWT,
    DWPT,
    WaveletRegressionModel,
    modwt_matrix,
    imodwt_matrix,
    dwt_matrix,
    idwt_matrix,
    predict,
    predict_wavespace,
    coef,
    ncoef,
    dof,
    nlevels,
    residuals,
    regression_stderr,
    coef_stderr,
    coef_shrunk,
    shrink

export wavelet, WT

abstract type AbstractWaveletTransform end

for transformtype in [:MODWT, :DWT, :WPT]
    @eval begin
        struct $transformtype <: AbstractWaveletTransform
            wavelet::DiscreteWavelet
            nlevels::Integer
        end
        Wavelets.wavelet(trans::$(transformtype)) = trans.wavelet
        nlevels(trans::$(transformtype)) = trans.nlevels
    end
end

struct WaveletRegressionModel
    X::AbstractArray
    Y::AbstractArray
    n::Integer
    mx::Integer
    my::Integer
    Xw::AbstractArray
    Yw::AbstractArray
    trans::AbstractWaveletTransform
    coefficients
end
Wavelets.wavelet(mod::WaveletRegressionModel) = wavelet(mod.trans)
nlevels(mod::WaveletRegressionModel) = nlevels(mod.trans)
coef(mod::WaveletRegressionModel) = mod.coefficients
ncoef(mod::WaveletRegressionModel) = sum(length(B) for B in mod.coefficients)
dof(mod::WaveletRegressionModel) = mod.n - ncoef(mod) - 1

function predict_wavespace(mod::WaveletRegressionModel, newXw::AbstractArray, shrink=false)
    n = size(newXw, 1)
    if shrink
        BB = coef_shrunk(mod)
    else
        BB = mod.coefficients
    end
    Yw = cat([reshape(newXw[:, j, :] * BB[j], (n, 1, mod.my))
        for j in 1:nlevels(mod)+1]..., dims=2)
    return Yw
end

function predict(mod::WaveletRegressionModel, newX::AbstractVecOrMat, shrink=false)
    @assert size(newX, 2) == size(mod.X, 2)
    newXw = modwt_matrix(newX, mod.trans)
    Yw = predict_wavespace(mod, newXw, shrink)
    Y = imodwt_matrix(Yw, mod.trans)
    return Y
end
predict(mod::WaveletRegressionModel, shrink=false) = predict(mod, mod.X, shrink)

function residuals(mod::WaveletRegressionModel, shrink=false)
    Yw_pred = predict_wavespace(mod, mod.Xw, shrink)
    return mod.Yw .- Yw_pred
end

function regression_stderr(mod::WaveletRegressionModel; unbiased=true, shrink=false)
    resid = residuals(mod, shrink)
    s2 = reshape(sum(abs2 , resid, dims=1), (mod.my, :))
    if unbiased
        return s2 / dof(mod)
    else
        return s2 / mod.n
    end
end

"""
Calculate shrinkage factor for regression coefficients.
J.B. Copas (1983). Regression, prediction, and shrinkage. J. Royal Statistical
Society B 45(3), 311-354.  https://www.jstor.org/stable/2345402
"""
function coef_shrunk(mod::WaveletRegressionModel)
    BBshrunk = deepcopy(coef(mod))
    for j in 1:mod.mx
        Xw = mod.Xw[:, j, :]
        n = mod.n
        p = mod.mx * mod.my
        v = n - p - 1
        σ2 = regression_stderr(mod)[:, j]
        B = coef(mod)[j]
        V = Xw'Xw/n
        M = cholesky(V).U

        shrinkage = zeros(1, mod.my)
        for k in 1:mod.my
            β = B[:, k]
            ξ = M * β
            shrinkage[k] = 1 - ((p-2) * σ2[k] * v) / (n * (v+2) * ξ'ξ)
        end
        BBshrunk[j] = B .* shrinkage
    end
    return BBshrunk
end

function shrink(mod::WaveletRegressionModel)
    BB = coef_shrunk(mod)
    return WaveletRegressionModel(mod.X, mod.Y, mod.n, mod.mx, mod.my,
        mod.Xw, mod.Yw, mod.trans, BB)
end

function coef_stderr(mod::WaveletRegressionModel; unbiased=true, shrink=false)
    # https://en.wikipedia.org/wiki/Ordinary_least_squares
    s2 = regression_stderr(mod, unbiased=unbiased, shrink=shrink)
    L = mod.trans.nlevels + 1
    Qxx =  [mod.Xw[:, j, :]' * mod.Xw[:, j, :] for j in 1:L]
    SE = [similar(B) for B in coef(mod)]
    for j in 1:L
        for i in 1:mod.my
            SE[j][:, i] .= sqrt.(diag(s2[i, j] .* inv(Qxx[j])))
        end
    end
    return SE
end

function modwt_matrix(X::AbstractVecOrMat, trans::MODWT)
    return cat([modwt(X[:, i], trans.wavelet, trans.nlevels) for i in 1:size(X, 2)]..., dims=3)
end

function imodwt_matrix(Xw::AbstractArray, trans::MODWT)
    n, nlevels, nvars = size(Xw)
    hcat([imodwt(Xw[:, :, i], trans.wavelet) for i in 1:nvars]...)
end

function wavelm(X::AbstractVecOrMat, Y::AbstractVecOrMat, trans::MODWT)
    Xw = modwt_matrix(X, trans)
    Yw = modwt_matrix(Y, trans)
    BB = [Xw[:, j, :] \ Yw[:, j, :] for j in 1:(trans.nlevels+1)]
    n = size(X, 1)
    mx = div(length(X), n)
    my = div(length(Y), n)
    return WaveletRegressionModel(X, Y, n, mx, my, Xw, Yw, trans, BB)
end

function dwt_matrix(X::AbstractVecOrMat, trans::DWT)
    return hcat([dwt(X[:, i], trans.wavelet, trans.nlevels) for i in 1:size(X, 2)]...)
end

function idwt_matrix(Xw::AbstractArray, trans::DWT)
    n, nvars = size(Xw)
    hcat([idwt(Xw[:, i], trans.wavelet, trans.nlevels) for i in 1:nvars]...)
end

function wavelm(X::AbstractVecOrMat, Y::AbstractVecOrMat, trans::DWT)
    Xw = dwt_matrix(X, trans)
    Yw = dwt_matrix(Y, trans)
    n = size(Xw, 1)
    indices = [detailrange(n, j) for j in 1:trans.nlevels]
    push!(indices, 1:detailindex(n, trans.nlevels, 1)-1) # scaling coef indices
    BB = [Xw[ii, :] \ Yw[ii, :] for ii in indices]
    n = size(X, 1)
    mx = div(length(X), n)
    my = div(length(Y), n)
    return WaveletRegressionModel(X, Y, n, mx, my, Xw, Yw, trans, BB)
end

end # module


using ITensors
using ITensorNetworkAD
using OptimKit
using Random
using Zygote

Random.seed!(1243)

N = 6
s = siteinds("S=1/2", N; conserve_qns=true)
os = OpSum()
for n in 1:(N - 1)
  os .+= 0.5, "S+", n, "S-", n + 1
  os .+= 0.5, "S-", n, "S+", n + 1
  os .+= "Sz", n, "Sz", n + 1
end
Hmpo = MPO(os, s)
ψ₀mps = randomMPS(s, n -> isodd(n) ? "↑" : "↓")
H = prod(Hmpo)
ψ₀ = prod(ψ₀mps)

# The Rayleigh quotient to minimize
function E(H::ITensor, ψ::ITensor)
  ψdag = dag(ψ)
  return (ψdag' * H * ψ)[] / (ψdag * ψ)[]
end

@show flux(ψ₀)
@show E(H, ψ₀)

E(ψ::ITensor) = E(H, ψ)
∇E(ψ::ITensor) = E'(ψ)
fg(ψ::ITensor) = (E(ψ), ∇E(ψ))

linesearch = HagerZhangLineSearch(;
  c₁=0.1, c₂=0.9, ϵ=1e-6, θ=1 / 2, γ=2 / 3, ρ=5.0, verbosity=0
)
algorithm = LBFGS(3; maxiter=30, gradtol=1e-8, linesearch=linesearch)
ψ, fψ, gψ, numfg, normgradhistory = optimize(fg, ψ₀, algorithm)

display(normgradhistory)

@show E(H, ψ)

D, _ = eigen(H; ishermitian=true)
@show minimum(D)