using PyPlot, PyCall, IJulia

"""
    t = delaunay(p)

Delaunay triangulation `t` of N x 2 node array `p`.
"""
function delaunay(p)
    tri = pyimport("matplotlib.tri")
    t = tri[:Triangulation](p[:,1], p[:,2])
    return Int64.(t[:triangles] .+ 1)
end

"""
    edges, boundary_indices = all_edges(t)

Find all unique edges in the triangulation `t` (ne x 2 array)
Second output is indices to the boundary edges.
"""
function all_edges(t)
    etag = vcat(t[:,[1,2]], t[:,[2,3]], t[:,[3,1]])
    etag = hcat(sort(etag, dims=2), 1:3*size(t,1))
    etag = sortslices(etag, dims=1)
    dup = all(etag[2:end,1:2] - etag[1:end-1,1:2] .== 0, dims=2)[:]
    keep = .![false;dup]
    edges = etag[keep,1:2]
    emap = cumsum(keep)
    invpermute!(emap, etag[:,3])
    emap = reshape(emap,:,3)
    dup = [dup;false]
    dup = dup[keep]
    bndix = findall(.!dup)
    return edges, bndix, emap
end

"""
    e = boundary_nodes(t)

Find all boundary nodes in the triangulation `t`.
"""
function boundary_nodes(t)
    edges, boundary_indices = all_edges(t)
    return unique(edges[boundary_indices,:][:])
end

"""
    tplot(p, t, u=nothing)

If `u` == nothing: Plot triangular mesh with nodes `p` and triangles `t`.
If `u` == solution vector: Plot filled contour color plot of solution `u`.
"""
function tplot(p, t, u=nothing)
    clf()
    axis("equal")
    if u == nothing
        tripcolor(p[:,1], p[:,2], t .- 1, 0*t[:,1],
                  cmap="Set3", edgecolors="k", linewidth=1)
    else
        tricontourf(p[:,1], p[:,2], t .- 1, u, 20)
    end
    draw()
end

"""
    inside = inpolygon(p, pv)

Determine if each point in the N x 2 node array `p` is inside the polygon
described by the NE x 2 node array `pv`.
"""
function inpolygon(p::Array{Float64,2}, pv::Array{Float64,2})
    path = pyimport("matplotlib.path")
    poly = path[:Path](pv)
    inside = [poly[:contains_point](p[ip,:]) for ip = 1:size(p,1)]
end

function remove_outside_tris(p, t, pv)
    pmid = dropdims(sum(p[t,:], dims=2), dims=2) / 3
    is_inside = inpolygon(pmid, pv)
    t = t[is_inside,:]
end

function triarea(p, t)
    d12 = @. p[t[:,2],:] - p[t[:,1],:]
    d13 = @. p[t[:,3],:] - p[t[:,1],:]
    @. abs(d12[:,1] * d13[:,2] - d12[:,2] * d13[:,1]) / 2
end

function remove_tiny_tris(p, t)
    t = t[triarea(p,t) .> 1e-14,:]
end

function circumcenter(p)
    dp1 = @. p[2,:] - p[1,:]
    dp2 = @. p[3,:] - p[1,:]

    mid1 = @. (p[1,:] + p[2,:]) / 2
    mid2 = @. (p[1,:] + p[3,:]) / 2

    s = [ -dp1[2] dp2[2]; dp1[1] -dp2[1]] \ (-mid1 .+ mid2)
    pc = @. mid1' + s[1] * [-dp1[2] dp1[1]]
end

function edge_midpoints(p, t)
    pmid = reshape(p[t,:] + p[t[:,[2,3,1]],:], :, 2) / 2
    pmid = unique(pmid, dims=1)
end

function pmesh(pv, hmax, nref)
    p = zeros(Float64, 0, 2)
    for i = 1:size(pv,1) - 1
        pp = pv[i:i+1,:]
        L = sqrt(sum(diff(pp, dims=1).^2, dims=2))[1]
        if L > hmax
            n = ceil(Int, L / hmax)
            ss = (0:n) / n
            pp = [1 .- ss ss] * pp
        end
        p = [p; pp[1:end-1,:]]
    end

    t = zeros(Int64, 0, 3)
    while true
        t = delaunay(p)
        t = remove_tiny_tris(p, t)
        t = remove_outside_tris(p, t, pv)
        # tplot(p,t), pause(1e-3)

        area = triarea(p, t)
        maxarea, ix = findmax(area)
        if maxarea < hmax^2 / 2
            break
        end
        pc = circumcenter(p[t[ix,:],:])
        p = [p; pc]
    end

    for iref = 1:nref
        p = [p; edge_midpoints(p, t)]
        t = delaunay(p)
        t = remove_tiny_tris(p, t)
        t = remove_outside_tris(p, t, pv)
        # tplot(p, t), pause(1e-3)
    end

    e = boundary_nodes(t)
    p, t, e
end

function pmesh_test()
    pv = [0 0; 1 0; .5 .5; 1 1; 0 1; 0 0]
    p,t,e = pmesh(pv, 0.2, 1);
    tplot(p,t)
end

using SparseArrays
using LinearAlgebra
function coefficients(RHS, p, t, k)
    LHS = [1 p[t[k,1],1] p[t[k,1],2]; 1 p[t[k,2],1] p[t[k,2],2]; 1 p[t[k,3],1] p[t[k,3],2]]
    coeff = LHS\RHS'
    return coeff'
end

function element(coeffmatrix, a, b)
    return coeffmatrix[a,2]*coeffmatrix[b,2] + coeffmatrix[a,3]*coeffmatrix[b,3]
end

function fempoi(p, t, e)
    areas = triarea(p,t)
    n = size(p)[1]
    A = spzeros(n, n); b = zeros(n);
    for k = 1:size(t)[1]
        cm = [coefficients([1 0 0] , p, t, k) ;
              coefficients([0 1 0] , p, t, k) ;
              coefficients([0 0 1] , p, t, k) ]
        insert = [element(cm, 1, 1) element(cm, 1, 2) element(cm, 1, 3);
                  element(cm, 2, 1) element(cm, 2, 2) element(cm, 2, 3);
                  element(cm, 3, 1) element(cm, 3, 2) element(cm, 3, 3)]
        insert *=  areas[k]

        A[t[k,:],t[k,:]] = A[t[k,:],t[k,:]]+insert
        #aa = areas[k]/3.0
        b[t[k,:]] = b[t[k,:]].+ areas[k]/3.0
    end
    for k = 1:size(e)[1]
        i=e[k]
        A[i,:].=0.0
        A[i,i]=1.0
        b[i]=0.0
    end
    dropzeros!(A)
    println(b)
    println(A)

    return A \ b
end

function poiconv(pv, hmax, nrefmax)
   pmax, tmax, emax = pmesh(pv, hmax, nrefmax)
   umax = fempoi(pmax, tmax, emax)
   errors = zeros(nrefmax)
   for ncur = 0:(nrefmax-1)
       pprox, tprox, eprox = pmesh(pv, hmax, ncur)
       uprox = fempoi(pprox, tprox, eprox)
       errorarray = maximum(abs.(umax[1:size(uprox)[1]] - uprox))
       #errors = [errors ; errorarray]
       errors[ncur+1]=errorarray
   end

   return errors
end

function p2mesh(p,t)
    "npoints = edge_midpoints(p,t)"
    "p2 = [p; npoints]"
    "e2 = zeroes()"
    m = size(p)[1]

    elist, bedges, emap = all_edges(t)
    mid_point=Dict()
    for k = 1:size(elist)[1]
        x=(p[elist[k,1],1]+p[elist[k,2],1])/2.0
        y=(p[elist[k,1],2]+p[elist[k,2],2])/2.0
        p=[p; x y]
        mid_point[elist[k,2],elist[k,1]]=size(p)[1]
        mid_point[elist[k,1],elist[k,2]]=size(p)[1]
    end
    t2=zeros(Int64,0,6)
    for i = 1:size(t)[1]
       t12=mid_point[t[i,1],t[i,2]]
       t23=mid_point[t[i,2],t[i,3]]
       t31=mid_point[t[i,3],t[i,1]]
       t2=[t2; t[i,:]' t12 t23 t31]
    end

    e2 = boundary_nodes(t)
    for i=1:size(bedges)[1]
       e=elist[bedges[i],:]
       md=mid_point[e[1],e[2]]
       e2=vcat(e2,md)
    end
    return p,t2,e2
end

using SymPy

function gquad(k, i, j, cmx, p2, t2, ak)
    f(x,y) = 4x^2*(cmx[5,i]*cmx[5,j]) + 2x*y*(cmx[5,i]*cmx[4,j] + cmx[5,j]*cmx[4,i]) + y^2*(cmx[4,i]*cmx[4,j]) + 2x*(cmx[5,i]*cmx[2,j] + cmx[5,j]*cmx[2,i]) + y*(cmx[2,i]*cmx[4,j] + cmx[2,j]*cmx[4,i]) + cmx[2,i]*cmx[2,j]
    g(x,y) = 4y^2*(cmx[6,i]*cmx[6,j]) + 2x*y*(cmx[6,i]*cmx[4,j] + cmx[6,j]*cmx[4,i]) + x^2*(cmx[4,i]*cmx[4,j]) + 2y*(cmx[6,i]*cmx[3,j] + cmx[6,j]*cmx[3,i]) + x*(cmx[3,i]*cmx[4,j] + cmx[3,j]*cmx[4,i]) + cmx[3,i]*cmx[3,j]

    result = 0
    for n = 1:3
	base = (p2[t2[k,1],:] + p2[t2[k,2],:] + p2[t2[k,3],:])/6
        num = base + p2[t2[k,n],:]/2
        result = result + f(num[1],num[2]) + g(num[1],num[2])
    end
    result = (result*ak)/3
    return result
end

function gquad2(k, i, cmx, p2, t2, ak)
    h(x,y) = cmx[1,i] + x*cmx[2,i] + y*cmx[3,i] + x*y*cmx[4,i] + x^2*cmx[5,i] + y^2*cmx[6,i]
    result = 0
    for n = 1:3
	base = (p2[t2[k,1],:] + p2[t2[k,2],:] + p2[t2[k,3],:])/6
        num = base + p2[t2[k,n],:]/2
        result = result + h(num[1],num[2])
    end
    result = (result*ak)/3
    return result
end

function coeff2(RHS, p2, t2, k)
    nm = p2[t2[k,1:6],:]
    LHS = zeros(6,6)
    for n = 1:6
       LHS[n,:] = [1 nm[n,1] nm[n,2] nm[n,1]*nm[n,2] nm[n,1]^2 nm[n,2]^2]
    end
    return (LHS\RHS), LHS
end

function fempoi2(p2, t2, e2)
    areas = triarea(p2,t2)
    n = size(p2)[1]
    A = spzeros(n,n); b = zeros(n);

    for k = 1:size(t2)[1]
        insert = zeros(6,6)
        insert2 = zeros(6)
        cmx, lhss = coeff2(Matrix{Float64}(I, 6, 6), p2, t2, k)
        for i = 1:6
            for j = 1:6
                insert[i,j] = gquad(k,i,j,cmx,p2,t2,areas[k])
            end
            insert2[i] = gquad2(k,i,cmx,p2,t2,areas[k])
        end
        A[t2[k,:],t2[k,:]] = A[t2[k,:],t2[k,:]]+insert
        b[t2[k,:]] = b[t2[k,:]]+ insert2
    end
    for k = 1:size(e2)[1]
        i=e2[k]
        A[i,:].=0.0
        A[i,i]=1.0
        b[i]=0.0
    end

    return A\b

end

function convtest(pv, hmax, nrefmax)
    p, t, e = pmesh(pv,hmax,nrefmax)
    p2, t2, e2 = p2mesh(p,t)
    umax = fempoi2(p2,t2,e2)
    errors = zeros(nrefmax)
    for ncur = 0:(nrefmax-1)
       pprox, tprox, eprox = pmesh(pv, hmax, ncur)
       pprox2, tprox2, eprox2 = p2mesh(pprox, tprox)
       uprox = fempoi2(pprox2, tprox2, eprox2)
       errorarray = maximum(abs.(umax[1:size(uprox)[1]] - uprox))
       #errors = [errors ; errorarray]
       errors[ncur+1]=errorarray
    end

    return errors
end

"errors = convtest(pv,hmax,nrefmax)

clf()
loglog(hmax ./ [1,2,4,8], errors)
rates = @. log2(errors[end-1,:]) - log2(errors[end,:])

@. log2(errors[end-1,:])"


#-------------------------------------------------------------------------------
# Set of functions useful for the execution of the main algorithm.
#-------------------------------------------------------------------------------

"""

    predefined_sample_radius(a::Vector{Float64}, b::Vector{Float64})

Calculates the initial value of the radius of the sample region 'δ' based on the lower and upper limits 'a' and 'b'.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

Returns the initial value of 'δ'.

"""
function predefined_sample_radius(
                                    a::Vector{Float64},
                                    b::Vector{Float64}
                                    )

    n = min( length(a), length(b) )
    aux = b[1] - a[1]
    aux2 = 0.0

    for i = 2:n

        aux2 = b[i] - a[i]

        if aux2 < aux

            aux = aux2

        end

    end

    return min( aux / 2.0, 1.0 )

end

"""

    verify_initial_room(n::Int64, δ::Float64, a::Vector{Float64},
                        b::Vector{Float64})

Checks whether the bounds satisfy the conditions 'b' >= 'a' + 2 * 'δ'.

    - 'n': dimension of the search space.

    - 'δ': radius of the sample set.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

Returns a boolean value.

"""
function verify_initial_room(
                                n::Int64,
                                δ::Float64,
                                a::Vector{Float64},
                                b::Vector{Float64}
                                )

    for i = 1:n

        if ( b[i] - a[i] ) < ( 2.0 * δ )

            return false

        end

    end

    return true

end

"""

    correct_guess_bounds!(n::Int64, δ::Float64, a::Vector{Float64},
                            b::Vector{Float64}, x::Vector{Float64},
                            ao::Vector{Float64}, bo::Vector{Float64})

Fixes initial guess 'x' and sets the lower and upper bound vectors 'ao' and 'bo'
shifted to the origin 'x'.

    - 'n': dimension of the search space.

    - 'δ': radius of the sample set.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

The function modifies the argument:

    - 'x': n-dimensional vector (initial guess).

    - 'ao': n-dimensional vector with the shifted lower bounds.

    - 'bo': n-dimensional vector with the shifted upper bounds.

"""
function correct_guess_bounds!(
                                n::Int64,
                                δ::Float64,
                                a::Vector{Float64},
                                b::Vector{Float64},
                                x::Vector{Float64},
                                ao::Vector{Float64},
                                bo::Vector{Float64}
                                )

    for i=1:n

        ao[i] = a[i] - x[i]
        bo[i] = b[i] - x[i]

        if ao[i] >= - δ

            if ao[i] >= 0.0

                x[i] = a[i]
                ao[i] = 0.0
                bo[i] = b[i] - a[i]

            else

                x[i] = a[i] + δ
                ao[i] = - δ
                bo[i] = max( b[i] - x[i], δ )

            end

        elseif bo[i] <= δ

            if bo[i] <= 0.0

                x[i] = b[i]
                ao[i] = a[i] - b[i]
                bo[i] = 0.0

            else

                x[i] = b[i] - δ
                ao[i] = min( a[i] - x[i], - δ )
                bo[i] = δ

            end

        end

    end

end

"""

    relative_reduction(model::AbstractModel, func_list::Array{Function, 1},
                        pred_red::Float64, y::Vector{Float64})

Computes the simplified relative reduction (using f_{i} instead of fmin) in the point 'y'.

    - 'model': model of LinearModel or QuadraticModel type.

    - 'func_list': list containing the functions that determine the objective
    function fmin.

    - 'pred_red': precalculated predicted reduction.

    - 'y': n-dimensional vector (point of interest).

Returns the value of the simplified relative reduction, the real reduction, the value of f_i(y) and the index 'model.imin'.

"""
function relative_reduction(
                            model::AbstractModel,
                            func_list::Array{Function, 1},
                            pred_red::Float64,
                            y::Vector{Float64}
                            )

    f_y = fi_eval(func_list, model.imin[], y)
    real_red = model.fval[model.kopt[] + 1] - f_y
    ρ = real_red / pred_red

    return ρ, real_red, f_y, model.imin[]

end

"""

    relative_reduction(model::AbstractModel, func_list::Array{Function, 1}, r::Int64,
                        pred_red::Float64, y::Vector{Float64}, imin_set::Vector{Bool})

Computes the relative reduction obtained in the point 'y'.

    - 'model': model of LinearModel or QuadraticModel type.

    - 'func_list': list containing the functions that determine the objective
    function fmin.

    - 'r': number of functions that make up the objective function fmin.

    - 'pred_red': precalculated predicted reduction.

    - 'y': n-dimensional vector (point of interest).

The function modifies the argument:

    - 'imin_set': boolean vector with the indexes belonging to the I_{min}(y) set.

Returns the value of the relative reduction, the real reduction, the value of fmin(y) and an index 'idx_y' belonging to Imin(y).

"""
function relative_reduction!(
                            model::AbstractModel,
                            func_list::Array{Function, 1},
                            r::Int64,
                            pred_red::Float64,
                            y::Vector{Float64},
                            imin_set::Vector{Bool}
                            )

    fmin_y, idx_y = fmin_eval!(func_list, r, y, imin_set)
    real_red = model.fval[model.kopt[] + 1] - fmin_y
    ρ = real_red / pred_red

    return ρ, real_red, fmin_y, idx_y

end

"""

    reconstruct_original_point!(model::AbstractModel, idx::Int64, a::Vector{Float64}, b::Vector{Float64},
                                ao::Vector{Float64}, bo::Vector{Float64}, x::Vector{Float64})

Constructs the model based in the given information.

    - 'model': model of LinearModel or QuadraticModel type.

    - 'idx': position of the point in 'Y' set.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

    - 'ao': n-dimensional vector with the shifted lower bounds.

    - 'bo': n-dimensional vector with the shifted upper bounds.

The function modifies the argument:

    - 'x': n-dimensional vector (point of interest).

"""
function reconstruct_original_point!(
                                    model::AbstractModel,
                                    idx::Int64,
                                    a::Vector{Float64},
                                    b::Vector{Float64},
                                    ao::Vector{Float64},
                                    bo::Vector{Float64},
                                    x::Vector{Float64}
                                    )

    for i = 1:model.n

        if model.Y[idx, i] == ao[i]

            x[i] = a[i]

        elseif model.Y[idx, i] == bo[i]

            x[i] = b[i]

        else

            x[i] = min( max( a[i], model.xbase[i] + model.Y[idx, i] ), b[i] )

        end

    end

end

"""

    print_warning(flag::Int64)

Prints information about the exit flag.

    - 'flag': exit flag.

"""
function print_warning(
                        exit_flag::Symbol
                        )

    if exit_flag == :success

        printstyled("Success: ", bold=true, color=:light_green)
        println("The algorithm succeeded.")

    elseif exit_flag == :max_iterations

        printstyled("Warning: ", bold=true, color=:light_red)
        println("The maximum number of iterations has been reached.")

    elseif exit_flag == :max_evaluations

        printstyled("Warning: ", bold=true, color=:light_red)
        println("The maximum number of function evaluations has been reached.")

    elseif exit_flag == :nondescent

        printstyled("Warning: ", bold=true, color=:light_red)
        println("The calculated direction is not descent for the model.")

    elseif exit_flag == :better_point

        printstyled("Notice: ", bold=true, color=:light_yellow)
        println("A better value will be obtained when evaluating the function on 'xopt'.")

    else

        printstyled("Warning: ", bold=true, color=:light_red)
        println("Exit flag not specified.")

    end
    println("--------------------------------------------------------------------------------")

end

"""

    print_iteration(it_flag::Symbol, status_flag::Symbol, full_calc::Bool, nit::Int64,
                        nf::Int64, imin_idx::Int64, δ::Float64, Δ::Float64, π::Float64,
                        ρ::Float64, pred_red::Float64, real_red::Float64, fopt::Float64,
                        xopt::Vector{Float64}, d::Vector{Float64})

Prints information about the iterations.

    - 'it_flag': iteration flag.

    - 'status_flag': status flag.

    - 'full_calc': indicates if ρ was full calculated.

    - 'nit': number of iterations.

    - 'nf': number of function evaluations.

    - 'imin_idx': index corresponding to the available function value.

    - 'δ': sample set radius.

    - 'Δ': trust-region radius.

    - 'π': stationarity measure.

    - 'ρ': relative reduction.

    - 'pred_red': predicted reduction.

    - 'real_red': real reduction.

    - 'fopt': function value in 'xopt'

    - 'xopt': best point so far.

    - 'd': last computed direction.

"""
function print_iteration(
                            it_flag::Symbol,
                            status_flag::Symbol,
                            full_calc::Bool,
                            nit::Int64,
                            nf::Int64,
                            imin_idx::Int64,
                            δ::Float64,
                            Δ::Float64,
                            π::Float64,
                            ρ::Float64,
                            pred_red::Float64,
                            real_red::Float64,
                            fopt::Float64,
                            xopt::Vector{Float64},
                            d::Vector{Float64}
                            )

    if nit == 0

        println("--------------------------------------------------------------------------------")
        println("--------------------------------------------------------------------------------")

    end

    if ( nit == 0 && it_flag == :max_evaluations )

        println("Iteration     : $( nit )")
        println("Func. eval.   : $( nf )")
        println("Radius δ      : $( δ )")
        println("Radius Δ      : $( Δ )")
        println("I_min index   : $( imin_idx )")
        println("Best point    : $( xopt )")
        println("Func. val.    : $( fopt )")
        println("Iter. type    : $( it_flag )")
        println("--------------------------------------------------------------------------------")

    elseif it_flag == :nonspecified

        println("Iteration     : $( nit )")
        println("Func. eval.   : $( nf )")
        println("Radius δ      : $( δ )")
        println("Radius Δ      : $( Δ )")
        println("Stationarity π: $( π )")
        println("I_min index   : $( imin_idx )")
        println("Best point    : $( xopt )")
        println("Func. val.    : $( fopt )")
        println("Iter. type    : $( it_flag )")
        println("--------------------------------------------------------------------------------")

    elseif it_flag == :criticality

        println("Iteration     : $( nit )")
        println("Func. eval.   : $( nf )")
        println("Radius δ      : $( δ )")
        println("Radius Δ      : $( Δ )")
        println("Stationarity π: $( π )")
        println("I_min index   : $( imin_idx )")
        println("Best point    : $( xopt )")
        println("Func. val.    : $( fopt )")
        println("Iter. type    : $( it_flag )")
        println("Step status   : $( status_flag )")
        println("Direction d   : $( d )")
        println("--------------------------------------------------------------------------------")

    else

        println("Iteration     : $( nit )")
        println("Func. eval.   : $( nf )")
        println("Radius δ      : $( δ )")
        println("Radius Δ      : $( Δ )")
        println("Stationarity π: $( π )")
        println("I_min index   : $( imin_idx )")
        println("Best point    : $( xopt )")
        println("Func. val.    : $( fopt )")
        println("Iter. type    : $( it_flag )")
        println("Step status   : $( status_flag )")
        println("Direction d   : $( d )")
        println("Full ρ        : $( full_calc )")
        println("Rel. reduc. ρ : $( ρ )")
        println("Pred. reduc.  : $( pred_red )")
        println("Real reduc.   : $( real_red )")
        println("--------------------------------------------------------------------------------")

    end

end

"""

    print_info(model::AbstractModel, output::LOWDEROutput, exit_flag::Symbol, it_flag::Symbol,
                status_flag::Symbol, verbose::Int64, nit::Int64, nf::Int64, δ::Float64, Δ::Float64,
                π::Float64, full_calc::Bool, pred_red::Float64, real_red::Float64, ρ::Float64,
                d::Vector{Float64})

Prints information about the last iteration of LOWDER.

    - 'model': model of LinearModel or QuadraticModel type.

    - 'output': object of LOWDEROutput type.

    - 'it_flag': iteration flag.

    - 'status_flag': status flag.

    - 'full_calc': indicates if fmin was calculated at 'xopt'.

    - 'verbose': level of vebosity.

    - 'δ': sample set radius.

    - 'Δ': trust-region radius.

    - 'π': stationarity measure.

    - 'ρ': relative reduction.

    - 'pred_red': predicted reduction.

    - 'real_red': real reduction.

    - 'd': last computed direction.

"""
function print_info(
                    model::AbstractModel,
                    output::LOWDEROutput,
                    it_flag::Symbol,
                    status_flag::Symbol,
                    full_calc::Bool,
                    verbose::Int64,
                    δ::Float64,
                    Δ::Float64,
                    π::Float64,
                    ρ::Float64,
                    pred_red::Float64,
                    real_red::Float64,
                    d::Vector{Float64}
                    )

    if verbose != 0

        print_iteration(it_flag, status_flag, full_calc, output.iter, output.nf, output.index, δ, Δ, π, ρ, pred_red, real_red, output.f, output.solution, d)

        if verbose ≥ 2

            print_warning(output.status)

            if !( model.kopt[] == 0 || full_calc )

                print_warning(:better_point)

            end

            if verbose == 3

                show_output(output)

            end

        end

    end

end

"""

    projection_active_set!(v::Vector{Float64}, active_set::Vector{Bool},
                            proj_v::Vector{Float64}; sym::Bool=false)

Computes the projection of the vector 'v' on the set of active constraints

    - 'v': n-dimensional vector (point of interest).

    - 'active_set': boolean n-dimensional vector with the active constraints.

The function modifies the argument:

    - 'proj_v': n-dimensional vector (projection of 'v').

The optional argument is:

    - 'sym': indicates if it is calculated based on the symmetric direction of 'v'. Set to 'false' by default.

"""
function projection_active_set!(
                                v::Vector{Float64},
                                active_set::Vector{Bool},
                                proj_v::Vector{Float64};
                                sym::Bool=false
                                )

    for i=1:length(v)

        if active_set[i]

            proj_v[i] = 0.0

        else

            if sym

                proj_v[i] = - v[i]

            else

                proj_v[i] = v[i]

            end

        end

    end

end

"""

    step_projection!(model::AbstractModel, a::Vector{Float64},
                        b::Vector{Float64}, x::Vector{Float64},
                        d::Vector{Float64})

Projects 'x' := 'xopt' + 'd' over the box constraints and
modifies 'd' to conform to the new point 'x'.

    - 'model': model of LinearModel or QuadraticModel type.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

The function modifies the argument:

    - 'x': n-dimensional vector (point of interest).

    - 'd': n-dimensional vector (direction).

"""
function step_projection!(
                            model::AbstractModel,
                            a::Vector{Float64},
                            b::Vector{Float64},
                            x::Vector{Float64},
                            d::Vector{Float64}
                            )

    for i = 1:model.n

        x[i] = max( a[i], min( model.xopt[i] + d[i], b[i] ) )
        d[i] = x[i] - model.xopt[i]

    end

end

"""

    compute_alpha_linear(Δ::Float64, a::Vector{Float64}, b::Vector{Float64},
                            x::Vector{Float64}, d::Vector{Float64}, s::Vector{Float64})

Calculates the step length α for 'trsbox!' routine if the model is linear.

    - 'Δ': trust-region radius.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

    - 'x': n-dimensional vector (current point).

    - 'd': n-dimensional vector (first direction).

    - 's': n-dimensional vector (second direction).

Returns the value of α, a boolean indicating whether α_Δ is chosen,
and an index indicating which box constraint is met in the case where α_B is chosen.

"""
function compute_alpha_linear(
                                Δ::Float64,
                                a::Vector{Float64},
                                b::Vector{Float64},
                                x::Vector{Float64},
                                d::Vector{Float64},
                                s::Vector{Float64}
                                )

    roots = []
    α_j = Inf
    α_B = Inf
    idx = 0

    # Computes α_B, the largest number such that 'a' ≤ 'x' + 'd' + 'α'*'s' ≤ 'b'.
    for j=1:length(x)

        if s[j] < 0.0

            α_j = ( a[j] - x[j] - d[j] ) / s[j]

        elseif s[j] > 0.0

            α_j = ( b[j] - x[j] - d[j] ) / s[j]

        end

        if α_j < α_B

            α_B = α_j
            idx = j

        end

    end

    # Computes α_Δ, the largest number such that ||'d' + 'α'*'s'|| ≤ 'Δ'.

    a_2 = dot(s, s)
    a_1 = 2.0 * dot(d, s)
    a_0 = dot(d, d) - Δ ^ 2.0

    solve_quadratic!(a_2, a_1, a_0, roots)

    α_Δ = maximum(roots)

    if ( α_Δ == NaN ) || ( α_Δ < 0.0 )

        return α_B, false, idx

    else

        α = min(α_B, α_Δ)

        if α == α_Δ

            return α, true, 0

        else

            return α, false, idx

        end

    end

end

"""

    new_search_direction!(pdTpd::Float64, pdTpg::Float64, pgTpg::Float64,
                            proj_d::Vector{Float64}, proj_grad::Vector{Float64})

Calculates a new search direction for the 'trsbox!' routine.

    - 'pdTpd': product of the direction 'd' by itself.

    - 'pdTpg': product of the projection of the direction 'd' by the gradient vector
                of the model evaluated in 'xopt'.

    - 'pgTpg': product of the projection of the gradient vector of the model evaluated in 'xopt' by itself.

    - 'proj_d': projection of the search direction 'd'

The function modifies the argument:

    - 'proj_grad': projection of the gradient of the model in 'xopt'.

"""
function new_search_direction!(
                                pdTpd::Float64,
                                pdTpg::Float64,
                                pgTpg::Float64,
                                proj_d::Vector{Float64},
                                proj_grad::Vector{Float64}
                                )

    roots = []
    α = 0.0
    β = 0.0

    if pdTpg ≈ 0.0

        # If P_{I}(d)'P_{I}(∇model(xk+d)) = 0, then the new direction is a multiple of P_{I}(∇Q(xk+d)).

        solve_quadratic!( pgTpg, 0.0, pdTpd, roots)

        β = minimum(roots)

        if ( β == NaN ) || ( β > 0.0 )

            @. proj_grad = 0.0

        else

            @. proj_grad *= β

        end

    else

        # Otherwise, the new search direction is a linear combination of P_{I}(d) and P_{I}(∇model(xk+d)).

        aux = ( pdTpd ^ 2.0 * pgTpg ) / pdTpg ^ 2.0 - pdTpd

        solve_quadratic!( aux, 0.0, - pdTpd, roots )

        α = maximum(roots)

        if ( α == NaN ) || ( α * pdTpd ≈ 0.0 )

            @. proj_grad = 0.0

        else

            β = - pgTpg / ( α * pdTpd )

            if ( α * pdTpg + β * pgTpg ) < 0.0

                @. proj_grad = α * proj_d + β * proj_grad

            else

                α = minimum(roots)

                if ( α == NaN ) || ( α * pdTpd ≈ 0.0 )

                    @. proj_grad = 0.0

                else

                    β = - pgTpg / ( α * pdTpd )

                    if ( α * pdTpg + β * pgTpg ) < 0.0

                        @. proj_grad = α * proj_d + β * proj_grad

                    else

                        @. proj_grad = 0.0

                    end

                end

            end

        end

    end

end

"""

    solve_quadratic!(a::Float64, b::Float64, c::Float64,
                        roots::Vector{Any})

Calculates the roots of the quadratic equation a * x ^ 2 + b * x + c = 0.

    - 'a': quadratic term.

    - 'b': linear term.

    - 'c': constant.

The function modifies the argument:

    - 'roots': vector with the roots of the quadratic equation.

"""
function solve_quadratic!(
                            a::Float64,
                            b::Float64,
                            c::Float64,
                            roots::Vector{Any}
                            )

    if a ≈ 0.0

        root = - c / b

        if !( isinf(root) )

            append!(roots, root)

        elseif ( b ≈ 0.0 ) && ( c ≈ 0.0 )

            append!(roots, 0.0)

        else

            append!(roots, NaN)

        end

    else

        b = b / a
        c = c / a

        if c ≈ 0.0

            append!(roots, 0.0)
            append!(roots, -b)

        else

            Δ = b ^ 2.0 - 4.0 * c

            if isinf(Δ)

                root_1 = -b
                root_2 = c / root_1

                if isinf(root_2)

                    append!(roots, root_1)

                else

                    append!(roots, root_1)
                    append!(roots, root_2)

                end

            else

                if Δ < 0.0

                    append!(roots, NaN)

                elseif Δ == 0.0

                    append!(roots, - 0.5 * b)

                else

                    if b < 0.0

                        root_1 = 0.5 * ( -b + sqrt(Δ) )

                    else

                        root_1 = 0.5 * ( - b - sqrt(Δ) )

                    end

                    root_2 = c / root_1

                    if isinf(root_2)

                        append!(roots, root_1)

                    else

                        append!(roots, root_1)
                        append!(roots, root_2)

                    end

                end

            end

        end

    end

end

"""

    binary_search(lower_value::Float64, upper_value::Float64,
                    stop_condition::Function, ε::Float64)

Performs a binary search on the range [low_value, high_value] to
satisfy 'stop_condition' with tolerance 'ϵ'.

    - 'lower_value': lower value.

    - 'upper_value': upper value.

    - 'stop_condition': stop condition.

    - 'ϵ': tolerance.

Returns a estimate that satisfy the 'stop_condition'.

"""
function binary_search(
                        lower_value::Float64,
                        upper_value::Float64,
                        stop_condition::Function,
                        ε::Float64
                        )

    if !( stop_condition(lower_value) )

        return NaN

    elseif stop_condition(upper_value)

        return upper_value

    else

        while true

            mean_value = (lower_value + upper_value) / 2.0

            if stop_condition(mean_value)

                if (upper_value - mean_value) ≤ ε

                    return mean_value

                else

                    lower_value = mean_value

                end

            else

                if (mean_value - lower_value) ≤ ε

                    return lower_value

                else

                    upper_value = mean_value

                end

            end

        end

    end

end

"""

    cond_θB(θ::Float64, a::Vector{Float64}, b::Vector{Float64}, xopt::Vector{Float64},
        d::Vector{Float64}, proj_d::Vector{Float64}, s::Vector{Float64})

Verifies in 'Θ' satisfies the ΘB condition for 'trsbox!' routine.

    - 'Θ': value that will be tested.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

    - 'xopt': best sample point so far.

    - 'd': first search direction 'd'.

    - 'proj_d': projection of the direction 'd'.

    - 's': second search direction 's'.

Returns a boolean.

"""
function cond_θB(
                    θ::Float64,
                    a::Vector{Float64},
                    b::Vector{Float64},
                    xopt::Vector{Float64},
                    d::Vector{Float64},
                    proj_d::Vector{Float64},
                    s::Vector{Float64}
                    )

    for i=1:length(a)

        xnewi = xopt[i] + d[i] - proj_d[i] + cos(θ) * proj_d[i] + sin(θ) * s[i]

        if ( xnewi < a[i] ) || ( xnewi > b[i] )

            return false

        end

    end

    return true

end

"""

    cond_θQ_linear(θ::Float64, gTd::Float64, gTpd::Float64, gTs::Float64)

Verifies in 'Θ' satisfies the ΘQ condition for 'trsbox!' routine, in the liner case.

    - 'Θ': value that will be tested.

    - 'gTd': product of the direction 'd' by the gradient vector of the model evaluated in 'xopt'.

    - 'gTd': product of the projection of the direction 'd' by the gradient vector of the
    model evaluated in 'xopt'.

    - 'gTs': product of the direction 's' by the gradient vector of the model evaluated in 'xopt'.

Returns a boolean.

"""
function cond_θQ_linear(
                        θ::Float64,
                        gTd::Float64,
                        gTpd::Float64,
                        gTs::Float64
                        )

    if ( gTd - gTpd + cos(θ) * gTpd + sin(θ) * gTs ) < 0.0

        return true

    else

        return false

    end

end

"""

    update_active_set!(a::Vector{Float64}, b::Vector{Float64},
                        xopt::Vector{Float64}, d::Vector{Float64},
                        active_set::Vector{Bool})

Updates the active set.

    - 'a': n-dimensional vector with the lower bounds.

    - 'b': n-dimensional vector with the upper bounds.

    - 'xopt': best sample point so far.

    - 'd': search direction 'd'.

The function modifies the argument:

    - 'active_set': boolean n-dimensional vector with the active constraints.

"""
function update_active_set!(
                            a::Vector{Float64},
                            b::Vector{Float64},
                            xopt::Vector{Float64},
                            d::Vector{Float64},
                            active_set::Vector{Bool}
                            )

    for i=1:length(a)

        if !( active_set[i] )

            xnewi = xopt[i] + d[i]

            if ( a[i] == xnewi ) || ( xnewi == b[i] )

                active_set[i] == true

            end

        end

    end

end

"""

    save_info!(model::AbstractModel, it_flag::Symbol, status_flag::Symbol,
                full_calc::Bool, nit::Int64, nf::Int64, δ::Float64,
                Δ::Float64, π::Float64, ρ::Float64, pred_red::Float64,
                real_red::Float64, d::Vector{Float64}, file::IO)

Saves information about the iterations.

    - 'model': model of LinearModel or QuadraticModel type.

    - 'it_flag': iteration flag.

    - 'status_flag': status flag.

    - 'full_calc': indicates if ρ was full calculated.

    - 'nit': number of iterations.

    - 'nf': number of function evaluations.

    - 'δ': sample set radius.

    - 'Δ': trust-region radius.

    - 'π': stationarity measure.

    - 'ρ': relative reduction.

    - 'pred_red': predicted reduction.

    - 'real_red': real reduction.

    - 'd': last computed direction.

    - 'file': IO.

"""
function save_info!(
                    model::AbstractModel,
                    it_flag::Symbol,
                    status_flag::Symbol,
                    full_calc::Bool,
                    nit::Int64,
                    nf::Int64,
                    δ::Float64,
                    Δ::Float64,
                    π::Float64,
                    ρ::Float64,
                    pred_red::Float64,
                    real_red::Float64,
                    d::Vector{Float64},
                    file::IO
                    )

    text = "$( nit ) $( nf ) $( model.imin[] ) $( model.kopt[] ) $( δ ) $( Δ ) $( π ) $( it_flag ) $( status_flag ) $( full_calc ) $( ρ ) $( pred_red ) $( real_red ) $( model.c[] )"

    print( file, text )

    for i = 1:model.n

        print( file, " $( model.g[i] )" )

    end

    for i = 1:model.n

        print( file, " $( model.xbase[i] )" )

    end

    for i = 1:model.n

        print( file, " $( model.xopt[i] )" )

    end

    for i = 1:model.m

        print( file, " $( model.fval[i] )" )

    end

    for i = 1:(model.m - 1)

        for j = 1:model.n

            print( file, " $( model.Y[i, j] )" )

        end

    end

    for i = 1:(model.n - 1)

        print( file, " $( d[i] )" )

    end

    println( file, " $( d[model.n] )" )

end

"""

    choose_index_altmov(model::LinearModel)

Chooses the point that must leave the sampling set for 'altmov' steps.

    - 'model': model of LinearModel or QuadraticModel type.

Returns an index 't' which indicates the point that must leave the sample set.

"""
function choose_index_altmov(
                                model::AbstractModel
                            )

    ( val, idx_t ) = findmax( model.dst )

    if idx_t == model.kopt[]

        idx_t = 0

    end

    return idx_t

end


using ModelingToolkit, StaticArrays, LinearAlgebra, SparseArrays
using DiffEqBase
using Test

canonequal(a, b) = isequal(simplify(a), simplify(b))

# Calculus
@parameters t σ ρ β
@variables x y z
@test isequal((Differential(z) * Differential(y) * Differential(x))(t),
              Differential(z)(Differential(y)(Differential(x)(t))))

@test canonequal(ModelingToolkit.derivative(sin(cos(x)), x),
                 -sin(x) * cos(cos(x)))

@register_symbolic no_der(x)
@test canonequal(ModelingToolkit.derivative([sin(cos(x)), hypot(x, no_der(x))], x),
                 [
                     -sin(x) * cos(cos(x)),
                     x / hypot(x, no_der(x)) +
                     no_der(x) * Differential(x)(no_der(x)) / hypot(x, no_der(x)),
                 ])

@register_symbolic intfun(x)::Int
@test ModelingToolkit.symtype(intfun(x)) === Int

eqs = [σ * (y - x),
    x * (ρ - z) - y,
    x * y - β * z]

simpexpr = [:($(*)(σ, $(+)(y, $(*)(-1, x))))
            :($(+)($(*)(x, $(+)(ρ, $(*)(-1, z))), $(*)(-1, y)))
            :($(+)($(*)(x, y), $(*)(-1, z, β)))]

σ, β, ρ = 2 // 3, 3 // 4, 4 // 5
x, y, z = 6 // 7, 7 // 8, 8 // 9
for i in 1:3
    @test eval(ModelingToolkit.toexpr.(eqs)[i]) == eval(simpexpr[i])
    @test eval(ModelingToolkit.toexpr.(eqs)[i]) == eval(simpexpr[i])
end

@parameters t σ ρ β
@variables x y z
∂ = ModelingToolkit.jacobian(eqs, [x, y, z])
for i in 1:3
    ∇ = ModelingToolkit.gradient(eqs[i], [x, y, z])
    @test canonequal(∂[i, :], ∇)
end

@test all(canonequal.(ModelingToolkit.gradient(eqs[1], [x, y, z]), [σ * -1, σ, 0]))
@test all(canonequal.(ModelingToolkit.hessian(eqs[1], [x, y, z]), 0))

du = [x^2, y^3, x^4, sin(y), x + y, x + z^2, z + x, x + y^2 + sin(z)]
reference_jac = sparse(ModelingToolkit.jacobian(du, [x, y, z]))

@test findnz(ModelingToolkit.jacobian_sparsity(du, [x, y, z]))[[1, 2]] ==
      findnz(reference_jac)[[1, 2]]

let
    @variables t x(t) y(t) z(t)
    @test ModelingToolkit.exprs_occur_in([x, y, z], x^2 * y) == [true, true, false]
end

@test isequal(ModelingToolkit.sparsejacobian(du, [x, y, z]), reference_jac)

using ModelingToolkit

rosenbrock(X) =
    sum(1:(length(X) - 1)) do i
        100 * (X[i + 1] - X[i]^2)^2 + (1 - X[i])^2
    end

@variables a, b
X = [a, b]

spoly(x) = simplify(x, expand = true)
rr = rosenbrock(X)

reference_hes = ModelingToolkit.hessian(rr, X)
@test findnz(sparse(reference_hes))[1:2] ==
      findnz(ModelingToolkit.hessian_sparsity(rr, X))[1:2]

sp_hess = ModelingToolkit.sparsehessian(rr, X)
@test findnz(sparse(reference_hes))[1:2] == findnz(sp_hess)[1:2]
@test isequal(map(spoly, findnz(sparse(reference_hes))[3]), map(spoly, findnz(sp_hess)[3]))

Joop, Jiip = eval.(ModelingToolkit.build_function(∂, [x, y, z], [σ, ρ, β], t))
J = Joop([1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J isa Matrix
J2 = copy(J)
Jiip(J2, [1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J2 == J

Joop, Jiip = eval.(ModelingToolkit.build_function(vcat(∂, ∂), [x, y, z], [σ, ρ, β], t))
J = Joop([1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J isa Matrix
J2 = copy(J)
Jiip(J2, [1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J2 == J

Joop, Jiip = eval.(ModelingToolkit.build_function(hcat(∂, ∂), [x, y, z], [σ, ρ, β], t))
J = Joop([1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J isa Matrix
J2 = copy(J)
Jiip(J2, [1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J2 == J

∂3 = cat(∂, ∂, dims = 3)
Joop, Jiip = eval.(ModelingToolkit.build_function(∂3, [x, y, z], [σ, ρ, β], t))
J = Joop([1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test size(J) == (3, 3, 2)
J2 = copy(J)
Jiip(J2, [1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J2 == J

s∂ = sparse(∂)
@test nnz(s∂) == 8
Joop, Jiip = eval.(ModelingToolkit.build_function(s∂, [x, y, z], [σ, ρ, β], t,
                                                  linenumbers = true))
J = Joop([1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test length(nonzeros(s∂)) == 8
J2 = copy(J)
Jiip(J2, [1.0, 2.0, 3.0], [1.0, 2.0, 3.0], 1.0)
@test J2 == J

# Function building

@parameters σ ρ β
@variables x y z
eqs = [σ * (y - x),
    x * (ρ - z) - y,
    x * y - β * z]
f1, f2 = ModelingToolkit.build_function(eqs, [x, y, z], [σ, ρ, β])
f = eval(f1)
out = [1.0, 2, 3]
o1 = f([1.0, 2, 3], [1.0, 2, 3])
f = eval(f2)
f(out, [1.0, 2, 3], [1.0, 2, 3])
@test all(o1 .== out)

function test_worldage()
    @parameters σ ρ β
    @variables x y z
    eqs = [σ * (y - x),
        x * (ρ - z) - y,
        x * y - β * z]
    f, f_iip = ModelingToolkit.build_function(eqs, [x, y, z], [σ, ρ, β];
                                              expression = Val{false})
    out = [1.0, 2, 3]
    o1 = f([1.0, 2, 3], [1.0, 2, 3])
    f_iip(out, [1.0, 2, 3], [1.0, 2, 3])
end
test_worldage()

## No parameters
@variables x y z
eqs = [(y - x)^2,
    x * (x - z) - y,
    x * y - y * z]
f1, f2 = ModelingToolkit.build_function(eqs, [x, y, z])
f = eval(f1)
out = zeros(3)
o1 = f([1.0, 2, 3])
f = eval(f2)
f(out, [1.0, 2, 3])
@test all(out .== o1)

# y ^ -1 test
g = let
    f(x, y) = x / y
    @variables x y
    ex = expand_derivatives(Differential(x)(f(x, y)))
    func_ex = build_function(ex, x, y)
    eval(func_ex)
end

@test g(42, 4) == 1 / 4

function test_worldage()
    @variables x y z
    eqs = [(y - x)^2,
        x * (x - z) - y,
        x * y - y * z]
    f, f_iip = ModelingToolkit.build_function(eqs, [x, y, z]; expression = Val{false})
    out = zeros(3)
    o1 = f([1.0, 2, 3])
    f_iip(out, [1.0, 2, 3])
end
test_worldage()

@test_nowarn muladd(x, y, 0)
@test promote(x, 0) == (x, identity(0))
@test_nowarn [x, y, z]'

let
    @register_symbolic foo(x)
    @variables t
    D = Differential(t)

    @test isequal(expand_derivatives(D(foo(t))), D(foo(t)))
    @test isequal(expand_derivatives(D(sin(t) * foo(t))),
                  cos(t) * foo(t) + sin(t) * D(foo(t)))
end

foo(; kw...) = kw
foo(args...; kw...) = args, kw
pp = :name => :cool_name

@named cool_name = foo()
@test collect(cool_name) == [pp]

@named cool_name = foo(42)
@test cool_name[1] == (42,)
@test collect(cool_name[2]) == [pp]

@named cool_name = foo(42; a = 2)
@test cool_name[1] == (42,)
@test collect(cool_name[2]) == [pp; :a => 2]

@named cool_name = foo(a = 2)
@test collect(cool_name) == [pp; :a => 2]

@named cool_name = foo(; a = 2)
@test collect(cool_name) == [pp; :a => 2]

@named cool_name = foo(name = 2)
@test collect(cool_name) == [:name => 2]

@named cool_name = foo(42; name = 3)
@test cool_name[1] == (42,)
@test collect(cool_name[2]) == [:name => 3]

kwargs = (; name = 3)
@named cool_name = foo(42; kwargs...)
@test cool_name[1] == (42,)
@test collect(cool_name[2]) == [:name => 3]

if VERSION >= v"1.5"
    name = 3
    @named cool_name = foo(42; name)
    @test cool_name[1] == (42,)
    @test collect(cool_name[2]) == [:name => name]
    @named cool_name = foo(; name)
    @test collect(cool_name) == [:name => name]

    ff = 3
    @named cool_name = foo(42; ff)
    @test cool_name[1] == (42,)
    @test collect(cool_name[2]) == [pp; :ff => ff]

    @named cool_name = foo(; ff)
    @test collect(cool_name) == [pp; :ff => ff]
end

foo(i; name) = i, name
@named goo[1:3] = foo(10)
@test isequal(goo, [(10, Symbol(:goo_, i)) for i in 1:3])
@named koo 1:3 i->foo(10i)
@test isequal(koo, [(10i, Symbol(:koo_, i)) for i in 1:3])


# From JuliaLang/Combinatorics

immutable Combinations{T}
    a::T
    t::Int
end

start(c::Combinations) = [1:c.t;]
function next(c::Combinations, s)
    comb = [c.a[si] for si in s]
    if c.t == 0
        # special case to generate 1 result for t==0
        return (comb,[length(c.a)+2])
    end
    s = copy(s)
    for i = length(s):-1:1
        s[i] += 1
        if s[i] > (length(c.a) - (length(s)-i))
            continue
        end
        for j = i+1:endof(s)
            s[j] = s[j-1]+1
        end
        break
    end
    (comb,s)
end
done(c::Combinations, s) = !isempty(s) && s[1] > length(c.a)-c.t+1

length(c::Combinations) = binomial(length(c.a),c.t)

eltype{T}(::Type{Combinations{T}}) = Vector{eltype(T)}

"Generate all combinations of `n` elements from an indexable object. Because the number of combinations can be very large, this function returns an iterator object. Use `collect(combinations(array,n))` to get an array of all combinations.
"
function combinations(a, t::Integer)
    if t < 0
        # generate 0 combinations for negative argument
        t = length(a)+1
    end
    Combinations(a, t)
end

"""
generate combinations of all orders, chaining of order iterators is eager,
but sequence at each order is lazy
"""
combinations(a) = chain([combinations(a,k) for k=1:length(a)]...)


using Test
include("Utilities.jl")

@testset "Utilities" begin
    G1 = MatrixNetwork(load_matrix_network("cores_example")) # Simple networks
    G2 = MatrixNetwork([1,2,2,3,3], [2,4,5,4,5])
    uG2 = make_undirected(G2)

    M1 = MatrixNetwork([1,1,3,3,3,3,3,4,4,6,6,6], [3,1,1,2,2,4,6,6,6,1,2,7])
    M2 = MatrixNetwork([1,1,2,2,2,3,3,4,4,5,5,6,6,6,7,7,7],
                       [2,6,1,1,4,2,5,1,3,3,4,1,6,7,4,4,5])

    @testset "Degree Distribution" begin
        for i = 2:10 # Testing common subgraphs: path, cycle, clique
            @test deg_distr(make_undirected(path(i))) == [1; [2 for j = 1:i-1]; 1]
            @test deg_distr(make_undirected(cycle(i+1))) == [2 for j = 1:i+1]
            @test deg_distr(clique(i)) == [i-1 for j = 1:i]
        end

        @test deg_distr(G1) == [1,4,2,6,5,3,3,4,3,5,4,1,1,3,3,0,2,2,2,3,1]
        @test deg_distr(uG2) == [1,3,2,2,2]
    end

    @testset "Nodes by Degree" begin # Getting sorted list of nodes by their degree
        @test nodes_by_deg(G1) == [16, 1, 12, 13, 21, 3, 17, 18, 19, 6, 7, 9, 14, 15, 20, 2, 8, 11, 5, 10, 4]
        @test nodes_by_deg(G1; descending=true) == [4, 5, 10, 2, 8, 11, 6, 7, 9, 14, 15, 20, 3, 17, 18, 19, 1, 12, 13, 21, 16]

        @test nodes_by_deg(uG2) == [1, 3, 4, 5, 2]
        @test nodes_by_deg(uG2; descending=true) == [2, 3, 4, 5, 1]
    end

    @testset "Get Edges" begin
        function edge_compare(A::MatrixNetwork)
            ei, ej = directed_edges(A)
            return get_edges(A) == [[ei[x],ej[x]] for x = 1:length(ei)]
        end
        @test edge_compare(G1)
        @test edge_compare(G2)
        @test edge_compare(uG2)
        @test edge_compare(M1)
        @test edge_compare(M2)
    end

    @testset "Simplify" begin
        simplify_test_simple(A::MatrixNetwork) = get_edges(simplify(A)) == get_edges(A) # Checks that simplifying a simple graph keeps it simple
        @test simplify_test_simple(G1)
        @test simplify_test_simple(G2)

        function simplify_test(A::MatrixNetwork)
            M = Array(sparse(simplify(A)))
            return Diagonal(M) == zeros(Int64, (A.n, A.n)) && minimum(M) == 0 && maximum(M) == 1
        end

        @test simplify_test(M1)
        @test simplify_test(M2)
    end

    @testset "Directed -> Undirected" begin
        function undirected_test(A::MatrixNetwork)
            U = sparse(make_undirected(A))
            return U == U' == sparse(A) + sparse(A)'
        end

        @test undirected_test(G1)
        @test undirected_test(G2)
        @test undirected_test(uG2)

        @test undirected_test(M1)
        @test undirected_test(M2)
    end

    @testset "Clustering Coefficients" begin
        @test clustering(G1) == (0.3555555555555555, 0.44)
        @test_throws ErrorException clustering(G2)

        @test clustering(make_undirected(M1)) == (0.5898904756952629, 0.585695488070208)
        @test clustering(make_undirected(M2)) == (0.4950303252150675, 0.48426879640604437)
    end

end


# # "Point exponential decay" time-stepper convergence test

using PyPlot

# Define a few utilities for running tests and unpacking and plotting results

include("ConvergenceTests/ConvergenceTests.jl") # we use the GaussianAdvectionDiffusion module here.

defaultcolors = plt.rcParams["axes.prop_cycle"].by_key()["color"]
removespine(side) = gca().spines[side].set_visible(false)
removespines(sides...) = [removespine(side) for side in sides]

run_single_test = ConvergenceTests.PointExponentialDecay.run_test

""" Run advection-diffusion test for all Nx in resolutions. """
function run_convergence_test(stop_time, proposal_Δt...)

    # Adjust time-step
    stop_iterations = [round(Int, stop_time / dt) for dt in proposal_Δt]
                 Δt = [stop_time / stop_iter for stop_iter in stop_iterations]

    # Run the tests
    results = [run_single_test(Δt=dt, stop_iteration=stop_iter)
               for (dt, stop_iter) in zip(Δt, stop_iterations)]

    return results, Δt
end

unpack_errors(results) = map(r -> r.L₁, results)

stop_time = 3
Δt = vcat(1 ./ collect(1000:-100:100), 1 ./ collect(90:-10:10), 1 ./ collect(9:-1:1))
results, Δt = run_convergence_test(stop_time, Δt...)
L₁ = unpack_errors(results)

fig, axs = subplots()

loglog(Δt, L₁, "o--", alpha=0.8, linewidth=1, label="Measured error")

# Guide line to confirm second-order scaling
loglog(Δt, L₁[1] .* Δt / Δt[1], "k-", alpha=0.4, linewidth=2, label=L"\sim \Delta t")

xlabel(L"\Delta t")
ylabel("\$L_1\$-norm of \$ | c_\\mathrm{simulated}(t) - \\mathrm{e}^{-t} | \$")
title("Oceananigans time-stepper convergence for \$ c(t) = \\mathrm{e}^{-t} \$")
removespines("top", "right")
legend()

savefig("figs/point_exponential_decay_time_stepper_convergence.png", dpi=480)


using Plots     # importing the package
gr()            # faster backend

# defining the range
x = 0:0.005:1

m = 1
b = 0
f = m * x .+ b  # linear
g = x .^ 2      # quadratic
h = x .^ 3      # cubic

# plot out the first function
# place legend in top left corner
plot(x, f, label="linear", legend=:topleft)
# edit existing plot
plot!(x, g, label="quadratic")
# increase resolution with dpi parameter
plot!(x, h, label="cubic", dpi=300)

# adding axis and title labels
xaxis!("x [-]")
yaxis!("f(x) [-]")
title!("polynomials")

# save the figure with string name
savefig("myawesomeplot.png")


import Base: rand,
             split

export Zonotope,
       scale,
       reduce_order

"""
    Zonotope{N<:Real} <: AbstractZonotope{N}

Type that represents a zonotope.

### Fields

- `center`     -- center of the zonotope
- `generators` -- matrix; each column is a generator of the zonotope

### Notes

Mathematically, a zonotope is defined as the set

```math
Z = \\left\\{ x ∈ \\mathbb{R}^n : x = c + ∑_{i=1}^p ξ_i g_i,~~ ξ_i \\in [-1, 1]~~ ∀ i = 1,…, p \\right\\},
```
where ``c \\in \\mathbb{R}^n`` is its *center* and ``\\{g_i\\}_{i=1}^p``,
``g_i \\in \\mathbb{R}^n``, is the set of *generators*.
This characterization defines a zonotope as the finite Minkowski sum of line
segments.
Zonotopes can be equivalently described as the image of a unit infinity-norm
ball in ``\\mathbb{R}^n`` by an affine transformation.

Zonotopes can be constructed in two different ways: either passing the generators as a matrix, where
each column represents a generator, or passing a list of vectors where each vector represents a generator.
Below we illustrate both ways.

The optional argument `remove_zero_generators` controls whether we remove zero
columns from the `generators` matrix. This option is active by default.

### Examples

A two-dimensional zonotope with given center and set of generators:

```jldoctest zonotope_label
julia> Z = Zonotope([1.0, 0.0], [0.1 0.0; 0.0 0.1])
Zonotope{Float64}([1.0, 0.0], [0.1 0.0; 0.0 0.1])

julia> dim(Z)
2
```
Here, each column of the second input corresponds to a generator.

We can collect its vertices using `vertices_list`:

```jldoctest zonotope_label
julia> vertices_list(Z)
4-element Array{Array{Float64,1},1}:
 [1.1, 0.1]
 [0.9, 0.1]
 [0.9, -0.1]
 [1.1, -0.1]
```

The support vector along a given direction can be computed using `σ`
(resp. the support function can be computed using `ρ`):

```jldoctest zonotope_label
julia> σ([1., 1.], Z)
2-element Array{Float64,1}:
 1.1
 0.1
```

Zonotopes admit an alternative constructor that receives a list of
vectors, each vector representing a generator:

```jldoctest
julia> Z = Zonotope(ones(2), [[1., 0.], [0., 1.], [1., 1.]])
Zonotope{Float64}([1.0, 1.0], [1.0 0.0 1.0; 0.0 1.0 1.0])

julia> Z.generators
2×3 Array{Float64,2}:
 1.0  0.0  1.0
 0.0  1.0  1.0
```
"""
struct Zonotope{N<:Real} <: AbstractZonotope{N}
    center::AbstractVector{N}
    generators::AbstractMatrix{N}

    function Zonotope(center::AbstractVector{N}, generators::AbstractMatrix{N};
                      remove_zero_generators::Bool=true) where {N<:Real}
        @assert length(center) == size(generators, 1) "the dimension of the " *
            "center ($(length(center))) and the generators " *
            "($(size(generators, 1))) need to match"
        if remove_zero_generators
            generators = delete_zero_columns!(generators)
        end
        new{N}(center, generators)
    end
end

isoperationtype(::Type{<:Zonotope}) = false

# constructor from center and list of generators
Zonotope(center::AbstractVector{N}, generators_list::AbstractVector{VN};
         remove_zero_generators::Bool=true
        ) where {N<:Real, VN<:AbstractVector{N}} =
    Zonotope(center, hcat(generators_list...);
             remove_zero_generators=remove_zero_generators)

# --- AbstractCentrallySymmetric interface functions ---

"""
    center(Z::Zonotope{N})::Vector{N} where {N<:Real}

Return the center of a zonotope.

### Input

- `Z` -- zonotope

### Output

The center of the zonotope.
"""
function center(Z::Zonotope{N})::Vector{N} where {N<:Real}
    return Z.center
end

# --- AbstractZonotope interface functions ---

"""
   genmat(Z::Zonotope)

Return the generator matrix of a zonotope.

### Input

- `Z` -- zonotope

### Output

A matrix where each column represents one generator of the zonotope `Z`.
"""
function genmat(Z::Zonotope)
    return Z.generators
end

"""
    generators(Z::Zonotope)

Return an iterator over the generators of a zonotope.

### Input

- `Z` -- zonotope

### Output

An iterator over the generators of `Z`.
"""
function generators(Z::Zonotope)
    return generators_fallback(Z)
end

"""
    ngens(Z::Zonotope)::Int

Return the number of generators of a zonotope.

### Input

- `Z` -- zonotope

### Output

Integer representing the number of generators.
"""
ngens(Z::Zonotope)::Int = size(Z.generators, 2)

# --- LazySet interface functions ---

"""
    rand(::Type{Zonotope}; [N]::Type{<:Real}=Float64, [dim]::Int=2,
         [rng]::AbstractRNG=GLOBAL_RNG, [seed]::Union{Int, Nothing}=nothing
        )::Zonotope{N}

Create a random zonotope.

### Input

- `Zonotope`       -- type for dispatch
- `N`              -- (optional, default: `Float64`) numeric type
- `dim`            -- (optional, default: 2) dimension
- `rng`            -- (optional, default: `GLOBAL_RNG`) random number generator
- `seed`           -- (optional, default: `nothing`) seed for reseeding
- `num_generators` -- (optional, default: `-1`) number of generators of the
                      zonotope (see comment below)

### Output

A random zonotope.

### Algorithm

All numbers are normally distributed with mean 0 and standard deviation 1.

The number of generators can be controlled with the argument `num_generators`.
For a negative value we choose a random number in the range `dim:2*dim` (except
if `dim == 1`, in which case we only create a single generator).
"""
function rand(::Type{Zonotope};
              N::Type{<:Real}=Float64,
              dim::Int=2,
              rng::AbstractRNG=GLOBAL_RNG,
              seed::Union{Int, Nothing}=nothing,
              num_generators::Int=-1
             )::Zonotope{N}
    rng = reseed(rng, seed)
    center = randn(rng, N, dim)
    if num_generators < 0
        num_generators = (dim == 1) ? 1 : rand(dim:2*dim)
    end
    generators = randn(rng, N, dim, num_generators)
    return Zonotope(center, generators)
end

# --- Zonotope functions ---

"""
    scale(α::Real, Z::Zonotope)

Concrete scaling of a zonotope.

### Input

- `α` -- scalar
- `Z` -- zonotope

### Output

The zonotope obtained by applying the numerical scale to the center and
generators of ``Z``.
"""
function scale(α::Real, Z::Zonotope)
    c = α .* Z.center
    gi = α .* Z.generators
    return Zonotope(c, gi)
end

"""
    reduce_order(Z::Zonotope, r::Union{Integer, Rational})::Zonotope

Reduce the order of a zonotope by overapproximating with a zonotope with less
generators.

### Input

- `Z` -- zonotope
- `r` -- desired order

### Output

A new zonotope with less generators, if possible.

### Algorithm

See `overapproximate(Z::Zonotope{N}, ::Type{<:Zonotope}, r::Union{Integer, Rational}) where {N<:Real}` for details.
"""
function reduce_order(Z::Zonotope{N}, r::Union{Integer, Rational})::Zonotope{N} where {N<:Real}
    return overapproximate(Z, Zonotope, r)
end

"""
    split(Z::Zonotope, j::Int)

Return two zonotopes obtained by splitting the given zonotope.

### Input

- `Z` -- zonotope
- `j` -- index of the generator to be split

### Output

The zonotope obtained by splitting `Z` into two zonotopes such that
their union is `Z` and their intersection is possibly non-empty.

### Algorithm

This function implements [Prop. 3, 1], that we state next. The zonotope
``Z = ⟨c, g^{(1, …, p)}⟩`` is split into:

```math
Z₁ = ⟨c - \\frac{1}{2}g^{(j)}, (g^{(1, …,j-1)}, \\frac{1}{2}g^{(j)}, g^{(j+1, …, p)})⟩ \\\\
Z₂ = ⟨c + \\frac{1}{2}g^{(j)}, (g^{(1, …,j-1)}, \\frac{1}{2}g^{(j)}, g^{(j+1, …, p)})⟩,
```
such that ``Z₁ ∪ Z₂ = Z`` and ``Z₁ ∩ Z₂ = Z^*``, where

```math
Z^* = ⟨c, (g^{(1,…,j-1)}, g^{(j+1,…, p)})⟩.
```

[1] *Althoff, M., Stursberg, O., & Buss, M. (2008). Reachability analysis of
nonlinear systems with uncertain parameters using conservative linearization.
In Proc. of the 47th IEEE Conference on Decision and Control.*
"""
function split(Z::Zonotope, j::Int)
    @assert 1 <= j <= ngens(Z) "cannot split a zonotope with $(ngens(Z)) generators along index $j"
    c, G = Z.center, Z.generators
    Gj = G[:, j]
    Gj_half = Gj / 2

    c₁ = c - Gj_half
    c₂ = c + Gj_half

    G₁ = copy(G)
    G₁[:, j] = Gj_half
    G₂ = copy(G₁)

    Z₁ = Zonotope(c₁, G₁)
    Z₂ = Zonotope(c₂, G₂)
    return Z₁, Z₂
end


include("../aimajulia.jl");

using Test;

using Main.aimajulia;

#The following logic tests are from the aima-python doctests

x = Expression("x");

y = Expression("y");

z = Expression("z");

@test variables(expr("F(x, x) & G(x, y) & H(y, z) & R(A, z, z)")) == Set([x, y, z]);

@test variables(expr("F(x, A, y)")) == Set([x, y]);

@test variables(expr("F(G(x), z)")) == Set([x, z]);

@test show(expr("P & Q ==> Q")) == "((P & Q) ==> Q)";

@test show(expr("P ==> Q(1)")) == "(P ==> Q(1))";

@test show(expr("P & Q | ~R(x, F(x))")) == "((P & Q) | ~(R(x, F(x))))";

@test show(expr("P & Q ==> R & S")) == "(((P & Q) ==> R) & S)";

@test tt_entails(expr("P & Q"), expr("Q")) == true;

@test tt_entails(expr("P | Q"), expr("Q")) == false;

@test tt_entails(expr("A & (B | C) & E & F & ~(P | Q)"), expr("A & E & F & ~P & ~Q")) == true;

@test proposition_symbols(expr("x & y & z | A")) == [Expression("A")];

@test proposition_symbols(expr("(x & B(z)) ==> Farmer(y) | A")) == [Expression("A"), expr("Farmer(y)"), expr("B(z)")];

@test tt_true("(P ==> Q) <=> (~P | Q)") == true;

@test typeof(pl_true(Expression("P"))) <: Nothing;

@test typeof(pl_true(expr("P | P"))) <: Nothing;

@test pl_true(expr("P | Q"), model=Dict([Pair(Expression("P"), true)])) == true;

@test pl_true(expr("(A | B) & (C | D)"),
                model=Dict([Pair(Expression("A"), false),
                            Pair(Expression("B"), true),
                            Pair(Expression("C"), true)])) == true;

@test pl_true(expr("(A & B) & (C | D)"),
                model=Dict([Pair(Expression("A"), false),
                            Pair(Expression("B"), true),
                            Pair(Expression("C"), true)])) == false;

@test pl_true(expr("(A & B) | (A & C)"),
                model=Dict([Pair(Expression("A"), false),
                            Pair(Expression("B"), true),
                            Pair(Expression("C"), true)])) == false;

@test typeof(pl_true(expr("(A | B) & (C | D)"),
                model=Dict([Pair(Expression("A"), true),
                            Pair(Expression("D"), false)]))) <: Nothing;

@test pl_true(Expression("P"), model=Dict([Pair(Expression("P"), false)])) == false;

@test typeof(pl_true(expr("P | ~P"))) <: Nothing;

@test eliminate_implications(expr("A ==> (~B <== C)")) == expr("(~B | ~C) | ~A");

@test eliminate_implications(expr("A ^ B")) == expr("(A & ~B) | (~A & B)");

@test move_not_inwards(expr("~(A | B)")) == expr("~A & ~B");

@test move_not_inwards(expr("~(A & B)")) == expr("~A | ~B");

@test move_not_inwards(expr("~(~(A | ~B) | ~(~C))")) == expr("(A | ~B) & ~C");

@test distribute_and_over_or(expr("(A & B) | C")) == expr("(A | C) & (B | C)");

@test associate("&", (expr("A & B"), expr("B | C"), expr("B & C"))) == expr("&(A, B, (B | C), B, C)");

@test associate("|", (expr("A | (B | (C | (A & B)))"),)) == expr("|(A, B, C, (A & B))");

@test conjuncts(expr("A & B")) == [Expression("A"), Expression("B")];

@test conjuncts(expr("A | B")) == [expr("A | B")];

@test disjuncts(expr("A | B")) == [Expression("A"), Expression("B")];

@test disjuncts(expr("A & B")) == [expr("A & B")];

@test repr(to_conjunctive_normal_form(Expression("&",
                                                aimajulia.wumpus_world_inference,
                                                Expression("~", expr("~P12"))))) ==
                                                "((~(P12) | B11) & (~(P21) | B11) & (P12 | P21 | ~(B11)) & ~(B11) & P12)";

@test to_conjunctive_normal_form(expr("~(B | C)")) == expr("~B & ~C");

@test repr(to_conjunctive_normal_form(expr("~(B | C)"))) == "(~(B) & ~(C))";

@test to_conjunctive_normal_form(expr("(P & Q) | (~P & ~Q)")) == expr("&((~P | P), (~Q | P), (~P | Q), (~Q | Q))");

@test repr(to_conjunctive_normal_form(expr("(P & Q) | (~P & ~Q)"))) == "((~(P) | P) & (~(Q) | P) & (~(P) | Q) & (~(Q) | Q))";

@test to_conjunctive_normal_form(expr("B <=> (P1 | P2)")) == expr("&((~P1 | B), (~P2 | B), |(P1, P2, ~B))");

@test repr(to_conjunctive_normal_form(expr("B <=> (P1 | P2)"))) == "((~(P1) | B) & (~(P2) | B) & (P1 | P2 | ~(B)))";

@test to_conjunctive_normal_form(expr("a | (b & c) | d")) == expr("|(b, a, d) & |(c, a, d)");

@test repr(to_conjunctive_normal_form(expr("a | (b & c) | d"))) == "((b | a | d) & (c | a | d))";

@test to_conjunctive_normal_form(expr("A & (B | (D & E))")) == expr("&(A, (D | B), (E | B))");

@test repr(to_conjunctive_normal_form(expr("A & (B | (D & E))"))) == "(A & (D | B) & (E | B))";

@test to_conjunctive_normal_form(expr("A | (B | (C | (D & E)))")) == expr("|(D, A, B, C) & |(E, A, B, C)");

@test repr(to_conjunctive_normal_form(expr("A | (B | (C | (D & E)))"))) == "((D | A | B | C) & (E | A | B | C))";

prop_kb = PropositionalKnowledgeBase();

@test count((function(item)
                if (typeof(item) <: Bool)
                    return item;
                else
                    return true;
                end
            end), collect(ask(prop_kb, e) for e in map(expr, ["A", "C", "D", "E", "Q"]))) == 0;

tell(prop_kb, expr("A & E"));

@test ask(prop_kb, expr("A")) == Dict([]);

@test ask(prop_kb, expr("E")) == Dict([]);

tell(prop_kb, expr("E ==> C"));

@test ask(prop_kb, expr("C")) == Dict([]);

retract(prop_kb, expr("E"));

@test ask(prop_kb, expr("E")) == false;

@test ask(prop_kb, expr("C")) == false;

plr_results = pl_resolve(to_conjunctive_normal_form(expr("A | B | C")),
                        to_conjunctive_normal_form(expr("~B | ~C | F")));

@test pretty_set(Set{Expression}(disjuncts(plr_results[1]))) == "Set(Expression[A, C, F, ~(C)])";

@test pretty_set(Set{Expression}(disjuncts(plr_results[2]))) == "Set(Expression[A, B, F, ~(B)])";

# Use PropositionalKnowledgeBase to represent the Wumpus World (Fig. 7.4)

kb_wumpus = PropositionalKnowledgeBase();
tell(kb_wumpus, expr("~P11"));
tell(kb_wumpus, expr("B11 <=> (P12 | P21)"));
tell(kb_wumpus, expr("B21 <=> (P11 | P22 | P31)"));
tell(kb_wumpus, expr("~B11"));
tell(kb_wumpus, expr("B21"));

# Can't find a pit at location (1, 1).
@test ask(kb_wumpus, expr("~P11")) == Dict([]);

# Can't find a pit at location (1, 2).
@test ask(kb_wumpus, expr("~P12")) == Dict([]);

# Found pit at location (2, 2).
@test ask(kb_wumpus, expr("P22")) == false;

# Found pit at location (3, 1).
@test ask(kb_wumpus, expr("P31")) == false;

# Locations (1, 2) and (2, 1) do not contain pits.
@test ask(kb_wumpus, expr("~P12 & ~P21")) == Dict([]);

# Found a pit in either (3, 1) or (2,2).
@test ask(kb_wumpus, expr("P22 | P31")) == Dict([]);

@test pl_fc_entails(aimajulia.horn_clauses_kb, Expression("Q")) == true;

@test pl_fc_entails(aimajulia.horn_clauses_kb, Expression("SomethingSilly")) == false;

@test inspect_literal(Expression("P")) == (Expression("P"), true);

@test inspect_literal(Expression("~", Expression("P"))) == (Expression("P"), false);

@test unit_clause_assign(expr("A | B | C"), Dict([Pair(Expression("A"), true)])) == (nothing, nothing);

@test unit_clause_assign(expr("B | ~C"), Dict([Pair(Expression("A"), true)])) == (nothing, nothing);

@test unit_clause_assign(expr("B | C"), Dict([Pair(Expression("A"), true)])) == (nothing, nothing);

@test unit_clause_assign(expr("~A | ~B"), Dict([Pair(Expression("A"), true)])) == (Expression("B"), false);

@test unit_clause_assign(expr("B | ~A"), Dict([Pair(Expression("A"), true)])) == (Expression("B"), true);

@test find_unit_clause(map(expr, ["A | B | C", "B | ~C", "~A | ~B"]), Dict([Pair(Expression("A"), true)])) == (Expression("B"), false);

@test find_pure_symbol(map(expr, ["A", "B", "C"]), map(expr, ["A | ~B", "~B | ~C", "C | A"])) == (Expression("A"), true);

@test find_pure_symbol(map(expr, ["A", "B", "C"]), map(expr, ["~A | ~B", "~B | ~C", "C | A"])) == (Expression("B"), false);

@test find_pure_symbol(map(expr, ["A", "B", "C"]), map(expr, ["~A | B", "~B | ~C", "C | A"])) == (nothing, nothing);

@test dpll_satisfiable(expr("A & ~B")) == Dict([Pair(Expression("A"), true),
                                                Pair(Expression("B"), false),]);

@test dpll_satisfiable(expr("P & ~P")) == false;

@test (dpll_satisfiable(expr("A & ~B & C & (A | ~D) & (~E | ~D) & (C | ~D) & (~A | ~F) & (E | ~F) & (~D | ~F) & (B | ~C | D) & (A | ~E | F) & (~A | E | D)"))
        == Dict([Pair(Expression("A"), true),
                Pair(Expression("B"), false),
                Pair(Expression("C"), true),
                Pair(Expression("D"), true),
                Pair(Expression("E"), false),
                Pair(Expression("F"), false),]));

function walksat_test(clauses::Array{Expression, 1}; solutions::Dict=Dict())
    local sln = walksat(clauses);
    if (!(typeof(sln) <: Nothing)) #found a satisfiable solution
        @test all(collect(pl_true(clause, model=sln) for clause in clauses));
        if (length(solutions) != 0)
            @test all(collect(pl_true(clause, model=solutions) for clause in clauses));
            @test sln == solutions;
        end
    end
    nothing;
end

walksat_test(map(expr, ["A & B", "A & C"]));

walksat_test(map(expr, ["A | B", "P & Q", "P & B"]));

walksat_test(map(expr, ["A & B", "C | D", "~(D | P)"]), solutions=Dict([Pair(Expression("A"), true),
                                                            Pair(Expression("B"), true),
                                                            Pair(Expression("C"), true),
                                                            Pair(Expression("D"), false),
                                                            Pair(Expression("P"), false),]));

@test (typeof(walksat(map(expr, ["A & ~A"]), p=0.5, max_flips=100)) <: Nothing);

@test (typeof(walksat(map(expr, ["A | B", "~A", "~(B | C)", "C | D", "P | Q"]), p=0.5, max_flips=100)) <: Nothing);

@test (typeof(walksat(map(expr, ["A | B", "B & C", "C | D", "D & A", "P", "~P"]), p=0.5, max_flips=100)) <: Nothing);

transition = Dict([Pair("A", Dict([Pair("Left", "A"), Pair("Right", "B")])),
                    Pair("B", Dict([Pair("Left", "A"), Pair("Right", "C")])),
                    Pair("C", Dict([Pair("Left", "B"), Pair("Right", "C")]))]);

@test (typeof(sat_plan("A", transition,"C", 2)) <: Nothing);

@test sat_plan("A", transition, "B", 3) == ["Right"];

@test sat_plan("C", transition, "A", 3) == ["Left", "Left"];

transition = Dict([Pair((0, 0), Dict([Pair("Right", (0, 1)), Pair("Down", (1, 0))])),
                    Pair((0, 1), Dict([Pair("Left", (1, 0)), Pair("Down", (1, 1))])),
                    Pair((1, 0), Dict([Pair("Right", (1, 0)), Pair("Up", (1, 0)), Pair("Left", (1, 0)), Pair("Down", (1, 0))])),
                    Pair((1, 1), Dict([Pair("Left", (1, 0)), Pair("Up", (0, 1))]))]);

@test sat_plan((0, 0), transition, (1, 1), 4) == ["Right", "Down"];

@test unify(expr("x + y"), expr("y + C"), Dict([])) == Dict([Pair(Expression("x"), Expression("y")),
                                                            Pair(Expression("y"), Expression("C"))]);

@test unify(expr("x"), expr("3"), Dict([])) == Dict([Pair(Expression("x"), Expression("3"))]);

@test unify(expr("x"), expr("x"), Dict([])) == Dict([]);

@test extend(Dict([Pair(Expression("x"), 1)]), Expression("y"), 2) == Dict([Pair(Expression("x"), 1),
                                                                            Pair(Expression("y"), 2)]);

@test repr(substitute(Dict([Pair(Expression("x"), Expression("42")),
                            Pair(Expression("y"), Expression("0"))]),
                    expr("F(x) + y"))) == "(F(42) + 0)";

function fol_bc_ask_query(q::Expression; kb::Union{Nothing, AbstractKnowledgeBase}=nothing)
    local answers::Tuple;
    if (typeof(kb) <: Nothing)
        answers = fol_bc_ask(aimajulia.test_fol_kb, q);
    else
        answers = fol_bc_ask(kb, q);
    end
    local test_vars = variables(q);
    return sort(collect(Dict(collect(Pair(k, v) for (k, v) in answer if (k in test_vars))) for answer in answers),
            lt=(function(d1::Dict, d2::Dict)
                    return isless(repr(d1), repr(d2));
                end));
end

@test fol_bc_ask_query(expr("Farmer(x)")) == [Dict([Pair(Expression("x"), Expression("Mac"))])];

@test fol_bc_ask_query(expr("Human(x)")) == [Dict([Pair(Expression("x"), Expression("Mac"))]),
                                            Dict([Pair(Expression("x"), Expression("MrsMac"))])];

@test fol_bc_ask_query(expr("Rabbit(x)")) == [Dict([Pair(Expression("x"), Expression("MrsRabbit"))]),
                                            Dict([Pair(Expression("x"), Expression("Pete"))])];

@test fol_bc_ask_query(expr("Criminal(x)"), kb=aimajulia.crime_kb) == [Dict([Pair(Expression("x"), Expression("West"))])];

@test differentiate(expr("x * x"), expr("x")) == expr("(x * 1) + (x * 1)");

@test differentiate_simplify(expr("x * x"), expr("x")) == expr("2 * x");

@test (constant_symbols(expr("x & y & z | A")) == Set([expr("A")]));

@test (constant_symbols(expr("(x & B(z)) & Father(John) ==> Farmer(y) | A")) == Set(map(expr, ["A", "John"])));

@test (predicate_symbols(expr("x & y & z | A")) == Set());

@test (predicate_symbols(expr("(x & B(z)) & Father(John) ==> Farmer(y) | A")) == Set([("B", 1), ("Father", 1), ("Farmer", 1)]));

@test (predicate_symbols(expr("(x & B(x, y, z)) & F(G(x, y), x) ==> P(Q(R(x, y)), x, y, z)")) == Set([("G", 2), ("P", 4), ("R", 2), ("B", 3), ("Q", 1), ("F", 2)]));


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ da4b6f2a-923e-11eb-3f34-cd40c2f87cd5
begin
	using Distributions
	using PlutoUI
	using Statistics
	using StatsPlots
end;

# ╔═╡ c032f824-923e-11eb-2e29-29847cbf1063
md"# A Central Limit Theorem"

# ╔═╡ 0f1865b4-93e3-11eb-38a1-a11e4048d827
TableOfContents()

# ╔═╡ 1c8c2488-93e3-11eb-1864-a7f2a7454e1b
md"## Words"

# ╔═╡ 1a896920-9307-11eb-3a80-63564275f568
md"The Central Limit Theorem discussed here describes the distribution of the random variable, the sample mean.  We don't usually think about a sample mean as a random variable, but that's the first major hurdle to understanding any Central Limit Theorem, functions of random variables are random variables.  Any statistic can be thought of as a random variable."

# ╔═╡ 80a3f330-9308-11eb-0ca5-77b0627d968f
md"Since statistics are random variables, they often follow some distribution.  Most times, we don't know the distribution associated with a statistic.  When the statistic is the mean, then this Central Limit Theorem tells us that the distribution of the sample mean is approximately Normal."

# ╔═╡ b64df9c2-9308-11eb-27f5-3d230dc92312
md"According to this Central Limit Theorem, **the distribution of the sample mean is approximately Normal**.  Therefore the density function of the sample mean describes which values any random mean is most likely to take on."

# ╔═╡ debdea5e-9308-11eb-29f7-71208f83e317
md"Note that we will have to massage the math behind this Central Limit Theorem to better visualize what is going on."

# ╔═╡ 4e184eaa-9307-11eb-1af5-4b2fdc6bbc6b
md"We usually think about one dataset being randomly sampled from a distribution.  Here, we'll consider a sample mean as coming from a distribution.  To help us imagine this abstraction, consider you and $$R - 1$$ of your friends each going out to collect your own random sample from the same process.  By the time each of you collects your own sample of size $$N$$ and then calculates your own mean (from your own data), there will be $$R$$ means."

# ╔═╡ 3e9630c6-9309-11eb-3cf5-73b109c4b412
md"This Central Limit Theorem says that the shape of these $$R$$ means will be approximately Normal, and more over that the width of this Normal distribution depends on $$N$$ (and the variance of the theoretical distribution from which you and your friends sampled from) and the approximation will improve as $$R$$ goes off to infinity."

# ╔═╡ 75dab232-9309-11eb-0f2a-ed982699b343
md"This last bit is where the math and our vizualization below differ.  Mathematically, the approximation and the width of the distribution of the sample mean both only depend on the sample size $$N$$.  The mathematical theory says that as $$N \to \infty$$, then the Normal approximation improves and the width of the distribution will decrease with the square root of the sample size $$N$$."

# ╔═╡ dc33209e-9309-11eb-099b-0f6bc414032a
md"To see visualize this process, pick a distribution below that is supposed to represent the distribution you and your $$R-1$$ friends are sampling from.  Then adjust the sample size and the number of friends $$R$$ you have to see the effect of the sampling distribution of the sample mean.  **Notice that this Central Limit Theorem is indifferent to the shape of the theoretical distribution; whatever shape it has, the sampling distribution is still approximately Normal.**"

# ╔═╡ 36c19130-93e3-11eb-0e54-f7c547f3b438
md"## Pictures"

# ╔═╡ 22658db8-923f-11eb-21ed-0bd2f08bf338
md"""Pick a distribution
$(@bind distribution Select([
	"Gamma" => "Gamma",
	"Normal" => "Normal",
	"Uniform" => "Uniform"]))
"""

# ╔═╡ f2d7090a-9301-11eb-06c8-0fa5fc97ce84
md"""
 $$N$$ = $(@bind N Slider(1:301, default = 27, show_value = true)) $$\quad$$
 $$R$$ = $(@bind R Slider(1:1001, default = 101, show_value = true))
"""

# ╔═╡ f69147d2-930a-11eb-3757-07178c0dadc8
md"## Math"

# ╔═╡ 7b0e370a-930a-11eb-2e33-0bed472a2026
html"

"

# ╔═╡ 1f93e9d0-930b-11eb-2220-332796c4523e
md"Assume you and your $$R-1$$ friends are randomly sampling from distribution $$F$$ with mean $$\mu$$ and variance $$\sigma^2$$.  In a simple version of the Central Limit Theorem of sample means, we require $$\sigma^2 < \infty$$ and the original data to be independent and identically distributed, $$X_1, \ldots, X_N \sim_{iid} F$$."

# ╔═╡ ff9123be-93e3-11eb-3fc5-076772232b87
md"Denote the sample mean as $$\hat{\mu} = N^{-1} \sum_{n = 1}^N X_N$$.  Then by this Central Limit Theorem we have that

$\frac{\hat{\mu} - \mu}{\sigma / \sqrt{N}} \sim N(0, 1)$

as $$N \rightarrow \infty$$."

# ╔═╡ bb1c25f4-9301-11eb-26ce-158f5bb94eae
D =
if distribution == "Gamma"
    Gamma(1, 10)
elseif distribution == "Normal"
	Normal(0, 10)
elseif distribution == "Uniform"
	Uniform(1, 6)
end;

# ╔═╡ aa79d4ac-923f-11eb-111d-3dceaf63f63d
begin
	p = plot(D, label = "", linewidth = 2, title = "Theoretical distribution: $(distribution)")
	plot!(p, [mean(D)], [0], t = :scatter, color = :black, label = "mean", markersize = 15, markershape = :utriangle)
end

# ╔═╡ a44bae34-9302-11eb-3367-cb12648d16be
samplemeans = [mean(rand(D, N)) for _ in 1:R]

# ╔═╡ 1966e0fe-9302-11eb-3ec2-0515c69711a1
begin
	h = histogram(samplemeans, legend = false,
		title = "Sampling distribution \n R = $(R) sample means \n each with N = $(N) observations")
	plot!(h, [mean(D)], [0], t = :scatter, color = :black, label = "mean", markersize = 15, markershape = :utriangle)
end

# ╔═╡ 86fde510-9300-11eb-2dd9-c1dc68850b33
onedataset = rand(D, N)

# ╔═╡ Cell order:
# ╟─c032f824-923e-11eb-2e29-29847cbf1063
# ╟─0f1865b4-93e3-11eb-38a1-a11e4048d827
# ╟─1c8c2488-93e3-11eb-1864-a7f2a7454e1b
# ╟─1a896920-9307-11eb-3a80-63564275f568
# ╟─80a3f330-9308-11eb-0ca5-77b0627d968f
# ╟─b64df9c2-9308-11eb-27f5-3d230dc92312
# ╟─debdea5e-9308-11eb-29f7-71208f83e317
# ╟─4e184eaa-9307-11eb-1af5-4b2fdc6bbc6b
# ╟─3e9630c6-9309-11eb-3cf5-73b109c4b412
# ╟─75dab232-9309-11eb-0f2a-ed982699b343
# ╟─dc33209e-9309-11eb-099b-0f6bc414032a
# ╟─36c19130-93e3-11eb-0e54-f7c547f3b438
# ╟─22658db8-923f-11eb-21ed-0bd2f08bf338
# ╟─aa79d4ac-923f-11eb-111d-3dceaf63f63d
# ╟─f2d7090a-9301-11eb-06c8-0fa5fc97ce84
# ╟─1966e0fe-9302-11eb-3ec2-0515c69711a1
# ╟─a44bae34-9302-11eb-3367-cb12648d16be
# ╟─86fde510-9300-11eb-2dd9-c1dc68850b33
# ╟─f69147d2-930a-11eb-3757-07178c0dadc8
# ╟─7b0e370a-930a-11eb-2e33-0bed472a2026
# ╟─1f93e9d0-930b-11eb-2220-332796c4523e
# ╟─ff9123be-93e3-11eb-3fc5-076772232b87
# ╟─bb1c25f4-9301-11eb-26ce-158f5bb94eae
# ╟─da4b6f2a-923e-11eb-3f34-cd40c2f87cd5


"""
    imon2005(setting)

Perform the Imon 2005 algorithm for a given regression setting.

# Arguments
- `setting::RegressionSetting`: A regression setting.

# Notes
The implementation uses LTS rather than LMS as suggested in the paper.

# Reference
A. H. M. Rahmatullah Imon (2005) Identifying multiple influential observations in linear regression,
Journal of Applied Statistics, 32:9, 929-946, DOI: 10.1080/02664760500163599
 """
function imon2005(setting::RegressionSetting)
    X, y = @extractRegressionSetting setting
    return imon2005(X, y)
end

function imon2005(X::Array{Float64,2}, y::Array{Float64,1})

    function SigmaWithoutIndex(X, y, R, i)
        n, p = size(X)
        Ri = filter(x -> x != i, R)
        XRi = X[Ri,:]
        yRi = y[Ri]
        reg = ols(XRi, yRi)
        betas = coef(reg)
        res = y .- X * betas
        return sqrt(sum(res.^2.0) / (n - p))
    end

    n, p = size(X)
    allindex = collect(1:n)
    ltsreg = lts(X, y)
    R::Array{Int,1} = ltsreg["hsubset"]
    XR = X[R,:]
    yR = y[R]
    XRXR = transpose(XR) * XR
    invXRXR = inv(XRXR)
    wiiR = [X[i,:]' * invXRXR * X[i,:] for i in allindex]
    wiiRAsterix = zeros(Float64, n)
    @inbounds for i in allindex
        if i in R
            wiiRAsterix[i] = wiiR[i] / (1.0 - wiiR[i])
        else
            wiiRAsterix[i] = wiiR[i] / (1.0 + wiiR[i])
        end
    end
    RegR = ols(XR, yR)
    BetaHatR = coef(RegR)
    resR = y - X * BetaHatR
    sigmaR = sum(resR.^2.0) / (n - p)
    tAsterix = zeros(Float64, n)
    @inbounds for i in allindex
        if i in R
            tAsterix[i] = resR[i] / (SigmaWithoutIndex(X, y, R, i) * sqrt(1.0 - wiiR[i]))
        else
            tAsterix[i] = resR[i] / (SigmaWithoutIndex(X, y, R, i) * sqrt(1.0 + wiiR[i]))
        end
    end

    GDFFITS = zeros(Float64, n)
    @inbounds for i in allindex
        GDFFITS[i] = sqrt(wiiRAsterix[i]) * tAsterix[i]
    end
    crit = 3.0 * sqrt(p / length(R))
    outlyingindex = filter(i -> abs(GDFFITS[i]) >= crit, allindex)
    result::Dict{String,Any} = Dict()
    result["crit"] = crit
    result["gdffits"] = GDFFITS
    result["outliers"] = outlyingindex
    return result
end


using HighestDensityRegions
using HighestDensityRegions:
    getindex_or_zero, interpolation_coefficient_and_index, interpolate_in
using Test

using Distributions
using DocStringExtensions

@testset "utilities" begin
    # getindex
    v = 1:3
    @test getindex_or_zero(v, 3) ≡ 3
    @test getindex_or_zero(v, 0) ≡ 0

    # interpolation index: arbitrary cumprobs
    c = [0.1, 0.4, 1.0]
    @test interpolation_coefficient_and_index(c, 0.1) == (1.0, 1)
    @test interpolation_coefficient_and_index(c, 0.2) == (1/3, 2)
    for _ in 1:20
        α = rand()
        γ, i = interpolation_coefficient_and_index(c, α)
        @test interpolate_in(c, γ, i) ≈ α
    end
    for (i, α) in enumerate(c)
        @test interpolation_coefficient_and_index(c, α) == (1.0, i)
    end

    # interpolation index: integer cumprobs
    N = 8
    cN = Vector((1:N) ./ N)
    for (i, α) in enumerate(cN)
        @test interpolation_coefficient_and_index(cN, α) == (1.0, i)
    end
end

"""
    $SIGNATURES

Calculate densities and counts using the cumulative density function for a
univariate distribution.
"""
function analytical_densities_and_counts(distribution, breaks)
    @assert issorted(breaks)
    counts = diff(cdf.(Ref(distribution), breaks))
    densities = counts ./ diff(breaks)
    densities, counts
end

"""
    $SIGNATURES

Calculate the empirical coverage probability of a (HDR) density threshold.
"""
coverage(threshold, densities, counts) =
    sum(counts[densities .≥ threshold]) / sum(counts)

"""
    $SIGNATURES

Test empirical coverage of HDR thresholds. `atol` is used for comparison, use
dense breaks for more precision.
"""
function test_coverage(αs, thresholds, densities, counts, atol = 0.01)
    for (i, α) in enumerate(αs)
        @test coverage(thresholds[i], densities, counts) ≈ α atol = atol
    end
end

@testset "normal thresholds with evenly spaced breaks" begin
    N = 1000
    breaks = range(-4; stop = 4, length = N)
    densities, counts =
        analytical_densities_and_counts(Normal(0, 1), breaks)
    αs = [0.25, 0.5, 0.75, 0.9]
    thresholds = hdr_thresholds(αs, densities, counts)
    test_coverage(αs, thresholds, densities, counts)
    thresholds_even = hdr_thresholds(αs, densities)
    @test thresholds_even ≈ thresholds
end

@testset "gamma thresholds with log breaks" begin
    breaks = exp.(range(0; stop = 2, length = 1000)) .- 1
    densities, counts = analytical_densities_and_counts(Gamma(1, 1), breaks)
    αs = [0.25, 0.5, 0.75, 0.9]
    thresholds = hdr_thresholds(αs, densities, counts)
    test_coverage(αs, thresholds, densities, counts)
end


# Kernel definition is taken from here: https://en.wikipedia.org/wiki/Kernel_(statistics)#Kernel_functions_in_common_use

abstract type CollocationKernel end
struct EpanechnikovKernel <: CollocationKernel end
struct UniformKernel <: CollocationKernel end
struct TriangularKernel <: CollocationKernel end
struct QuarticKernel <: CollocationKernel end
struct TriweightKernel <: CollocationKernel end
struct TricubeKernel <: CollocationKernel end
struct GaussianKernel <: CollocationKernel end
struct CosineKernel <: CollocationKernel end
struct LogisticKernel <: CollocationKernel end
struct SigmoidKernel <: CollocationKernel end
struct SilvermanKernel <: CollocationKernel end

function calckernel(::EpanechnikovKernel,t)
    if abs(t) > 1
        return 0
    else
        return 0.75*(1-t^2)
    end
end

function calckernel(::UniformKernel,t)
    if abs(t) > 1
        return 0
    else
        return 0.5
    end
end

function calckernel(::TriangularKernel,t)
    if abs(t) > 1
        return 0
    else
        return (1-abs(t))
    end
end

function calckernel(::QuarticKernel,t)
  if abs(t)>0
    return 0
  else
    return (15*(1-t^2)^2)/16
  end
end

function calckernel(::TriweightKernel,t)
  if abs(t)>0
    return 0
  else
    return (35*(1-t^2)^3)/32
  end
end

function calckernel(::TricubeKernel,t)
  if abs(t)>0
    return 0
  else
    return (70*(1-abs(t)^3)^3)/80
  end
end

function calckernel(::GaussianKernel,t)
  exp(-0.5*t^2)/(sqrt(2*π))
end

function calckernel(::CosineKernel,t)
  if abs(t)>0
    return 0
  else
    return (π*cos(π*t/2))/4
  end
end

function calckernel(::LogisticKernel,t)
  1/(exp(t)+2+exp(-t))
end

function calckernel(::SigmoidKernel,t)
  2/(π*(exp(t)+exp(-t)))
end

function calckernel(::SilvermanKernel,t)
  sin(abs(t)/2+π/4)*0.5*exp(-abs(t)/sqrt(2))
end


using FMMLIB2D
using SpecialFunctions
using LinearAlgebra
using Random
using Test

Random.seed!(0)

# TODO: Compute gradient and hessian

function direct_self(source, charge, dipvec, dipstr, zk)
    pot = zero(charge)
    N = size(source, 2)
    for i=1:N
        for j=1:N
            if i == j
                continue
            end
            r1 = source[1,j]-source[1,i]
            r2 = source[2,j]-source[2,i]
            r = sqrt(r1^2 + r2^2)
            rdotq = r1*dipvec[1,j] + r2*dipvec[2,j]
            pot[i] += charge[j]*besselh(0, zk*r)*im/4 -
                dipstr[j]*rdotq*1im/4*besselh(1, zk*r)*zk/r
        end
    end
    return pot
end
function direct_targ(source, charge, dipvec, dipstr, target, zk)
    M = size(target, 2)
    N = size(source, 2)
    pot = zeros(typeof(charge[1]), M)
    for i=1:M
        for j=1:N
            r1 = source[1,j]-target[1,i]
            r2 = source[2,j]-target[2,i]
            r = sqrt(r1^2 + r2^2)
            rdotq = r1*dipvec[1,j] + r2*dipvec[2,j]
            pot[i] += charge[j]*besselh(0, zk*r)*im/4 -
                dipstr[j]*rdotq*1im/4*besselh(1, zk*r)*zk/r
        end
    end
    return pot
end

@testset "Helmholtz FMM 2D (complex)" begin
    iprec = 5
    N = 40
    M = 30
    zk = rand() + 1im*rand()
    source = rand(2, N)
    charge = rand(N) + 1im*rand(N)
    dipstr = rand(N) + 1im*rand(N)
    dipvec = rand(2, N)
    target = rand(2, M)
    ifcharge = 1
    ifdipole = 1
    ifpot = 1
    ifgrad = 1
    ifhess = 1
    ifpottarg = 1
    ifgradtarg = 1
    ifhesstarg = 1

    refpottarg = direct_targ(source, charge, dipvec, dipstr, target, zk)
    refpot = direct_self(source, charge, dipvec, dipstr, zk)

    @testset "Direct interface: hfmm2dparttarg" begin
        pot, grad, hess, pottarg, gradtarg, hesstarg =
            hfmm2dparttarg(iprec, zk, source, ifcharge, charge, ifdipole, dipstr, dipvec, ifpot, ifgrad, ifhess,
                           target, ifpottarg, ifgradtarg, ifhesstarg)
        pot_relerr = norm(refpot-pot, Inf) / norm(refpot, Inf)
        pottarg_relerr = norm(refpottarg-pottarg, Inf) / norm(refpottarg, Inf)
        @test pot_relerr < 1e-15
        @test pottarg_relerr < 1e-15
    end

    @testset "Keyword interface" begin
        U = hfmm2d(zk=zk, source=source, charge=charge, dipstr=dipstr, dipvec=dipvec)
        @test norm(U.pot-refpot, Inf) / norm(refpot, Inf) < 1e-14
        U = hfmm2d(zk=zk, source=source, charge=charge, dipstr=dipstr, dipvec=dipvec, target=target)
        @test norm(U.pottarg-refpottarg, Inf) / norm(refpot, Inf) < 1e-14
    end

    @testset "Tolerance test" begin
        N = 1000
        source = rand(2, N)
        charge = rand(N) + 1im*rand(N)
        dipstr = rand(N) + 1im*rand(N)
        dipvec = rand(2, N)
        refpot = direct_self(source, charge, dipvec, dipstr, zk)
        for tolexp=-14:-1
            TOL = 0.49*10.0^tolexp
            U = hfmm2d(zk=zk, source=source, charge=charge, tol=TOL, dipstr=dipstr, dipvec=dipvec)
            relerr = norm(U.pot-refpot) / norm(refpot)
            @test relerr < TOL
        end
    end
end


############################################################################################
# Higher Order Data Memory
HO_latent = randn(_rng, N)
HO_data = randn(_rng, size(HO_latent, 1))
HO_param = (
    μ₀=Param(0.0, Normal(0.0, 10.0)),
    μ=Param(-0.50, Normal(0.0, 10.0)),
    σ=Param(0.250, Gamma(0.5, 1.0)),
    ϕ=Param(0.90, Uniform(-1.0, 1.0)),
    latent=Param(HO_latent, [Normal(0.0, 10) for i in Base.OneTo(N)]),
)
struct Markov_HO <: ModelName end
markov_HO = ModelWrapper(Markov_HO(), HO_param)
markov_HO_objective = Objective(markov_HO, HO_data, :latent)

function BaytesFilters.dynamics(objective::Objective{<:ModelWrapper{<:Markov_HO}})
    @unpack model, data = objective
    @unpack μ₀, μ, σ, ϕ = model.val

    initialˢ = Normal(μ, σ / sqrt((1 - ϕ^2)))
    function transition(particles, iter)
        return Normal(μ + ϕ * (mean(view(particles, (iter - 2):(iter - 1))) - μ), σ)
    end
    function evidence(particles, iter)
        return Normal(μ₀, sqrt(exp(mean(view(particles, (iter - 3):iter)))))
    end
    return Markov(initialˢ, transition, evidence)
end
############################################################################################