function
directRP!(
    RP::Vector,
    y::Vector,
    m::Int,
    h::Real,
    N::Int,
    ε::Real,
    ulₘ::Vector,
    urₘ::Vector
)
    # Функция для внутреннего использования, поэтому входные массивы размерности N-1
    @assert length(y)   == N-1  "Вектор искомой функции размерности N-1"
    @assert length(RP)  == N-1  "Вектор функции правой части размерности N-1"

    @assert length(ulₘ) > m > 0
    @assert length(urₘ) > m > 0

    @assert N > 4
    @assert ε > 0

    RP[1] = ε * (y[2] - 2 * y[1] + ulₘ[m]) / h^2 +
        y[1] * (y[2] - ulₘ[m]) / (2 * h) - y[1];

    for n in 2:N-2
        RP[n] = ε * (y[n+1] - 2 * y[n] + y[n-1]) / h^2 +
            y[n] * (y[n+1] - y[n-1]) / (2 * h) - y[n];
    end

    RP[N-1] = ε * (urₘ[m] - 2 * y[N-1] + y[N-2]) / h^2 +
        y[N-1] * (urₘ[m] - y[N-2]) / (2 * h) - y[N-1];

    return RP;
end

function
DRP!(
    dl::Vector,
    dd::Vector,
    du::Vector,
    y::Vector,
    m::Int,
    h::Real,
    N::Int,
    ε::Real,
    ulₘ::Vector,
    urₘ::Vector
)
    # Функция для внутреннего использования, поэтому входные массивы размерности N-1
    @assert length(y)   == N-1  "Вектор искомой функции размерности N-1"

    @assert length(ulₘ) > m > 0
    @assert length(urₘ) > m > 0

    @assert m > 0
    @assert h > 0
    @assert N > 3
    @assert ε > 0

    @assert length(dl) == N-2   "Вектор поддиагональных элементов размерностью N-2"
    @assert length(dd) == N-1   "Вектор    диагональных элементов размерностью N-1"
    @assert length(du) == N-2   "Вектор поддиагональных элементов размерностью N-2"

    dd[1] = -2 * ε / h^2 + (y[2] - ulₘ[m]) / (2 * h) - 1;
    du[1] = ε / h^2 - y[1] / (2 * h);

    for n in 2:N-2
        dl[n - 1] = ε / h^2 - y[n] / (2 * h);
        dd[n] = - 2 * ε / h^2 + (y[n+1] - y[n-1])/ (2 * h) - 1;
        du[n] = ε / h^2 + y[n] / (2 * h);
    end

    dl[N-2] = ε / h^2 + y[N-1] / (2 * h);
    dd[N-1] = - 2 * ε / h^2 + (urₘ[m] - y[N-2])/ (2 * h) - 1;

    return dl, dd, du;
end

function
solve!(
    u::Matrix,
    u₀::Vector,
    N::Int,
    h::Real,
    Tₘ::Vector,
    M::Int,
    τ::Real,
    ε::Real,
    ulₘ::Vector,
    urₘ::Vector;
    α::Complex = complex(0.5, 0.5))

    if size(u) != (N+1, M+1)
        throw(ArgumentError("""size(u) == $(size(u)), N == $(N), M == $(M)
        Матрица `u` должена иметь размерность (N+1, M+1)."""))
    end

    if length(u₀) != N+1
        throw(ArgumentError("""length(u₀) == $(length(u₀)), N == $(N)
        Массив `u₀` должен иметь размерность N+1."""))
    end

    if length(Tₘ) != M+1
        throw(ArgumentError("""length(Tₘ) == $(length(Tₘ)), M == $(M)
        Массив `Tₘ` должен иметь размерность M+1"""))
    end
    if length(ulₘ) != M+1
        throw(ArgumentError("""length(ulₘ) == $(length(ulₘ)), M == $(M)
        Массив `ulₘ` должен иметь размерность M+1"""))
    end
    if length(urₘ) != M+1
        throw(ArgumentError("""length(y) == $(length(y)), M == $(M)
        Массив `ulₘ` должен иметь размерность M+1"""))
    end

    u[1  , :]   = ulₘ;                  # Запись левого  ГУ на каждом шаге
    u[N+1, :]   = urₘ;                  # Запись правого ГУ на каждом шаге
    u[:, 1] = u₀;                       # Запись начального условия, это перепишет ГУ.

    y = u₀[2:end-1];
    rp = zero(y);
    dl = zeros(N-2);
    dd = zeros(N-1);
    du = zeros(N-2);
    j = Tridiagonal(zeros(N-2), zeros(N-1), zeros(N-2));

    for m in 1:M

        directRP!(rp, y, m, h, N, ε, ulₘ, urₘ);
        DRP!(dl, dd, du, y, m, h, N, ε, ulₘ, urₘ);

        j = Tridiagonal(dl, dd, du);

        W = (I - α  * τ * j) \ rp;
        y = y .+ τ * real(W);

        u[2:N, m+1] = y;
    end

    return u;
end


using TensorFlow
using Distributions
include("mnist_loader.jl")

loader = DataLoader()

session = Session(Graph())

function weight_variable(shape)
    initial = map(Float32, rand(Normal(0, .001), shape...))
    return Variable(initial)
end

function bias_variable(shape)
    initial = fill(Float32(.1), shape...)
    return Variable(initial)
end

function conv2d(x, W)
    nn.conv2d(x, W, [1, 1, 1, 1], "SAME")
end

function max_pool_2x2(x)
    nn.max_pool(x, [1, 2, 2, 1], [1, 2, 2, 1], "SAME")
end

@tf begin

    x = placeholder(Float32)
    y_ = placeholder(Float32)

    W_conv1 = weight_variable([5, 5, 1, 32])
    b_conv1 = bias_variable([32])

    x_image = reshape(x, [-1, 28, 28, 1])

    h_conv1 = nn.relu(conv2d(x_image, W_conv1) + b_conv1)
    h_pool1 = max_pool_2x2(h_conv1)

    W_conv2 = weight_variable([5, 5, 32, 64])
    b_conv2 = bias_variable([64])

    h_conv2 = nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
    h_pool2 = max_pool_2x2(h_conv2)

    W_fc1 = weight_variable([7*7*64, 1024])
    b_fc1 = bias_variable([1024])

    h_pool2_flat = reshape(h_pool2, [-1, 7*7*64])
    h_fc1 = nn.relu(h_pool2_flat * W_fc1 + b_fc1)

    keep_prob = placeholder(Float32)
    h_fc1_drop = nn.dropout(h_fc1, keep_prob)

    W_fc2 = weight_variable([1024, 10])
    b_fc2 = bias_variable([10])

    global y_conv = nn.softmax(h_fc1_drop * W_fc2 + b_fc2)

    global cross_entropy = reduce_mean(-reduce_sum(y_.*log(y_conv), axis=[2]))
end

train_step = train.minimize(train.AdamOptimizer(1e-4), cross_entropy)

correct_prediction = indmax(y_conv, 2) .== indmax(y_, 2)

accuracy = reduce_mean(cast(correct_prediction, Float32))

run(session, global_variables_initializer())

for i in 1:200
    batch = next_batch(loader, 50)
    if i%100 == 1
        train_accuracy = run(session, accuracy, Dict(x=>batch[1], y_=>batch[2], keep_prob=>1.0))
        info("step $i, training accuracy $train_accuracy")
    end
    run(session, train_step, Dict(x=>batch[1], y_=>batch[2], keep_prob=>.5))
end

testx, testy = load_test_set()
test_accuracy = run(session, accuracy, Dict(x=>testx, y_=>testy, keep_prob=>1.0))
info("test accuracy $test_accuracy")

visualize()


module GPML
using QuadGK
using StatsFuns
using LinearAlgebra

function post_funs(datalik, Σ)
    function f(u)
        datalik.f(u) - 1 / 2 * u' * (Σ \ u) - 1 / 2 * logdet(Σ) - length(u) / 2 * log(2π)
    end
    function gr(u)
        datalik.gr(u) - Σ \ u
    end
    function he(u)
        datalik.he(u) - inv(Σ)
    end
    (f = f, gr = gr, he = he)
end

function normal_funs(y, σobs)
    n = length(y)
    function f(u)
        sum(normlogpdf.(u, σobs, y))
    end
    function gr(u)
        @. (y - u) / σobs^2
    end
    function he(u)
        Diagonal(-ones(n) / σobs^2)
    end
    function ∇³(u)
        zeros(n)
    end

    (f = f, gr = gr, he = he, ∇³ = ∇³)
end

function logistic_funs(y)
    # Convert data to [-1, 1] GPML format
    y2 = 2y .- 1
    # Data likelihood
    function f(u)
        -sum(log.(1 .+ exp.(-y2 .* u)))
    end
    # Gradient
    function gr(u)
        p = logistic.(u)
        y .- p
    end
    # Diagonal Hessian
    function he(u)
        p = logistic.(u)
        Diagonal(-p .* (1 .- p))
    end

    (f = f, gr = gr, he = he)
end

function probit_funs(y)
    y2 = 2y .- 1
    # Data likelihood
    function f(u)
        sum(normlogcdf.(y2 .* u))
    end
    # Gradient
    function gr(u)
        @. y2 * normpdf(u) / normcdf(y2 * u)
    end
    # Hessian
    function he(u)
        u_pdf = normpdf.(u)
        yu_cdf = normcdf.(y2 .* u)
        d = @. -(u_pdf / yu_cdf)^2 - y2 * u * u_pdf / yu_cdf
        Diagonal(d)
    end

    (f = f, gr = gr, he = he)
end
"""
    gpml_laplace(datalik, Σ, u0; utol = 1e-8, maxit = 10)

- `datalik` Named tuple with observation likelihood (`f`), gradient (`gr`), and
    Hessian (`he`).
- `Σ` Covariance matrix of `u`.
- `u` Initial value for Newton iteration of `u`.
- `utol` Difference between successive `u` values that ends Newton steps.
- `maxit` Maximum number of Newton iterations.

Approximate a marginal likelihood using the Laplace approximation. Adapted from
Algorithm 3.1 of Gaussian Processes for Machine Learning (p. 46).
"""
function gpml_laplace(datalik, Σ, u0; utol = 1e-8, maxit = 10)
    u = copy(u0)
    it = 0
    while true
        it += 1
        W = -datalik.he(u)
        Wsqrt = sqrt(W)
        global L = cholesky(Hermitian(I + Wsqrt * Σ * Wsqrt))
        b = W * u + datalik.gr(u)
        global a = b - Wsqrt * (L \ (Wsqrt * Σ * b))
        unew = Σ * a
        if maximum(unew .- u) < utol
            u .= unew
            break
        elseif it ≥ maxit
            break
        end
        u .= unew
    end
    logq = -1 / 2 * a' * u + datalik.f(u) - logdet(L) / 2
    logq, u, a
end

"""
    gpml_pred(xpred, datalik, uhat, xobs, Σ, covfun, sigmoidfun)

Provide classification predictions at locations `xpred` with observation
likelihood `datalik` and `uhat` estimated by `gpml_laplace` with observations at
`xobs`, covariance matrix `Σ`, covariance function `covfun`, and sigmoid (e.g.
`logistic`) `signmoidfun`. Algorithm 3.2 of Gaussian Processes for Machine
Learning (p. 47).
"""
function gpml_pred(xpred, datalik, uhat, xobs, Σ, covfun, sigmoidfun)
    W = -datalik.he(uhat)
    Wsqrt = sqrt(W)
    L = cholesky(Hermitian(I + Wsqrt * Σ * Wsqrt))
    kcross = covfun.(xobs, xpred')
    upred = kcross' * datalik.gr(uhat)
    v = L.L \ (Wsqrt * kcross)
    Vpred = covfun.(xpred, xpred') - v' * v
    ppred = quadgk(z -> sigmoidfun(z) * normpdf(upred, sqrt(Vpred), z), -Inf, Inf)[1]
    ppred, upred, sqrt(Vpred)
end

function gpml_grad(covpars, covfun, datalik, u0)
    K = covfun.f(covpars)
    logq, uhat, a = gpml_laplace(datalik, K, u0)
    W = -datalik.he(uhat)
    Wsqrt = sqrt(W)
    L = cholesky(Hermitian(I + Wsqrt * K * Wsqrt))
    R = Wsqrt * (L \ Wsqrt)
    C = L.L \ (Wsqrt * K)
    s2 = -Diagonal(diag(K) - diag(C'C)) / 2 * datalik.∇³(uhat)
    grad_covpars = similar(covpars)
    C = covfun.gr(covpars)
    for j in 1:length(covpars)
        s1 = (a' * C[j] * a - tr(R * C[j])) / 2
        b = C[j] * datalik.gr(uhat)
        s3 = b - K * R * b
        grad_covpars[j] = s1 + s2's3
    end
    logq, grad_covpars, uhat
end

end # module GPML


using MinFEM
using LinearAlgebra

function semilinear(mesh::Mesh, L::AbstractMatrix, M::AbstractMatrix,
                    s::AbstractVector, BoundaryIndices::Set{Int64}=[], tol=1e-10)

  y = zeros(mesh.nnodes)

  pde = PDESystem(A=L, b=M*s, bc=zeros(mesh.nnodes), DI=BoundaryIndices)

  res = Inf
  while res > tol
    pde.A = L + asmCubicDerivativeMatrix(mesh, y)
    pde.b = -L*y + M*s - asmCubicTerm(mesh, y)
    refresh(pde)
    solve(pde)

    y += pde.state
    res = norm(pde.state)
  end
  return y
end

function solve_semilinear(file_name::String)

  mesh = import_mesh(file_name)

  L = asmLaplacian(mesh)
  M = asmMassMatrix(mesh)

  f(x) = 100.0*x[1]*x[2]
  s = evaluateMeshFunction(mesh, f)

  boundary = union!(mesh.Boundaries[1001].Nodes,
                    mesh.Boundaries[1002].Nodes,
                    mesh.Boundaries[1003].Nodes,
                    mesh.Boundaries[1004].Nodes);

  y = semilinear(mesh, L, M, s, boundary);
  return true
end

@test solve_semilinear("test_v4.msh")


function getAreaKernel{T <: FloatingPoint}(:: Type{T})
        nType = T == Float64 ? "double" : "float"
        nPi = T == Float64 ? "M_PI" : "M_PI_F"
        nPi4 = T == Float64 ? "M_PI_4" : "M_PI_4_F"

        return "
        #if defined(cl_khr_fp64)  // Khronos extension available?
        #pragma OPENCL EXTENSION cl_khr_fp64 : enable
        #elif defined(cl_amd_fp64)  // AMD extension available?
        #pragma OPENCL EXTENSION cl_amd_fp64 : enable
        #endif
        #define number $nType
        #define number4 $(nType)4
        #define PI $nPi
        #define PI_4 $nPi4

        __kernel void area(
                      __global const number *d,
                      __global number4 *dout,
                      const number LONG,
                      const number SHORT) {

            int i = get_global_id(0);

            const number dir = d[i];

            const number area1 = LONG*SHORT/2 * (   PI_4/2-dir - atan((SHORT-LONG)*sin(2*(  PI_4/2-dir)) / (SHORT+LONG + (SHORT-LONG)*cos(2*(  PI_4/2-dir)))));
            const number area2 = LONG*SHORT/2 * ( 3*PI_4/2-dir - atan((SHORT-LONG)*sin(2*(3*PI_4/2-dir)) / (SHORT+LONG + (SHORT-LONG)*cos(2*(3*PI_4/2-dir)))));
            const number area3 = LONG*SHORT/2 * ( 5*PI_4/2-dir - atan((SHORT-LONG)*sin(2*(5*PI_4/2-dir)) / (SHORT+LONG + (SHORT-LONG)*cos(2*(5*PI_4/2-dir)))));
            const number area4 = LONG*SHORT/2 * ( 7*PI_4/2-dir - atan((SHORT-LONG)*sin(2*(7*PI_4/2-dir)) / (SHORT+LONG + (SHORT-LONG)*cos(2*(7*PI_4/2-dir)))));
            const number area5 = LONG*SHORT/2 * ( 9*PI_4/2-dir - atan((SHORT-LONG)*sin(2*(9*PI_4/2-dir)) / (SHORT+LONG + (SHORT-LONG)*cos(2*(9*PI_4/2-dir)))));

            const number lps = SHORT * LONG * PI;

            number4 a;

            a.s0 = (area2-area1)/lps;
            a.s1 = (area3-area2)/lps;
            a.s2 = (area4-area3)/lps;
            a.s3 = (area5-area4)/lps;

            dout[i] = a;
        }
"
end

function areaCL!{T <: FloatingPoint}(
    d_buff,
    area_buff,
    long :: Real,
    d1 :: Int, d2 :: Int,
    ctx, queue, program, :: Type{T})

    k_a = Kernel(program, "area")

    short = one(T)
    call(queue, k_a, d1*d2, nothing, d_buff, area_buff, convert(T, long), short)
end


using JuMP, Ipopt
using Plots, LinearAlgebra
default(dpi=100)
using BenchmarkTools
using Random
using StatsBase, StatsModels

clearconsole()

N = 15 #data points
x = collect(range(0,1,length=N)) #training points and data
t = sin.(2*pi.*x) + 0.2 .* randn(size(x))
X = collect(range(-0.5,1.5,length=100)) #test points and data
T = sin.(2*pi.*X)

a = 5e-3
b = 11.1
D = 8 #poly degree
M = D+1 #model param num
W = 0.5.*[1 -1]

s = zeros(size(X))
m = s

for j = 1:N

end

t2 = t
x2 = x
ϕ = broadcast(^, x2',0:D)
Sinv = a.*I + b.*(ϕ*ϕ')
Φ = broadcast(^, X', 0:D)
for k = 1:length(X)
    m[k] = b * Φ[:,k]'*(inv(Sinv)*ϕ*t2)
    s[k] = 1/b + Φ[:,k]'*(inv(Sinv)*Φ[:,k])
end #end for

scatter(x2,t2);
plot!(X,T)
plot!(X,m)


"""
基本的点
"""

using Printf

struct Point2D
    x :: Float64
    y :: Float64
end

#function Base.show(io::IO, pt::Point2D)
#    @printf(io, "2D Point (%.6f, %.6f)\n", pt.x, pt.y)
#end

function shift_point(pt1::Point2D, pt2::Point2D)
    return Point2D(pt1.x + pt2.x, pt1.y + pt2.y)
end

Base.:+(pt1::Point2D, pt2::Point2D) = shift_point(pt1, pt2)

Base.:-(pt1::Point2D) = Point2D(-pt1.x, -pt1.y)

Base.:-(pt1::Point2D, pt2::Point2D) = shift_point(pt1, -pt2)

Base.:*(scal::N, pt1::Point2D) where N <: Real = Point2D(scal*pt1.x, scal*pt1.y)

"""
两个点的中点
"""
function middle_point(pt1::Point2D, pt2::Point2D; sc1=0.5, sc2=0.5) :: Point2D
    scs = sc1 + sc2
    xpt = pt1.x * sc1 + pt2.x * sc2
    xpt /= scs
    ypt = pt1.y * sc1 + pt2.y * sc2
    ypt /= scs
    return Point2D(xpt, ypt)
end

"""获取坐标系中点的角度，从x轴逆时针开始算"""
function absolute_angle(pt::Point2D)
    ang = atan(pt.y, pt.x)
    #负半轴加2pi
    if pt.y < 0
        ang = ang + 2pi
    end
    return ang
end


# ---
# title: 728. Self Dividing Numbers
# id: problem728
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Math
# link: <https://leetcode.com/problems/self-dividing-numbers/description/>
# hidden: true
# ---
#
# A _self-dividing number_ is a number that is divisible by every digit it
# contains.
#
# For example, 128 is a self-dividing number because `128 % 1 == 0`, `128 % 2 ==
# 0`, and `128 % 8 == 0`.
#
# Also, a self-dividing number is not allowed to contain the digit zero.
#
# Given a lower and upper number bound, output a list of every possible self
# dividing number, including the bounds if possible.
#
# **Example 1:**
#
#
#
#     Input:
#     left = 1, right = 22
#     Output: [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 15, 22]
#
#
# **Note:**
#
# * The boundaries of each input argument are `1 <= left <= right <= 10000`.
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


@testset "Box" begin
    b = Box([0,0], [2,4])  # specify center and radii
    @test b == deepcopy(b)
    @test hash(b) == hash(deepcopy(b))
    @test [0.3,-1.5] ∈ b
    @test [0.3,-2.5] ∉ b

    @test ((x,nout) = surfpt_nearby([0,0],b); (x≈[1,0] && nout≈[1,0]) || (x≈[0,2] && nout≈[0,1]))  # handle point at center properly
    @test all([(p = [1sx,2sy]; surfpt_nearby(1.1p,b) == (p, normalize(1.1p-p))) for sx = (-1,1), sy = (-1,1)])  # outside corners
    @test all([((x, nout) = surfpt_nearby([1sx,2sy],b); x≈[1sx,2sy] && all([sx 0; 0 sy]*nout.≥0) && norm(nout)≈1) for sx = (-1,1), sy = (-1,1)])  # at corners
    @test all([surfpt_nearby([ρ*1sx,1sy],b) == ([1sx,1sy], [sx,0]) for ρ = (one⁻⁻,1,one⁺⁺), sx = (-1,1), sy = (-1,0,1)])  # around faces
    @test all([surfpt_nearby([0.5sx,ρ*2sy],b) == ([0.5sx,2sy], [0,sy]) for ρ = (one⁻⁻,1,one⁺⁺), sx = (-1,0,1), sy = (-1,1)])  # around faces

    @test normal([1.1,0],b) == [1,0]
    @test normal([-1.1,0],b) == [-1,0]
    @test normal([1.1,2.1],b) == [1,1]/√2
    @test bounds(b) == ([-1,-2],[1,2])
    @test bounds(Box([0,0], [2,4], [1 1; 1 -1])) ≈ ([-3*√0.5,-3*√0.5], [3*√0.5,3*√0.5])
    @test checkbounds(b)
    @test checkbounds(Box([0,0], [2,4], [1 1; 1 -1]))

    @test (∆ = rand(2); translate(b,∆) ≈ Box([0,0]+∆, [2,4]))
end  # @testset "Box"

@testset "Box, rotated" begin
    ax1, ax2 = normalize.(([1,-1], [1,1]))
    r1, r2 = 1, 2  # "radii"
    br = Box([0,0], [2r1, 2r2], [ax1 ax2])

    R = [ax1 ax2]  # rotation matrix

    Cin = R * (GeometryPrimitives.signmatrix(br) .* (one⁻ .* [r1,r2]))  # around corners, inside
    Cout = R * (GeometryPrimitives.signmatrix(br) .* (one⁺ .* [r1,r2]))  # around corners, outside
    for j = 1:2; @test Cin[:,j] ∈ br; end
    for j = 1:2; @test Cout[:,j] ∉ br; end

    @test br == deepcopy(br)
    @test hash(br) == hash(deepcopy(br))

    @test ((x,nout) = surfpt_nearby([0,0],br); (x≈r1*ax1 && nout≈ax1) || (x≈r2*ax2 && nout≈ax2))  # handle point at center properly
    @test all([(p = (s1*r1*ax1+s2*r2*ax2); surfpt_nearby(1.1p,br) ≈ (p,normalize(1.1p-p))) for s1 = (-1,1), s2 = (-1,1)])  # outside corners
    @test all([(p = (s1*r1*ax1+s2*r2*ax2); (x,nout) = surfpt_nearby(p,br); x≈p && all([s1*ax1 s2*ax2]'*nout.≥0) && norm(nout)≈1) for s1 = (-1,1), s2 = (-1,1)])  # at corners
    @test all([(p1 = s1*r1*ax1; p2 = s2*r2/2*ax2; surfpt_nearby(ρ*p1+p2,br) ≈ (p1+p2, s1*ax1)) for ρ = (one⁻⁻,1,one⁺⁺), s1 = (-1,1), s2 = (-1,0,1)])  # around faces
    @test all([(p1 = s1*r1/2*ax1; p2 = s2*r2*ax2; surfpt_nearby(p1+ρ*p2,br) ≈ (p1+p2, s2*ax2)) for ρ = (one⁻⁻,1,one⁺⁺), s1 = (-1,0,1), s2 = (-1,1)])  # around faces

    @test normal(R*[1.1r1, 0], br) ≈ R*[1,0]
    @test normal(R*[-1.1r1, 0], br) ≈ R*[-1,0]
    @test normal(R*[0, 1.1r2], br) ≈ R*[0,1]
    @test normal(R*[0, -1.1r2], br) ≈ R*[0,-1]
    @test normal(R*[1.1r1, 1.01r2], br) ≈ R*(normalize([1.1r1, 1.01r2]-[r1,r2]))

    xmax = (R*[r1,r2])[1]
    ymax = (R*[-r1,r2])[2]
    @test bounds(br) ≈ (-[xmax,ymax], [xmax,ymax])
    @test checkbounds(br)

    @test (∆ = rand(2); translate(br,∆) ≈ Box([0,0]+∆, [2r1, 2r2], [ax1 ax2]))
end  # @testset "Box, rotated"

@testset "Box, skewed" begin
    ax1, ax2 = normalize.(([1,-1], [0,1]))
    r1, r2 = 1, 1  # "radii"
    bs = Box([0,0], [2r1, 2r2], [ax1 ax2])

    @test bs == deepcopy(bs)
    @test hash(bs) == hash(deepcopy(bs))

    n1, n2 = normalize.(([1,0], [1,1]))
    @test ((x,nout) = surfpt_nearby([0,0],bs); (x⋅n1≈1/√2 && nout≈n1) || (x⋅n2≈1/√2 && nout≈n2))  # handle point at center properly
    @test all([(p = (s1*r1*ax1+s2*r2*ax2); (x,nout) = surfpt_nearby(1.1p,bs); nout⋅(s1*n1+s2*n2)>0 && norm(nout)≈1) for s1 = (-1,1), s2 = (-1,1)])  # outside corners
    @test all([(p = (s1*r1*ax1+s2*r2*ax2); (x,nout) = surfpt_nearby(p,bs); x≈p && nout⋅(s1*n1+s2*n2)>0 && norm(nout)≈1) for s1 = (-1,1), s2 = (-1,1)])  # at corners
    # Use the above, less demanding tests instead of the below two; surfpt_nearby suffers from some inaccuracy around corners of skewed boxes.
    # @test all([(p = (s1*r1*ax1+s2*r2*ax2); (x,nout) = surfpt_nearby(1.1p,bs); all([s1*n1 s2*n2]'*nout.>0) && norm(nout)≈1) for s1 = (-1,1), s2 = (-1,1)])  # outside corners
    # @test all([(p = (s1*r1*ax1+s2*r2*ax2); (x,nout) = surfpt_nearby(p,bs); all([s1*n1 s2*n2]'*nout.≥0) && norm(nout)≈1) for s1 = (-1,1), s2 = (-1,1)])  # at corners
    @test all([(p1 = s1*r1*ax1; p2 = s2*r2/2*ax2; surfpt_nearby(ρ*p1+p2,bs) ≈ (p1+p2, s1*n1)) for ρ = (one⁻,1,one⁺), s1 = (-1,1), s2 = (-1,0,1)])  # around faces
    @test all([(p1 = s1*r1/2*ax1; p2 = s2*r2*ax2; surfpt_nearby(p1+ρ*p2,bs) ≈ (p1+p2, s2*n2)) for ρ = (one⁻,1,one⁺), s1 = (-1,0,1), s2 = (-1,1)])  # around faces

    @test norm(normal([0,1], bs)) ≈ 1

    xmax = (r1*ax1+r2*ax2)[1]
    ymax = (r2*ax2-r1*ax1)[2]
    @test bounds(bs) ≈ (-[xmax,ymax],[xmax,ymax])
    @test checkbounds(bs)

    @test (∆ = rand(2); translate(bs,∆) ≈ Box([0,0]+∆, [2r1, 2r2], [ax1 ax2]))
end  # @testset "Box, skewed"


using Test
using IntervalArithmetic

@testset "construction" begin
@testset "Interval() strict w.r.t. invalid inputs" begin
    @test_throws AssertionError Interval(3.0, 2.0)
    @test_throws AssertionError Interval(-1.0, -2.0)
    @test_throws AssertionError Interval(Inf, Inf)
    @test_throws AssertionError Interval(-Inf, -Inf)
    @test_throws AssertionError Interval(NaN, 1.0)
end

@testset "interval() flexible w.r.t. invalid inputs" begin
    @test interval(3, 2) |> isempty
    @test interval(NaN) |> isnai
    @test interval(Inf, Inf) |> isempty
    @test interval(-Inf, -Inf) |> isempty
    @test interval(1.0, 2.0) isa Interval
end
end # construction

@testset "isempty() distinguishes empty and non-empty" begin
    @test emptyinterval() |> isempty
    @test interval(1, 2) |> !isempty
end

@testset "isnai() distinguishes intervals and non-intervals" begin
    @test nai() |> isnai
    @test interval(1, 2) |> !isnai
    @test emptyinterval() |> !isnai
end

@testset "unit ball" begin
    a = unit_ball(Float64)
    @test diam(a) == 2
    @test radius(a) == 1
    @test mid(a) == 0
    @test zero_in(a)
end

@testset "nonnegatives" begin
    a = nonnegatives(Float64)
    @test diam(a) |> isinf
    @test inf(a) == 0
    @test sup(a) |> isinf
    @test isunbounded(a)
    @test zero_in(a)
    @test intersect(a, -a) == interval(0)
end

@testset "entire real line" begin
    a = entireinterval(Float64)
    @test diam(a) |> isinf
    @test inf(a) |> isinf
    @test sup(a) |> isinf
    @test isunbounded(a)
    @test zero_in(a)
    @test intersect(a, -a) == a
end


using Pkg
Pkg.activate(".")

using Revise
using ExponentialFamilies
using BenchmarkTools
using Distributions
@btime logpdf(normal(0,1), 0.0)

const x = randn(10000);

logpdf(iid(normal(0,1)),x)

dist = zeros(10000)
expf = zeros(10000)

for j=1:10000
    @views dist[j] = @elapsed sum(Distributions.logpdf.(Normal(),x[1:j]))
    @views expf[j] = @elapsed logpdf(iid(normal(0,1)),x[1:j])
end

using Plots

js = 1:10000
plot(js, dist ./ js * 1e9, label="Using Distributions.jl", legend=:bottomleft)
plot!(js, expf ./ js * 1e9, label="Using ExponentialFamilies.jl")
ylims!(0,10) #quantile(dist ./ js * 1e9, 0.99))
xlabel!("Number of Observations")
ylabel!("Nanoseconds per Observation")
# using Cthulhu


@testset "transformedkernel" begin
    rng = MersenneTwister(123456)
    x = rand(rng)*2
    v1 = rand(rng, 3)
    v2 = rand(rng, 3)

    s = rand(rng)
    s2 = rand(rng)
    v = rand(rng, 3)
    k = SqExponentialKernel()
    kt = TransformedKernel(k,ScaleTransform(s))
    ktard = TransformedKernel(k,ARDTransform(v))
    @test kt(v1, v2) == transform(k, ScaleTransform(s))(v1, v2)
    @test kt(v1, v2) == transform(k, s)(v1,v2)
    @test kt(v1, v2) ≈ k(s * v1, s * v2) atol=1e-5
    @test ktard(v1, v2) ≈ transform(k, ARDTransform(v))(v1, v2) atol=1e-5
    @test ktard(v1, v2) == transform(k,v)(v1, v2)
    @test ktard(v1, v2) == k(v .* v1, v .* v2)
    @test transform(kt, s2)(v1, v2) ≈ kt(s2 * v1, s2 * v2)
    @test KernelFunctions.kernel(kt) == k
    @test repr(kt) == repr(k) * "\n\t- " * repr(ScaleTransform(s))

    @testset "kernelmatrix" begin
        rng = MersenneTwister(123456)

        Nx = 5
        Ny = 4
        D = 3

        k = SqExponentialKernel()
        t = ScaleTransform(randn(rng))
        kt = TransformedKernel(k, t)

        @testset "$(typeof(x))" for (x, y) in [
            (ColVecs(randn(rng, D, Nx)), ColVecs(randn(rng, D, Ny))),
            (RowVecs(randn(rng, Nx, D)), RowVecs(randn(rng, Ny, D))),
        ]
            @test kernelmatrix(kt, x, y) ≈ kernelmatrix(k, map(t, x), map(t, y))

            @test kernelmatrix(kt, x) ≈ kernelmatrix(k, map(t, x))

            @test kerneldiagmatrix(kt, x) ≈ kerneldiagmatrix(k, map(t, x))

            tmp = Matrix{Float64}(undef, length(x), length(y))
            @test kernelmatrix!(tmp, kt, x, y) ≈ kernelmatrix(kt, x, y)

            tmp_square = Matrix{Float64}(undef, length(x), length(x))
            @test kernelmatrix!(tmp_square, kt, x) ≈ kernelmatrix(kt, x)

            tmp_diag = Vector{Float64}(undef, length(x))
            @test kerneldiagmatrix!(tmp_diag, kt, x) ≈ kerneldiagmatrix(kt, x)
        end
    end
    test_ADs(x->transform(SqExponentialKernel(), x[1]), rand(1))# ADs = [:ForwardDiff, :ReverseDiff])
end


module TestLQNash

using QuantEcon
using Base.Test
using FactCheck

# set up for when agents don't interact with each other
a = [.95 0.
     0 .95]
b1 = [.95; 0.]
b2 = [0.; .95]
r1 = [-.25 0.
      0. 0.]
r2 = [0. 0.
      0. -.25]
q1 = [-.15]
q2 = [-.15]

f1, f2, p1, p2 = nnash(a, b1, b2, r1, r2, q1, q2, 0, 0, 0, 0, 0, 0,
                       tol=1e-8, max_iter=10000)

alq = .95
blq = .95
rlq = -.25
qlq = -.15

lq_obj = LQ(qlq, rlq, alq, blq, bet=1)

p, f, d = stationary_values(lq_obj)

facts("Testing lqnash.jl") do

    context("Checking the policies") do

        @fact sum(f1) --> roughly(sum(f2))
        @fact sum(f1) --> roughly(sum(f))
    end

    context("Checking the Value Function") do

        @fact p1[1, 1] --> roughly(p2[2, 2])
        @fact p1[1, 1] --> roughly(p[1])
    end

    context("Judd test case")  do
        # Define Parameters
        delta = 0.02
        d = [-1 0.5
           0.5 -1]
        B = [25; 25]
        c1 = [1; -2; 1]
        c2 = [1; -2; 1]
        e1 = [10; 10; 3]
        e2 = [10; 10; 3]
        delta_1 = 1 - delta

        ## Define matrices
        a = [delta_1 0 -delta_1*B[1]
           0 delta_1 -delta_1*B[2]
           0 0 1]

        b1 = delta_1 * [1 -d[1, 1]
                      0 -d[2, 1]
                      0 0]
        b2 = delta_1 * [0 -d[1, 2]
                      1 -d[2, 2]
                      0 0]

        r1 = -[0.5*c1[3] 0 0.5*c1[2]
             0 0 0
             0.5*c1[2] 0 c1[1]]
        r2 = -[0 0 0
             0 0.5*c2[3] 0.5*c2[2]
             0 0.5*c2[2] c2[1]]

        q1 = [-0.5*e1[3] 0
            0 d[1, 1]]
        q2 = [-0.5*e2[3] 0
            0 d[2, 2]]

        s1 = zeros(2, 2)
        s2 = copy(s1)

        w1 = [0 0
            0 0
           -0.5*e1[2] B[1]/2.]
        w2 = [0 0
            0 0
           -0.5*e2[2] B[2]/2.]

        m1 = [0 0
            0 d[1, 2]/2.]
        m2 = copy(m1)

        # build model and solve it
        f1, f2, p1, p2 = nnash(a, b1, b2, r1, r2, q1, q2, s1, s2, w1, w2, m1, m2)

        aaa = a - b1*f1 - b2*f2
        aa = aaa[1:2, 1:2]
        tf = eye(2) - aa
        tfi = inv(tf)
        xbar = tfi*aaa[1:2, 3]

        # Define answers from matlab. TODO: this is ghetto
        f1_ml = [0.243666582208565   0.027236062661951 -6.827882928738190
                 0.392370733875639   0.139696450885998 -37.734107291009138]

        f2_ml = [0.027236062661951   0.243666582208565  -6.827882928738186
                 0.139696450885998   0.392370733875639 -37.734107291009131]

        xbar_ml = [1.246871007582702, 1.246871007582685]

        @fact f1 --> roughly(f1_ml)
        @fact f2 --> roughly(f2_ml)
        @fact xbar --> roughly(xbar_ml)

    end
end
end  # module


struct KroneckerMap{T, As<:Tuple{Vararg{LinearMap}}} <: LinearMap{T}
    maps::As
    function KroneckerMap{T, As}(maps::As) where {T, As}
        for A in maps
            promote_type(T, eltype(A)) == T || throw(InexactError())
        end
        return new{T,As}(maps)
    end
end

KroneckerMap{T}(maps::As) where {T, As<:Tuple{Vararg{LinearMap}}} = KroneckerMap{T, As}(maps)

"""
    kron(A::LinearMap, B::LinearMap)
    kron(A, B, Cs...)

Construct a `KroneckerMap <: LinearMap` object, a (lazy) representation of the
Kronecker product of two `LinearMap`s. One of the two factors can be an `AbstractMatrix`,
which is then promoted to a `LinearMap` automatically.

To avoid fallback to the generic [`Base.kron`](@ref) in the multi-map case,
there must be a `LinearMap` object among the first 8 arguments in usage like
`kron(A, B, Cs...)`.

For convenience, one can also use `A ⊗ B` or `⊗(A, B, Cs...)` (typed as `\\otimes+TAB`) to construct the
`KroneckerMap`, even when all arguments are of `AbstractMatrix` type.

If `A`, `B`, `C` and `D` are linear maps of such size that one can form the matrix
products `A*C` and `B*D`, then the mixed-product property `(A⊗B)*(C⊗D) = (A*C)⊗(B*D)`
holds. Upon vector multiplication, this rule is checked for applicability.

# Examples
```jldoctest; setup=(using LinearAlgebra, SparseArrays, LinearMaps)
julia> J = LinearMap(I, 2) # 2×2 identity map
LinearMaps.UniformScalingMap{Bool}(true, 2)

julia> E = spdiagm(-1 => trues(1)); D = E + E' - 2I;

julia> Δ = kron(D, J) + kron(J, D); # discrete 2D-Laplace operator

julia> Matrix(Δ)
4×4 Array{Int64,2}:
 -4   1   1   0
  1  -4   0   1
  1   0  -4   1
  0   1   1  -4
```
"""
Base.kron(A::LinearMap, B::LinearMap) = KroneckerMap{promote_type(eltype(A), eltype(B))}((A, B))
Base.kron(A::LinearMap, B::KroneckerMap) = KroneckerMap{promote_type(eltype(A), eltype(B))}(tuple(A, B.maps...))
Base.kron(A::KroneckerMap, B::LinearMap) = KroneckerMap{promote_type(eltype(A), eltype(B))}(tuple(A.maps..., B))
Base.kron(A::KroneckerMap, B::KroneckerMap) = KroneckerMap{promote_type(eltype(A), eltype(B))}(tuple(A.maps..., B.maps...))
Base.kron(A::LinearMap, B::LinearMap, Cs::LinearMap...) = KroneckerMap{promote_type(eltype(A), eltype(B), map(eltype, Cs)...)}(tuple(A, B, Cs...))
Base.kron(A::AbstractMatrix, B::LinearMap) = kron(LinearMap(A), B)
Base.kron(A::LinearMap, B::AbstractMatrix) = kron(A, LinearMap(B))
# promote AbstractMatrix arguments to LinearMaps, then take LinearMap-Kronecker product
for k in 3:8 # is 8 sufficient?
    Is = ntuple(n->:($(Symbol(:A,n))::AbstractMatrix), Val(k-1))
    # yields (:A1, :A2, :A3, ..., :A(k-1))
    L = :($(Symbol(:A,k))::LinearMap)
    # yields :Ak::LinearMap
    mapargs = ntuple(n -> :(LinearMap($(Symbol(:A,n)))), Val(k-1))
    # yields (:LinearMap(A1), :LinearMap(A2), ..., :LinearMap(A(k-1)))

    @eval Base.kron($(Is...), $L, As::MapOrMatrix...) =
        kron($(mapargs...), $(Symbol(:A,k)), convert_to_lmaps(As...)...)
end

struct KronPower{p}
    function KronPower(p::Integer)
        p > 1 || throw(ArgumentError("the Kronecker power is only defined for exponents larger than 1, got $k"))
        return new{p}()
    end
end

"""
    ⊗(k::Integer)

Construct a lazy representation of the `k`-th Kronecker power
`A^⊗(k) = A ⊗ A ⊗ ... ⊗ A`, where `A` can be an `AbstractMatrix` or a `LinearMap`.
"""
⊗(k::Integer) = KronPower(k)

⊗(A, B, Cs...) = KroneckerMap{promote_type(eltype(A), eltype(B), map(eltype, Cs)...)}(convert_to_lmaps(A, B, Cs...))

Base.:(^)(A::MapOrMatrix, ::KronPower{p}) where {p} =
    ⊗(ntuple(n -> convert_to_lmaps_(A), Val(p))...)

Base.size(A::KroneckerMap) = map(*, size.(A.maps)...)

LinearAlgebra.issymmetric(A::KroneckerMap) = all(issymmetric, A.maps)
LinearAlgebra.ishermitian(A::KroneckerMap{<:Real}) = issymmetric(A)
LinearAlgebra.ishermitian(A::KroneckerMap) = all(ishermitian, A.maps)

LinearAlgebra.adjoint(A::KroneckerMap{T}) where {T} = KroneckerMap{T}(map(adjoint, A.maps))
LinearAlgebra.transpose(A::KroneckerMap{T}) where {T} = KroneckerMap{T}(map(transpose, A.maps))

Base.:(==)(A::KroneckerMap, B::KroneckerMap) = (eltype(A) == eltype(B) && A.maps == B.maps)

#################
# multiplication helper functions
#################

@inline function _kronmul!(y, B, X, At, T)
    na, ma = size(At)
    mb, nb = size(B)
    v = zeros(T, ma)
    temp1 = similar(y, na)
    temp2 = similar(y, nb)
    @views @inbounds for i in 1:ma
        v[i] = one(T)
        A_mul_B!(temp1, At, v)
        A_mul_B!(temp2, X, temp1)
        A_mul_B!(y[((i-1)*mb+1):i*mb], B, temp2)
        v[i] = zero(T)
    end
    return y
end
@inline function _kronmul!(y, B::Union{MatrixMap,UniformScalingMap}, X, At::Union{MatrixMap,UniformScalingMap}, T)
    na, ma = size(At)
    mb, nb = size(B)
    if nb*ma < mb*na
        mul!(reshape(y, (mb, ma)), B, convert(Matrix, X*At))
    else
        mul!(reshape(y, (mb, ma)), convert(Matrix, B*X), At isa MatrixMap ? At.lmap : At.λ)
    end
    return y
end

#################
# multiplication with vectors
#################

Base.@propagate_inbounds function A_mul_B!(y::AbstractVector, L::KroneckerMap{T,<:NTuple{2,LinearMap}}, x::AbstractVector) where {T}
    require_one_based_indexing(y)
    @boundscheck check_dim_mul(y, L, x)
    A, B = L.maps
    X = LinearMap(reshape(x, (size(B, 2), size(A, 2))); issymmetric=false, ishermitian=false, isposdef=false)
    _kronmul!(y, B, X, transpose(A), T)
    return y
end
Base.@propagate_inbounds function A_mul_B!(y::AbstractVector, L::KroneckerMap{T}, x::AbstractVector) where {T}
    require_one_based_indexing(y)
    @boundscheck check_dim_mul(y, L, x)
    A = first(L.maps)
    B = kron(Base.tail(L.maps)...)
    X = LinearMap(reshape(x, (size(B, 2), size(A, 2))); issymmetric=false, ishermitian=false, isposdef=false)
    _kronmul!(y, B, X, transpose(A), T)
    return y
end
# mixed-product rule, prefer the right if possible
# (A₁ ⊗ A₂ ⊗ ... ⊗ Aᵣ) * (B₁ ⊗ B₂ ⊗ ... ⊗ Bᵣ) = (A₁B₁) ⊗ (A₂B₂) ⊗ ... ⊗ (AᵣBᵣ)
Base.@propagate_inbounds function A_mul_B!(y::AbstractVector, L::CompositeMap{<:Any,<:Tuple{KroneckerMap,KroneckerMap}}, x::AbstractVector)
    B, A = L.maps
    if length(A.maps) == length(B.maps) && all(M -> check_dim_mul(M[1], M[2]), zip(A.maps, B.maps))
        A_mul_B!(y, kron(map(*, A.maps, B.maps)...), x)
    else
        A_mul_B!(y, LinearMap(A)*B, x)
    end
end
# mixed-product rule, prefer the right if possible
# (A₁ ⊗ B₁)*(A₂⊗B₂)*...*(Aᵣ⊗Bᵣ) = (A₁*A₂*...*Aᵣ) ⊗ (B₁*B₂*...*Bᵣ)
Base.@propagate_inbounds function A_mul_B!(y::AbstractVector, L::CompositeMap{T,<:Tuple{Vararg{KroneckerMap{<:Any,<:Tuple{LinearMap,LinearMap}}}}}, x::AbstractVector) where {T}
    As = map(AB -> AB.maps[1], L.maps)
    Bs = map(AB -> AB.maps[2], L.maps)
    As1, As2 = Base.front(As), Base.tail(As)
    Bs1, Bs2 = Base.front(Bs), Base.tail(Bs)
    apply = all(A -> check_dim_mul(A...), zip(As1, As2)) && all(A -> check_dim_mul(A...), zip(Bs1, Bs2))
    if apply
        A_mul_B!(y, kron(prod(As), prod(Bs)), x)
    else
        A_mul_B!(y, CompositeMap{T}(map(LinearMap, L.maps)), x)
    end
end

Base.@propagate_inbounds At_mul_B!(y::AbstractVector, A::KroneckerMap, x::AbstractVector) =
    A_mul_B!(y, transpose(A), x)

Base.@propagate_inbounds Ac_mul_B!(y::AbstractVector, A::KroneckerMap, x::AbstractVector) =
    A_mul_B!(y, adjoint(A), x)

###############
# KroneckerSumMap
###############
struct KroneckerSumMap{T, As<:Tuple{LinearMap,LinearMap}} <: LinearMap{T}
    maps::As
    function KroneckerSumMap{T, As}(maps::As) where {T, As}
        for A in maps
            size(A, 1) == size(A, 2) || throw(ArgumentError("operators need to be square in Kronecker sums"))
            promote_type(T, eltype(A)) == T || throw(InexactError())
        end
        return new{T,As}(maps)
    end
end

KroneckerSumMap{T}(maps::As) where {T, As<:Tuple{LinearMap,LinearMap}} = KroneckerSumMap{T, As}(maps)

"""
    kronsum(A, B)
    kronsum(A, B, Cs...)

Construct a `KroneckerSumMap <: LinearMap` object, a (lazy) representation of the
Kronecker sum `A⊕B = A ⊗ Ib + Ia ⊗ B` of two square linear maps of type
`LinearMap` or `AbstractMatrix`. Here, `Ia` and `Ib` are identity operators of
the size of `A` and `B`, respectively. Arguments of type `AbstractMatrix` are
automatically promoted to `LinearMap`s.

For convenience, one can also use `A ⊕ B` or `⊕(A, B, Cs...)` (typed as
`\\oplus+TAB`) to construct the `KroneckerSumMap`.

# Examples
```jldoctest; setup=(using LinearAlgebra, SparseArrays, LinearMaps)
julia> J = LinearMap(I, 2) # 2×2 identity map
LinearMaps.UniformScalingMap{Bool}(true, 2)

julia> E = spdiagm(-1 => trues(1)); D = LinearMap(E + E' - 2I);

julia> Δ₁ = kron(D, J) + kron(J, D); # discrete 2D-Laplace operator, Kronecker sum

julia> Δ₂ = kronsum(D, D);

julia> Δ₃ = D^⊕(2);

julia> Matrix(Δ₁) == Matrix(Δ₂) == Matrix(Δ₃)
true
```
"""
kronsum(A::MapOrMatrix, B::MapOrMatrix) =
    KroneckerSumMap{promote_type(eltype(A), eltype(B))}(convert_to_lmaps(A, B))
kronsum(A::MapOrMatrix, B::MapOrMatrix, C::MapOrMatrix, Ds::MapOrMatrix...) =
    kronsum(A, kronsum(B, C, Ds...))

struct KronSumPower{p}
    function KronSumPower(p::Integer)
        p > 1 || throw(ArgumentError("the Kronecker sum power is only defined for exponents larger than 1, got $k"))
        return new{p}()
    end
end

"""
    ⊕(k::Integer)

Construct a lazy representation of the `k`-th Kronecker sum power `A^⊕(k) = A ⊕ A ⊕ ... ⊕ A`,
where `A` can be a square `AbstractMatrix` or a `LinearMap`.
"""
⊕(k::Integer) = KronSumPower(k)

⊕(a, b, c...) = kronsum(a, b, c...)

Base.:(^)(A::MapOrMatrix, ::KronSumPower{p}) where {p} = kronsum(ntuple(n -> convert_to_lmaps_(A), Val(p))...)

Base.size(A::KroneckerSumMap, i) = prod(size.(A.maps, i))
Base.size(A::KroneckerSumMap) = (size(A, 1), size(A, 2))

LinearAlgebra.issymmetric(A::KroneckerSumMap) = all(issymmetric, A.maps)
LinearAlgebra.ishermitian(A::KroneckerSumMap{<:Real}) = all(issymmetric, A.maps)
LinearAlgebra.ishermitian(A::KroneckerSumMap) = all(ishermitian, A.maps)

LinearAlgebra.adjoint(A::KroneckerSumMap{T}) where {T} = KroneckerSumMap{T}(map(adjoint, A.maps))
LinearAlgebra.transpose(A::KroneckerSumMap{T}) where {T} = KroneckerSumMap{T}(map(transpose, A.maps))

Base.:(==)(A::KroneckerSumMap, B::KroneckerSumMap) = (eltype(A) == eltype(B) && A.maps == B.maps)

Base.@propagate_inbounds function A_mul_B!(y::AbstractVector, L::KroneckerSumMap, x::AbstractVector)
    @boundscheck check_dim_mul(y, L, x)
    A, B = L.maps
    ma, na = size(A)
    mb, nb = size(B)
    X = reshape(x, (nb, na))
    Y = reshape(y, (nb, na))
    mul!(Y, X, convert(AbstractMatrix, transpose(A)))
    mul!(Y, B, X, true, true)
    return y
end

Base.@propagate_inbounds At_mul_B!(y::AbstractVector, A::KroneckerSumMap, x::AbstractVector) =
    A_mul_B!(y, transpose(A), x)

Base.@propagate_inbounds Ac_mul_B!(y::AbstractVector, A::KroneckerSumMap, x::AbstractVector) =
    A_mul_B!(y, adjoint(A), x)


### A Pluto.jl notebook ###
# v0.16.4

using Markdown
using InteractiveUtils

# ╔═╡ d80342ae-522b-11eb-19c0-cf931fb4ae81
begin
	import Pkg
	Pkg.activate(Base.current_project())
	using Statistics
end

# ╔═╡ 9c2ef32a-522d-11eb-29e6-29da29e4dec7
using Plots

# ╔═╡ 96096b41-fc25-43f9-ab75-50c5196e9604
md"# Chapter 8.5 Expected vs. Sample Updates"

# ╔═╡ 88b47ee6-522d-11eb-1a98-ed4dd43dfd11
begin
	Base.@kwdef mutable struct SampleAvg
		t::Int = 0
		avg::Float64 = 0.0
	end
	function (s::SampleAvg)(x)
		s.t += 1
		s.avg += (x - s.avg) / s.t
		s.avg
	end
end

# ╔═╡ 8fde026e-522d-11eb-2aa1-f3e28c238ea6
function run_once(b)
    rms = Float64[]
    distribution = randn(b)
    expectation = mean(distribution)
    sample_avg = SampleAvg()

    for i in 1:2*b
        avg = sample_avg(distribution[rand(1:b)])
        push!(rms, abs(avg - expectation))
    end
    rms
end

# ╔═╡ 39865f6a-8500-4277-884f-81b7de151fc6
md"## Figure 8.7"

# ╔═╡ 969b48aa-522d-11eb-2039-61916ddf87d4
begin
	n_runs = 1000
	p = plot(legend=:topright)

	for b in [2, 10, 100, 1000]
		rms = mean(run_once(b) for _ in 1:n_runs)
		xs = (1:2*b) ./ b
		plot!(p, xs, rms, label="b=$b", ylabel="RMS error in value estimate")
	end

	p
end

# ╔═╡ Cell order:
# ╟─96096b41-fc25-43f9-ab75-50c5196e9604
# ╠═d80342ae-522b-11eb-19c0-cf931fb4ae81
# ╠═88b47ee6-522d-11eb-1a98-ed4dd43dfd11
# ╠═8fde026e-522d-11eb-2aa1-f3e28c238ea6
# ╟─39865f6a-8500-4277-884f-81b7de151fc6
# ╠═9c2ef32a-522d-11eb-29e6-29da29e4dec7
# ╠═969b48aa-522d-11eb-2039-61916ddf87d4


## Make an animation of a random walk

using Plots
using GraphRecipes
using SparseArrays
using DelimitedFiles
## Load the graph
ei,ej = copy.(eachcol(Int.(readdlm("undir-graph-14.edges"))))
xy = readdlm("undir-graph-14.xy")'
## A simple simulation
using Random
Random.seed!(1)
X = 1 # start on node 1
A = sparse(ei,ej,1,14,14) # make the sparse matrix
A = A+A' # make it symmetric (we only store half)
for i=1:10
  global X
  Xn = rand(findnz(A[:,X])[1])
  println("Walk moves from $X to $Xn")
  X = Xn
end
## A simple simulation of where it spends it's time.
Random.seed!(1)
function rw_time(A, start, nsteps)
  x = zeros(size(A,1))
  X = start
  x[X] += 1
  for i=1:(nsteps-1) # to account for start
    X = rand(findnz(A[:,X])[1])
    x[X] += 1
  end
  x ./= nsteps
end
x1000 = rw_time(A,1,1000)
## A visual simulation of the random walk
gr(size=(300,300))
using Random
Random.seed!(1)
function rw_on_graph(ei, ej, xy, start, nstep, nframes)
  eirw = [ei; ej]
  ejrw = [ej; ei]

  X = start
  # random walk step
  p = graphplot(ei, ej, x =xy[1,:], y=xy[2,:],
    markercolor=:white, markerstrokecolor=:black,
    markersize=4, linecolor=3, linealpha=1.0, linewidth=1.0,
    axis_buffer=0.02, background=:black,dpi=300)
  #p1 = scatter!([xy[1,X]],[xy[2,X]],markersize=4, color=1)
  p2 = scatter!([xy[1,X]],[xy[2,X]],markersize=8, color=:orange)

  Xn = X
  anim = @animate for i=0:(nframes*nstep)
    stepi = mod(i,nframes) # this is the frame number in the step
    if stepi == 0
      X = Xn
      Xn = rand(ejrw[eirw .== X])
    end
    # interpolate between the two positions
    pos = min(stepi/(nframes-2),1)
    p[3] = [(1-pos)*xy[1,X] + pos*xy[1,Xn]],[(1-pos)*xy[2,X] + pos*xy[2,Xn]]
    ##p[4] = [xy[1,Xn]],[xy[2,Xn]]
    title!("$i $stepi", textcolor=:white)
  end
  return anim
end
anim = rw_on_graph(ei,ej,xy,1,100,20)
gif(anim, "rw-on-undir-14.gif", fps=40)

## Next we want to show the frequency that we visit places.
gr(size=(400,300),dpi=300)
using Random
Random.seed!(1)
function rw_on_graph_with_dist(ei, ej, xy, start, nstep)
  eirw = [ei; ej]
  ejrw = [ej; ei]
  N = maximum(eirw)
  X = start
  # random walk step
  ## Simulate the random walk and show the distribution

  l = @layout([a{0.75w} b])
  x = ones(N)
  ps = plot(graphplot(ei, ej, x =xy[1,:], y=xy[2,:],
    markercolor=:white, markerstrokecolor=:black,
    markersize=4, linecolor=3, linealpha=1.0, linewidth=1.0,
    axis_buffer=0.02, background=:black,dpi=300),
    scatter(x/N, 1:N, label="", framestyle=:none,background=:black), layout=l)
  p1 = ps[1]
  p2 = ps[2]
  xlims!(ps[2], (-0.1, 1.1))
  p1a = scatter!(p1, [xy[1,X]],[xy[2,X]],markersize=4, color=1)
  p1b = scatter!(p1, [xy[1,X]],[xy[2,X]],markersize=8, color=:orange)
  Xn = X # initialize
  nframes = 3
  anim = @animate for i=0:nframes*nstep
    stepi = mod(i,3)
    if stepi == 0
      X = Xn
      Xn = rand(ejrw[eirw .== X])
    end
    x[Xn] += 1/nframes

    pos = stepi/nframes

    ps[4] = [xy[1,X]],[xy[2,X]]
    ps[5] = [pos*xy[1,Xn] + (1-pos)*xy[1,X]],[pos*xy[2,Xn]+ (1-pos)*xy[2,X]]
    ps[3] = x/(N+i/nframes),collect(1:N)
  end
  return anim
end

anim = rw_on_graph_with_dist(ei, ej, xy, 1, 500)
gif(anim, "rw-on-undir-with-dist.gif", fps=25)


module Day06

using DataStructures: counter

export day06

const TIME_TO_REPRODUCE = 7
const TIME_TO_MATURITY = 9

function readInput(inputfile="inputs/input06.txt")
    return parse.(UInt8, split(readline(inputfile), ","))
end

# Let L(x,t) be the population resulting starting with a SINGLE lanternfish with counter x
# after t days
# Use relations for L(x,t)
# - L(x, t) = L(0, t-x)
# - L(0, k) = 1 (for k ≤ 0)
# - L(0, t) = L(0, t-TIME_TO_REPRODUCE) + L(0, t-TIME_TO_MATURITY)

################################## more Julian solution ##################################
## a bit slower than the lower one
## but that's just due to the use of DataStructures.counter
## without that the 2 solution run in pretty much identical time

struct LaternfishPopulationPrediction
    precomputed::Vector{Int64}
    upto::Int64
    function LaternfishPopulationPrediction(upto)
        new(_precompute_lanternfish_popuplation(upto), upto)
    end
end

function _precompute_lanternfish_popuplation(upto)
    populations = Vector{Int64}(undef, upto+TIME_TO_MATURITY)
    populations[1:TIME_TO_MATURITY] .= 1
    for index in TIME_TO_MATURITY .+ (1:upto)
        populations[index] = populations[index-TIME_TO_REPRODUCE] + populations[index-TIME_TO_MATURITY]
    end
    return populations
end

function Base.getindex(pop::LaternfishPopulationPrediction, time)
    time <= pop.upto || error("time larger than simulated! ($time > $(pop.upto))")
    return pop.precomputed[time+9]
end

function population(counts, time, precomputed=nothing)
    precomputed === nothing && (precomputed = LaternfishPopulationPrediction(time))
    total = 0
    for (ini, count) in counts
        total += precomputed[time-ini]*count
    end
    return total
end

function day06(inputdata = readInput("inputs/input06.txt"))
    pop = LaternfishPopulationPrediction(256)
    counts = counter(inputdata)
    return population(counts, 80, pop), population(counts, 256, pop)
end

######################## using Memoize.jl and optimizing counter ########################

using Memoize

L(x,t) = _L(t-x)
@memoize function _L(t)
    if t <= 0
        return 1
    else
        return _L(t-TIME_TO_REPRODUCE) + _L(t-TIME_TO_MATURITY)
    end
end

_L(256) # warm cache

function population_alt(counts, time)
    total = 0
    for (ini, count) in counts
        total += L(ini-1, time)*count
    end
    return total
end

function day06_alt(inputdata = readInput("inputs/input06.txt"))
    counts = enumerate(_mycount(inputdata)) # ~4x faster than counter
    return population_alt(counts, 80), population_alt(counts, 256)
end

function _mycount(inputdata)
    ret = zeros(Int64, maximum(inputdata)+1)
    for i in inputdata
        ret[i+1] += 1
    end
    return ret
end

end #module


function lanczosIterations(A, stepsize, αs, βs, vs, reorthogonalize)

    iter = length(αs)
    v1 = vs[iter]
    v = vs[iter + 1]
    β = βs[iter]

    for i = iter .+ (1:stepsize)
        w = A*v
        α = real(dot(w,v))

        axpy!(-α, v, w)
        axpy!(-β, v1, w)

        # Orthogonlize (FixMe! consider a partial reorthogonal strategy here)
        if reorthogonalize
            for j = 1:i - 1
                axpy!(-dot(vs[j], w), vs[j], w)
            end
        end

        v1 = v
        v, β = qr!(w)

        push!(αs, α)
        push!(βs, β)
        push!(vs, v)
    end
    return αs, βs, vs
end

function _teig(A,
    nvals = 1;
    maxiter = size(A, 1),
    initvec = convert(Vector{eltype(A)}, randn(size(A, 1))),
    tolconv = sqrt(eps(real(eltype(A)))),
    reorthogonalize = true,
    stepsize = max(1, div(nvals, 10)),
    debug = false)

    # LinAlg.chksquare(A)

    ### initialize αs, βs and vs ###
    # run a single iteration of the Lanczos procedure. Note that this step handles promotion Lanczos vectors as well. They will in in general be different from `initvec`.
    v1 = A*initvec
    v0 = convert(typeof(v1), initvec)
    v0, nm0 = qr!(v0)
    rmul!(v1, inv(nm0)) # because we don't assume that initvec is normalized
    α = real(dot(v1, v0))
    axpy!(-α, v0, v1)
    v1, β = qr!(v1)
    αs = [α]
    βs = [β]
    vs = typeof(v1)[v0, v1]

    ### initial iterations ###
    lanczosIterations(A, nvals - 1, αs, βs, vs, reorthogonalize)

    ### Iteration count ###
    iter = nvals

    # values0 = sort!(abs(eigvals(SymTridiagonal(αs, βs[1:end - 1]))), rev = true)
    # values1 = values0

    z = zeros(eltype(αs), 1)

    ### run itertions ###
    # continue until convergence or the maximum number of iterations has been reached.
    converged = UInt[]
    while iter <= maxiter
        lanczosIterations(A, stepsize, αs, βs, vs, reorthogonalize)
        # values, vectors = LinearAlgebra.EigenSelfAdjoint.eigQL!(SymTridiagonal(copy(αs), βs[1:end - 1]), [zeros(length(αs) - 1); ones(eltype(αs), 1)]')
        values, vectors = eigen(SymTridiagonal(αs, βs[1:end-1]))

        ### Use criteria (9.18) of Peter Arbenz' lecture notes ###
        βj = βs[end]
        converged = Int[]
        for i = 1:length(αs)
            if βj*abs(vectors[end,i]) < abs(values[i])*tolconv
                push!(converged, i)
            end
        end

        debug && @show converged

        # if length(converged) >= nvals
        #     break
        # end

        # For now we hardcode convergence for largest positive value. This should be a choice later on
        if all([length(αs) - i + 1 in converged for i = 1:nvals])
            break
        end

        iter += stepsize
        debug && @show iter
    end
    if length(converged) < nvals
        error("no convergence")
    end

    S = SymTridiagonal(αs, βs[1:end-1])
    values, vectors = eigen(S)
    return values[converged], vectors[:, converged], S, vs, βs[end]
end

teig(A,
    nvals = 1;
    maxiter = size(A, 1),
    initvec = convert(Vector{eltype(A)}, randn(size(A, 1))),
    tolconv = 1e-12,
    reorthogonalize = true,
    stepsize = max(1, div(nvals, 10)),
    debug = false) = _teig(A, nvals, maxiter = maxiter, initvec = initvec,
        tolconv = tolconv, reorthogonalize = reorthogonalize, stepsize = stepsize, debug = debug)


# print information over iterations of nndescent
using NearestNeighborDescent
using LightGraphs
using Distances
using MLDatasets
using Statistics

include("../benchmark/benchutils.jl")

function diameter_stats(graph)
    diameters = [knn_diameter(graph, v) for v in vertices(graph)]
    min_diam, med_diam, max_diam = minimum(diameters), median(diameters), maximum(diameters)
    mean_diam = mean(diameters)
    return min_diam, mean_diam, med_diam, max_diam
end

num_new_edges(graph) = sum(KNNGraphs.flag(e) for e in edges(graph))

function do_iter(graph, data, metric, sample_rate, true_indices)
    c = NearestNeighborDescent.local_join!(graph, data, metric; sample_rate=sample_rate)
    min_diam, mean_diam, med_diam, max_diam = diameter_stats(graph)
    nne = num_new_edges(graph)
    indices = knn_matrices(graph)[1]
    rec = recall(indices, true_indices)
    return c, (min_diam, mean_diam, med_diam, max_diam), nne, rec
end

function join_with_history(data, k, metric, sample_rate, true_indices)
    graph = HeapKNNGraph(data, k, metric)
    indices = knn_matrices(graph)[1]
    count_history = []
    diam_history = []
    new_edge_history = []
    recall_history = []

    count = ne(graph)
    push!(count_history, "iter_0"=>count)
    push!(diam_history, "iter_0"=>diameter_stats(graph))
    push!(new_edge_history, "iter_0"=>num_new_edges(graph))
    recall₀ = recall(indices, true_indices)
    push!(recall_history, "iter_0"=>recall₀)
    iter = 1
    while count > 0
        count, diams, nne, rec = do_iter(graph, data, metric, sample_rate, true_indices)
        push!(count_history, "iter_$iter"=>count)
        push!(diam_history, "iter_$iter"=>diams)
        push!(new_edge_history, "iter_$iter"=>nne)
        push!(recall_history, "iter_$iter"=>rec)
        iter += 1
    end
    return count_history, diam_history, new_edge_history, recall_history
end

mnist_x_arr = [col for col in
    eachcol(MNIST.convert2features(MNIST.traintensor(Float64, 1:10000)))]
k = 20
metric = SqEuclidean()

true_graph = NearestNeighborDescent.brute_knn(mnist_x_arr, metric, k)
true_indices = getindex.(true_graph, 1)

c_hist_100, d_hist_100, e_hist_100, r_hist_100 =
    join_with_history(mnist_x_arr, k, metric, 1, true_indices)

c_hist_25, d_hist_25, e_hist_25, r_hist_25 =
    join_with_history(mnist_x_arr, k, metric, .25, true_indices)


@doc raw"""
    truncated_conjugate_gradient_descent(M, F, gradF, x, η, HessF, trust_region_radius)

solve the trust-region subproblem

```math
\operatorname*{arg\,min}_{η ∈ T_xM}
m_x(η) \quad\text{where}
m_x(η) = F(x) + ⟨\operatorname{grad}F(x),η⟩_x + \frac{1}{2}⟨\operatorname{Hess}F(x)[η],η⟩_x,
```

```math
\text{such that}\quad ⟨η,η⟩_x ≤ Δ^2
```

with the [`truncated_conjugate_gradient_descent`](@ref).
For a description of the algorithm and theorems offering convergence guarantees,
see the reference:

* P.-A. Absil, C.G. Baker, K.A. Gallivan,
    Trust-region methods on Riemannian manifolds, FoCM, 2007.
    doi: [10.1007/s10208-005-0179-9](https://doi.org/10.1007/s10208-005-0179-9)
* A. R. Conn, N. I. M. Gould, P. L. Toint, Trust-region methods, SIAM,
    MPS, 2000. doi: [10.1137/1.9780898719857](https://doi.org/10.1137/1.9780898719857)

# Input

* `M` – a manifold ``\mathcal M``
* `F` – a cost function ``F: \mathcal M → ℝ`` to minimize
* `gradF` – the gradient ``\operatorname{grad}F: \mathcal M → T\mathcal M`` of `F`
* `HessF` – the hessian ``\operatorname{Hess}F(x): T_x\mathcal M → T_x\mathcal M``, ``X ↦ \operatoname{Hess}F(x)[X] = ∇_ξ\operatorname{grad}f(x)``
* `x` – a point on the manifold ``x ∈ \mathcal M``
* `η` – an update tangential vector ``η ∈ T_x\mathcal M``
* `trust_region_radius` – a trust-region radius

# Optional

* `evaluation` – ([`AllocatingEvaluation`](@ref)) specify whether the gradient and hessian work by
   allocation (default) or [`MutatingEvaluation`](@ref) in place
* `preconditioner` – a preconditioner for the hessian H
* `θ` – 1+θ is the superlinear convergence target rate. The algorithm will
    terminate early if the residual was reduced by a power of 1+theta.
* `κ` – the linear convergence target rate: algorithm will terminate
    early if the residual was reduced by a factor of kappa.
* `randomize` – set to true if the trust-region solve is to be initiated with a
    random tangent vector. If set to true, no preconditioner will be
    used. This option is set to true in some scenarios to escape saddle
    points, but is otherwise seldom activated.
* `project_vector!` : (`copyto!`) specify a projection operation for tangent vectors
    for numerical stability. A function `(M, Y, p, X) -> ...` working in place of `Y`.
    per default, no projection is perfomed, set it to `project!` to activate projection.
* `stopping_criterion` – ([`StopWhenAny`](@ref), [`StopAfterIteration`](@ref),
    [`StopIfResidualIsReducedByFactor`](@ref), [`StopIfResidualIsReducedByPower`](@ref),
    [`StopWhenCurvatureIsNegative`](@ref), [`StopWhenTrustRegionIsExceeded`](@ref) )
    a functor inheriting from [`StoppingCriterion`](@ref) indicating when to stop,
    where for the default, the maximal number of iterations is set to the dimension of the
    manifold, the power factor is `θ`, the reduction factor is `κ`.
    .
* `return_options` – (`false`) – if activated, the extended result, i.e. the
    complete [`Options`](@ref) re returned. This can be used to access recorded values.
    If set to false (default) just the optimal value `x_opt` is returned

and the ones that are passed to [`decorate_options`](@ref) for decorators.

# Output

* `η` – an approximate solution of the trust-region subproblem in ``T_{x}\mathcal M``.
OR
* `options` - the options returned by the solver (see `return_options`)

# see also
[`trust_regions`](@ref)
"""
function truncated_conjugate_gradient_descent(
    M::AbstractManifold,
    F::TF,
    gradF::TG,
    x,
    η,
    H::TH,
    trust_region_radius::Float64;
    kwargs...,
) where {TF,TG,TH}
    x_res = allocate(x)
    copyto!(M, x_res, x)
    return truncated_conjugate_gradient_descent!(
        M, F, gradF, x_res, η, H, trust_region_radius; kwargs...
    )
end
@doc raw"""
    truncated_conjugate_gradient_descent!(M, F, gradF, x, η, HessF, trust_region_radius; kwargs...)

solve the trust-region subproblem in place of `x`.

# Input
# Input
* `M` – a manifold ``\mathcal M``
* `F` – a cost function ``F: \mathcal M → ℝ`` to minimize
* `gradF` – the gradient ``\operatorname{grad}F: \mathcal M → T\mathcal M`` of `F`
* `HessF` – the hessian ``\operatorname{Hess}F(x): T_x\mathcal M → T_x\mathcal M``, ``X ↦ \operatoname{Hess}F(x)[X] = ∇_ξ\operatorname{grad}f(x)``
* `x` – a point on the manifold ``x ∈ \mathcal M``
* `η` – an update tangential vector ``η ∈ T_x\mathcal M``
* `trust_region_radius` – a trust-region radius

For more details and all optional arguments, see [`truncated_conjugate_gradient_descent`](@ref).
"""
function truncated_conjugate_gradient_descent!(
    M::AbstractManifold,
    F::TF,
    gradF::TG,
    x,
    η,
    H::TH,
    trust_region_radius::Float64;
    evaluation=AllocatingEvaluation(),
    preconditioner::Tprec=(M, x, ξ) -> ξ,
    θ::Float64=1.0,
    κ::Float64=0.1,
    randomize::Bool=false,
    stopping_criterion::StoppingCriterion=StopWhenAny(
        StopAfterIteration(manifold_dimension(M)),
        StopWhenAll(StopIfResidualIsReducedByPower(θ), StopIfResidualIsReducedByFactor(κ)),
        StopWhenTrustRegionIsExceeded(),
        StopWhenCurvatureIsNegative(),
        StopWhenModelIncreased(),
    ),
    project_vector!::Proj=copyto!,
    return_options=false,
    kwargs..., #collect rest
) where {TF,TG,TH,Tprec,Proj}
    p = HessianProblem(M, F, gradF, H, preconditioner; evaluation=evaluation)
    o = TruncatedConjugateGradientOptions(
        p,
        x,
        η,
        trust_region_radius,
        randomize;
        θ=θ,
        κ=κ,
        stop=stopping_criterion,
        (project_vector!)=project_vector!,
    )
    o = decorate_options(o; kwargs...)
    resultO = solve(p, o)
    if return_options
        resultO
    else
        return get_solver_result(resultO)
    end
end
function initialize_solver!(p::HessianProblem, o::TruncatedConjugateGradientOptions)
    (o.randomize) || zero_vector!(p.M, o.η, o.x)
    o.Hη = o.randomize ? get_hessian(p, o.x, o.η) : zero_vector(p.M, o.x)
    o.gradient = get_gradient(p, o.x)
    o.residual = o.randomize ? o.gradient + o.Hη : o.gradient
    o.z = o.randomize ? o.residual : get_preconditioner(p, o.x, o.residual)
    o.δ = -deepcopy(o.z)
    o.Hδ = zero_vector(p.M, o.x)
    o.δHδ = inner(p.M, o.x, o.δ, o.Hδ)
    o.ηPδ = o.randomize ? inner(p.M, o.x, o.η, o.δ) : zero(o.δHδ)
    o.δPδ = inner(p.M, o.x, o.residual, o.z)
    o.ηPη = o.randomize ? inner(p.M, o.x, o.η, o.η) : zero(o.δHδ)
    if o.randomize
        o.model_value = inner(p.M, o.x, o.η, o.gradient) + 0.5 * inner(p.M, o.x, o.η, o.Hη)
    else
        o.model_value = 0
    end
    o.z_r = inner(p.M, o.x, o.z, o.residual)
    o.initialResidualNorm = sqrt(inner(p.M, o.x, o.residual, o.residual))
    return o
end
function step_solver!(
    p::P, o::O, ::Int
) where {P<:HessianProblem,O<:TruncatedConjugateGradientOptions}
    # Updates
    get_hessian!(p, o.Hδ, o.x, o.δ)
    o.δHδ = inner(p.M, o.x, o.δ, o.Hδ)
    α = o.z_r / o.δHδ
    ηPη_new = o.ηPη + 2 * α * o.ηPδ + α^2 * o.δPδ
    # Check against negative curvature and trust-region radius violation.
    if o.δHδ <= 0 || ηPη_new >= o.trust_region_radius^2
        τ = (-o.ηPδ + sqrt(o.ηPδ^2 + o.δPδ * (o.trust_region_radius^2 - o.ηPη))) / o.δPδ
        o.η = o.η + τ * o.δ
        o.Hη = o.Hη + τ * o.Hδ
        o.ηPη = ηPη_new
        return o
    end
    o.ηPη = ηPη_new
    new_η = o.η + α * o.δ
    new_Hη = o.Hη + α * o.Hδ
    # No negative curvature and o.η - α * (o.δ) inside TR: accept it.
    o.new_model_value =
        inner(p.M, o.x, new_η, o.gradient) + 0.5 * inner(p.M, o.x, new_η, new_Hη)
    o.new_model_value >= o.model_value && return o
    copyto!(p.M, o.η, o.x, new_η)
    o.model_value = o.new_model_value
    copyto!(p.M, o.Hη, o.x, new_Hη)
    o.residual = o.residual + α * o.Hδ
    # Precondition the residual.
    o.z = o.randomize ? o.residual : get_preconditioner(p, o.x, o.residual)
    zr = inner(p.M, o.x, o.z, o.residual)
    # Compute new search direction.
    β = zr / o.z_r
    o.z_r = zr
    o.δ = -o.z + β * o.δ
    o.project_vector!(p.M, o.δ, o.x, o.δ)
    o.ηPδ = β * (α * o.δPδ + o.ηPδ)
    o.δPδ = o.z_r + β^2 * o.δPδ
    return o
end
get_solver_result(o::O) where {O<:TruncatedConjugateGradientOptions} = o.η


export gmres

function gmres{T1,T2}(A::SparseMatrixCSC{T1,Int},b::Array{T2,1},restrt::Int; kwargs...)
	Ax = zeros(promote_type(T1,T2),size(A,1))
	return gmres(x -> A_mul_B!(1.0,A,x,0.0,Ax),b,restrt;kwargs...)
end

gmres(A,b::Vector,restrt;kwargs...) = gmres(x -> A*x ,b,restrt;kwargs...)

"""
x,flag,err,iter,resvec = gmres(A,b,restrt,tol=1e-2,maxIter=100,M=1,x=[],out=0)

Generalized Minimal residual ( GMRESm ) method with restarts applied to A*x = b.

Input:

  A       - function computing A*x
  b       - right hand side vector
  restrt  - number of iterations between restarts
  tol     - error tolerance
  maxIter - maximum number of iterations
  M       - preconditioner, function computing M\\x
  x       - starting guess
  out     - flag for output (0 : only errors, 1 : final status, 2: error at each iteration)

Output:

  x       - approximate solution
  flag    - exit flag (  0 : desired tolerance achieved,
                        -1 : maxIter reached without converging
                        -9 : right hand side was zero)
  err     - norm of relative residual, i.e., norm(A*x-b)/norm(b)
  iter    - number of iterations
  resvec  - norm of relative residual at each iteration
"""
function gmres(A::Function,b::Vector,restrt::Int; tol::Real=1e-2,maxIter::Int=100,M::Function=identity,x::Vector=[],out::Int=0,storeInterm::Bool=false)

    # initialization
    n  = length(b)
	if norm(b)==0; return zeros(eltype(b),n),-9,0.0,0,[0.0]; end
    if isempty(x)
        x = zeros(n)
        r = M(b)
    else
        r = M(b-A(x))
    end
	if storeInterm
        X = zeros(n,maxIter)	# allocate space for intermediates
    end

    bnrm2 = norm(b)
    if bnrm2 == 0.0; bnrm2 = 1.0; end

    err = norm( r ) / bnrm2
    if err < tol; return x, err; end

    restrt = min(restrt,n-1)
    V     = zeros(n,restrt+1)
    H     = zeros(restrt+1,restrt)
    cs    = zeros(restrt)
    sn    = zeros(restrt)
    e1    = zeros(n)
    e1[1] = 1.0

    if eltype(b) <: Complex || eltype(r) <: Complex || eltype(A) <: Complex
        e1 = complex(e1)
        b  = complex(b)
        r  = complex(r)
        x  = complex(x)
        V  = complex(V)
        H  = complex(H)
        cs = complex(cs)
        sn = complex(sn)
    end

    resvec = zeros((1+restrt)*maxIter)
    if out==2
    	println(@sprintf("=== gmres ===\n%4s\t%7s\n","iter","relres"))
    end

    iter = 0
    flag = -1
    cnt  = 1
    for iter = 1:maxIter
        V[:,1] = r / norm( r )
        s      = norm( r )*e1;

        if out==2;; print(@sprintf("%3d\t", iter));end

        for i = 1:restrt
            w = A(V[:,i])
            w = M(w)

            for k = 1:i # basis using Gram-Schmidt
                H[k,i] = dot(w,V[:,k])
                w     -= H[k,i]*V[:,k]
            end
            H[i+1,i] = norm( w )
            V[:,i+1] = w / H[i+1,i]

            for k = 1:i-1 # apply Givens rotation
                temp     =  cs[k]*H[k,i] + sn[k]*H[k+1,i]
                H[k+1,i] = -sn[k]*H[k,i] + cs[k]*H[k+1,i]
                H[k,i]   = temp
            end

            # Approximate residual norm
            cs[i],sn[i], = symOrtho( H[i,i], H[i+1,i] )
            s[i+1] = -sn[i]*s[i]
            s[i]   = cs[i]*s[i]
            H[i,i] = cs[i]*H[i,i] + sn[i]*H[i+1,i]
            H[i+1,i] = 0.0
            err  = abs(s[i+1]) / bnrm2

            if out==2; print(@sprintf("%1.1e ", err));end

            resvec[cnt] = err

            if err <= tol
                y  = H[1:i,1:i] \ s[1:i]
                x += V[:,1:i]*y
                if out==2; print("\n"); end
                flag = 0; break
            end
            cnt = cnt+1
        end
        if  err <= tol
            flag = 0
            break
        end
        y  = H[1:restrt,1:restrt]\s[1:restrt]
        x += V[:,1:restrt]*y
        if storeInterm; X[:,iter] = x; end

        r = b - A(x)
        r = M(r)

        s[restrt+1] = norm(r)
        resvec[cnt] = abs(s[restrt+1]) / bnrm2

        if out==2; print(@sprintf("\t %1.1e\n", err)); end
    end

    if out>=0
        if flag==-1
            println(@sprintf("gmres iterated maxIter (=%d) times without achieving the desired tolerance.",maxIter))
        elseif flag==0 && out>=1
            println(@sprintf("gmres achieved desired tolerance at iteration %d. Residual norm is %1.2e.",iter,resvec[cnt]))
        end
    end
    if storeInterm
	    return X[:,1:iter],flag,resvec[cnt],iter,resvec[1:cnt]
	else
	    return x,flag,resvec[cnt],iter,resvec[1:cnt]
	end
end


## extend logical operators so that we can have a notation like

#f(x,y) < 0 or f(x,y) == 0

#import Base: <, <=, ==, !=, !==, >=, >
import Base: &, |, !

abstract type Predicate end

"""

A predicate is defined in terms of a function of two variables, an
inquality, and either another function or a real number.  They are
conveniently created by the functions `Lt`, `Le`, `Eq`, `Ne`, `Ge`,
and `Gt`. The equivalent unicode operators:

* `≪` (`\\ll[tab]`),
* `≦` (`\\leqq[tab]`),
* `⩵` (`\\Equal[tab]`),
* `≶` (`\\lessgtr[tab]`)  or `≷` (`\\gtrless[tab]`),
* `≧` (`\\geqq[tab]`),
* `≫` (`\\leqq[tab]`) may also be used.

The use of Julia's usual comparison operators is no longer supported.

To combine predicates, `&` and `|` can be used.

To negate a predicate, `!` is used.

"""
mutable struct Pred <: Predicate
    f
    op
    val
end

## meta these
preds = [(:Lt, :≪, :<), # \ll
         (:Le, :≦, :<=), # \leqq
         (:Eq, :⩵, :(==)), # \Equal
         (:Ge, :≧, :>=), # \gegg
         (:Gt, :≫, :>) # \gg
         ]

    # type piracy of CommonEq
    for (fn,uop, op) ∈ (
        (:Lt, :≪, :<), # \ll
        (:Le, :≦, :<=), # \leqq
        (:Eq, :⩵, :(==)), # \Equal
        (:Ge, :≧, :>=), # \gegg
        (:Gt, :≫, :>) # \gg
    )
    fnname =  string(fn)
    @eval begin
        (CommonEq.$fn)(f::Function, x::Real) = Pred(f, $op, x)
        (CommonEq.$fn)(f::Function, g::Function) = $(fn)((x,y) -> f(x,y) - g(x,y), 0)
    end
    eval(Expr(:export, fn))
end

CommonEq.Ne(f::Function, x::Real) = Pred(f, !== , x)
CommonEq.Ne(f::Function, g::Function) = Ne((x,y) -> f(x,y) - g(x,y), 0)
const Neq = Ne # deprecate Neq

"""

Predicates can be joined together with either `&` or `|`. Individual
predicates can be negated with `!`. The parsing rules require the
individual predicates to be enclosed with parentheses, as in `(f==0) | (g==0)`.

"""
mutable struct Preds <: Predicate
    ps
    ops
end

## Some algebra for Pred and Preds
(&)(r1::Pred, r2::Pred) = Preds([r1,r2], Any[&])
(|)(r1::Pred, r2::Pred) = Preds([r1,r2], Any[|])

(&)(ps::Preds, r1::Pred) = Preds(vcat(ps.ps, r1), vcat(ps.ops, &))
(&)(r1::Pred, ps::Preds) = ps & r1
(|)(ps::Preds, r1::Pred) = Preds(vcat(ps.ps, r1), vcat(ps.ops, |))
(|)(r1::Pred, ps::Preds) = ps | r1


struct Octonion{T<:Real} <: Number
  s::T
  v1::T
  v2::T
  v3::T
  v4::T
  v5::T
  v6::T
  v7::T
  norm::Bool
end

Octonion(s::Real, v1::Real, v2::Real, v3::Real, v4::Real, v5::Real, v6::Real, v7::Real, n::Bool = false) =
  Octonion(promote(s, v1, v2, v3, v4, v5, v6, v7)..., n)
Octonion(x::Real) = Octonion(x, zero(x), zero(x), zero(x), zero(x), zero(x), zero(x), zero(x), abs(x) == one(x))
Octonion(z::Complex) = Octonion(z.re, z.im, zero(z.re), zero(z.re), zero(z.re), zero(z.re), zero(z.re), zero(z.re), abs(z) == one(z.re))
Octonion(q::Quaternion) = Octonion(q.s, q.v1, q.v2, q.v3, zero(q.s), zero(q.s), zero(q.s), zero(q.s), q.norm)
Octonion(s::Real, a::Vector) = Octonion(s, a[1], a[2], a[3], a[4], a[5], a[6], a[7])
Octonion(a::Vector) = Octonion(0, a[1], a[2], a[3], a[4], a[5], a[6], a[7])

convert(::Type{Octonion{T}}, x::Real) where {T} =
  Octonion(convert(T, x), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0))
convert(::Type{Octonion{T}}, z::Complex) where {T} =
  Octonion(convert(T, real(z)), convert(T, imag(z)), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0))
convert(::Type{Octonion{T}}, q::Quaternion) where {T} =
  Octonion(convert(T, real(q)), convert(T, q.v1), convert(T, q.v2), convert(T, q.v3), convert(T, 0), convert(T, 0), convert(T, 0), convert(T, 0))
convert(::Type{Octonion{T}}, o::Octonion{T}) where {T <: Real} = o
convert(::Type{Octonion{T}}, o::Octonion) where {T} =
  Octonion(convert(T, o.s), convert(T, o.v1), convert(T, o.v2), convert(T, o.v3), convert(T, o.v4), convert(T, o.v5), convert(T, o.v6), convert(T, o.v7), o.norm)

promote_rule(::Type{Octonion{T}}, ::Type{T}) where {T <: Real} = Octonion{T}
promote_rule(::Type{Octonion}, ::Type{T}) where {T <: Real} = Octonion
promote_rule(::Type{Octonion{T}}, ::Type{S}) where {T <: Real, S <: Real} = Octonion{promote_type(T, S)}
promote_rule(::Type{Complex{T}}, ::Type{Octonion{S}}) where {T <: Real, S <: Real} = Octonion{promote_type(T, S)}
promote_rule(::Type{Quaternion{T}}, ::Type{Octonion{S}}) where {T <: Real, S <: Real} = Octonion{promote_type(T, S)}
promote_rule(::Type{Octonion{T}}, ::Type{Octonion{S}}) where {T <: Real, S <: Real} = Octonion{promote_type(T, S)}

octo(p, v1, v2, v3, v4, v5, v6, v7, n = false) = Octonion(p, v1, v2, v3, v4, v5, v6, v7, n)
octo(x) = Octonion(x)
octo(s, a) = Octonion(s, a)

function show(io::IO, o::Octonion)
  pm(x) = x < 0 ? " - $(-x)" : " + $x"
  print(io, o.s, pm(o.v1), "im", pm(o.v2), "jm", pm(o.v3), "km", pm(o.v4), "ilm", pm(o.v5), "jlm", pm(o.v6), "klm", pm(o.v7), "lm")
end

real(o::Octonion) = o.s
imag(o::Octonion) = [o.v1, o.v2, o.v3, o.v4, o.v5, o.v6, o.v7]

(/)(o::Octonion, x::Real) = Octonion(o.s / x, o.v1 / x, o.v2 / x, o.v3 / x, o.v4 / x, o.v5 / x, o.v6 / x, o.v7 / x)

conj(o::Octonion) = Octonion(o.s, -o.v1, -o.v2, -o.v3, -o.v4, -o.v5, -o.v6, -o.v7, o.norm)
abs(o::Octonion) = sqrt(o.s * o.s + o.v1 * o.v1 + o.v2 * o.v2 + o.v3 * o.v3 + o.v4 * o.v4 + o.v5 * o.v5 + o.v6 * o.v6 + o.v7 * o.v7)
float(q::Octonion{T}) where T = convert(Octonion{float(T)}, q)
abs2(o::Octonion) = o.s * o.s + o.v1 * o.v1 + o.v2 * o.v2  + o.v3 * o.v3 + o.v4 * o.v4 + o.v5 * o.v5 + o.v6 * o.v6 + o.v7 * o.v7
inv(o::Octonion) = o.norm ? conj(o) : conj(o) / abs2(o)

function normalize(o::Octonion)
  if (o.norm)
    return o
  end
  o = o / abs(o)
  Octonion(o.s, o.v1, o.v2, o.v3, o.v4, o.v5, o.v6, o.v7, true)
end

function normalizea(o::Octonion)
  if (o.norm)
    return (o, one(o.s))
  end
  a = abs(o)
  o = o / a
  (Octonion(o.s, o.v1, o.v2, o.v3, o.v4, o.v5, o.v6, o.v7, true), a)
end

(-)(o::Octonion) = Octonion(-o.s, -o.v1, -o.v2, -o.v3, -o.v4, -o.v5, -o.v6, -o.v7, o.norm)

(+)(o::Octonion, w::Octonion) = Octonion(o.s + w.s,
                                            o.v1 + w.v1,
                                            o.v2 + w.v2,
                                            o.v3 + w.v3,
                                            o.v4 + w.v4,
                                            o.v5 + w.v5,
                                            o.v6 + w.v6,
                                            o.v7 + w.v7)

(-)(o::Octonion, w::Octonion) = Octonion(o.s - w.s,
                                            o.v1 - w.v1,
                                            o.v2 - w.v2,
                                            o.v3 - w.v3,
                                            o.v4 - w.v4,
                                            o.v5 - w.v5,
                                            o.v6 - w.v6,
                                            o.v7 - w.v7)

macro ocmul(i, j)
  si = Symbol("v$i")
  sj = Symbol("v$j")
  esc(:(o.$si * w.$sj - o.$sj * w.$si))
end

(*)(o::Octonion, w::Octonion) =
  Octonion(
            o.s * w.s - o.v1 * w.v1 - o.v2 * w.v2 - o.v3 * w.v3 - o.v4 * w.v4 - o.v5 * w.v5 - o.v6 * w.v6 - o.v7 * w.v7,
            o.s * w.v1 + o.v1 * w.s + @ocmul( 2, 3 ) + @ocmul( 6, 5 ) + @ocmul( 7, 4 ),
            o.s * w.v2 + o.v2 * w.s + @ocmul( 3, 1 ) + @ocmul( 4, 6 ) + @ocmul( 7, 5 ),
            o.s * w.v3 + o.v3 * w.s + @ocmul( 1, 2 ) + @ocmul( 5, 4 ) + @ocmul( 7, 6 ),
            o.s * w.v4 + o.v4 * w.s + @ocmul( 1, 7 ) + @ocmul( 3, 5 ) + @ocmul( 6, 2 ),
            o.s * w.v5 + o.v5 * w.s + @ocmul( 1, 6 ) + @ocmul( 2, 7 ) + @ocmul( 4, 3 ),
            o.s * w.v6 + o.v6 * w.s + @ocmul( 2, 4 ) + @ocmul( 3, 7 ) + @ocmul( 5, 1 ),
            o.s * w.v7 + o.v7 * w.s + @ocmul( 4, 1 ) + @ocmul( 5, 2 ) + @ocmul( 6, 3 )
          )

(/)(o::Octonion, w::Octonion) = o * inv(w)

function exp(o::Octonion)
  s = o.s
  se = exp(s)
  scale = se
  th = abs(Octonion(imag(o)))
  if th > 0
    scale *= sin(th) / th
  end
  Octonion(se * cos(th),
            scale * o.v1,
            scale * o.v2,
            scale * o.v3,
            scale * o.v4,
            scale * o.v5,
            scale * o.v6,
            scale * o.v7,
            abs(s) == 0)
end

function log(o::Octonion)
  o, a = normalizea(o)
  s = o.s
  M = abs(Octonion(imag(o)))
  th = atan(M, s)
  if M > 0
    M = th / M
    return Octonion(log(a),
                     o.v1 * M,
                     o.v2 * M,
                     o.v3 * M,
                     o.v4 * M,
                     o.v5 * M,
                     o.v6 * M,
                     o.v7 * M)
  else
    return Octonion(log(a), th, 0.0, 0.0)
  end
end

(^)(o::Octonion, w::Octonion) = exp(w * log(o))

function sqrt(o::Octonion)
  exp(0.5 * log(o))
end

octorand() = octo(randn(), randn(), randn(), randn(), randn(), randn(), randn(), randn())


module MultiArmedBanditEnv

using ..Reinforce
using Distributions

export MultiArmedBandit

"""
The multi-armed bandit environment.

    MultiArmedBandit(k, n = 100; σ = 1)

- `k` is the number of available arms.
  The reward distribution for each arms is a Normal distribution centered
  at the range of [-1, 1] and the standard deviation is 1.
- `n` is the max steps for each episode.
- `σ` controls the the standard deviation for all Normal distribution.

    MultiArmedBandit(x::Vector{<:Distribution}...)

In case that you want to other distributions as the reward distribution.
"""
mutable struct MultiArmedBandit{K,D<:Vector} <: AbstractEnvironment
  arms::D
  n::Int  # max steps
  r::Float64
end

function MultiArmedBandit(k::Int, n::Int = 1000; σ::Real = 1)
  k ≤ 0 && throw(ArgumentError("k must > 0"))
  arms = map(i -> Normal(rand(Uniform(-1, 1)), σ), 1:k)
  MultiArmedBandit{k,typeof(arms)}(arms, n, 0)
end

function MultiArmedBandit(x::Vararg{Distribution,N}) where {N}
  y = collect(x)
  MultiArmedBandit{N,typeof(y)}(y, N, 0)
end

Reinforce.state(::MultiArmedBandit) = nothing
Reinforce.reward(env::MultiArmedBandit) = env.r
Reinforce.reset!(env::MultiArmedBandit) = (env.r = 0; env)
Reinforce.actions(::MultiArmedBandit{K}, s) where {K} = Base.OneTo(K)
Reinforce.step!(env::MultiArmedBandit, s, a::Int) = (env.r = rand(env.arms[a]); (env.r, nothing))
Reinforce.maxsteps(env::MultiArmedBandit) = env.n
Reinforce.ismdp(::MultiArmedBandit) = true

end  # module MultiArmedBanditEnv


using Flux, DiffEqFlux, StochasticDiffEq

function lotka_volterra(du,u,p,t)
  x, y = u
  α, β, δ, γ = p
  du[1] = dx = α*x - β*x*y
  du[2] = dy = -δ*y + γ*x*y
end
function lotka_volterra_noise(du,u,p,t)
  du[1] = 0.01u[1]
  du[2] = 0.01u[2]
end
prob = SDEProblem(lotka_volterra,lotka_volterra_noise,[1.0,1.0],(0.0,10.0))
p = param([2.2, 1.0, 2.0, 0.4])
function predict_fd_sde()
  diffeq_fd(p,sol->sol[1,:],101,prob,SOSRI(),saveat=0.1)
end
loss_fd_sde() = sum(abs2,x-1 for x in predict_fd_sde())
loss_fd_sde()
Flux.back!(loss_fd_sde())

prob = SDEProblem(lotka_volterra,lotka_volterra_noise,[1.0,1.0],(0.0,0.5))
function predict_rd_sde()
  Tracker.collect(diffeq_rd(p,prob,SOSRI(),saveat=0.0:0.1:0.5))
end
loss_rd_sde() = sum(abs2,x-1 for x in predict_rd_sde())
loss_rd_sde()
Flux.back!(loss_rd_sde())


include("poly_factorization_project.jl")

"""
Takes some input polynomial (of type Polynomial) and converts it to a string whilst also simplifying it to make it more human readable.
"""
function pretty_printer(input::Polynomial)::String
    manipulated_string = replace(string(input), "+ -" => "- ")
    manipulated_string = replace(manipulated_string, "x^1" => "x")
    return replace(manipulated_string, "⋅x^0" => "")
end

x = x_poly()

p1 = 11x^2 + 52x^5
p2 = 103x^5 + 7x^3
prime = 5

println("To showcase some of the functions within this repo, we will use the two polynomials p1 = ", pretty_printer(p1), " and p2 = ", pretty_printer(p2), "\n")

println("The scripts in this repo allow you to:\n")
println("Add polynomials together:")
println("p1 + p2 = ", pretty_printer(p1+p2), "\n")

println("Multiply polynomials and raise them by a power:")
println("p1 * p2 = ", pretty_printer(p1*p2))
println("p1^3 = ", pretty_printer(p1^prime), "\n")

println("Divide one polynomial by another:")
println("p1/p2 = ", pretty_printer((p2÷p1)(prime)), "\n")

println("Find the remainder of this polynomial division")
println(rem(p1,p2)(prime), "\n")

println("Take the derivative of a polynomial:")
println("p1' * p2 = ", pretty_printer(derivative(p1*p2)))
println("p1' * p2 + p1 * p2' = ", derivative(p1)*p2 + p1*derivative(p2), "\n")

println("Find the GCD of two polynomials modulo prime:")
println("GCD(p1,p2)(mod 3) = ", pretty_printer(gcd(p1,p2,prime)), "\n")

println("Factorize polynomials:")
p = mod(p1,prime)
println("Will factor p1(mod $prime): ", p)
factorization = factor(p,prime)
println("Here is the factorization: ", factorization)

pr = mod(expand_factorization(factorization),prime)
println("Reconstructing: ", pretty_printer(pr), "\n")

println("Make a polynomial square free:")
println("The square free version of the polynomial, ", pretty_printer(p1), " is, ", pretty_printer(square_free(p1, prime)), "\n")

println("Construct a cyclotonic polynomial given a prime integer input:")
println("The following is a cyclotonic polynomial, ", pretty_printer(cyclotonic_polynomial(prime)), "\n")

println("Construct a linear monic polynomial given an integer input:")
println("The following is a linear monic polynomial generated with input $prime, ", pretty_printer(linear_monic_polynomial(prime)), "\n")

println("Make a polynomial primitive:")
println("The primitive part of the polynomial ", pretty_printer(p1), " is ", pretty_printer(prim_part(p1)(prime)), "\n")

println("Take the modulus of a polynomial:")
println(pretty_printer(p1), " modulo $prime is congruent to ", pretty_printer(mod(p1, prime)), "\n")

println("Find the degree of a polynomial:")
println("The degree of the polynomial ", pretty_printer(p1), " is equal to ", degree(p1), "\n")

println("Find the leading term of any given polynomial:")
println("The leading term of the polynomial ", pretty_printer(p1), " is equal to ", pretty_printer(Polynomial(leading(p1))), "\n")

println("Aside from the functions showcased, this software package has much more functionality! \nIt can also perform arithmetic on individual polynomial terms, manipulate polynomials \nprogrammatically with pop! and push! and much, much more!")


# Tests for Revenue Hölder DEA Models
@testset "RevenueHölderDEAModel" begin

    # Test using Book data
    X = [1; 1; 1; 1; 1; 1; 1; 1];
    Y = [7 7; 4 8; 8 4; 3 5; 3 3; 8 2; 6 4; 1.5 5];
    P = [1 1; 1 1; 1 1; 1 1; 1 1; 1 1; 1 1; 1 1];

    # Revenue Hölder L1
    revenueholderl1 = dearevenueholder(X, Y, P, l = 1)

    @test typeof(revenueholderl1) == RevenueHolderDEAModel

    @test nobs(revenueholderl1) == 8
    @test ninputs(revenueholderl1) == 1
    @test noutputs(revenueholderl1) == 2
    @test ismonetary(revenueholderl1) == false

    @test efficiency(revenueholderl1) == efficiency(revenueholderl1, :Economic)
    @test efficiency(revenueholderl1, :Economic)   ≈ [0.0; 2.0; 2.0; 6.0; 8.0; 4.0; 4.0; 7.5]
    @test efficiency(revenueholderl1, :Technical)  ≈ [0.0; 0.0; 0.0; 3.0; 5.0; 0.0; 2.0; 3.0]
    @test efficiency(revenueholderl1, :Allocative) ≈ [0.0; 2.0; 2.0; 3.0; 3.0; 4.0; 2.0; 4.5]
    @test normfactor(revenueholderl1) ≈ [1.0; 1.0; 1.0; 1.0; 1.0; 1.0; 1.0; 1.0]

    # With monetary = true
    revenueholderl1mom = dearevenueholder(X, Y, P, l = 1, monetary = true)

    @test ismonetary(revenueholderl1mom) == true

    @test efficiency(revenueholderl1mom) ≈ efficiency(revenueholderl1) .* normfactor(revenueholderl1)
    @test efficiency(revenueholderl1mom, :Technical) ≈ efficiency(revenueholderl1, :Technical) .* normfactor(revenueholderl1)
    @test efficiency(revenueholderl1mom, :Allocative) ≈ efficiency(revenueholderl1, :Allocative) .* normfactor(revenueholderl1)

    # With CRS (Same result in this example)
    revenueholderl1crs = dearevenueholder(X, Y, P, l = 1, rts = :CRS)

    @test efficiency(revenueholderl1crs, :Economic)   ≈ [0.0; 2.0; 2.0; 6.0; 8.0; 4.0; 4.0; 7.5]
    @test efficiency(revenueholderl1crs, :Technical)  ≈ [0.0; 0.0; 0.0; 3.0; 5.0; 0.0; 2.0; 3.0]
    @test efficiency(revenueholderl1crs, :Allocative) ≈ [0.0; 2.0; 2.0; 3.0; 3.0; 4.0; 2.0; 4.5]

    # Revenue Hölder L1 Weighted (weakly)
    revenueholderl1weight = dearevenueholder(X, Y, P, l = 1, weight = true)

    @test efficiency(revenueholderl1weight, :Economic)   ≈ [0.0; 0.25; 0.25; 1.2; 8/3; 0.5; 2/3; 1.5]
    @test efficiency(revenueholderl1weight, :Technical)  ≈ [0.0; 0.0; 0.0; 0.6; 5/3; 0.0; 1/3; 0.6]
    @test efficiency(revenueholderl1weight, :Allocative) ≈ [0.0; 0.25; 0.25; 0.6; 1.0; 0.5; 1/3; 0.9]

    # Revenue Hölder L2
    logs, value = Test.collect_test_logs() do
        revenueholderl2 = dearevenueholder(X, Y, P, l = 2, optimizer = DEAOptimizer(:NLP))

        @test normfactor(revenueholderl2) ≈ sqrt(2) .* ones(8)
        show(IOBuffer(), revenueholderl2)
    end

    # Revenue Hölder L2 Weighted (weakly)
    logs, value = Test.collect_test_logs() do
        revenueholderl2weight = dearevenueholder(X, Y, P, l = 2, weight = true, optimizer = DEAOptimizer(:NLP))

        @test normfactor(revenueholderl2weight) ≈ sqrt.(sum((P .* Y).^2, dims = 2))
        show(IOBuffer(), revenueholderl2weight)
    end

    # Revenue Hölder LInf
    revenueholderlInf = dearevenueholder(X, Y, P, l = Inf)

    @test efficiency(revenueholderlInf, :Economic)   ≈ [0.0; 1.0; 1.0; 3.0; 4.0; 2.0; 2.0; 3.75]
    @test efficiency(revenueholderlInf, :Technical)  ≈ [0.0; 0.0; 0.0; 2.5; 4.0; 0.0; 1.5; 2.875]
    @test efficiency(revenueholderlInf, :Allocative) ≈ [0.0; 1.0; 1.0; 0.5; 0.0; 2.0; 0.5; 0.875]
    @test normfactor(revenueholderlInf) ≈ [2.0; 2.0; 2.0; 2.0; 2.0; 2.0; 2.0; 2.0]

    # Revenue Hölder LInf Weighted (weakly)
    revenueholderlInfweight = dearevenueholder(X, Y, P, l = Inf, weight = true)

    @test efficiency(revenueholderlInfweight, :Economic)   ≈ [0.0; 1/6; 1/6; 0.75; 4/3; 0.4; 0.4; 15/13]
    @test efficiency(revenueholderlInfweight, :Technical)  ≈ [0.0; 0.0; 0.0; 5/9; 4/3; 0.0; 3/11; 0.6]
    @test efficiency(revenueholderlInfweight, :Allocative) ≈ [0.0; 1/6; 1/6; 7/36; 0.0; 0.4; 7/55; 36/65]
    @test normfactor(revenueholderlInfweight) ≈ [14.0; 12.0; 12.0; 8.0; 6.0; 10.0; 10.0; 6.5]

    # Print
    show(IOBuffer(), revenueholderl1)
    show(IOBuffer(), revenueholderl1weight)
    show(IOBuffer(), revenueholderlInf)
    show(IOBuffer(), revenueholderlInfweight)

    # Test errors
    @test_throws DimensionMismatch dearevenueholder([1; 2 ; 3], [4 ; 5], [1; 1; 1], l = 1) #  Different number of observations
    @test_throws DimensionMismatch dearevenueholder([1; 2; 3], [4; 5; 6], [1; 2; 3; 4], l = 1) # Different number of observation in prices
    @test_throws DimensionMismatch dearevenueholder([1 1; 2 2; 3 3 ], [4; 5; 6], [1 1 1; 2 2 2; 3 3 3], l = 1) # Different number of input prices and inputs
    @test_throws ArgumentError dearevenueholder([1; 2; 3], [4; 5; 6], [1; 2; 3], l = 5) # Invalid l
    @test_throws ArgumentError dearevenueholder([1; 2; 3], [4; 5; 6], [1; 2; 3], l = 1, rts = :Error) # Invalid returns to scale

end


# Implementation of Armijo backtracking linesearch

"""Armijo backtracking linesearch
Armijo(model, x, s, fx, g, t, α) computes a step t satisfying Armijo condition
The function returns (xtrial, fxtrial) : the new iterate for the linesearch and
the value of the objective in xtrial
"""
function Armijo(model, x, s, fx=obj(model, x), g=grad(model, x), t=1., α=1e-4)
    @info(loggerArm, @sprintf("Armijo backtracking linesearch on problem %s", model.meta.name))
    (t ≤ 0) && (t = 1.)
    @debug(loggerArm, @sprintf("initial step t = %7.1e", t))
    (α ≤ 0 || α ≥ 1) && (α = 1e-4)
    @debug(loggerArm, @sprintf("α = %8.1e", α))
    xtrial = x + t * s
    fxtrial = obj(model, xtrial) # f(x + t * s)
    @debug(loggerArm, @sprintf("f(x) = %8.1e", fxtrial))
    αgs = α * dot(g, s)
    β = fx + t * αgs # f(x) + t * α * gᵀs
    @debug(loggerArm, @sprintf("β = %8.1e", β))

    while fxtrial > β && t > eps()
        @debug(loggerArm, @sprintf("f(x) = %8.1e > β = %8.1e", fxtrial, β))
        t = 0.5 * t
        @debug(loggerArm, @sprintf("new t = %7.1e", t))
        xtrial = x + t * s
        fxtrial = obj(model, xtrial)
        β = fx + t * αgs
    end
    @debug(loggerArm, @sprintf("final values : f(x+t*s) = %8.1e ≤ β = %8.1e", fxtrial, β))
    return xtrial, fxtrial
end


"""
    rodrigues(nv, Θ)

Create a rotation matrix from a normalized axis and an angle (in radian).
"""
function rodrigues(nv, Θ)
    et = eltype(nv)
    #R = nv*nv' + cos(Θ).*(Matrix{et}(I, 3,3) - nv*nv') + sin(Θ).*crossprodtensor(nv)
    R = nv*nv' + cos(Θ).*(Matrix(I, 3,3) - nv*nv')
    pluscrossprod!(R, sin(Θ), nv)
    return R
end

"""
    rodrigues(nv, Θ)

Create a rotation matrix from a normalized axis and an angle (in radian).
"""
function rodrigues(nv::SA, Θ) where {SA<:StaticArray}
    #R = nv*nv' + cos(Θ).*(Matrix{et}(I, 3,3) - nv*nv') + sin(Θ).*crossprodtensor(nv)
    R = MMatrix{3,3}(nv*nv' + cos(Θ).*(Matrix(I, 3,3) - nv*nv'))
    pluscrossprod!(R, sin(Θ), nv)
    return SMatrix{3,3}(R)
end

"""
    pluscrossprod!(A, value, v)

Add a crossproduct tensor to `A`.
Equals to: `A+value .* crossprodtensor(v)`
"""
function pluscrossprod!(A, value, v)
    A[1,2] -= value*v[3]
    A[1,3] += value*v[2]

    A[2,1] += value*v[3]
    A[2,3] -= value*v[1]

    A[3,1] -= value*v[2]
    A[3,2] += value*v[1]

    return A
end

"""
    crossprodtensor(v)

Create the cross product tensor of 3 dimensional vector.
"""
function crossprodtensor(v)
    [0 -v[3] v[2];
    v[3] 0 -v[1];
    -v[2] v[1] 0]
end

"""
    rodriguesrad(nv, ϑ)

Create a rotation matrix from an axis (`nv`) and an angle (`ϑ`) in radians.
"""
function rodriguesrad(nv, ϑ)
    nvn = normalize(nv)
    return rodrigues(nvn, ϑ)
end

"""
    rodriguesdeg(nv, ϑ)

Create a rotation matrix from an axis (`nv`) and an angle (`ϑ`) in degrees.
"""
function rodriguesdeg(nv, ϑ)
    nvn = normalize(nv)
    return rodrigues(nvn, deg2rad(ϑ))
end

## Collection of useful functions.

"""
    arbitrary_orthogonal(vec)

Create an arbitrary orthogonal vector to `vec`.
"""
function arbitrary_orthogonal(vec)
    @assert size(vec,1) == 3 "Implemented only for 3 dimensional."
    v = normalize(vec)
    b0 = (v[1] <  v[2]) && (v[1] <  v[3]);
    b1 = (v[2] <= v[1]) && (v[2] <  v[3]);
    b2 = (v[3] <= v[1]) && (v[3] <= v[2]);
    rv = normalize!([convert(Int, b0), convert(Int, b1), convert(Int, b2)])
    return convert(typeof(vec), cross(v, rv))
end

"""
    arbitrary_orthogonal2(vec)

Create an arbitrary orthogonal vector to `vec`.
"""
function arbitrary_orthogonal2(vec)
    @assert size(vec,1) == 3 "Implemented only for 3 dimensional."
    v = normalize(vec)
    rv = normalize(rand(3))
    return convert(typeof(vec), cross(v, rv))
end

"""
    isparallel(v1, v2, alpharad)

Check if `v1` and `v2` are parallel in an `alpharad` angle
    and point towards the same direction.

The vectors considered to be normalized.
"""
function isparallel(v1, v2, alpharad)
    dot(v1, v2) > cos(alpharad)
end

"""
    findAABB(points)

Find the axis aligned minimum bounding box of the given points.
"""
function findAABB(points)
    minV = convert(Array, deepcopy(points[1]))
    maxV = convert(Array, deepcopy(points[1]))
    for i in 1:length(points)
        a = points[i]
        for j in 1:length(minV)
            minV[j] = minV[j] > a[j] ? a[j] : minV[j]
            maxV[j] = maxV[j] < a[j] ? a[j] : maxV[j]
        end
    end
    return convert(eltype(points), minV), convert(eltype(points), maxV)
end

"""
    chopzaxis(points)

Chop the 3rd element of every point.
"""
function chopzaxis(points)
    @assert length(points[1]) == 3 "Implemented only for 3 long vectors."
    [SVector(a[1], a[2]) for a in points]
end

"""
    unitdisk2square(p)

Transform point on unit disk to unit sqaure.

Maps a unit circle to a unit sqaure [0,1]^2.
Source: shirley1997 - A low distortion map between disk and square
"""
function unitdisk2square(p)
    r = norm(p)
    phi = atan(p[2], p[1])
    if phi < -π/4
        phi += 2*π # in range [-pi/4,7pi/4]
    end
    if phi < π/4 # region 1
        a = r
        b = phi*a/(π/4)
    elseif phi < 3*π/4 # region 2
        b = r
        a = -(phi-π/2)*b/(π/4)
    elseif phi < 5*π/4 # region 3
        a = -r
        b = (phi - π)*a/(π/4)
    else # region 4
        b = -r
        a = -(phi-3*π/2)*b/(π/4)
    end
    # TODO:
    # decision: squre: [-1,1]^2 or [0,1]^2
    convert(typeof(p), [(a+1)/2, (b+1)/2])
end

"""
    smallestdistance(points)

Find the smallest distance between the points.
"""
function smallestdistance(points)
    @assert length(points) > 1 "At least two point is needed for that."
    ld = norm(points[2]-points[1])
    for i in eachindex(points)
        for j in eachindex(points)
            if i!=j
                d = norm(points[i]-points[j])
                ld = d < ld ? d : ld
            end
        end
    end
    ld
end

"""
    minmaxdistance(points)

Find the smallest, largest and average distance between the points.
"""
function minmaxdistance(points)
    @assert length(points) > 1 "At least two point is needed for that."
    ld = norm(points[2]-points[1])
    maxid = norm(points[2]-points[1])
    sumd = 0.
    sumi = 0
    for i in eachindex(points)
        for j in eachindex(points)
            if i!=j
                d = norm(points[i]-points[j])
                sumd += d
                sumi += 1
                ld = d < ld ? d : ld
                maxid = d > maxid ? d : maxid
            end
        end
    end
    avg_d = sumd/sumi
    return (mind=ld, maxd=maxid, avgd=avg_d)
end

"""
    not0minmaxdistance(points)

Find the smallest and largest distance between the points.
Zero distance is not taken into account.
"""
function not0minmaxdistance(points)
    @assert length(points) > 1 "At least two point is needed for that."
    ld = norm(points[2]-points[1])
    maxid = norm(points[2]-points[1])
    for i in eachindex(points)
        for j in eachindex(points)
            if i!=j
                d = norm(points[i]-points[j])
                d == 0 && continue
                ld = d < ld ? d : ld
                maxid = d > maxid ? d : maxid
            end
        end
    end
    return (mind=ld, maxd=maxid)
end

"""
    prob(n, s, N, k)

The probability of successful detection of a shape sized `n`,
from a point cloud size of `N`, with `k` size of minimal sets and `s` draws.

# Arguments:
- `n`: size of the shape.
- `s`: number of candidates that have been drawn.
- `N`: size of the point cloud.
- `k`: size of the minimal set required to define a shape candidate.
"""
prob(n, s, N, k) = 1-(1-(n/N)^k)^s

"""
    havesameelement(A, B)

Return true if `A` and `B` have at least one common element.
"""
function havesameelement(A, B)
    for a in A
        for b in B
            a == b && return true
        end
    end
    false
end

"""
    allisdifferent(a::AbstractArray{T, 1}) where {T}

Check if all the elements of an array are different.
Supports only one dimensional arrays.
"""
function allisdifferent(a::AbstractArray{T, 1}) where {T}
    #TODO: make it more abstract with:
    #https://docs.julialang.org/en/v1/base/iterators/
    size(a, 1) < 2 && return true
    for i in 2:size(a, 1)
        for j in 1:i-1
            a[i] == a[j] && return false
        end
    end
    return true
end

function chooseS(A, k)
    symtoint = Dict(:lengthC=>1, :allcand=>2, :nofminset=>3)
    return A[symtoint[k]]
end

"""
    defaultiterationparameters(shape_types)

Construct a named tuple with the default iteration parameters.
`shape_types` is an array of `FittedShape`s,
that controls which primitives you want to fit to the point cloud.

# Examples

```julia-repl
julia> RANSAC.defaultiterationparameters([FittedPlane])
(iteration = (drawN = 3, minsubsetN = 15, prob_det = 0.9,
shape_types = UnionAll[FittedPlane], τ = 900, itermax = 1000,
extract_s = :nofminset, terminate_s = :nofminset),)

julia> RANSAC.defaultiterationparameters([FittedPlane, FittedSphere, FittedCone])
(iteration = (drawN = 3, minsubsetN = 15, prob_det = 0.9,
shape_types = UnionAll[FittedPlane, FittedSphere, FittedCone], τ = 900, itermax = 1000,
extract_s = :nofminset, terminate_s = :nofminset),)
```

# Implementation
- `drawN`: number of points to be sampled (length of a minimal subset).
- `minsubsetN`: number of minimal sets sampled in one iteration.
- `prob_det`: probability of detection.
- `τ`: minimal shape size.
- `itermax`: maximum number of iteration.
- `shape_types`: shapes that are fitted to the point cloud (array of types).
- `extract_s`, `terminate_s`: they are for easier testing, do not delete or modify it.
"""
function defaultiterationparameters(shape_types)
    # `drawN`: number of points to be sampled (length of a minimal subset)
    # `minsubsetN`: number of minimal sets sampled in one iteration
    # `prob_det`: probability of detection
    # `τ`: minimal shape size
    # `itermax`: maximum number of iteration
    # `shape_types`: shapes that are fitted to the point cloud (array of types)
    # `extract_s`, `terminate_s`:
        # track the number of candidates for the probabilities
        # which to use to terminate the algorithm
    # 1. :lengthC - number of candidates in a given iteration
    # 2. :allcand - number of candidates that have been ever scored
    # 3. :nofminset - number of minimal sets that have been drawn so far
    ip = (drawN=3, minsubsetN=15, prob_det=0.9, shape_types=shape_types, τ=900, itermax=1000, extract_s=:nofminset, terminate_s=:nofminset)
    return (iteration=ip,)
end

"""
    defaultcommonparameters()

Construct a `NamedTuple` with the default common parameters.

# Examples
```julia-repl
julia> defaultcommonparameters()
(common = (collin_threshold = 0.2, parallelthrdeg = 1.0),)
```

# Implementation
This section describes the role of the common parameters.
- `collin_threshold`: 3 points can be nearly collinear,
    in some cases they must be filtered. See the code of:
    ` fit(::Type{FittedPlane}, p, n, params)`.
- `parallelthrdeg`: threshold for two vectos being parallel, in degrees.
    If `abs(dot(a,b))>cosd(parallelthrdeg)`, `a` and `b` are considered to be parallel.
"""
function defaultcommonparameters()
    #`collin_threshold`: threshold of points being collinear
    # `parallelthrdeg`: threshold of two vectors being parallel (in degrees)
    cp = (collin_threshold = 0.2, parallelthrdeg = 1.)
    return (common=cp,)
end

"""
    defaultparameters(shape_types::Vector{T}) where {T}

Construct a `NamedTuple` with the given shape types and the default parameters.

# Examples
```julia-repl
julia> defaultparameters([FittedSphere, FittedPlane])
(iteration = (drawN = 3, minsubsetN = 15, prob_det = 0.9,
shape_types = UnionAll[FittedSphere, FittedPlane], τ = 900, itermax = 1000,
extract_s = :nofminset, terminate_s = :nofminset),
common = (collin_threshold = 0.2, parallelthrdeg = 1.0),
sphere = (ϵ = 0.3, α = 0.08726646259971647, sphere_par = 0.02),
plane = (ϵ = 0.3, α = 0.08726646259971647))
```
"""
function defaultparameters(shape_types::Vector{T}) where {T}
    def_iter_pars = defaultiterationparameters(shape_types)
    def_iter_pars = merge(def_iter_pars, defaultcommonparameters())
    def_pars = defaultshapeparameters.(shape_types)
    for a in def_pars
        def_iter_pars = merge(def_iter_pars, a)
    end
    def_iter_pars
end

"""
    ransacparameters(p::T=DEFAULT_PARAMETERS; kwargs...) where {T<:NamedTuple}

Construct a `NamedTuple` based on a previous one,
defaulting to `DEFAULT_PARAMETERS` and override it with the kwargs.
Check the docs and examples for more.

# Examples
```julia-repl
julia> p1 = ransacparameters()
(iteration = (drawN = 3, minsubsetN = 15, prob_det = 0.9),
plane = (ϵ = 0.3, α = 0.08726646259971647),
cone = (ϵ = 0.3, α = 0.08726646259971647, minconeopang = 0.03490658503988659),
cylinder = (ϵ = 0.3, α = 0.08726646259971647),
sphere = (ϵ = 0.3, α = 0.08726646259971647, sphere_par = 0.1))

julia> p2 = ransacparameters(p1; sphere=(ϵ=0.9, α=deg2rad(1),), plane=(ϵ=1.0,))
(iteration = (drawN = 3, minsubsetN = 15, prob_det = 0.9),
plane = (ϵ = 1.0, α = 0.08726646259971647),
cone = (ϵ = 0.3, α = 0.08726646259971647, minconeopang = 0.03490658503988659),
cylinder = (ϵ = 0.3, α = 0.08726646259971647),
sphere = (ϵ = 0.9, α = 0.017453292519943295, sphere_par = 0.1))
```
"""
function ransacparameters(p::T=DEFAULT_PARAMETERS; kwargs...) where {T<:NamedTuple}
    newp = p
    for a in keys(kwargs)
        oldpar = get(p, a, kwargs[a])
        newpar = merge(oldpar, kwargs[a])
        newp = merge(newp, (a=>newpar,))
    end
    return newp
end

"""
    ransacparameters(p::AbstractArray; kwargs...)

Construct a `NamedTuple` for a given types of shapes
using [`defaultparameters`](@ref) and override it with the kwargs.
Check the docs and examples for more.

# Examples
```julia-repl
julia> p1 = ransacparameters([FittedSphere, FittedCylinder])
(iteration = (drawN = 3, minsubsetN = 15, prob_det = 0.9,
shape_types = UnionAll[FittedSphere, FittedCylinder], τ = 900, itermax = 1000,
extract_s = :nofminset, terminate_s = :nofminset),
common = (collin_threshold = 0.2, parallelthrdeg = 1.0),
sphere = (ϵ = 0.3, α = 0.08726646259971647, sphere_par = 0.02),
cylinder = (ϵ = 0.3, α = 0.08726646259971647))

julia> p2 = ransacparameters([FittedSphere, FittedCylinder], sphere=(ϵ=0.01,), cylinder=(α=0.02,))
(iteration = (drawN = 3, minsubsetN = 15, prob_det = 0.9,
shape_types = UnionAll[FittedSphere, FittedCylinder], τ = 900, itermax = 1000,
extract_s = :nofminset, terminate_s = :nofminset),
common = (collin_threshold = 0.2, parallelthrdeg = 1.0),
sphere = (ϵ = 0.01, α = 0.08726646259971647, sphere_par = 0.02),
cylinder = (ϵ = 0.3, α = 0.02))
```
"""
function ransacparameters(p::AbstractArray; kwargs...)
    dpars = defaultparameters(p)
    return ransacparameters(dpars; kwargs...)
end

"""
    push2candidatesandlevels!(candidates, candidate, levels, current_level)

Push `candidate` to `candidates` and `current_level` to `levels`.
Handle the case, when `candidate` is an array of candidates.
Used in [`forcefitshapes!`](@ref).
"""
function push2candidatesandlevels!(candidates, candidate, levels, current_level)
    push!(candidates, candidate)
    push!(levels, current_level)
end

function push2candidatesandlevels!(candidates, candidate::AbstractArray, levels, current_level)
    append!(candidates, candidate)
    append!(levels, fill(current_level, size(candidate)))
end

"""
    setfloattype(nt, T)

Convert every `Real` but not `Integer` value to type `T` in a `NamedTuple` recursively.
"""
function setfloattype(nt, T)
    Keys = []
    Values = []
    for (k,v) in pairs(nt)
        push!(Keys, k)
        v = nt[k]
        if v isa NamedTuple
            push!(Values, setfloattype(v,T))
        elseif (v isa Number) && ! (v isa Integer)
            # v is a Real or a float
            push!(Values, convert(T, v))
        else
            push!(Values, v)
        end
    end
    return (; zip(Keys,Values)...)
end