#=
  Test of cumulative constraint in Julia ConstraintSolver.jl

  Model created by Hakan Kjellerstrand, hakank@gmail.com
  See also my Julia page: http://www.hakank.org/julia/

=#
using ConstraintSolver, JuMP, Printf
using Cbc, GLPK, Ipopt
const CS = ConstraintSolver
include("constraints_utils.jl")

#
# The cumulative constraint is in constraints_utils.jl
#
function cumulative_test(problem, print_solutions=true,all_solutions=false,time_limit=600)

    cbc_optimizer = optimizer_with_attributes(Cbc.Optimizer, "logLevel" => 0)
    glpk_optimizer = optimizer_with_attributes(GLPK.Optimizer)
    ipopt_optimizer = optimizer_with_attributes(Ipopt.Optimizer)

    model = Model(optimizer_with_attributes(CS.Optimizer,   "all_solutions"=> all_solutions,
                                                            # "all_optimal_solutions"=>true,
                                                            "logging"=>[],

                                                            "traverse_strategy"=>:BFS,
                                                            # "traverse_strategy"=>:DFS, # <-
                                                            # "traverse_strategy"=>:DBFS,

                                                            # "branch_split"=>:Smallest,
                                                            # "branch_split"=>:Biggest,
                                                            "branch_split"=>:InHalf, # <-

                                                            # "simplify"=>false,
                                                            # "simplify"=>true, # default

                                                            "time_limit"=>time_limit,

                                                            # "lp_optimizer" => cbc_optimizer,
                                                            "lp_optimizer" => glpk_optimizer,
                                                            # "lp_optimizer" => ipopt_optimizer,
                                        ))
    n             = problem[:n]
    problem_name  = problem[:name]
    durations     = problem[:durations]
    resources     = problem[:resources]
    start_time_lb = problem[:start_time_domain][1]
    start_time_ub = problem[:start_time_domain][2]
    end_time_lb   = 0
    end_time_ub   = start_time_ub+maximum(durations)
    limit_lb  = problem[:limit_domain][1]
    limit_ub  = problem[:limit_domain][2]

    println("$problem_name (n:$n)")
    println("durations:$durations")
    println("resources:$resources")
    println("start_time_lb:$start_time_lb")
    println("start_time_ub:$start_time_ub")
    println("end_time_lb:$end_time_lb")
    println("end_time_ub:$end_time_ub")
    println("limit_lb:$limit_lb")
    println("limit_ub:$limit_ub")

    @variable(model, start_time_lb <= start_times[1:n] <= start_time_ub, Int)
    @variable(model, end_time_lb   <= end_times[1:n]   <= end_time_ub,   Int)
    @variable(model, limit_lb      <= limit            <= limit_ub,      Int)
    @variable(model, start_time_lb <= max_time         <= end_time_ub,   Int)

    for i in 1:n
        @constraint(model,end_times[i] == start_times[i] + durations[i])
    end
    @constraint(model, end_times .<= max_time)
    cumulative(model, start_times, durations, resources, limit)

    # This don't work: MethodError: no method matching iterate(::Nothing)
    # @objective(model, Min, max_time)
    # @objective(model,Min,limit)

    # Solve the problem
    optimize!(model)

    status = JuMP.termination_status(model)
    # println("status:$status")
    if status == MOI.OPTIMAL
        num_sols = MOI.get(model, MOI.ResultCount())
        println("num_sols:$num_sols\n")
        if print_solutions
            for sol in 1:num_sols
                println("solution #$sol")
                start_timesx = convert.(Integer,JuMP.value.(start_times; result=sol))
                end_timesx = convert.(Integer,JuMP.value.(end_times; result=sol))
                max_timex = convert.(Integer,JuMP.value.(max_time; result=sol))
                limitx = convert.(Integer,JuMP.value.(limit; result=sol))
                println("start_times:$start_timesx")
                println("durations  :$durations")
                println("end_times  :$end_timesx")
                println("resources  :$resources")
                # println("max_time   :$max_timex")
                println("limit      :$limitx")

                for i in 1:n
                    println("Task $i: $(start_timesx[i])..$(end_timesx[i]) (duration:$(durations[i]) resource:$(resources[i]))")
                end

                time_line = minimum(start_timesx):maximum(end_timesx)
                println("\nTime line: (tasks active (resource))")
                for t in time_line
                    @printf "time %3d: " t
                    total_resources = 0
                    for i in 1:length(start_timesx)
                        if start_timesx[i] <= t && t < start_timesx[i]+durations[i]
                            total_resources += resources[i]
                            @printf " %3d (%2d) " i resources[i]
                        else
                            @printf " %3s  %3s " "" ""
                        end
                    end
                    println("  total_resources: $total_resources")
                end
            end
        end
    else
        println("status:$status")
    end

    return status
end

furniture_moving = Dict(
                        :name               => "Furniture moving",
                        :tasks              => ["piano", "chair", "bed", "table"],
                        :n                  => 4,
                        :durations          => [30,10,15,15],
                        :resources          => [3,1,3,2],
                        :start_time_domain  => [0,60],
                        :limit_domain       => [1,3]
)

# From my Picat model cumulative_decomp.pi
test1 = Dict(
             :name => "cumulative_decomp.pi",
             :tasks => [1,2,3,4,5],
             :n     => 5,
             :durations => [3,9,10,6,2],
             :resources => [1,2,1,1,3],
             :start_time_domain => [1,10],
             :limit_domain => [1,3] # [1,8]
)

# From cumulative_test2.pi
# Originally from (simplified)
# http://stackoverflow.com/questions/25170868/search-heuristics-for-cumulatives
# Note: This is currently (ConstraintSolver.jl 0.5.1) too hard for a 10 min time limit
test2 = Dict(
            :name => "cumulative_test2.pi",
            :n     => 20,
            :tasks => collect(1:20),
            :durations => [318, 246, 797, 238, 251, 279, 987, 847, 426, 787,6, 681,465,46, 3, 427,956,657,113,251],
            :resources => [1 for _ in 1:20],
            :start_time_domain => [1,1000],
            :limit_domain => [1,1]

)

# instances = [furniture_moving,test1,test2]
instances = [furniture_moving,test1]
for instance in instances
    println("Problem $(instance[:name])")
    @time cumulative_test(instance)
    println("\n")
end


# Common regression problems

abstract Problem{T}

### Univariate Prediction Problem

immutable UnivariateRegression{L<:UnivariateLoss,
                               T<:AbstractFloat,
                               XT<:StridedMatrix,
                               YT<:StridedVector} <: Problem{T}
    loss::L
    d::Int
    n::Int
    bias::T
    X::XT
    Y::YT
end

function UnivariateRegression{T<:AbstractFloat}(loss::UnivariateLoss,
                                                X::StridedMatrix{T},
                                                Y::StridedVector;
                                                bias::Real=0.0)
    d, n = size(X)
    length(Y) == n || throw(DimensionMismatch())
    UnivariateRegression{typeof(loss), T, typeof(X), typeof(Y)}(
        loss, d, n, convert(T, bias), X, Y)
end

nsamples(pb::UnivariateRegression) = pb.n
inputs(pb::UnivariateRegression) = pb.X
outputs(pb::UnivariateRegression) = pb.Y

loss(pb::UnivariateRegression) = pb.loss

has_bias(pb::UnivariateRegression) = (pb.bias != 0)
pred_without_bias(pb::UnivariateRegression) = LinearPred(pb.d)
pred_with_bias(pb::UnivariateRegression) = AffinePred(pb.d, pb.bias)

initsol{L,T}(pb::UnivariateRegression{L,T}) = zeros(T, has_bias(pb) ? pb.d + 1: pb.d)

# specific problems

linearreg{T<:AbstractFloat}(X::StridedMatrix{T}, y::StridedVector{T}; bias::Real=0.0) =
    UnivariateRegression(SqrLoss(), X, y; bias=bias)

logisticreg{T<:AbstractFloat}(X::StridedMatrix{T}, y::StridedVector; bias::Real=0.0) =
    UnivariateRegression(LogisticLoss(), X, convert(Vector{T}, y); bias=bias)

### Multivariate Prediction Problem

immutable MultivariateRegression{L<:MultivariateLoss,
                                 T<:AbstractFloat,
                                 XT<:StridedMatrix,
                                 YT<:StridedVecOrMat} <: Problem{T}
    loss::L
    d::Int
    k::Int
    n::Int
    bias::T
    X::XT
    Y::YT
end

function MultivariateRegression{T<:AbstractFloat}(loss::MultivariateLoss,
                                                  X::StridedMatrix{T},
                                                  Y::StridedVecOrMat,
                                                  k::Int;
                                                  bias::Real=0.0)
    d, n = size(X)
    size(Y, ndims(Y)) == n || throw(DimensionMismatch())
    MultivariateRegression{typeof(loss), T, typeof(X), typeof(Y)}(
        loss, d, k, n, convert(T, bias), X, Y)
end

nsamples(pb::MultivariateRegression) = pb.n
inputs(pb::MultivariateRegression) = pb.X
outputs(pb::MultivariateRegression) = pb.Y

loss(pb::MultivariateRegression) = pb.loss

has_bias(pb::MultivariateRegression) = (pb.bias != 0)
pred_without_bias(pb::MultivariateRegression) = MvLinearPred(pb.d, pb.k)
pred_with_bias(pb::MultivariateRegression) = MvAffinePred(pb.d, pb.k, pb.bias)

initsol{L,T}(pb::MultivariateRegression{L,T}) = zeros(T, pb.k, has_bias(pb) ? pb.d + 1: pb.d)

# specific problems

linearreg{T}(X::StridedMatrix{T}, Y::StridedMatrix{T}; bias::Real=0.0) =
    MultivariateRegression(SumSqrLoss(), X, Y, size(Y,1); bias=bias)

mlogisticreg{T}(X::StridedMatrix{T}, Y::StridedVector{Int}, k::Int; bias::Real=0.0) =
    MultivariateRegression(MultiLogisticLoss(), X, Y, k; bias=bias)


module CoordinateDescentTest

const  NUMBER_REPEAT = 1

using CoordinateDescent
using Test
using ProximalBase
using Random
using LinearAlgebra
using SparseArrays

# test |Y - β|_2^2 + λ⋅|β|_1
@testset "Small ProxL1" begin
  Y = [1., 1.5]

  λ = 1.2
  g = ProximalBase.ProxL1(λ)
  f = CDQuadraticLoss(Matrix(1.0I, 2, 2), -Y)

  opt = CDOptions(;maxIter=100, optTol=1e-8, warmStart=true, randomize=false)
  x = SparseIterate(2)
  coordinateDescent!(x, f, g, opt)

  @test Vector(x) ≈ Vector([0., 0.3])
end

# check that the warm start and non warm start produce the same result
@testset "ProxL1" begin

    for i=1:NUMBER_REPEAT
      n = 500
      p = 50
      s = 5

      X = randn(n, p)
      β = randn(s)
      Y = X[:,1:s] * β + randn(n)

      λ = 0.01
      g = ProximalBase.ProxL1(λ)
      f = CDLeastSquaresLoss(Y, X)

      opt1 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=true, randomize=false)
      opt2 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=true, randomize=true)
      opt3 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=false, randomize=false)
      opt4 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=false, randomize=true)

      x1 = SparseIterate(sprand(p, 0.6))
      x2 = SparseIterate(sprand(p, 0.6))
      x3 = SparseIterate(sprand(p, 0.6))
      x4 = SparseIterate(sprand(p, 0.6))

      coordinateDescent!(x1, f, g, opt1)
      coordinateDescent!(x2, f, g, opt2)
      coordinateDescent!(x3, f, g, opt3)
      coordinateDescent!(x4, f, g, opt4)

      @test Vector(x1) ≈ Vector(x2) atol=1e-5
      @test Vector(x3) ≈ Vector(x2) atol=1e-5
      @test Vector(x4) ≈ Vector(x2) atol=1e-5
    end
end

@testset "AProxL1" begin

    for i=1:NUMBER_REPEAT
      n = 500
      p = 50
      s = 10

      X = randn(n, p)
      β = randn(s)
      Y = X[:,1:s] * β + randn(n)

      λ = 0.01
      g = ProximalBase.ProxL1(λ, rand(p))
      f = CDLeastSquaresLoss(Y, X)

      opt1 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=true, randomize=false)
      opt2 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=true, randomize=true)
      opt3 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=false, randomize=false)
      opt4 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=false, randomize=true)

      x1 = SparseIterate(sprand(p, 0.6))
      x2 = SparseIterate(sprand(p, 0.6))
      x3 = SparseIterate(sprand(p, 0.6))
      x4 = SparseIterate(sprand(p, 0.6))

      coordinateDescent!(x1, f, g, opt1)
      coordinateDescent!(x2, f, g, opt2)
      coordinateDescent!(x3, f, g, opt3)
      coordinateDescent!(x4, f, g, opt4)

      @test Vector(x1) ≈ Vector(x2) atol=1e-5
      @test Vector(x3) ≈ Vector(x2) atol=1e-5
      @test Vector(x4) ≈ Vector(x2) atol=1e-5
    end
end

# @testset "weighted least squares loss" begin
#
#   n, s, p = 500, 2, 50
#   gk = GaussianKernel(0.3)
#   w = zeros(n)
#   sw = zeros(n)
#
#   for i=1:NUMBER_REPEAT
#     for degree=0:2
#       Y, X, Z, betaMat = genData(n, s)
#       X = [X zeros(n, p-s)]
#       z0 = rand()
#
#       cp = p*(degree+1)
#       eX = zeros(n, cp)
#       _expand_X!(eX, X, Z, z0, degree)
#
#       @. w = evaluate(gk, Z, z0)
#       @. sw = sqrt(w)
#
#       λ = 0.001
#       g = ProximalBase.ProxL1(λ)
#       f1 = CDLeastSquaresLoss(diagm(sw) * Y, diagm(sw)*X)
#       f2 = CDWeightedLSLoss(Y, X, w)
#
#       opt1 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=true, randomize=false)
#       opt2 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=true, randomize=true)
#       opt3 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=false, randomize=false)
#       opt4 = CDOptions(;maxIter=5000, optTol=1e-8, warmStart=false, randomize=true)
#
#       x1 = SparseIterate(sprand(p, 0.6))
#       x2 = SparseIterate(sprand(p, 0.6))
#       x3 = SparseIterate(sprand(p, 0.6))
#       x4 = SparseIterate(sprand(p, 0.6))
#       x5 = SparseIterate(sprand(p, 0.6))
#       x6 = SparseIterate(sprand(p, 0.6))
#       x7 = SparseIterate(sprand(p, 0.6))
#       x8 = SparseIterate(sprand(p, 0.6))
#
#       coordinateDescent!(x1, f1, g, opt1)
#       coordinateDescent!(x2, f1, g, opt2)
#       coordinateDescent!(x3, f1, g, opt3)
#       coordinateDescent!(x4, f1, g, opt4)
#
#       coordinateDescent!(x5, f2, g, opt1)
#       coordinateDescent!(x6, f2, g, opt2)
#       coordinateDescent!(x7, f2, g, opt3)
#       coordinateDescent!(x8, f2, g, opt4)
#
#
#       @test Vector(x1) ≈ Vector(x2) atol=1e-5
#       @test Vector(x3) ≈ Vector(x2) atol=1e-5
#       @test Vector(x4) ≈ Vector(x2) atol=1e-5
#       @test Vector(x5) ≈ Vector(x2) atol=1e-5
#       @test Vector(x6) ≈ Vector(x2) atol=1e-5
#       @test Vector(x7) ≈ Vector(x2) atol=1e-5
#       @test Vector(x8) ≈ Vector(x2) atol=1e-5
#     end
#   end
#
# end

end


"""
``MolecularGeometry`` provides methods for
measuring the geometric properties of molecules.
The following quantities are supported:

+-----------------------------------+--------------------------------+
| Quantity                          |Method                          |
+===================================+================================+
|geometric center                   |gage(GeometricCenter, array)    |
|                                   |                                |
|                                   |gage(GeometricCenter, obj)      |
+-----------------------------------+--------------------------------+

"""
module MolecularGeometry
include(joinpath("..", "deps", "Distances", "src", "Distances.jl"))

macro debug(expression)
    return :($expression)
end

export GeometricCenter, MinDistance, PrincipalAxes
export RotationAngle, ProjectedRotationAngle
export gage, nearly_zero, smaller_vector

# types
include(joinpath("type", "all.jl"))

# functions
include(joinpath("function", "all.jl"))

end


nums = open("inputs/day09.txt") do f
    parse.(Int64, readlines(f))
end

function solve(nums, step=25)::Union{Int64,Nothing}
    start = 1
    for i in start + step:length(nums)
        slice = view(nums, i:(i + step))
        last = nums[i + step]
        search = function ()
            for a in slice
                for b in slice
                    if last == a + b
                        return true
                    end
                end
            end
            return false
        end
        if !search()
            return last
        end
    end
end

function solve2(nums, target)::Union{Int64,Nothing}
    for start in 1:length(nums)
        mx = nums[start]
        mn = nums[start]
        acc = nums[start]
        for i in (start + 1):length(nums)
            acc += nums[i]
            mx = max(mx, nums[i])
            mn = min(mn, nums[i])
            if acc > target
                break
            end
            if acc == target
                return mn + mx
            end
        end
    end
end

function main()
    part1 = solve(nums)
    println("Part 1: ", part1)
    println("Part 2: ", solve2(nums, part1))
end

main()


using Flux
using Flux: logitcrossentropy, onehotbatch, onecold, @epochs
using Flux.Data: DataLoader
using MLDatasets: MNIST
using Random
using Zygote

function get_dataloaders(batch_size::Int, shuffle::Bool)
    train_x, train_y = MNIST.traindata(Float32)
    test_x, test_y = MNIST.testdata(Float32)

    train_y, test_y = onehotbatch(train_y, 0:9), onehotbatch(test_y, 0:9)

    train_loader = DataLoader(train_x, train_y, batchsize=batch_size, shuffle=shuffle)
    test_loader = DataLoader(test_x, test_y, batchsize=batch_size, shuffle=shuffle)

    return train_loader, test_loader
end

function accuracy(data_loader, model)
    acc_correct = 0
    for (x_batch, y_batch) in data_loader
        batch_size = size(x_batch)[end]
        acc_correct += sum(onecold(model(x_batch)) .== onecold(y_batch)) / batch_size
    end
    return acc_correct / length(data_loader)
end

struct FFNetwork
    fc_1
    dropout
    fc_2
    FFNetwork(
        input_dim::Int, hidden_dim::Int, dropout_ratio::Float32, num_classes::Int
    ) = new(
        Dense(input_dim, hidden_dim, relu),
        Dropout(dropout_ratio),
        Dense(hidden_dim, num_classes),
    )
end

function (net::FFNetwork)(x)
    x = Flux.flatten(x)
    return net.fc_2(net.dropout(net.fc_1(x)))
end

function cross_entropy_loss(model, x, y)
    ŷ = model(x)
    return logitcrossentropy(model(x), y)
end

function main(num_epochs, batch_size, shuffle, η)
    train_loader, test_loader = get_dataloaders(batch_size, shuffle)

    model = FFNetwork(28*28, 128, 0.2f0, 10)
    trainable_params = Flux.params(model.fc_1, model.fc_2)
    optimiser = ADAM(η)

    for epoch = 1:num_epochs
        acc_loss = 0.0
        for (x_batch, y_batch) in train_loader
            loss, back = pullback(trainable_params) do
                cross_entropy_loss(model, x_batch, y_batch)
            end
            # Feed the pullback 1 to obtain the gradients and update then model parameters
            gradients = back(1f0)
            Flux.Optimise.update!(optimiser, trainable_params, gradients)
            acc_loss += loss
        end
        avg_loss = acc_loss / length(train_loader)
        println("Epoch: $epoch - loss: $avg_loss")
    end

    testmode!(model)
    @show accuracy(train_loader, model)
    @show accuracy(test_loader, model)
    println("Complete!")
end

if abspath(PROGRAM_FILE) == @__FILE__
    batch_size = 64
    shuffle_data = true
    η = 0.0001
    num_epochs = 5
    main(num_epochs, batch_size, shuffle_data, η)
end


using SimulatedNeuralMoments, Flux, MCMCChains, StatsPlots, DelimitedFiles
using BSON:@save
using BSON:@load

# get the things to define the structure for the model
# For your own models, you will need to supply the functions
# found in MNlib.jl, using the same formats
include("MNlib.jl")

function main()
lb, ub = PriorSupport()

# fill in the structure that defines the model
model = SNMmodel("Mixture of Normals example model", lb, ub, InSupport, Prior, PriorDraw, auxstat)
# train the net, and save it and the transformation info
# nnmodel, nninfo = MakeNeuralMoments(model; Epochs=100)
# @save "neuralmodel.bson" nnmodel nninfo  # use this line to save the trained neural net
@load "neuralmodel.bson" nnmodel nninfo # use this to load a trained net
# illustrate basic NN point estimation
θ = TrueParameters()
m = NeuralMoments(θ, 1, model, nnmodel, nninfo)
cnames = ["true", "estimate"]
println("Basic NN estimation, true parameters and estimates")
prettyprint([θ m], cnames)
# draw a chain of length 10000 plus 500 burnin
chain, junk, junk = MCMC(m, 10500, model, nnmodel, nninfo, verbosity=true)
chain = chain[501:end,:]
# visualize results
chn = Chains(chain, ["μ₁","μ₂","σ₁","σ₂","p"])
display(chn)
plot(chn)
savefig("chain.png")
println("SNM estimation: true params and posterior median")
cnames = ["true", "pos. median"]
prettyprint([θ median(chain,dims=1)[:]], cnames)
end
main()


###
# UDY Oracle Tests
###
#doesn't use any bounds
facts("portTest UDY") do
	srand(8675309); data = randn(500, 2)
	w = UDYOracle(data, .1, .2)
	portTest(w, -2.450295808139833, [0.5372084714704519, 0.4627915285295481], TOL=1e-6,
			unc_lower=[-10, -10], unc_upper=[10, 10])
end


struct Knnda
    X::Array{Float64}
    y::AbstractMatrix
    fm
    nlvdis::Int
    metric::String
    h::Real
    k::Int
    tol::Real
    lev::AbstractVector
    ni::AbstractVector
end

"""
    knnda(X, y; nlvdis = 0, metric = "eucl", h = Inf, k = 1, tol = 1e-4)
k-Nearest-Neighbours weighted discrimination (kNN-DA).
* `X` : X-data.
* `y` : y-data (class membership).
* `nlvdis` : Number of latent variables (LVs) to consider in the
    global PLS used for the dimension reduction before
    calculating the dissimilarities. If `nlvdis = 0`, there is no dimension reduction.
* `metric` : Type of dissimilarity used to select the neighbors.
    Possible values are "eucl" (default; Euclidean distance)
    and "mahal" (Mahalanobis distance).
* `h` : A scalar defining the shape of the weight function. Lower is h,
    sharper is the function. See function `wdist`.
* `k` : The number of nearest neighbors to select for each observation to predict.
* `tol` : For stabilization when very close neighbors.

For each new observation to predict, i) a number of
`k` nearest neighbors is selected (= "weighting 1") and ii) a weigthed vote (= "weighting 2")
is then computed to select the most frequent class in this neighborhood.

Weightings 1 and 2 are computed from the dissimilarities between the observation
to predict and the training observations. Depending on argument `nlvdis`,
the computation is done from the raw X-data or after a dimension reduction.
In the last case, global PLS2 scores (LVs) are
computed from {`X`, Y-dummy} (where Y-dummy is the dummy table build from `y`),
and the dissimilarities are computed over these scores.

In general, for high dimensional X-data, using the Mahalanobis distance requires
preliminary dimensionality reduction of the data.
"""
function knnda(X, y; nlvdis = 0, metric = "eucl", h = Inf, k = 1, tol = 1e-4)
    X = ensure_mat(X)
    y = ensure_mat(y)
    ztab = tab(y)
    if nlvdis == 0
        fm = nothing
    else
        fm = plskern(X, dummy(y).Y; nlv = nlvdis)
    end
    return Knnda(X, y, fm, nlvdis, metric, h, k, tol,
        ztab.keys, ztab.vals)
end

"""
    predict(object::Knnda1, X)
Compute the y-predictions from the fitted model.
* `object` : The fitted model.
* `X` : X-data for which predictions are computed.
"""
function predict(object::Knnda, X)
    X = ensure_mat(X)
    m = size(X, 1)
    # Getknn
    if isnothing(object.fm)
        res = getknn(object.X, X; k = object.k, metric = object.metric)
    else
        res = getknn(object.fm.T, transform(object.fm, X); k = object.k, metric = object.metric)
    end
    listw = copy(res.d)
    for i = 1:m
        w = wdist(res.d[i]; h = object.h)
        w[w .< object.tol] .= object.tol
        listw[i] = w
    end
    # End
    pred = similar(object.y, m, 1)
    @inbounds for i = 1:m
        s = res.ind[i]
        pred[i, :] .= findmax_cla(object.y[s], listw[i])
    end
    (pred = pred, listnn = res.ind, listd = res.d, listw = listw)
end


### A Pluto.jl notebook ###
# v0.12.17

using Markdown
using InteractiveUtils

# ╔═╡ ff81125a-086d-11eb-2226-eddf1955c141
begin
	# Activate environment. You do not need to do this if you are not using
	# separate environment
	push!(LOAD_PATH, "$(homedir())/pCloudDrive/julia_project")
	using Pkg
	Pkg.activate("/home/yusri/pCloudDrive/julia_project/MovingFrame/")
end

# ╔═╡ 5df5c832-ee72-11ea-20bf-7b42b7e25571
begin
	using GeometryBasics
	using FileIO
	using LinearAlgebra;
	using GaussianProcesses;
	using PlutoUI
	using MovingFrame;
	import Plots;
	const mva = MovingFrame
	const plt = Plots
end

# ╔═╡ 67beff68-ee73-11ea-3b77-fbf331f8ce9c
# Create MovingCells Type
struct MovingCells
    coordinates::Matrix{Float64}
    time::Vector{Float64}
    smooth_n_points::Int64
    smooth_coordinates::Matrix{Float64}
    smooth_time::Vector{Float64}
    T::Matrix{Float64}
    N::Matrix{Float64}
    B::Matrix{Float64}
    curvature::Vector{Float64}
    torsion::Vector{Float64}
    speed::Vector{Float64}
    original_shapes::Vector{GeometryBasics.Mesh}
    reoriented_shapes::Vector{GeometryBasics.Mesh}
    Spharm_Cx::Matrix{Float64}
    Spharm_Cy::Matrix{Float64}
    Spharm_Cz::Matrix{Float64}
end

# ╔═╡ 7139c988-ee73-11ea-1127-25bfb372000d
begin
	const base_dir = pwd();
	# You need to extract the cell_shape_final.zip first
	const input_data_dir = joinpath(base_dir, "cell_shape_final/")
	const input_files = readdir(input_data_dir);
	const n_points = length(input_files)
end

# ╔═╡ 9ebba1d6-ee73-11ea-1de2-d78152ef6b5e
begin
	# Calculating the cell trajectory
	coordinate = Matrix{Float64}(undef, n_points, 3)
	time = collect(1.0:n_points)
	shapes = Vector{GeometryBasics.Mesh}(undef, n_points)
	for i = 1:length(input_files)
		shape = load(joinpath(input_data_dir, input_files[i]))
		shapes[i] = shape
		verts = decompose(Point3f0, shape)
		# The trajectory is the mass centre of the cell
		coordinate[i, :] .= reduce(+, verts) / length(verts)
	end
	# Normalizing volume and translation
	shapes = (volume_normalizing ∘ mesh_centering).(shapes)
	new_coordinate = coordinate .- coordinate[1, :]'
end

# ╔═╡ 7d1cf63e-ee83-11ea-33e9-651efeecaf00
begin
	exp_lengthscale = 3.4 # lengthscale = e^3.4
	exp_variance= 0.0 #variance = e^0.0

	# Gaussian Processs using Zero mean function and
	# Squared exponential kernel (note that hyperparameters are on the log scale)
	smoother =  GaussianProcessSmoother(MeanZero(), SE(exp_lengthscale, exp_variance))
	smooth_coordinates = smoothing(new_coordinate, smoother, step=1.)

	smooth_time = range(1.0, stop=time[end], length=size(smooth_coordinates, 1)) |> collect
	T, N, B, curvature, torsion, speed = parallel_transport(smooth_coordinates)
end

# ╔═╡ 5b0d2618-0878-11eb-0fe0-69b9357b91e9
# Reoriented the cell shape to the Moving Frame basis
reoriented_shape = [
    mesh_reorientation(shapes[i], T[i, :], N[i, :], B[i, :])
    for i in eachindex(shapes)
]

# ╔═╡ 29c8a4b8-087a-11eb-3780-4fa5a2c12d0e
# Spherical parameterization
spheres = MovingFrame.spherization.(reoriented_shape, n_step=100)

# ╔═╡ ed0b2ee6-090d-11eb-3516-9b7788a1859b
begin
	# Calculating spherical harmonics coefficients
	spharm_coefs = [
	    MovingFrame.spharm_descriptor(
	        reoriented_shape[i],
	        GeometryBasics.coordinates(spheres[i]),
	    ) for i in eachindex(reoriented_shape)
	]
	Cx = MovingFrame.array_of_array_to_matrix([coef[:Cx] for coef in spharm_coefs])
	Cy = MovingFrame.array_of_array_to_matrix([coef[:Cy] for coef in spharm_coefs])
	Cz = MovingFrame.array_of_array_to_matrix([coef[:Cz] for coef in spharm_coefs])
end

# ╔═╡ 18462938-090e-11eb-2154-45881e7ef9f1
begin
	# Calculating shape eccentricity
	xy_ecc = Cx[:, 4] ./ Cy[:, 2]
	xz_ecc = Cx[:, 4] ./ Cz[:, 3]
	yz_ecc = Cy[:, 2] ./ Cz[:, 3]
end

# ╔═╡ cd0cd780-ee89-11ea-0e25-45fdbb85346d
begin
plt.plot(smooth_coordinates[:,1],smooth_coordinates[:,2],smooth_coordinates[:,3],label ="SMOOTH_TRAJECTORY")
plt.scatter!(new_coordinate[:,1],new_coordinate[:,2],new_coordinate[:,3],label ="OBSERVATION",markersize=1.0)

end

# ╔═╡ 46ff40b6-090e-11eb-1216-b1a6a3af5695
plt.plot(smooth_time,[xy_ecc xz_ecc],label=["XY Eccentricity" "XZ Eccentricity"])

# ╔═╡ 04cdf2de-3dcd-11eb-2123-9fef1cb5553e
begin
	p1 = plt.plot(smooth_time,speed,ylabel = "Speed")
	p2 = plt.plot(smooth_time,curvature,ylabel = "Curvature",)
	p3 = plt.plot(smooth_time,torsion,ylabel="Torsion",)
	plt.plot(p1,p2,p3,legend=false)
end

# ╔═╡ Cell order:
# ╠═ff81125a-086d-11eb-2226-eddf1955c141
# ╠═5df5c832-ee72-11ea-20bf-7b42b7e25571
# ╠═67beff68-ee73-11ea-3b77-fbf331f8ce9c
# ╠═7139c988-ee73-11ea-1127-25bfb372000d
# ╠═9ebba1d6-ee73-11ea-1de2-d78152ef6b5e
# ╠═7d1cf63e-ee83-11ea-33e9-651efeecaf00
# ╠═5b0d2618-0878-11eb-0fe0-69b9357b91e9
# ╠═29c8a4b8-087a-11eb-3780-4fa5a2c12d0e
# ╠═ed0b2ee6-090d-11eb-3516-9b7788a1859b
# ╠═18462938-090e-11eb-2154-45881e7ef9f1
# ╠═cd0cd780-ee89-11ea-0e25-45fdbb85346d
# ╠═46ff40b6-090e-11eb-1216-b1a6a3af5695
# ╠═04cdf2de-3dcd-11eb-2123-9fef1cb5553e


"""
    pathfinder(logp, ∇logp, θ₀::AbstractVector{<:Real}, ndraws::Int; kwargs...)

Find the best multivariate normal approximation encountered while optimizing `logp`.

The multivariate normal approximation returned is the one that maximizes the evidence lower
bound (ELBO), or equivalently, minimizes the KL divergence between

# Arguments
- `logp`: a callable that computes the log-density of the target distribution.
- `∇logp`: a callable that computes the gradient of `logp`.
- `θ₀`: initial point of length `dim` from which to begin optimization
- `ndraws`: number of approximate draws to return

# Keywords
- `rng::Random.AbstractRNG`: The random number generator to be used for drawing samples
- `optimizer::Optim.AbstractOptimizer`: Optimizer to be used for constructing trajectory.
Defaults to `Optim.LBFGS(; m=$DEFAULT_HISTORY_LENGTH, linesearch=$DEFAULT_LINE_SEARCH)`.
- `history_length::Int=$DEFAULT_HISTORY_LENGTH`: Size of the history used to approximate the
inverse Hessian. This should only be set when `optimizer` is not an `Optim.LBFGS`.
- `ndraws_elbo::Int=5`: Number of draws used to estimate the ELBO
- `kwargs...` : Remaining keywords are forwarded to `Optim.Options`.

# Returns
- `q::Distributions.MvNormal`: ELBO-maximizing multivariate normal distribution
- `ϕ::AbstractMatrix{<:Real}`: draws from multivariate normal with size `(dim, ndraws)`
- `logqϕ::Vector{<:Real}`: log-density of multivariate normal at columns of `ϕ`
"""
function pathfinder(
    logp,
    ∇logp,
    θ₀,
    ndraws;
    rng::Random.AbstractRNG=Random.default_rng(),
    optimizer=DEFAULT_OPTIMIZER,
    history_length::Int=optimizer isa Optim.LBFGS ? optimizer.m : DEFAULT_HISTORY_LENGTH,
    ndraws_elbo::Int=5,
    kwargs...,
)
    # compute trajectory
    θs, logpθs, ∇logpθs = maximize_with_trace(logp, ∇logp, θ₀, optimizer; kwargs...)
    L = length(θs) - 1
    @assert L + 1 == length(logpθs) == length(∇logpθs)
    # fit mv-normal distributions to trajectory
    qs = fit_mvnormals(θs, ∇logpθs; history_length=history_length)
    # find ELBO-maximizing distribution
    lopt, elbo, ϕ, logqϕ = maximize_elbo(rng, logp, qs[2:end], ndraws_elbo)
    @info "Optimized for $L iterations. Maximum ELBO of $(round(elbo; digits=2)) reached at iteration $lopt."

    # get parameters of ELBO-maximizing distribution
    q = qs[lopt + 1]

    # reuse existing draws; draw additional ones if necessary
    if ndraws_elbo < ndraws
        ϕnew, logqϕnew = rand_and_logpdf(rng, q, ndraws - ndraws_elbo)
        ϕ = hcat(ϕ, ϕnew)
        append!(logqϕ, logqϕnew)
    elseif ndraws < ndraws_elbo
        ϕ = ϕ[:, 1:ndraws]
        logqϕ = logqϕ[1:ndraws]
    end

    return q, ϕ, logqϕ
end


@doc raw"""
    AbstractMetric

Abstract type for the pseudo-Riemannian metric tensor ``g``, a family of smoothly
varying inner products on the tangent space. See [`inner`](@ref).

# Functor

    (metric::Metric)(M::Manifold)

Generate the `MetricManifold` that wraps the manifold `M` with given `metric`.
This works for both a variable containing the metric as well as a subtype `T<:AbstractMetric`,
where a zero parameter constructor `T()` is availabe.
"""
abstract type AbstractMetric end

# piping syntax for decoration
(metric::AbstractMetric)(M::AbstractManifold) = MetricManifold(M, metric)
(::Type{T})(M::AbstractManifold) where {T<:AbstractMetric} = MetricManifold(M, T())

"""
    MetricManifold{𝔽,M<:AbstractManifold{𝔽},G<:AbstractMetric} <: AbstractDecoratorManifold{𝔽}

Equip a [`AbstractManifold`](@ref) explicitly with a [`AbstractMetric`](@ref) `G`.

For a Metric AbstractManifold, by default, assumes, that you implement the linear form
from [`local_metric`](@ref) in order to evaluate the exponential map.

If the corresponding [`AbstractMetric`](@ref) `G` yields closed form formulae for e.g.
the exponential map and this is implemented directly (without solving the ode),
you can of course still implement that directly.

# Constructor

    MetricManifold(M, G)

Generate the [`AbstractManifold`](@ref) `M` as a manifold with the [`AbstractMetric`](@ref) `G`.
"""
struct MetricManifold{𝔽,M<:AbstractManifold{𝔽},G<:AbstractMetric} <:
       AbstractConnectionManifold{𝔽}
    manifold::M
    metric::G
end

@doc raw"""
    RiemannianMetric <: AbstractMetric

Abstract type for Riemannian metrics, a family of positive definite inner
products. The positive definite property means that for ``X  ∈ T_p \mathcal M``, the
inner product ``g(X, X) > 0`` whenever ``X`` is not the zero vector.
"""
abstract type RiemannianMetric <: AbstractMetric end

@doc raw"""
    christoffel_symbols_first(
        M::MetricManifold,
        p,
        B::AbstractBasis;
        backend::AbstractDiffBackend = diff_backend(),
    )

Compute the Christoffel symbols of the first kind in local coordinates of basis `B`.
The Christoffel symbols are (in Einstein summation convention)

```math
Γ_{ijk} = \frac{1}{2} \Bigl[g_{kj,i} + g_{ik,j} - g_{ij,k}\Bigr],
```

where ``g_{ij,k}=\frac{∂}{∂ p^k} g_{ij}`` is the coordinate
derivative of the local representation of the metric tensor. The dimensions of
the resulting multi-dimensional array are ordered ``(i,j,k)``.
"""
christoffel_symbols_first(::AbstractManifold, ::Any, B::AbstractBasis)
function christoffel_symbols_first(
    M::AbstractManifold,
    p,
    B::AbstractBasis;
    backend::AbstractDiffBackend=diff_backend(),
)
    ∂g = local_metric_jacobian(M, p, B; backend=backend)
    n = size(∂g, 1)
    Γ = allocate(∂g, Size(n, n, n))
    @einsum Γ[i, j, k] = 1 / 2 * (∂g[k, j, i] + ∂g[i, k, j] - ∂g[i, j, k])
    return Γ
end
@decorator_transparent_signature christoffel_symbols_first(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis;
    kwargs...,
)

function christoffel_symbols_second(
    M::AbstractManifold,
    p,
    B::AbstractBasis;
    backend::AbstractDiffBackend=diff_backend(),
)
    Ginv = inverse_local_metric(M, p, B)
    Γ₁ = christoffel_symbols_first(M, p, B; backend=backend)
    Γ₂ = allocate(Γ₁)
    @einsum Γ₂[l, i, j] = Ginv[k, l] * Γ₁[i, j, k]
    return Γ₂
end

"""
    connection(::MetricManifold)

Return the [`LeviCivitaConnection`](@ref) for a metric manifold.
"""
connection(::MetricManifold) = LeviCivitaConnection()

@doc raw"""
    det_local_metric(M::AbstractManifold, p, B::AbstractBasis)

Return the determinant of local matrix representation of the metric tensor ``g``, i.e. of the
matrix ``G(p)`` representing the metric in the tangent space at ``p`` with as a matrix.

See also [`local_metric`](@ref)
"""
det_local_metric(::AbstractManifold, p, ::AbstractBasis)
function det_local_metric(M::AbstractManifold, p, B::AbstractBasis)
    return det(local_metric(M, p, B))
end
@decorator_transparent_signature det_local_metric(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis,
)
"""
    einstein_tensor(M::AbstractManifold, p, B::AbstractBasis; backend::AbstractDiffBackend = diff_backend())

Compute the Einstein tensor of the manifold `M` at the point `p`, see [https://en.wikipedia.org/wiki/Einstein_tensor](https://en.wikipedia.org/wiki/Einstein_tensor)
"""
einstein_tensor(::AbstractManifold, ::Any, ::AbstractBasis)
function einstein_tensor(
    M::AbstractManifold,
    p,
    B::AbstractBasis;
    backend::AbstractDiffBackend=diff_backend(),
)
    Ric = ricci_tensor(M, p, B; backend=backend)
    g = local_metric(M, p, B)
    Ginv = inverse_local_metric(M, p, B)
    S = sum(Ginv .* Ric)
    G = Ric - g .* S / 2
    return G
end
@decorator_transparent_signature einstein_tensor(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis;
    kwargs...,
)

@doc raw"""
    flat(N::MetricManifold{M,G}, p, X::FVector{TangentSpaceType})

Compute the musical isomorphism to transform the tangent vector `X` from the
[`AbstractManifold`](@ref) `M` equipped with [`AbstractMetric`](@ref) `G` to a cotangent by
computing

````math
X^♭= G_p X,
````
where ``G_p`` is the local matrix representation of `G`, see [`local_metric`](@ref)
"""
flat(::MetricManifold, ::Any...)

@decorator_transparent_fallback function flat!(
    M::MetricManifold,
    ξ::CoTFVector,
    p,
    X::TFVector,
)
    g = local_metric(M, p, ξ.basis)
    copyto!(ξ.data, g * X.data)
    return ξ
end

@doc raw"""
    inverse_local_metric(M::AbstractcManifold, p, B::AbstractBasis)

Return the local matrix representation of the inverse metric (cometric) tensor, usually
written ``g^{ij}``.

See also [`local_metric`](@ref)
"""
inverse_local_metric(::AbstractManifold, ::Any, ::AbstractBasis)
function inverse_local_metric(M::AbstractManifold, p, B::AbstractBasis)
    return inv(local_metric(M, p, B))
end
@decorator_transparent_signature inverse_local_metric(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis,
)

default_decorator_dispatch(M::MetricManifold) = default_metric_dispatch(M)

"""
    is_default_metric(M, G)

Indicate whether the [`AbstractMetric`](@ref) `G` is the default metric for
the [`AbstractManifold`](@ref) `M`. This means that any occurence of
[`MetricManifold`](@ref)(M,G) where `typeof(is_default_metric(M,G)) = true`
falls back to just be called with `M` such that the [`AbstractManifold`](@ref) `M`
implicitly has this metric, for example if this was the first one implemented
or is the one most commonly assumed to be used.
"""
function is_default_metric(M::AbstractManifold, G::AbstractMetric)
    return _extract_val(default_metric_dispatch(M, G))
end

default_metric_dispatch(::AbstractManifold, ::AbstractMetric) = Val(false)
function default_metric_dispatch(M::MetricManifold)
    return default_metric_dispatch(base_manifold(M), metric(M))
end

"""
    is_default_metric(MM::MetricManifold)

Indicate whether the [`AbstractMetric`](@ref) `MM.G` is the default metric for
the [`AbstractManifold`](@ref) `MM.manifold,` within the [`MetricManifold`](@ref) `MM`.
This means that any occurence of
[`MetricManifold`](@ref)`(MM.manifold, MM.G)` where `is_default_metric(MM.manifold, MM.G)) = true`
falls back to just be called with `MM.manifold,` such that the [`AbstractManifold`](@ref) `MM.manifold`
implicitly has the metric `MM.G`, for example if this was the first one
implemented or is the one most commonly assumed to be used.
"""
function is_default_metric(M::MetricManifold)
    return _extract_val(default_metric_dispatch(M))
end

function Base.convert(::Type{MetricManifold{𝔽,MT,GT}}, M::MT) where {𝔽,MT,GT}
    return _convert_with_default(M, GT, default_metric_dispatch(M, GT()))
end

function _convert_with_default(
    M::MT,
    T::Type{<:AbstractMetric},
    ::Val{true},
) where {MT<:AbstractManifold}
    return MetricManifold(M, T())
end
function _convert_with_default(
    M::MT,
    T::Type{<:AbstractMetric},
    ::Val{false},
) where {MT<:AbstractManifold}
    return error(
        "Can not convert $(M) to a MetricManifold{$(MT),$(T)}, since $(T) is not the default metric.",
    )
end

@doc raw"""
    inner(N::MetricManifold{M,G}, p, X, Y)

Compute the inner product of `X` and `Y` from the tangent space at `p` on the
[`AbstractManifold`](@ref) `M` using the [`AbstractMetric`](@ref) `G`. If `G` is the default
metric (see [`is_default_metric`](@ref)) this is done using `inner(M, p, X, Y)`,
otherwise the [`local_metric`](@ref)`(M, p)` is employed as

````math
g_p(X, Y) = ⟨X, G_p Y⟩,
````
where ``G_p`` is the loal matrix representation of the [`AbstractMetric`](@ref) `G`.
"""
inner(::MetricManifold, ::Any, ::Any, ::Any)

@decorator_transparent_fallback :intransparent function inner(
    M::MetricManifold,
    p,
    X::TFVector,
    Y::TFVector,
)
    X.basis === Y.basis ||
        error("calculating inner product of vectors from different bases is not supported")
    return dot(X.data, local_metric(M, p, X.basis) * Y.data)
end

@doc raw"""
    local_metric(M::AbstractManifold, p, B::AbstractBasis)

Return the local matrix representation at the point `p` of the metric tensor ``g`` with
respect to the [`AbstractBasis`](@ref) `B` on the [`AbstractManifold`](@ref) `M`, usually written ``g_{ij}``.
The matrix has the property that ``g(X, Y)=X^\mathrm{T} [g_{ij}] Y = g_{ij} X^i Y^j``,
where the latter expression uses Einstein summation convention.
The metric tensor is such that the formula works for the given [`AbstractBasis`](@ref) `B`.
"""
local_metric(::AbstractManifold, ::Any, ::AbstractBasis)
@decorator_transparent_signature local_metric(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis;
    kwargs...,
)

@doc raw"""
    local_metric_jacobian(
        M::AbstractManifold,
        p,
        B::AbstractBasis;
        backend::AbstractDiffBackend = diff_backend(),
    )

Get partial derivatives of the local metric of `M` at `p` in basis `B` with respect to the
coordinates of `p`, ``\frac{∂}{∂ p^k} g_{ij} = g_{ij,k}``. The
dimensions of the resulting multi-dimensional array are ordered ``(i,j,k)``.
"""
local_metric_jacobian(::AbstractManifold, ::Any, B::AbstractBasis)
function local_metric_jacobian(
    M::AbstractManifold,
    p,
    B::AbstractBasis;
    backend::AbstractDiffBackend=diff_backend(),
)
    n = size(p, 1)
    ∂g = reshape(_jacobian(q -> local_metric(M, q, B), p, backend), n, n, n)
    return ∂g
end
@decorator_transparent_signature local_metric_jacobian(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis;
    kwargs...,
)

@doc raw"""
    log(N::MetricManifold{M,G}, p, q)

Copute the logarithmic map on the [`AbstractManifold`](@ref) `M` equipped with the [`AbstractMetric`](@ref) `G`.

If the metric was declared the default metric using [`is_default_metric`](@ref), this method
falls back to `log(M,p,q)`. Otherwise, you have to provide an implementation for the non-default
[`AbstractMetric`](@ref) `G` metric within its [`MetricManifold`](@ref)`{M,G}`.
"""
log(::MetricManifold, ::Any...)

@doc raw"""
    log_local_metric_density(M::AbstractManifold, p, B::AbstractBasis)

Return the natural logarithm of the metric density ``ρ`` of `M` at `p`, which
is given by ``ρ = \log \sqrt{|\det [g_{ij}]|}`` for the metric tensor expressed in basis `B`.
"""
log_local_metric_density(::AbstractManifold, ::Any, ::AbstractBasis)
function log_local_metric_density(M::AbstractManifold, p, B::AbstractBasis)
    return log(abs(det_local_metric(M, p, B))) / 2
end
@decorator_transparent_signature log_local_metric_density(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis,
)

@doc raw"""
    metric(M::MetricManifold)

Get the metric ``g`` of the manifold `M`.
"""
metric(::MetricManifold)

function metric(M::MetricManifold)
    return M.metric
end

@doc raw"""
    ricci_curvature(M::AbstractManifold, p, B::AbstractBasis; backend::AbstractDiffBackend = diff_backend())

Compute the Ricci scalar curvature of the manifold `M` at the point `p` using basis `B`.
The curvature is computed as the trace of the Ricci curvature tensor with respect to
the metric, that is ``R=g^{ij}R_{ij}`` where ``R`` is the scalar Ricci curvature at `p`,
``g^{ij}`` is the inverse local metric (see [`inverse_local_metric`](@ref)) at `p` and
``R_{ij}`` is the Riccie curvature tensor, see [`ricci_tensor`](@ref). Both the tensor and
inverse local metric are expressed in local coordinates defined by `B`, and the formula
uses the Einstein summation convention.
"""
ricci_curvature(::AbstractManifold, ::Any, ::AbstractBasis)
function ricci_curvature(
    M::AbstractManifold,
    p,
    B::AbstractBasis;
    backend::AbstractDiffBackend=diff_backend(),
)
    Ginv = inverse_local_metric(M, p, B)
    Ric = ricci_tensor(M, p, B; backend=backend)
    S = sum(Ginv .* Ric)
    return S
end
@decorator_transparent_signature ricci_curvature(
    M::AbstractDecoratorManifold,
    p,
    B::AbstractBasis;
    kwargs...,
)

@doc raw"""
    sharp(N::MetricManifold{M,G}, p, ξ::FVector{CotangentSpaceType})

Compute the musical isomorphism to transform the cotangent vector `ξ` from the
[`AbstractManifold`](@ref) `M` equipped with [`AbstractMetric`](@ref) `G` to a tangent by
computing

````math
ξ^♯ = G_p^{-1} ξ,
````
where ``G_p`` is the local matrix representation of `G`, i.e. one employs
[`inverse_local_metric`](@ref) here to obtain ``G_p^{-1}``.
"""
sharp(::MetricManifold, ::Any, ::CoTFVector)

function sharp!(M::N, X::TFVector, p, ξ::CoTFVector) where {N<:MetricManifold}
    Ginv = inverse_local_metric(M, p, X.basis)
    copyto!(X.data, Ginv * ξ.data)
    return X
end

function Base.show(io::IO, M::MetricManifold)
    return print(io, "MetricManifold($(M.manifold), $(M.metric))")
end

#
# Introduce transparency
# (a) new functions & other parents
for f in [
    christoffel_symbols_first,
    det_local_metric,
    einstein_tensor,
    inverse_local_metric,
    local_metric,
    local_metric_jacobian,
    log_local_metric_density,
    ricci_curvature,
]
    eval(
        quote
            function decorator_transparent_dispatch(
                ::typeof($f),
                ::AbstractConnectionManifold,
                args...,
            )
                return Val(:parent)
            end
        end,
    )
end

function decorator_transparent_dispatch(
    ::typeof(christoffel_symbols_second),
    ::MetricManifold,
    args...,
)
    return Val(:parent)
end


# Faster B-spline manifold

"""
B-spline manifold for lower polynomial degree
TODO: make the field `bsplinespaces` to be conposite type, not abstract type, for performance
"""
struct FastBSplineManifold{T} <: AbstractBSplineManifold
    bsplinespaces::Vector{<:FastBSplineSpace}
    controlpoints::Array{T} where T<:Point
    function FastBSplineManifold(Ps::AbstractVector{<:AbstractBSplineSpace}, a::AbstractArray{T}) where T
        Ps = FastBSplineSpace.(Ps)
        if collect(size(a)) ≠ dim.(Ps)
            throw(DimensionMismatch())
        else
            P = convert(Vector{FastBSplineSpace}, Ps)
            new{T}(P, float(a))
        end
    end
end

struct BSplineCurve{p1,T} <: AbstractBSplineManifold
    bsplinespace1::FastBSplineSpace{p1}
    controlpoints::Array{T,1} where T<:Point
    function BSplineCurve(P1::AbstractBSplineSpace, a::AbstractArray{T,1}) where T
        p1 = degree(P1)
        if size(a) ≠ (dim(P1),)
            throw(DimensionMismatch())
        else
            new{p1,T}(P1, float(a))
        end
    end
end
function BSplineCurve(P::AbstractVector{<:AbstractBSplineSpace}, a::AbstractArray{<:Point,1})
    if length(P) ≠ 1
        throw(DimensionMismatch())
    else
        BSplineCurve(P[1], a)
    end
end

struct BSplineSurface{p1,p2,T} <: AbstractBSplineManifold
    bsplinespace1::FastBSplineSpace{p1}
    bsplinespace2::FastBSplineSpace{p2}
    controlpoints::Array{T,2} where T<:Point
    function BSplineSurface(P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, a::AbstractArray{T,2}) where T
        p1, p2 = degree(P1), degree(P2)
        if size(a) ≠ (dim(P1), dim(P2))
            throw(DimensionMismatch())
        else
            new{p1,p2,T}(P1, P2, float(a))
        end
    end
end
function BSplineSurface(P::AbstractVector{<:AbstractBSplineSpace}, a::AbstractArray{<:Point,2})
    if length(P) ≠ 2
        throw(DimensionMismatch())
    else
        BSplineSurface(P[1], P[2], a)
    end
end

struct BSplineSolid{p1,p2,p3,T} <: AbstractBSplineManifold
    bsplinespace1::FastBSplineSpace{p1}
    bsplinespace2::FastBSplineSpace{p2}
    bsplinespace3::FastBSplineSpace{p3}
    controlpoints::Array{T,3} where T<:Point
    function BSplineSolid(P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, P3::AbstractBSplineSpace, a::AbstractArray{T,3}) where T
        p1, p2, p3 = degree(P1), degree(P2), degree(P3)
        if size(a) ≠ (dim(P1), dim(P2), dim(P3))
            throw(DimensionMismatch())
        else
            new{p1,p2,p3,T}(P1, P2, P3, float(a))
        end
    end
end
function BSplineSolid(P::AbstractVector{<:AbstractBSplineSpace}, a::AbstractArray{<:Point,3})
    if length(P) ≠ 3
        throw(DimensionMismatch())
    else
        BSplineSolid(P[1], P[2], P[3], a)
    end
end

function BSplineManifold(M::BSplineCurve{p1}) where {p1}
    P1 = M.bsplinespace1
    a = M.controlpoints
    return BSplineManifold([P1], a)
end

function BSplineManifold(M::BSplineSurface{p1,p2}) where {p1} where {p2}
    k1 = M.bsplinespace1
    P2 = M.bsplinespace2
    a = M.controlpoints
    return BSplineManifold([P1, P2], a)
end

function BSplineManifold(M::BSplineSolid{p1,p2,p3}) where {p1} where {p2} where {p3}
    P1 = M.bsplinespace1
    P2 = M.bsplinespace2
    P3 = M.bsplinespace3
    a = M.controlpoints
    return BSplineManifold([P1, P2, P3], a)
end

"""
convert AbstractBSplineManifold to FastBSplineManifold
"""
function FastBSplineManifold(M::AbstractBSplineManifold)
    return FastBSplineManifold(BSplineSpace.(M.bsplinespaces), M.controlpoints)
end

function (M::FastBSplineManifold)(t::AbstractVector{<:Real})
    # TODO: faster
    return BSplineManifold(M)(t)
end

function (M::BSplineCurve{p1})(t1::Real) where {p1}
    P1, = bsplinespaces(M)
    a = controlpoints(M)
    n1 = dim(P1)
    k1 = P1.knots
    j1 = _knotindex(P1, t1)
    b1 = _bsplinebasis(P1,t1,j1)

    f1 = j1-p1
    l1 = j1
    S1 = b1[1]*a[f1]
    for i1 in f1+1:l1
        S1 += b1[i1-f1+1]*a[i1]
    end

    return S1
end

function (M::BSplineSurface{p1,p2})(t1::Real,t2::Real) where {p1} where {p2}
    P1, P2 = bsplinespaces(M)
    a = controlpoints(M)
    n1, n2 = dim(P1), dim(P2)
    k1, k2 = P1.knots, P2.knots
    j1, j2 = _knotindex(P1, t1), _knotindex(P2, t2)
    b1 = _bsplinebasis(P1,t1,j1)
    b2 = _bsplinebasis(P2,t2,j2)

    f1, f2 = j1-p1, j2-p2
    l1, l2 = j1, j2
    S2 = b2[1]*a[f1,f2]
    for i2 in f2+1:l2
        S2 += b2[i2-f2+1]*a[f1,i2]
    end
    S1 = b1[1]*S2
    for i1 in f1+1:l1
        S2 = b2[1]*a[i1,f2]
        for i2 in f2+1:l2
            S2 += b2[i2-f2+1]*a[i1,i2]
        end
        S1 += b1[i1-f1+1]*S2
    end

    return S1
end

function (M::BSplineSolid{p1,p2,p3})(t1::Real,t2::Real,t3::Real) where {p1} where {p2} where {p3}
    P1, P2, P3 = M.bsplinespace1, M.bsplinespace2, M.bsplinespace3
    a = controlpoints(M)
    n1, n2, n3 = dim(P1), dim(P2), dim(P3)
    k1, k2, k3 = P1.knots, P2.knots, P3.knots
    j1, j2, j3 = _knotindex(P1, t1), _knotindex(P2, t2), _knotindex(P3, t3)
    b1 = _bsplinebasis(P1,t1,j1)
    b2 = _bsplinebasis(P2,t2,j2)
    b3 = _bsplinebasis(P3,t3,j3)

    f1, f2, f3 = j1-p1, j2-p2, j3-p3
    l1, l2, l3 = j1, j2, j3
    S3 = b3[1]*a[f1,f2,f3]
    for i3 in f3+1:l3
        S3 += b3[i3-f3+1]*a[f1,f2,i3]
    end
    S2 = b1[1]*b2[1]*S3
    for i2 in f2+1:l2
        S3 = b3[1]*a[f1,i2,f3]
        for i3 in f3+1:l3
            S3 += b3[i3-f3+1]*a[f1,i2,i3]
        end
        S2 += b1[1]*b2[i2-f2+1]*S3
    end
    S1 = S2
    for i1 in f1+1:l1
        S3 = b3[1]*a[i1,f2,f3]
        for i3 in f3+1:l3
            S3 += b3[i3-f3+1]*a[i1,f2,i3]
        end
        S2 = b1[i1-f1+1]*b2[1]*S3
        for i2 in f2+1:l2
            S3 = b3[1]*a[i1,i2,f3]
            for i3 in f3+1:l3
                S3 += b3[i3-f3+1]*a[i1,i2,i3]
            end
            S2 += b1[i1-f1+1]*b2[i2-f2+1]*S3
        end
        S1 += S2
    end

    return S1
end

# function (M::Union{BSplineCurve,BSplineSurface,BSplineSolid})(t::AbstractVector{<:Real})
#     return M(t...)
# end

function (M::BSplineCurve)(t::AbstractVector{<:Real})
    return M(t...)
end

function (M::BSplineSurface)(t::AbstractVector{<:Real})
    return M(t...)
end

function (M::BSplineSolid)(t::AbstractVector{<:Real})
    return M(t...)
end

dim(M::FastBSplineManifold) = length(M.bsplinespaces)
dim(M::BSplineCurve) = 1
dim(M::BSplineSurface) = 2
dim(M::BSplineSolid) = 3

function bsplinespaces(M::FastBSplineManifold)
    return M.bsplinespaces
end

function bsplinespaces(M::BSplineCurve{p1}) where {p1}
    return (M.bsplinespace1, )
end

function bsplinespaces(M::BSplineSurface{p1,p2}) where {p1} where {p2}
    return (M.bsplinespace1, M.bsplinespace2)
end

function bsplinespaces(M::BSplineSolid{p1,p2,p3}) where {p1} where {p2} where {p3}
    return (M.bsplinespace1, M.bsplinespace2, M.bsplinespace3)
end

function controlpoints(M::FastBSplineManifold)
    return M.controlpoints
end

function controlpoints(M::BSplineCurve{p1}) where {p1}
    return M.controlpoints
end

function controlpoints(M::BSplineSurface{p1,p2}) where {p1} where {p2}
    return M.controlpoints
end

function controlpoints(M::BSplineSolid{p1,p2,p3}) where {p1} where {p2} where {p3}
    return M.controlpoints
end


module Interpolation

    using ..Utils

    export Interpolator, HermiteInterpolation

    include("interpolation/interpolation.jl")
    include("interpolation/hermite_interpolation.jl")

end


using OutOfSampleBootstraps
using Base.Test

n = 100
R = 30
P = n - R
k = 4

y = randn(n)
x = [ones(n) randn(n, k-1)]

estimates = Array(Float64, k, P)
errors = Array(Float64, P)

## Verify that recursive_ols produces the right estimates.
b1 = recursive_ols(y, x, R)
@test_approx_eq(b1[:,1], x[1:R,:] \ y[1:R])
@test_approx_eq(b1[:,end], x[1:end-1,:] \ y[1:end-1])
## results after reordering too
shuffle = rand(1:n, n)
b1_shuffled = recursive_ols(y, x, R, shuffle)
@test_approx_eq(b1_shuffled[:,1], x[shuffle[1:R],:] \ y[shuffle[1:R]])
@test_approx_eq(b1_shuffled[:,end], x[shuffle[1:end-1],:] \ y[shuffle[1:end-1]])

## Make sure that different functions to get the recursive OLS
## estimates return the same values.
recursive_ols!(estimates, y, x)
@test_approx_eq(estimates, b1)
recursive_ols!(estimates, y, x, shuffle)
@test_approx_eq(estimates, b1_shuffled)
recursive_ols!(estimates, errors, y, x)
@test_approx_eq(estimates, b1)
recursive_ols!(estimates, errors, y, x, shuffle)
@test_approx_eq(estimates, b1_shuffled)

## Make sure basic bootstraps run
b = Array(Float64, 199) ## Initialize storage. Each call will
                        ## overwrite the previous results
recursive_bootstrap!(b, :naive, Block(:moving, 15), y, x, R)
recursive_bootstrap!(b, :cs07_ols, Block(:moving, 15), y, x, R)
recursive_bootstrap!(b, :mine_ols, Block(:moving, 15), y, x, R)


firstpass(str) = match(r"^\s*(\w+)\s+(-?\d+)\s+(-?\d+)\s*$", str)

secondpass(::Nothing, str) = Symbol(strip(str)), nothing
function secondpass(m, _)
    poi, _x, _y = m.captures
    x = parse(Float64, _x)
    y = parse(Float64, _y)
    return Symbol(poi), Space(x, y)
end

parsepoi(str) = secondpass(firstpass(str), str)

function parsepois(row)
    poi_names = Symbol[]
    expected_locations = Dict{Symbol, Space}()
    for poi in split(row, ',')
        poi_name, xy = parsepoi(poi)
        push!(poi_names, poi_name)
        if !isnothing(xy)
            expected_locations[poi_name] = xy
        end
    end
    (; poi_names, expected_locations)
end

function _adjust_expected(pois, expected_locations)
    e = @LArray Matrix(hcat(values(expected_locations)...)) (; Dict(k => (2i-1:2i) for (i,k) in enumerate(keys(expected_locations)))...)
    r = Matrix(hcat((only(space(pois[k])) for k in keys(expected_locations))...))
    R, μ = getR(e, r)
    Dict(k => Space(R * e[k] + μ) for k in keys(expected_locations))
end

function getR(e, r)
    e .-= mean(e, dims = 2)
    μr = mean(r, dims = 2)
    r .-= μr
    F = svd(e * r')
    R = F.V * F.U'
    return R, μr
end

adjust_expected(pois, expected_locations) = length(expected_locations) ≤ 1 ?  Dict(k => only(space(pois[k])) for k in keys(expected_locations)) : _adjust_expected(pois, expected_locations)

function flipy!(pois)
    ymax = maximum(maximum(last, space(v)) for v in values(pois))
    for v in values(pois), row in LazyRows(v.xyt)
        x2, y = row.xy
        row.xy = Space(x2, ymax - y)
    end
end


Using APES

## Li & Stoica fig1
spectral_lines_f = [0.04, 0.05, 0.08, 0.09,    0.16:0.005:0.35]
spectral_lines_p = vec(repmat([pi/4], length(spectral_lines_f)))
spectral_lines_a = [10, 10, 10, 2,   vec(repmat([0.5], length(spectral_lines_f)-4, 1))]

## Simulation properties
N = 128
noise_mean = 0
noise_vari = 1

## Create example signal
z = vec(zeros(N, 1))
n = 1.0:N
for i in 1:length(spectral_lines_f)
    z = z + spectral_lines_a[i] * sin(2 * pi * spectral_lines_f[i] * n + spectral_lines_p[i])
end
srand(1) # For reproducible results
z += sqrt(noise_vari)*randn(size(z))

println("Prime timer")
@time apes(z, spectral_lines_f);
println()
println()

println("Time for 10 runs")
@time for n=1:10; apes(z, 0.0:0.001:0.5); end


using QPSReader, QuadraticModels, QuadraticModelsGurobi
using Test

@testset "QuadraticModelsGurobi.jl" begin
    qps1 = readqps("QAFIRO.SIF") #lower bounds
    stats1 = gurobi(QuadraticModel(qps1))
    @test isapprox(stats1.objective, -1.59078179, atol=1e-2)

    qps2 = readqps("HS21.SIF") # low/upp bounds
    stats2 = gurobi(QuadraticModel(qps2))
    @test isapprox(stats2.objective, -9.99599999e1, atol=1e-2)

    qps3 = readqps("HS52.SIF") # free bounds
    stats3 = gurobi(QuadraticModel(qps3))
    @test isapprox(stats3.objective, 5.32664756, atol=1e-2)
end


#
# Interface to DifferentialEquations.jl
#

#
# Evaluate function and Jacobian at u if they have not
# been evaluated before at u.
#
# See https://github.com/SciML/DifferentialEquations.jl/issues/521
# for discussion of another way to do this
#
function _eval_res_jac!(sys,u)
    uhash=hash(u)
    if uhash!=sys.uhash
        ur=reshape(u,sys)
        eval_and_assemble(sys,ur,ur,sys.residual,Inf)
        sys.uhash=uhash
    else
        global nd
        nd=nd+1
    end
end

"""
$(SIGNATURES)

Interpret the  discrete problem as an ODE/DAE problem. Provide the
rhs function for DifferentialEquations.jl.
"""
function eval_rhs!(du, u, sys,t)
    global nf
    nf=nf+1
    _eval_res_jac!(sys,u)
    du.=-vec(sys.residual)
    nothing
end

njac=0
nf=0
nd=0
"""
$(SIGNATURES)

Interpret the  discrete problem as an ODE/DAE problem. Provide the
jacobi matrix calculation function for DifferentialEquations.jl.
"""
function eval_jacobian!(J, u, sys,t)
    global njac
    njac=njac+1
    _eval_res_jac!(sys,u)
    # Need to implement broadcast for ExtendableSparse.
    J.=-sys.matrix.cscmatrix
    nothing
end

"""
$(SIGNATURES)

Calculate the mass matrix for use with DifferentialEquations.jl.
"""
function mass_matrix(system::AbstractSystem{Tv, Ti}) where {Tv, Ti}
    physics=system.physics
    grid=system.grid
    data=physics.data
    node=Node{Tv,Ti}(system)
    bnode=BNode{Tv,Ti}(system)
    nspecies=num_species(system)
    cellnodefactors=system.cellnodefactors
    bfacenodefactors=system.bfacenodefactors
    bgeom=grid[BFaceGeometries][1]
    bfacenodes=grid[BFaceNodes]
    bfaceregions=grid[BFaceRegions]
    nbfaces=num_bfaces(grid)

    if isdata(data)
        isbstorage=(physics.bstorage!=nofunc)

        storagewrap= function(y, u)
            y.=0
            physics.storage(y,u,node,data)
            nothing
        end

        bstoragewrap=function(y, u)
            y.=0
            physics.bstorage(y,u,bnode,data)
            nothing
        end

    else
        isbstorage=(physics.bstorage!=nofunc2)

        storagewrap= function(y, u)
            y.=0
            physics.storage(y,u,node)
            nothing
        end
        bstoragewrap=function(y, u)
            y.=0
            physics.bstorage(y,u,bnode)
            nothing
        end
    end

    result_s=DiffResults.DiffResult(Vector{Tv}(undef,nspecies),Matrix{Tv}(undef,nspecies,nspecies))
    Y=Array{Tv,1}(undef,nspecies)
    UK=Array{Tv,1}(undef,nspecies)
    cfg_s=ForwardDiff.JacobianConfig(storagewrap, Y, UK)
    cfg_bs=ForwardDiff.JacobianConfig(bstoragewrap, Y, UK)

    U=unknowns(system,inival=1)
    M=unknowns(system,inival=0)

    geom=grid[CellGeometries][1]
    cellnodes=grid[CellNodes]
    cellregions=grid[CellRegions]
    bfacenodes=grid[BFaceNodes]

    for icell=1:num_cells(grid)
        for inode=1:num_nodes(geom)
            _fill!(node,cellnodes,cellregions,inode,icell)
            for ispec=1:nspecies
                UK[ispec]=U[ispec,node.index]
            end
            ForwardDiff.jacobian!(result_s,storagewrap,Y,UK,cfg_s)
            res_stor=DiffResults.value(result_s)
            jac_stor=DiffResults.jacobian(result_s)
            K=node.index
            for idof=_firstnodedof(M,K):_lastnodedof(M,K)
                ispec=_spec(M,idof,K)
                M[ispec,K]+=cellnodefactors[inode,icell]*jac_stor[ispec,ispec]
            end
        end
    end

    if isbstorage
        for ibface=1:nbfaces
            for ibnode=1:num_nodes(bgeom)
                _fill!(bnode,bfacenodes,bfaceregions,ibnode,ibface)
                for ispec=1:nspecies
                    UK[ispec]=U[ispec,bnode.index]
                end
                ForwardDiff.jacobian!(result_s,bstoragewrap,Y,UK,cfg_bs)
                jac_bstor=DiffResults.jacobian(result_s)
                for idof=_firstnodedof(M,K):_lastnodedof(M,K)
                    ispec=_spec(M,idof,K)
                    M[ispec,K]+=bfacenodefactors[ibnode,ibface]*jac_bstor[ispec,ispec]
                end

            end
        end
    end

    return Diagonal(vec(M))
end

"""
$(SIGNATURES)

Complete the system and provide the jacobi matrix as prototype
for the jacobian for use with `DifferentialEquations.jl`.
"""
function jac_prototype(sys::AbstractSystem)
    _complete!(sys, create_newtonvectors=true)
    sys.matrix.cscmatrix
end

"""
        solve mit Union{Module, Nothing}
    package=DifferentialEquations
    solve(inival,sys,tspan,pkg=..., solver=...)

    saveat,tstops,tspan vs tsample
    => tstops=tsample
       tspan=[tstops[1],tstops[end]]
       save=:all, :sample,:nothing
    FunctionCallingCallback for collecting data
    SavingCallback
    TerminateSteadyState
    keep evolve ?
"""

"""
````
solve(DifferentialEquations, inival, system, tspan;
      solver=nothing,
      kwargs...)
````

Use a  timestepping scheme  from [DifferentialEquations.jl](https://github.com/SciML/DifferentialEquations.jl) to perform
transient solution  of the  system.

The system must have a constant diagonal mass matrix.

Any stiff solver  capable to work with mass matrices  is possible.  If
the system  contains elliptic equations, it  needs to be a  DAE solver
able to work with mass matrices.  See a list
[here](https://diffeq.sciml.ai/stable/solvers/dae_solve/#dae_solve_full).

The  constructed  solver should  be  passed  as `solver=`  kwarg.   By
default (if `nothing` is passed), the `Rosenbrock23` solver is used.

All other  kwargs are  passed to  `DifferentialEquations.solve()`, see
[here](https://diffeq.sciml.ai/stable/basics/common_solver_opts/)  for
an overview.

In order to avoid a package dependency on DifferentialEquations,  the module
corresponding to that package is passed as the first argument.

Returns a transient solution object `sol` containing stored solutions,
see [`TransientSolution`](@ref).

"""
function solve(DiffEq::Module,
               inival::AbstractArray,
               sys::AbstractSystem,
               tspan; solver=nothing, kwargs...)

    if isnothing(solver)
        solver=DiffEq.Rosenbrock23()
    end
    global njac
    global nf
    global nd

    njac=0
    nf=0
    nd=0
    f = DiffEq.ODEFunction(eval_rhs!,
                           jac=eval_jacobian!,
                           jac_prototype=jac_prototype(sys),
                           mass_matrix=mass_matrix(sys))

    prob = DiffEq.ODEProblem(f,vec(inival),tspan,sys)
    sol = DiffEq.solve(prob,solver; kwargs...)
    # Return solution as TransientSolution such that sol[i] adheres to the
    # different storage schemes for multispecies cases.
    TransientSolution([reshape(sol.u[i],sys) for i=1:length(sol.u)] ,sol.t)
end


using InvariantMeasures
using ValidatedNumerics

@testset "Ulam assembler" begin

D = Mod1Dynamic(x->2*x)
B = Ulam(8)
P = assemble(B, D)

Ptrue = [
		0.5 0.5 0 0 0 0 0 0;
		0 0 0.5 0.5 0 0 0 0;
		0 0 0 0 0.5 0.5 0 0;
		0 0 0 0 0 0 0.5 0.5;
		0.5 0.5 0 0 0 0 0 0;
		0 0 0.5 0.5 0 0 0 0;
		0 0 0 0 0.5 0.5 0 0;
		0 0 0 0 0 0 0.5 0.5;
		]
Ptrue = Ptrue'

@test all(contains_zero.(P-Ptrue))

# mod1_dynamic with a non-Markov dynamic
D = mod1_dynamic(x->x+0.5)
B = Ulam(8)
P = assemble(B, D)

Ptrue = [
0  0  0  0  1  0  0  0;
0  0  0  0  0  1  0  0;
0  0  0  0  0  0  1  0;
0  0  0  0  0  0  0  1;
1  0  0  0  0  0  0  0;
0  1  0  0  0  0  0  0;
0  0  1  0  0  0  0  0;
0  0  0  1  0  0  0  0;
		]

@test all(contains_zero.(P-Ptrue))

@test opnormbound(L1, DiscretizedOperator(B, D)) == 1
@test opnormbound(Linf, DiscretizedOperator(B, D)) == 1

end


using RobotDynamics
using LinearAlgebra
using StaticArrays
using Rotations
using BenchmarkTools
using Test

struct SatellitePair{R,T} <: LieGroupModel
    J1::SMatrix{3,3,T,9}   # inertia of satellite 1
    J2::SMatrix{3,3,T,9}   # inertia of satellite 2
end

function RobotDynamics.dynamics(model::SatellitePair, x, u)
    vs = RobotDynamics.vec_states(model, x)
    qs = RobotDynamics.rot_states(model, x)
    ω1 = vs[2]
    ω2 = vs[3]
    q1 = qs[1]
    q2 = qs[2]

    J1, J2 = model.J1, model.J2
    u1 = u[SA[1,2,3]]
    u2 = u[SA[4,5,6]]
    ω1dot = J1\(u1 - ω1 × (J1 * ω1))
    ω2dot = J2\(u2 - ω2 × (J2 * ω2))
    q1dot = Rotations.kinematics(q1, ω1)
    q2dot = Rotations.kinematics(q2, ω2)
    SA[
        q1dot[1], q1dot[2], q1dot[3], q1dot[4],
        ω1dot[1], ω1dot[2], ω1dot[3],
        q2dot[1], q2dot[2], q2dot[3], q2dot[4],
        ω2dot[1], ω2dot[2], ω2dot[3],
    ]
end

RobotDynamics.control_dim(::SatellitePair) = 6

RobotDynamics.LieState(::SatellitePair{R}) where R = RobotDynamics.LieState(R, (0,3,3))

J1 = SMatrix{3,3}(Diagonal(fill(1.0, 3)))
J2 = SMatrix{3,3}(Diagonal(fill(2.0, 3)))
model = SatellitePair{UnitQuaternion{Float64}, Float64}(J1, J2)
x,u = rand(model)
@test norm(x[1:4]) ≈ 1
@test norm(x[8:11]) ≈ 1
s = RobotDynamics.LieState(model)
@test length(s) == 14

ω1 = x[5:7]
ω2 = x[12:14]
q1 = UnitQuaternion(x[1:4],false)
q2 = UnitQuaternion(x[8:11],false)
@test all(RobotDynamics.vec_states(s, x)[2:3] .≈ (ω1, ω2))
@test all(RobotDynamics.rot_states(s, x) .≈ (q1,q2))

model = SatellitePair{MRP{Float64}, Float64}(J1, J2)
x,u = rand(model)
@test length(x) == 12
s = RobotDynamics.LieState(model)
@test length(s) == 12

ω1 = x[4:6]
ω2 = x[10:12]
q1 = MRP(x[1:3]...)
q2 = MRP(x[7:9]...)
@test all(RobotDynamics.vec_states(s, x)[2:3] .≈ (ω1, ω2))
@test all(RobotDynamics.rot_states(s, x) .≈ (q1,q2))


function calculate_propensity!(c::Cell, propRules::Vector{Function}, indice::Array{Int})
    for i in indice
        c.A[i] = (propRules[i](c))
    end
end

function initialize_propensities!(model::Model)
    for cell in model.cells
    # Calculate the propensity function, ak, for each reaction.
        calculate_propensity!(cell, model.propRules, collect(1:34))
        # Generate M independent, uniform (0,1) random numbers, rk, and for each k set Pk=ln1/rk.
        for i in 1:34
            cell.Pk[i] = log(1/rand())
            delT = cell.A[i] > 0 ? (cell.Pk[i]-cell.Tk[i])/cell.A[i] : Inf
            enqueue!(cell.pq, i, delT)
        end
    end
end

function min_delayed(cell::Cell, t::Float64)
    min = Inf
    sk = 0
    for i in 1:8
        if (first(cell.pq_delayed[i])-t) < min
            min = first(cell.pq_delayed[i])-t
            sk = i
        end
    end
    return (sk, min)
end

function jump(cell::Cell, t::Float64)
    sk, min = min_delayed(cell, t)
    if sk != 0
        reg = isless(peek(cell.pq)[2], min)
        dt = reg ? peek(cell.pq)[2] : min
        event = reg ? dequeue!(cell.pq) : sk + 34
        if !(reg)
            deleteat!(cell.pq_delayed[sk],1)
            cell.pos[sk] -= 1
        end
    else
        dt = peek(cell.pq)[2]
        event = dequeue!(cell.pq)
    end
    return (dt, event)
end

function update_tjump!(cell::Cell)
    for i in 1:34
        cell.pq[i] = cell.A[i] > 0 ? (cell.Pk[i]-cell.Tk[i])/cell.A[i] : Inf
    end
end

function update_tk!(cell::Cell,dt::Float64)
    for i in 1:34
        cell.Tk[i] += cell.A[i]*dt
    end
end


# ____________________________________________________________________
# powerlaw
#
mutable struct powerlaw <: AbstractComponent
    norm::Parameter
    x0::Parameter
    alpha::Parameter

    function powerlaw(x0::Number)
        out = new(Parameter(1),
                  Parameter(x0),
                  Parameter(-1))
        out.norm.low = 0
        out.x0.low = 0
        out.x0.fixed = true
        out.alpha.low = -5; out.alpha.high = 5
        return out
    end
end

function evaluate!(buffer, comp::powerlaw, x::Domain{1},
                   norm, x0, alpha)
    buffer .= norm .* (x ./ x0).^alpha
end


###########################
## Portfolio get moments ##
###########################

function getPMean(invs::Investments, mus::DataFrame)
    ## possibly multiple mu vectors
    (nMus, nAss) = size(mus)
    if nMus == 1
        pMean = weights(invs)*array(mus)'
    else                                # multiple mus
        pMean = sum(weights(invs).*array(mus), 2)
    end
    tm = Timematr(pMean, idx(invs))
    return tm
end

#################################
## Portfolio covariance matrix ##
#################################

function getPVar(wgts::Array{Float64, 2}, covMatr::Array{Float64, 2})
    nPfs = size(wgts, 1)
    pVars = Array(Float64, nPfs)
    for ii=1:nPfs
        pVars[ii, 1] = (wgts[ii, :]*covMatr*(wgts[ii, :])')[1]
    end
    return pVars
end

function getPVar(invs::Investments, covMatr::DataFrame)
    pVars = getPVar(core(invs), array(covMatr))
    return Timematr(pVars, idx(invs))
end


# # Gaussian Process Multi-Class Classification
#
# ## Preliminary steps
#
# ### Loading necessary packages
# ## Data generation and setting up packages
using Plots
using Distributions
using AugmentedGaussianProcesses

# ### Generate data from a mixture of gaussians (you can control the noise)
n_data = 300
n_dim = 2
n_grid = 100
minx = -2.5;
maxx = 3.5;
# ### We try different noises (different overlaps)
σs = [0.1, 0.2, 0.3, 0.4, 0.5, 0.8]
n_class = n_dim + 1;
# ### We create a function generating a mixture of Gaussians
function generate_mixture_data(σ)
    centers = zeros(n_class, n_dim)
    ## Create equidistant centers
    for i in 1:n_dim
        centers[i, i] = 1.0
    end
    centers[end, :] .= (1 + sqrt(n_class)) / n_dim
    centers ./= sqrt(n_dim)
    ## Generate distributions with desired noise
    distr = [MvNormal(centers[i, :], σ) for i in 1:n_class]
    X = zeros(Float64, n_data, n_dim)
    y = zeros(Int64, n_data)
    for i in eachindex(y)
        y[i] = rand(1:n_class)
        X[i, :] = rand(distr[y[i]])
    end
    return X, y
end
# ### And a function to plot the data
function plot_data(X, Y, σ)
    p = Plots.plot(size(300, 500); lab="", title="sigma = $σ")
    ys = unique(Y)
    Plots.scatter!(eachcol(X)...; group=Y, msw=0.0, lab="")
    return p
end

plot([plot_data(generate_mixture_data(σ)..., σ) for σ in σs]...)

# ## Model training
# ### Run sparse multiclass classification with different level of noise
models = Vector{AbstractGP}(undef, length(σs))
kernel = SqExponentialKernel()
num_inducing = 50
for (i, σ) in enumerate(σs)
    @info "Training with data with noise $σ"
    m = SVGP(
        generate_mixture_data(σ)...,
        kernel,
        LogisticSoftMaxLikelihood(n_class),
        AnalyticVI(),
        num_inducing;
        optimiser=false,
        Zoptimiser=false,
    )
    @time train!(m, 20)
    models[i] = m
end

# ### Function to create predictions and plot them
function compute_grid(model, n_grid=50)
    xlin = range(minx, maxx; length=n_grid)
    ylin = range(minx, maxx; length=n_grid)
    x_grid = Iterators.product(xlin, ylin)
    y_p = proba_y(model, vec(collect.(x_grid)))
    y = predict_y(model, vec(collect.(x_grid)))
    return y_p, y, xlin, ylin
end;

function plot_contour(model, σ)
    n_grid = 100
    pred_proba, pred, x, y = compute_grid(model, n_grid)
    colors = reshape(
        [
            RGB([pred_proba[model.likelihood.ind_mapping[j]][i] for j in 1:n_class]...) for
            i in 1:(n_grid^2)
        ],
        n_grid,
        n_grid,
    ) # Convert the predictions into an RGB array
    Plots.contour(
        x,
        y,
        colors;
        cbar=false,
        fill=false,
        color=:black,
        linewidth=2.0,
        title="sigma = $σ",
    )
    return Plots.contour!(
        x,
        y,
        reshape(pred, n_grid, n_grid);
        clims=(0, 100),
        colorbar=false,
        color=:gray,
        levels=10,
    )
end;
# ### Plot the final results
Plots.plot(plot_contour.(models, σs)...)


"""
    Exo1SplitBinion{T <: Real} = Parabolic{SplitBinion{T}}

An exo-perplex number is a parabolic Cayley-Dickson construct with perplex numbers.
"""
const Exo1SplitBinion{T <: Real} = Parabolic{SplitBinion{T}}

function Exo1SplitBinion(z::SplitBinion{T}) where T <: Real
    Exo1SplitBinion{T}(z, zero(SplitBinion{T}))
end

function Exo1SplitBinion(a::T, b::T, c::T, d::T) where T <: Real
    Exo1SplitBinion{T}(SplitBinion{T}(a, b), SplitBinion{T}(c, d))
end

function Exo1SplitBinion(a::T, b::T, c::T) where T <: Real
    Exo1SplitBinion{T}(SplitBinion{T}(a, b), SplitBinion{T}(c, zero(T)))
end

function Exo1SplitBinion(a::T, b::T) where T <: Real
    Exo1SplitBinion{T}(SplitBinion{T}(a, b), zero(SplitBinion{T}))
end

function Exo1SplitBinion(a::T) where T <: Real
    Exo1SplitBinion{T}(SplitBinion{T}(a, zero(T)), zero(SplitBinion{T}))
end

function Exo1SplitBinion(a::Real, b::Real)
    Exo1SplitBinion(promote(a, b)...)
end

function Exo1SplitBinion(a::Real, b::Real, c::Real)
    Exo1SplitBinion(promote(a, b, c)...)
end

function Exo1SplitBinion(a::Real, b::Real, c::Real, d::Real)
    Exo1SplitBinion(promote(a, b, c, d)...)
end

function Exo1SplitBinion(z::Exo1Real{T}) where T <: Real
    Exo1SplitBinion(z.l, zero(T), z.r, zero(T))
end

"""
    unreal(z::Exo1SplitBinion)

The unreal part of an exo-perplex number is a 3-vector.
"""
function unreal(z::Exo1SplitBinion)
    vcat(unreal(z.l), array(z.r))
end

function show(io::IO, z::Exo1SplitBinion)
    print(io, "[1: ")
    print(io, z.l.l)
    print(io, ", s: ")
    print(io, z.l.r)
    print(io, ", a: ")
    print(io, z.r.l)
    print(io, ", sa: ")
    print(io, z.r.r)
    print(io, "]")
end

function random(::Type{Exo1SplitBinion{T}}) where T <: Real
    Exo1SplitBinion{T}(random(SplitBinion{T}), random(SplitBinion{T}))
end


module VectorClouds

using LinearAlgebra
using NearestNeighbors
using ..Utils.Mathes

const VectorCloud = Matrix

export VectorCloud

@inline function to_kd_tree(self::VectorCloud; leafsize::Integer=1)
    if isempty(self)
        return nothing
    else
        return KDTree(self[1:3, :], leafsize=leafsize)
    end
end

@inline function to_kd_tree(self::Nothing)
    return nothing
end

"""
    query(self::VectorCloud{T}, targetVectorCloud::VectorCloud;
                targetTree::KDTree = to_kd_tree(targetVectorCloud))
query a VectorCloud to a target KDTree
"""
function query(self::VectorCloud{T}, targetVectorCloud::VectorCloud{T};
               targetTree::KDTree=to_kd_tree(targetVectorCloud)) where T
    queryVectorCloud = self

    idxs, dists = knn(targetTree, queryVectorCloud[1:3, :], 1, false)

    # distance matrix
    ret = Matrix{T}(undef, 2, size(queryVectorCloud, 2))

    for (i, nodeIndexList) in idxs |> enumerate
        targetNodeIndex = nodeIndexList[1]
        # physical distance
        dist = dists[i][1]

        queryVector = queryVectorCloud[4:6, i]
        targetVector = targetVectorCloud[4:6, targetNodeIndex]
        # absolute dot product
        adp = abs( dot( queryVector, targetVector ) )

        ret[:, i] = [dist, adp]
    end
    ret
end

end # end of module VectorClouds


mutable struct InfiniteMatrixProductState{T<:RLorCX}
    d  :: Int
    Γ  :: Array{T, 3}
    Λ  :: Vector{Float64}
end

function InfiniteMatrixProductState(M::Array{T, 3}) where{T<:RLorCX}
    diml, d, dimr = size(M)
    @assert (diml == dimr)
    fact = svd(reshape(M, diml, d*dimr), full=false)
    Vt = reshape(fact.Vt, diml, d, dimr)

    @tensor Γ[l,d,r] := Vt[l,d,m] * fact.U[m,r]

    InfiniteMatrixProductState{T}(d, (Γ,), fact.S/norm(fact.S))
end

"""
    poshermitsqrt(A)

return the square root of the positvie hermitian matrix `A` using
eigenvalue decomposition.

"""
function poshermitsqrt(A::Matrix{T}) where{T<:Number}
    !isapprox(A, A') && error("Matrix is not hermitian! ", norm(A-A'))
    S, U = eigen(Hermitian(A))
    if !all(S .> 0)
        !all(S .< 0) && error("Matrix is not positive!", S)
        S = -S
    end

    U * Diagonal(sqrt.(S)) * U'
end

function _transfermatvec(v::Matrix{T}, uc::Array{T, 4}, direction::Symbol) where{T<:Number}
    if direction == :LEFT
        @tensor w[u,d] := (v[u',d'] * uc[u',o1,o2,u]) * conj(uc)[d',o1,o2,d]
        w
    elseif direction == :RIGHT
        @tensor w[u,d] := (v[u',d'] * uc[u,o1,o2,u']) * conj(uc)[d,o1,o2,d']
        w
    else
        error("unknown direction!")
    end
end

function InfiniteMatrixProductState(A::Array{T, 3},
                                    Λ::Vector{Float64},
                                    B::Array{T, 3},
                                    Λex::Vector{Float64}) where {T<:RLorCX}
    n = size(A, 1)
    d = size(A, 2)
    @assert size(A) == (n, d, n)
    @assert length(Λ) == n
    @assert size(B) == (n, d, n)
    @assert length(Λex) == n

    # @tensor LB[l,o,r] := Diagonal(Λ)[l,m] * B[m,o,r]
    # Q, Λr = qr(reshape(LB, n*d, n))
    # Q = Matrix(Q)
    @tensor Luc[l,o1,o2,r] := (A[l,o1,ml] * Diagonal(Λ)[ml,mm]) *
        B[mm,o2,mr] * Diagonal(1 ./Λex)[mr,r]

    es, vs, info = eigsolve(v->_transfermatvec(v, Luc, :LEFT),
                            rand(n,n), 1, :LR)
    if !(es[1] ≈ 1.0)
       println("The largest left eigenvalue of transfer matrix is not unit : ", es[1])
    end
    v = vs[1]
    if !(v ≈ conj(v))
        println("The left eigenvector is not real!", norm(v-conj(v)))
        v = real.(v)
    end
    X = poshermitsqrt(real.(v))

    # @tensor AL[l,o,r] := A[l,o,m] * Diagonal(Λ)[m,r]
    # Q, Λl = qr(transpose(reshape(AL, n, d*n)))
    # Q = transpose(Matrix(Q))
    # Λl = transpose(ΛL)
    @tensor Ruc[l,o1,o2,r] := (Diagonal(1 ./Λex)[l,ml] * A[ml,o1,mm]) *
        Diagonal(Λ)[mm, mr] * B[mr,o2,r]

    es, vs, info = eigsolve(v->_transfermatvec(v, Ruc, :RIGHT),
                            rand(n,n), 1, :LR)
    if !(es[1] ≈ 1.0)
        println("The largest right eigenvalue of transfer matrix is not unit : ", es[1])
    end
    v = vs[1]
    if !(v ≈ conj(v))
        println("The right eigenvector is not real!", norm(v-conj(v)))
    end
    Y = poshermitsqrt(real.(v))

    ###NOTE:
    ## Left normalzied canonical unitcell is (XA) Λ (BY) inv(Y Λex X)
    ## Right noralized canonical unitcell is inv(Y Λex X) (XA) Λ (BY)
    ## so naming A <- XA and B <-BY and Λex <- Y Λex X works!

    #println(X - conj(X))
    @tensor XA[l,o,r] := X[l,m] * A[m,o,r]
    Q, R = qr(reshape(XA, n*d, n))
    Γ1 = reshape(Matrix(Q), n,d,n)

    @tensor BY[l,o,r] := (R[l,ml] * Diagonal(Λ)[ml,mm]) * B[mm,o,mr] * Y[mr,r]
    Q, R = qr(reshape(BY, n*d, n))
    Γ2 = reshape(Matrix(Q), n,d,n)

    @show norm(R - Matrix(I, n, n))

    L = Y*Diagonal(Λex)*X
    norm(L - Diagonal(diag(L)))
    InfiniteMatrixProductState{T}(d, Γ1, inv(Y*Diagonal(Λex)*X))
end

function ketstate2imps(vstate::Vector{T}, lx::Int, d::Int;
                       maxdim::Int=d^div(lx, 2)) where{T<:RLorCX}
    @assert lx % 2 == 1
    @assert length(vstate) == d^lx

    l = div(lx, 2)
    fact = svd(reshape(vstate, d^(l+1), d^l), full=false)

    S, n, ratio = truncate(fact.S, maxdim=maxdim)
    Vt = fact.Vt[1:n, :]

    ##NOTE: Using inversion symmetry the left and right l sites have
    ##the same vector, so we only need to put an identity in the
    ##middle, conjugate and then contract to get rid of all the left
    ##and right tensors except the one in the middle (which we are
    ##looking for)

    psi_b = reshape(Vt, n, 1, d*ones(Int, l)...)
    psi_a = permutedims(psi_b, [1,2,(l+2:-1:3)...])

    ten_b = reshape(conj(psi_b), n, 1, d^l)
    ten_a = reshape(conj(psi_a), n, 1, d^l)
    I_ext = reshape(Matrix(1.0I, d, d), 1, d, 1, d)
    vstate_aib = reshape(vstate, d^l, 2, d^l)

    @tensor M[a,d,b] := ((vstate_aib[a',d', b'] * ten_a[a, il, a']) *
                         ten_b[b,ir, b']) * I_ext[il, d, ir, d']

    InfiniteMatrixProductState(M)
end

function measure_bond(imps::InfiniteMatrixProductState{T},
                      bond::Array{T, 4}) where{T<:Number}
    d = imps.d
    @assert size(bond) == (d,d,d,d)

    Γ = imps.Γs#[1]
    @tensor sbb[l,d1,d2,r] :=
        Diagonal(imps.Λ)[l, ml] * Γ[ml, d1, mr] * Γ[mr, d2, r]
    @tensor v = scalar(sbb[l,d1',d2',r] * bond[d1,d2,d1',d2'] * conj(sbb)[l,d1,d2,r])
    real(v)
end


for N in [Float64, Rational{Int}, Float32]
    B1 = BallInf(zeros(N, 2), N(1))
    B2 = Ball1(ones(N, 2), N(1))
    B3 = Hyperrectangle(low=N[-1, -1], high=N[2, 2])
    S = Singleton(N[10, 10])
    UXY = UnionSet(B1, B2)

    # type alias
    @test B1 ∪ B2 == UXY

    # array type (union of a finite number of convex sets)
    Uarr = UnionSetArray([B1, B2])

    # constructor without argumenst
    UnionSetArray()

    # getindex & length
    @test Uarr[1] == B1 && Uarr[2] == B2
    @test length(Uarr) == 2

    # swap
    U2 = swap(UXY)
    @test UXY.X == U2.Y && UXY.Y == U2.X

    # neutral and absorbing elements
    E = EmptySet{N}(2)
    @test neutral(UnionSet) == neutral(UnionSetArray) == EmptySet
    @test UnionSet(B1, E) == UnionSet(E, B1) == B1
    U = Universe{N}(2)
    @test absorbing(UnionSet) == absorbing(UnionSetArray) == Universe
    @test UnionSet(B1, U) == UnionSet(U, B1) == U

    for U in [UXY, Uarr]
        # dimension
        @test dim(U) == dim(B1)

        # support vector (default algorithm)
        d = N[1, 0]
        @test σ(d, U) == [N(2), N(1)]
        @test σ(d, U, algorithm="support_vector") == [N(2), N(1)]

        # support vector (support function algorithm)
        @test σ(d, U, algorithm="support_function") == [N(2), N(1)]

        # support function
        @test ρ(d, U) == N(2)

        # an_element and membership
        @test an_element(U) ∈ U

        # emptiness
        @test !isempty(U)

        # boundedness
        @test isbounded(U) && isboundedtype(typeof(U))

        # inclusion
        subset, point = ⊆(U, B3, true)
        @test U ⊆ B3 && subset && point == N[]
        subset, point = ⊆(U, B2, true)
        @test U ⊈ B2 && !subset && point ∈ U && point ∉ B2

        # isdisjoint
        disjoint1, point1 = isdisjoint(U, B1, true)
        disjoint2, point2 = isdisjoint(B1, U, true)
        @test !isdisjoint(U, B1) && !isdisjoint(B1, U) && !disjoint1 &&
              !disjoint2 && point1 ∈ U && point1 ∈ B1 && point2 ∈ U &&
              point2 ∈ B1
        disjoint1, point1 = isdisjoint(U, S, true)
        disjoint2, point2 = isdisjoint(S, U, true)
        @test isdisjoint(U, S) && isdisjoint(S, U) && disjoint1 &&
              disjoint2 && point1 == point2 == N[]

        # vertices list
        @test ispermutation(vertices_list(U),
            [N[1, 1], N[1, -1], N[-1, 1], N[-1, -1],
            N[1, 2], N[1, 0], N[2, 1], N[0, 1]])
        @test ispermutation(vertices_list(U, apply_convex_hull=true),
            [N[1, -1], N[-1, 1], N[-1, -1], N[1, 2], N[2, 1]])
    end

    # linear map
    # (only a sufficient test because there is currently no equivalence check:
    #  test that the map of the union is equivalent to the union of each map)
    A = N[1 0; 0 1]
    Y = linear_map(A, UXY)
    @test isequivalent(Y.X, linear_map(A, UXY.X))
    @test isequivalent(Y.Y, linear_map(A, UXY.Y))
    Y = linear_map(A, Uarr)
    for k in 1:length(array(Uarr))
        @test isequivalent(array(Y)[k], linear_map(A, array(Uarr)[k]))
    end

    # an_element and membership
    X = HPolyhedron([HalfSpace(N[1, 0], N(0)), HalfSpace(N[-1, 0], N(-1))])
    UXY = UnionSet(X, B1)
    @test an_element(UXY) ∈ UXY
    Uarr = UnionSetArray([EmptySet{N}(2), B1])
    @test an_element(Uarr) ∈ Uarr

    # boundedness of unbounded unions
    @test !isbounded(UXY) && !isboundedtype(typeof(UXY))
    Uarr = UnionSetArray([X, B1])
    @test !isbounded(Uarr) && !isboundedtype(typeof(Uarr))

    # is_polyhedral
    @test is_polyhedral(UXY)
    if N isa AbstractFloat
        U2 = UnionSet(B1, Ball2(zeros(N, 2), N(1)))
        @test !is_polyhedral(U2)
    end

    # volume
    X = Interval(N(1), N(2))
    Y = Interval(N(3), N(4))
    @test volume(UnionSet(X, Y)) == N(2)
end

for N in [Float64]
    B1 = BallInf(zeros(N, 2), N(1))
    B2 = Ball1(ones(N, 2), N(1))
    B3 = Hyperrectangle(low=N[-1, -1], high=N[2, 2])
    S = Singleton(N[10, 10])
    UXY = UnionSet(B1, B2)
    Uarr = UnionSetArray([B1, B2])

    for U in [UXY, Uarr]
        # intersection
        @test !isempty(intersection(U, B3)) && !isempty(intersection(B3, U))
        @test isempty(intersection(U, S)) && isempty(intersection(S, U))
    end

    # emptiness
    emptyP = HPolyhedron([HalfSpace(N[1, 0], N(0)), HalfSpace(N[-1, 0], N(-1))])
    @test !isempty(emptyP ∪ B1) && !isempty(B1 ∪ emptyP) &&
          isempty(emptyP ∪ emptyP)
    @test !isempty(UnionSetArray([emptyP, B1])) &&
          !isempty(UnionSetArray([B1, B2, emptyP])) &&
          isempty(UnionSetArray([emptyP, emptyP]))

    # boundedness
    unboundedP = HPolyhedron([HalfSpace(N[1, 0], N(0))])
    @test !isbounded(unboundedP ∪ B1) && !isbounded(B1 ∪ unboundedP)
    @test !isbounded(UnionSetArray([unboundedP, B1])) &&
          !isbounded(UnionSetArray([B1, B2, unboundedP]))

    # isdisjoint
    disjoint1, point1 = isdisjoint(UXY, UXY, true)
    disjoint2, point2 = isdisjoint(UXY, Uarr, true)
    disjoint3, point3 = isdisjoint(Uarr, UXY, true)
    disjoint4, point4 = isdisjoint(Uarr, Uarr, true)
    @test !isdisjoint(UXY, UXY) && !isdisjoint(UXY, Uarr) &&
          !isdisjoint(Uarr, UXY) && !isdisjoint(Uarr, Uarr) && !disjoint1 &&
          !disjoint2 && !disjoint3 && !disjoint4 && point1 ∈ UXY &&
          point2 ∈ UXY && point2 ∈ Uarr && point3 ∈ UXY && point3 ∈ Uarr &&
          point4 ∈ Uarr
end


rules_text = readlines("day7/input.txt")

struct Bag
    adjective::String
    color::String
end

struct BagRule
    bag::Bag
    holds::Set{Tuple{Int, Bag}}
end

function parse_bag(line::String)
    adj, color = match(r"^(\w+) (\w+) bags", line).captures
    other_bags = Set{Tuple{Int, Bag}}()
    for match in eachmatch(r"(\d) (\w+) (\w+) bag[s]?", line)
        num, adj_inside, color_inside = match.captures
        push!(other_bags, (parse(Int, num), Bag(adj_inside, color_inside)))
    end
    return BagRule(Bag(adj, color), other_bags)
end

bags_rules = parse_bag.(rules_text)

function contains_bags(bag::Bag)
    global bags_rules
    bag_rule = bags_rules[findfirst(b -> b.bag == bag, bags_rules)]
    bags_inside = 0
    for (num, bag) in bag_rule.holds
        bags_inside += num*contains_bags(bag)
        bags_inside += num
    end
    return bags_inside
end

println(contains_bags(Bag("shiny", "gold")))

# that was harder, but we got through it!
# and it was fun! :D


#=
Visual illustration of when the law of large numbers fails

@author : Spencer Lyon <spencer.lyon@nyu.edu>
          Victoria Gregory <victoria.gregory@nyu.edu>

References
----------
Based off the original python file cauchy_samples.py
=#

using Plots
pyplot()
using Distributions
using LaTeXStrings

srand(12)  # reproducible results
n = 200
dist = Cauchy()
data = rand(dist, n)

function plot_draws()
    t = "$n observations from the Cauchy distribution"
    N = repmat(linspace(1, n, n), 1, 2)'
    heights = [zeros(1,n); data']
    plot(1:n, data, color=:blue, markershape=:circle,
         alpha=0.5, title=t, legend=:none, linewidth=0)
    plot!(N, heights, linewidth=0.5, color=:blue)
end

function plot_means()
    # == Compute sample mean at each n == #
    sample_mean = Array(Float64, n)
    for i=1:n
        sample_mean[i] = mean(data[1:i])
    end

    # == Plot == #
    plot(1:n, sample_mean, color=:red,
         alpha=0.6, label=L"$\bar{X}_n$",
         linewidth=3, legendfont=font(12))
    plot!(1:n, zeros(n), color=:black,
          linewidth=1, linestyle=:dash, label="", grid=false)
end

plot_draws()
plot_means()


using Plots
using GeometryTypes
gr()t
plot()

### SANITY CHECK
x = 1:10; y = rand(10,2) # 2 columns means two lines
plot(x,y)

#rect = HyperRectangle(Vec(0.5, 0.5), Vec(1., 1))

#module foo
#    struct Rect
#        NE::GeometryTypes.Point2f0
#        SW::GeometryTypes.Point2f0
#    end
#end

@recipe function f(r::HyperRectangle)
    points = decompose(Point{2,Float64}, r)
    rectpoints = points[[1,2,4,3],:]
    xs = [pt[1] for pt in rectpoints];
    ys = [pt[2] for pt in rectpoints];
    seriestype := :shape
	s = Shape(xs[:], ys[:])
end
sizePlot = (400,400)
r = HyperRectangle(Vec(1,1), Vec(2,4))
fig = plot(opacity=0.5, legend=false, size=sizePlot, yaxis=( (0,10), 0:1:10), xaxis=( (0,10), 0:1:10))
plot(r, opacity=0.5, legend=false, size=sizePlot, yaxis=( (0,10), 0:1:10), xaxis=( (0,10), 0:1:10))
plot!(r)


using ApproxFunOrthogonalPolynomials, ApproxFunBase, SpecialFunctions, Test
    import ApproxFunBase: testbandedoperator

@testset "Laguerre and WeightedLaguerre" begin

    @testset "General scaled rays" begin
        r = Ray(-1.0,0.0,2.0,true)
        L = Laguerre(1.0,r)
        f = x -> exp.(-x)
        F = Fun(f, L)
        @test F(.3) ≈ -F'(.3)

        r = Ray(1.0,0.0,2.0,false)
        L = Laguerre(1.0,r)
        f = x -> exp.(-x)
        F = Fun(f, L, 100)
        @test F(1.3) ≈ -F'(1.3)

        r = Ray(1.0,π,2.0,true)
        L = Laguerre(1.0,r)
        f = x -> exp.(x)
        F = Fun(f, L, 100)
        @test F(.3) ≈ F'(.3)

        r = Ray(-3.0,0.0,2.0,true)
        L = Laguerre(1.0,r)
        f = x -> exp.(-x)
        F = Fun(f, L)  # overflow/underflow issues beyond 190ish
        @test abs(F(0.0) - f(0.)) < 1e-5
        @test abs(F(-2.0) - f(-2.)) < 1e-5

        r = Ray(3.0,π,.1,false)
        L = Laguerre(1.0,r)
        f = x -> exp.(-x^2)
        F = Fun(f, L)  # overflow/underflow issues beyond 190ish
        @test abs(F(-1.0) - f(-1.0)) < 1e-5
    end

    @testset "Evaluation" begin
        f=Fun(Laguerre(0.), [1,2,3])
        @test f(0.1) ≈ 5.215

        f = Fun(Laguerre(0.1), ones(100))
        @test f(0.2) ≈ 8.840040924281498

        x = Fun(Laguerre())
        w = (1+x) * Fun(WeightedLaguerre(),[1.0])
        @test w(0.1) ≈ exp(-0.1)*(1+0.1)
        @test last(w) == 0.0
    end

    @testset "Derivative" begin
        f = Fun(Laguerre(0.1), ones(100))
        @test (Derivative(Laguerre(0.1)) * f)(0.2) ≈ -71.44556705957386

        f = Fun(Laguerre(0.2), ones(100))
        @test (Derivative(Laguerre(0.2)) * f)(0.3) ≈ -137.05785783078218
    end

    @testset "Conversion" begin
        f = Fun(Laguerre(0.2), ones(100))
        @test (Conversion(Laguerre(0.2), Laguerre(1.2)) * f)(0.1) ≈ f(0.1)
        @test (Conversion(Laguerre(0.2), Laguerre(2.2)) * f)(0.1) ≈ f(0.1)
    end

    @testset "Derivative" begin
        f = Fun(LaguerreWeight(0.0, Laguerre(0.1)), ones(100))
        @test f'(0.2) ≈ -65.7322962859456

        B = Evaluation(LaguerreWeight(0.0, Laguerre(0.1)), false)
        @test B*f ≈ 151.53223385808576

        x = Fun(Laguerre(0.0))
        S = WeightedLaguerre(0.0)
        D = Derivative(S)
        u = [ldirichlet(); D^2 - x] \ [airyai(0.0); 0.0]
        @test u(1.0) ≈ airyai(1.0)
    end

    @testset "Multiplication" begin
        w = Fun(WeightedLaguerre(), [1.0])
        @test sum(w) == 1
        t = Fun(identity, space(w))
        @test t(10.0) == 10.0
        M = Multiplication(t^2 + 1, space(w))
        @test (M \ w)(1.0) ≈ exp(-1)/2
    end

    @testset "Multiplication rangespace" begin
        x = Fun(Laguerre())
        w = Fun(WeightedLaguerre(-0.5),[1.0])
        M = Multiplication(w,space(1+x))
        @test rangespace(M) == LaguerreWeight(-0.5,Laguerre())
    end
end