using DrWatson
@quickactivate "ExercisesRepo"
include(srcdir("colorscheme.jl"))
using Makie, DynamicalSystems, InteractiveChaos
using AbstractPlotting.MakieLayout

# %% Animation of changing ε in curve of 1D energy balance model
function fitzhugh(x, p, t)
    u, w = x
    I, a, b = p
    udot = a*u*(u-b)*(1-u) - w + I
    wdot = 0.01*(u-w)
    return SVector(udot, wdot)
end

a = 3.0
b = 0.2
I = 0.0
p = [I, a, b]
fh = ContinuousDynamicalSystem(fitzhugh, rand(2), p)

# Layout plot:
scene, layout = layoutscene(resolution = (1000, 800))
ax = layout[1,1] = LAxis(scene)
display(scene)
ax.xlabel = "u"
ax.ylabel = "w"

# plot the two nullclines
us = -0.4:0.01:1.2
w1 = us
w2 = @. a*us*(us-b)*(1-us) + I
lines!(ax, us, w1; color = COLORS[1], linewidth = 2, linestyle = :dash)
lines!(ax, us, w2; color = COLORS[2], linewidth = 2, linestyle = :dot)

spoint = select_point(ax.scene)
on(spoint) do pos
    tr = trajectory(fh, 10000, SVector(pos...))
    lines!(ax.scene, columns(tr)...; color = InteractiveChaos.randomcolor())
end

# %% orbit evolution in a 2D torus
using OrdinaryDiffEq

R = 2.0
r = 0.5

function torus(u)
    θ, φ = u
    x = (R + r*cos(θ))*cos(φ)
    y = (R + r*cos(θ))*sin(φ)
    z = r*sin(θ)
    return SVector(x, y, z)
end

function quasiperiodic_f(u, p, t)
    # here we make the frequency ratio a state variable, because the
    # trajectory_animator application doesn't allow different parameters for different
    # initial conditions
    θ, φ, ω = u
    θdot = ω
    φdot = 1.0
    return SVector(θdot, φdot, 0.0)
end

ds = ContinuousDynamicalSystem(quasiperiodic_f, [0.0, 0.0, 0.0], nothing)
u0s = [[0, 0, ω] for ω ∈ [3, sqrt(7), 5]]
diffeq = (alg = Tsit5(), dtmax = 0.01)

lims = ((-R-r, R+r), (-R-r, R+r), (-3r, 3r))

scene, main, layout, obs = interactive_evolution(
    ds, u0s; tail = 20000, diffeq, colors = COLORS, transform = torus, lims,
    plotkwargs = (linewidth = 1.0,),
)


using PkgSRA, Test, Random
Random.seed!(13)

include("../utils/sindy_turing_utils.jl")
# include("../utils/sparse_regression_functions.jl")
# include("../utils/sindy_statistics_utils.jl")

# Generate test data: Lorenz
include("../examples/example_lorenz.jl")
dat = Array(solve_lorenz_system())
numerical_grad = numerical_derivative(dat, ts)

# Test new interface
alg = slstQuantile(4, 0.2) # Will get extra terms
sindy_library = Dict("cross_terms"=>2,"constant"=>nothing);
test_model = sindyc(dat, numerical_grad, nothing, ts,
                        library=sindy_library,
                        optimizer=alg)

# Built correct terms: names
named_terms = build_term_names(test_model)
@testset "Term Metadata" begin
    @test named_terms ==
        ["x", "y", "z", "", "xx", "xy", "xz", "yy", "yz", "zz"]
end

# Built correct terms: values
library_functions = convert_string2function(test_model.library)
terms = calc_augmented_data(dat, library_functions)

@testset "Data Augmentation" begin
    @test size(terms,1) == length(named_terms)
    @test terms[1:3,:] == dat
    @test terms[4,:] == ones(size(dat,2))
    @test terms[5:10,:] == calc_cross_terms(dat, 2)
end;

# Nonzero term and type tests
t1 = get_nonzero_terms(test_model)
t2 = get_nonzero_terms(core_dyn_true)
# typeof(test_model) <: sindycModel
# @testset "Type Tests" begin
@test typeof(test_model) <: sindycModel
@test issubset(t2, t1)
# end

# Also test cross validation function
null_model = sindyc(dat, 0.1.*randn(size(dat)), nothing, ts,
                        library=sindy_library,
                        optimizer=alg)
# null_model = sindyc(dat, 0.1.*randn(size(dat)), nothing, ts,
#                         library=sindy_library,
#                         use_lasso=true)
err_test = sindy_cross_validate(test_model, dat, numerical_grad)
err_null = sindy_cross_validate(null_model, dat, numerical_grad)

@testset "Cross Validation" begin
    @test err_test < err_null
end

# SINDY tests: up to cubed terms

# Generate test data: Van der Pol
include("../examples/example_vanDerPol.jl")
dat = Array(solve_vdp_system())
numerical_grad = numerical_derivative(dat, ts)

sindy_library = Dict("cross_terms"=>[2,3],
                    "constant"=>nothing);
# test_model2 = sindyc(dat, numerical_grad, nothing, ts,
#                         library=sindy_library,
#                         use_lasso=true,
#                         var_names=["x", "y"])
test_model = sindyc(dat, numerical_grad, nothing, ts,
                        library=sindy_library,
                        optimizer=alg,
                        var_names=["x", "y"])

named_terms = build_term_names(test_model2)
@testset "Cubic term names" begin
    @test named_terms ==
        ["x", "y", "", "xx", "xy", "yy", "xxx", "xxy", "xyy", "yyy"]
end


using NaturalES

sols=[]
cnt=0
function rosenbrock(x::AbstractVector{T}) where T
    global cnt=(cnt+1)%20
    cnt==10 && push!(sols,copy(x))
    s=(1.0 - x[1])^2
    for i in 1:(length(x)-1)
        s+=100.0 * (x[i+1] - x[i]^2)^2
    end
    return s
end

exponential_nes(rosenbrock,[-0.41,0.17],0.01)
sols
A=hcat(sols...)'|>collect

using PyPlot

pygui(true)
function rosenbrock2d(x,y)
  return (1.0 - x)^2 + 100.0 * (y - x^2)^2
end

x=-0.5:0.001:1.5
y=-0.5:0.001:1.5
M=rosenbrock2d.(x,y')
plt.style.use("dark_background")
plot(A[:,2],A[:,1],color="red")
contour(x,y,M,log.(1.01:0.5:10))
colorbar()
scatter(1,1,color="blue")

using NaturalES
function rosenbrock2d(x::AbstractVector{T}) where T
    s=(1.0 - x[1])^2
    for i in 1:(length(x)-1)
        s+=100.0 * (x[i+1] - x[i]^2)^2
    end
    return s
end

using BenchmarkTools
@btime exponential_nes($rosenbrock2d,$[0.0,0.0],$1.0)


randuniform(dims::Base.Dims) = randuniform(Float64, dims)
randuniform(::Type{T}, dims::Base.Dims) where {T<:Number} = rand(T, dims)

randnormal(dims::Base.Dims) = randnormal(Float64, dims)
randnormal(::Type{T}, dims::Base.Dims) where {T<:Number} = randn(T, dims)

randisometry(dims::Base.Dims{2}) = randisometry(Float64, dims)
randisometry(::Type{T}, dims::Base.Dims{2}) where {T<:Number} =
    dims[1] >= dims[2] ? _leftorth!(randnormal(T, dims), QRpos(), 0)[1] :
        throw(DimensionMismatch("cannot create isometric matrix with dimensions $dims; isometry needs to be tall or square"))

const randhaar = randisometry


using PyAMG
using Base.Test

L1d(N) = spdiagm((-ones(N-1), 2*ones(N), -ones(N-1)), (-1,0,1), N, N) * N^2
function L2d(N)
  L1 = L1d(N)
  A = kron(speye(N), L1) + kron(L1, speye(N))
  b = ones(size(A,1))
  return A, b
end
function L3d(N)
  L1 = L1d(N)
  I = speye(N)
  A = kron(L1, I, I) + kron(I, L1, I) + kron(I, I, L1)
  b = ones(size(A,1))
  return A, b
end

println("=================================================")
println("Test 1: Julia `\\` vs PyAMG Blackbox `solve` (2D) Laplacian")
A, b = L2d(100)
print(" \\ : ")
@time x1 = A \ b
print("PyAMG-Warmup: ")
@time x2 = PyAMG.solve(A, b; tol=1e-12, verb=false);
print("PyAMG: ")
@time x3 = PyAMG.solve(A, b; tol=1e-12, verb=false)
println("|x_\\ - x_amg|_∞ = ", norm(x3-x1, Inf))
println("|A x_amg - b|_∞ = ", norm(A*x2 - b, Inf))
@assert norm(x3-x1, Inf) < 1e-10

println("=================================================")
println("Test 2: Julia `\\` vs PyAMG Blackbox `solve` (3D)")
A, b = L3d(30)
print(" \\ : ")
@time x1 = A \ b
print("PyAMG-Warmup: ")
@time x2 = PyAMG.solve(A, b; tol=1e-12, verb=false)
print("PyAMG: ")
@time x3 = PyAMG.solve(A, b; tol=1e-12, verb=false)
println("|x_\\ - x_amg|_∞ = ", norm(x3-x1, Inf))
println("|A x_amg - b|_∞ = ", norm(A*x2 - b, Inf))
@assert norm(x3-x1, Inf) < 1e-9

println("=================================================")
println("Test 3: RugeStubenSolver (3D)")
println("        50 x 50 x 50 grid = 125k dofs ")
A, b = L3d(50)
print("Create solver: ")
@time amg = RugeStubenSolver(A)
print("First solve: ")
@time x1 = solve(amg, b)
print("Second solve: (tol 1e-9)")
@time x2 = solve(amg, b; tol=1e-9)
print("Third solve: (tol 1e-6)")
@time x3 = solve(amg, b; tol=1e-6)
println("|A x_amg - b|_∞ = ", norm(A * x3 - b, Inf))
@assert norm(A * x2 - b, Inf) < 1e-8

println("=================================================")
println("Test 4: \\, ldiv, and kwargs")
A, b = L2d(50)
amg = RugeStubenSolver(A)
x_solve = solve(amg, b, tol=1e-6, cycle="V", accel="cg")
set_kwargs!(amg, tol=1e-6, cycle="V", accel="cg")
x_bs = amg \ b
x_ldiv = A_ldiv_B!(zeros(x_solve), amg, b)
@assert x_solve == x_bs
@assert x_solve == x_ldiv

println("=================================================")
println("Test 5: AMG as a preconditioner")
println("        100 x 100 Dirichlet problem, TOL = 1e-4")
println("        PyAMG vs CG vs PCG  (using IterativeSolvers)")
A, b = L2d(100)
amg = RugeStubenSolver(A)
M = aspreconditioner(amg)
using IterativeSolvers
TOL = 1e-4
println("Plain CG:")
@time x_cg, _ = IterativeSolvers.cg(A, b, 1; tol=TOL)
@time x_cg, _ = IterativeSolvers.cg(A, b, 1; tol=TOL)
println("PyAMG-preconditionerd CG:  (see `aspreconditioner`)")
@time x_pcg, _ = IterativeSolvers.cg(A, b, M; tol=TOL)
@time x_pcg, _ = IterativeSolvers.cg(A, b, M; tol=TOL)
println("PyAMG solver")
@time x_pyamg = PyAMG.solve(amg, b; tol=TOL*1e-2, accel="cg")
@time x_pyamg = PyAMG.solve(amg, b; tol=TOL*1e-2, accel="cg")
x = A \ b
println("|x_cg-x| = ", norm(x_cg - x), " \n",
         "|x_pcg-x| = ", norm(x_pcg - x), "\n",
         "|x_pyamg-x| = ", norm(x_pyamg - x) )
@assert norm(x_cg - x) < 1e-5
@assert norm(x_pcg - x) < 1e-5
@assert norm(x_pyamg - x) < 1e-5


# --------- SOLUTION --------------------------------------------
# ----------------------------- LOGISIC ------------------------
using JuMP,Ipopt,DataFrames, GLM, ForwardDiff
m=nothing
m = Model(solver=IpoptSolver())  # sum( -1 * y .* (x*beta) + log(1+exp((x*beta))) )
dfd = readtable("/home/iriadmin/g/JLib/src/StatOpt/logreg.csv")
x = Array(dfd[:,2:end])
y = Array(dfd[:,1])
(rows, cols) = size(x)
g = glm(y ~ -1 + x0 + x1 + x2 + x3 + x4 + x6 + x7, dfd,  Bernoulli(), LogitLink())
@variable(m, B[z=1:cols], start=0)  # getvalue(B)  @defVar( m, 0.0 <= b[1:(K1-1)] <= 100.0 )
@NLobjective( m, Min, sum{ ( -1*y[r]*sum{x[r,c]*B[c],c in 1:cols} ) + log(1+exp(sum{x[r,cc]*B[cc],cc in 1:cols})) ,r in 1:rows} )

statusM = solve( m )
df = DataFrame( beta = getvalue( B ) )

function mynllf(beta::Vector)
	nll = 0.0
	for r in 1:rows
		dbl_part1 = 0.0
		for c in 1:cols
			dbl_part1 = dbl_part1 + x[r,c] * beta[c]
		end
		nll = nll -1.0 * y[r] * dbl_part1 + log(1+exp(dbl_part1))
	end
	return nll
end

sqrt(diag(inv(  ForwardDiff.hessian(mynllf,getvalue(B))  )))
getvalue(B)

# ----------------------- POISON --------------------------------
using JuMP,Ipopt,DataFrames, GLM
m=nothing
m = Model(solver=IpoptSolver())
dfd = readtable("/home/iriadmin/g/JLib/src/StatOpt/ZD/poison_gamma/y.csv")
names!(dfd,[:y,:x1, :x2, :x3,:x4,:x5])
x = Array(dfd[:,2:end])
y = Array(dfd[:,1])
(rows, cols) = size(x)
g = glm(y ~ -1 + x1 + x2 + x3 + x4 +x5 , dfd,  Poisson(), LogLink())
@variable(m, B[z=1:cols], start=0)
@NLobjective( m, Min, sum{ exp(sum{x[r,cc]*B[cc],cc in 1:cols}) - (y[r]*sum{x[r,c]*B[c],c in 1:cols}) ,r in 1:rows} )
statusM = solve( m )
df = DataFrame(beta=getvalue(B))
mynllf(beta::Vector) = sum(exp(x*beta) .- ((x*beta).*y))
sqrt(diag(inv(ForwardDiff.hessian(mynllf,getvalue(B)))))

# ----------------------- GAMMA - WORKING On THIS --------------------------------
using JuMP,Ipopt,DataFrames, GLM
m=nothing
m = Model(solver=IpoptSolver())
dfd = readtable("/home/iriadmin/g/JLib/src/StatOpt/ZD/poison_gamma/g.csv")
names!(dfd,[:y,:x1, :x2, :x3,:x4,:x5])
x = Array(dfd[:,2:end])
y = Array(dfd[:,1])
(rows, cols) = size(x)
g = glm(y ~ -1 + x1 + x2 + x3 + x4 +x5 , dfd,  Gamma(), LogLink())
@variable(m, B[z=1:cols], start=0)
@NLobjective( m, Min, sum{ exp(-1.0*sum{x[r,cc]*B[cc],cc in 1:cols}) + (sum{x[r,c]*B[c],c in 1:cols}) ,r in 1:rows} )
statusM = solve( m )
df = DataFrame(beta=getvalue(B))
mynllf(beta::Vector) = sum(exp(-1.0*(x*beta)) .+ ((x*beta)))
sqrt(diag(inv(ForwardDiff.hessian(mynllf,getvalue(B)))))

str_path <- "C:\\Users\\mdzhd\\Documents\\Bayesian_Approach_with_Lampros\\"

myd <- read.csv( paste0( str_path, "gamma_regression_ex2.csv" ) )

myglm <- glm( y ~ x1 + x2 + x3, data = myd, family = Gamma( link = log ) )

x <- as.matrix( myd[, -c(1)] )
y <- as.matrix( myd[, 1] )

NegLogLike <- function( beta )
{
	xbeta <- x %*% beta
	Li <- y * exp( -1.0 * xbeta ) +  xbeta
	sum( Li )
}

beta_initial <- as.matrix( rep( 0, ncol( x ) ), ncol(x), 1 )

myoptim <- optim( beta_initial, f = NegLogLike, g = NULL, method = "BFGS", hessian = TRUE)

u <- exp( x %*% as.matrix( myoptim$par , ncol( x ), 1 ) )

v <- ( y - u )/u

v2 <- v * v

phi <- sum( v2 ) / ( nrow( x ) - ncol(x ) )

myhessian <- myoptim$hessian / phi

var_cov_m <- solve( myhessian )

beta_se <- sqrt( diag( var_cov_m ) )

zvalue <- myoptim$par / beta_se

pvalue <- 2 * ( 1 - pnorm(abs( zvalue ) ) )

coeff_outputs <- cbind( myoptim$par, beta_se, zvalue, round( pvalue, 6 ) )

colnames( coeff_outputs ) <- c( "Estimates", "Std. Error", "z value", "Pr(>|z|)" )
rownames( coeff_outputs ) <- c()

print( coeff_outputs )
summary( myglm )$coefficients

# --------- SOLUTION END --------------------------------------------

# ----------------------------------------------------------------------
# -------- OLD WORKING.... but wrong -----------------------------------
# ----------------------------------------------------------------------

using JuMP;
using Ipopt;
using DataFrames;
my_table = readtable("/home/iriadmin/g/jlib/src/StatOpt/newdata.csv")
( N, K1 ) = size( my_table )
m = Model(solver=IpoptSolver())
@defVar( m, 0.0 <= b[1:(K1-1)] <= 100.0 )
@NLobjective( m, Min, sum{ ( my_table[i,1] - sum{ my_table[i,j] * b[ j - 1] , j in 2:K1} )^2 , i in 1:N} )
statusM = solve( m )
df = DataFrame( beta = getvalue( b ) )

# ----------------------- GT LOGISTIC --------------------------------
using JuMP,Ipopt,DataFrames, GLM
m=nothing
m = Model(solver=IpoptSolver())  # sum( -1 * y .* (x*beta) + log(1+exp((x*beta))) )
dfd = readtable("/home/iriadmin/g/jlib/src/StatOpt/logreg.csv")
x = Array(dfd[:,2:end])
y = Array(dfd[:,1])
(rows, cols) = size(x)
g = glm(y ~ -1 + x0 + x1 + x2 + x3 + x4 + x6 + x7, dfd,  Bernoulli(), LogitLink()) # X0 s the replacement for intercept - otherwise glm will generate a [1...] arr
@variable(m, B[z=1:cols], start=0)  # getvalue(B)  @defVar( m, 0.0 <= b[1:(K1-1)] <= 100.0 )
@NLobjective( m, Min, sum{ ( -1*y[r]*sum{x[r,c]*B[c],c in 1:cols} ) + log(1+exp(sum{x[r,cc]*B[cc],cc in 1:cols})) ,r in 1:rows} )
#..... sum( -1 * y .* (x*B) + log(1+exp((x*B))) )

d = JuMP.NLPEvaluator(m)
MathProgBase.initialize(d, [:Grad,:HessVec,:Hess])
statusM = solve( m )  #statusM = solve( m, IpoptOptions=[("tol",1e-6)] )
df = DataFrame( beta = getvalue( B ) )

d = JuMP.NLPEvaluator(m)
MathProgBase.initialize(d, [:Grad,:HessVec,:Hess])
hess = MathProgBase.hesslag_structure(d)
diag(hess)
beta_se <- sqrt( diag( var_cov_m ) )


using WAVEFORM, LinearAlgebra, Test

###################################################################################################
# wave_base.jl

# 2D model
n = 101 .* [1;1]
d = 10.0 .*[1;1]
o = 0.0 .* [1;1]

freqs = [5.0;10.0;15.0]
t0 = 0.0
f0 = 10.0
unit = "m/s"

vel_background = 2000
L = n.*d

xsrc = 0.0:100.0:L[2]
ysrc = [0.0]
zsrc = [10.0]

# Receiver grid definition
xrec = 0:10.0:L[2]
yrec = [0.0]
zrec = [950.0]

# Model type containing the domain geometry
model = Model{Int64,Float64}(n,d,o,t0,f0,unit,freqs,xsrc,ysrc,zsrc,xrec,yrec,zrec)

@test typeof(model) == Model{Int64, Float64}
@test model.n == n
@test model.d == d
@test model.o == o

# 2D: helm2d_chen9p or helm2d_std7; 3D: helm3d_operto27 or helm3d_std9
pde_scheme = WAVEFORM.helm2d_chen9p
cut_pml = true
implicit_matrix = true
npml = [1 1; 1 1]
misfit = least_squares;
srcfreqmask = trues(length(xsrc), length(freqs))
lsopts = LinSolveOpts(; tol=1e-9, maxit=1000, solver=:lufact, precond=:identity)

# Misfit function for the objective
pdeopts = PDEopts(pde_scheme, n, d, o, cut_pml, implicit_matrix, npml, minimum, srcfreqmask, lsopts)

@test typeof(pdeopts) == PDEopts{Int64, Float64}
@test pdeopts.comp_n == n
@test pdeopts.comp_d == d
@test pdeopts.comp_o == o
@test pdeopts.srcfreqmask == srcfreqmask

###################################################################################################
# wave_util.jl

idx = index_block([1,2,3,4], 2)
@test size(idx) == (2,)
@test idx[1] == [1,2]
@test idx[2] == [3,4]

w = fwi_wavelet(freqs, t0, f0)
@test typeof(w) == Vector{ComplexF64}
@test length(w) == length(freqs)

g1, g2 = odn_to_grid(o, d, n)
@test length(g1) == n[1]
@test length(g2) == n[2]
@test g1[1] == o[1]
@test g2[1] == o[2]
@test g1[end] == o[1] + (n[1]-1)*d[1]
@test g2[end] == o[2] + (n[2]-1)*d[2]

o_, d_, n_ = grid_to_odn([g1, g2])
@test o_ == o
@test d_ == d
@test n_ == n

###################################################################################################
# helm.jl

ndims=2
n = 101 .* [1;1]
npml = npml = [1 1; 1 1]
Pext, Ppad = WAVEFORM.get_pad_ext_ops(n,npml,ndims)

x = randn((n...))
y = randn(((n + sum(npml, dims=2))...))

@test dot(vec(y), Pext*vec(x)) - dot(vec(x), Pext'*vec(y)) < 1e-9
@test dot(vec(y), Ppad*vec(x)) - dot(vec(x), Ppad'*vec(y)) < 1e-9

phys_to_comp = Pext
comp_to_phys = Ppad'

# Param to wavenum
f, df, ddf = param_to_wavenum(vel_background,freqs[1],"m/s")
f, df, ddf = param_to_wavenum(vel_background,freqs[1],"s/m")
f, df, ddf = param_to_wavenum(vel_background,freqs[1],"s2/km2")


using HypothesisTests, Test
using StatsBase
using HypothesisTests: default_tail

@testset "Power Divergence" begin
#Example 1 in R
#Agresti (2007) p. 39

d = [[762,484] [327,239] [468,477]]

m = PowerDivergenceTest(d)

@test m.theta0 ≈ [0.25523082406125785,0.19670969099133556,0.11593952361049113,0.08935608756107216,0.1935739395970214,0.1491899341788219]
@test m.thetahat ≈ [0.2763873775843308,0.1755531374682626,0.11860718171926006,0.08668842945230323,0.16974972796517954,0.17301414581066377]

c = confint(m)
c0 = [(.25788900979, .29554669435),
      (.15705476968, .19471245423),
      (.10010881393, .13776649848),
      (.06819006166, .10584774622),
      (.15125136017, .18890904473),
      (.15451577802, .19217346257)]

for i = 1:length(c)
    @test c[i][1] ≈ c0[i][1]
    @test c[i][2] ≈ c0[i][2]
end

@test pvalue(m) ≈ 2.9535891832117357e-7
@test default_tail(m) == :right
@test m.stat ≈ 30.070149095754687
@test m.df ≈ 2
@test m.n ≈ 2757
@test m.residuals ≈ reshape([2.198855766015898,-2.504669492560728,0.4113701700566286,-0.46858294710127296,-2.8432397155451494,3.2386734435365825], size(d))
@test m.stdresiduals ≈ reshape([4.502053521086705,-4.502053521086705,0.6994517329844298,-0.6994517329844298,-5.315945542704929,5.315945542704929], size(d))

m = PowerDivergenceTest(d,lambda=0.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-1.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-2.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-0.5)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=2/3)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = ChisqTest(d)
m = MultinomialLRT(d)

confint(m, method = :bootstrap)
confint(m, method = :bootstrap, tail=:left)
confint(m, method = :bootstrap, tail=:right)

confint(m, method = :gold)
confint(m, method = :gold, tail=:left)
confint(m, method = :gold, tail=:right)

confint(m, method = :quesenberry_hurst)
confint(m, method = :quesenberry_hurst, tail=:left)
confint(m, method = :quesenberry_hurst, tail=:right)

confint(m, method = :sison_glaz)
confint(m, method = :sison_glaz, correct=false)
confint(m, method = :sison_glaz, tail=:left)
confint(m, method = :sison_glaz, tail=:right)

@test_throws ArgumentError confint(m, method=:FOO)
@test_throws ArgumentError confint(m, tail=:fox)

#Example 3 in R

d = [ 20, 15, 25 ]
m = PowerDivergenceTest(d)

@test m.theta0 ≈ [0.3333333333333333,0.3333333333333333,0.3333333333333333]
@test m.thetahat ≈ [0.3333333333333333,0.25,0.4166666666666667]

c = confint(m)
c0 = [(.21666666667, .48098082062),
      (.13333333333, .39764748728),
      (.30000000000, .56431415395)]

for i in 1:length(c)
    @test c[i][1] ≈ c0[i][1]
    @test c[i][2] ≈ c0[i][2]
end

@test pvalue(m) ≈ 0.2865047968601901
@test m.stat ≈ 2.5
@test m.df ≈ 2
@test m.n ≈ 60
@test m.residuals ≈ [0.0,-1.118033988749895,1.118033988749895]
@test m.stdresiduals ≈ [0.0,-1.3693063937629153,1.3693063937629153]

m = PowerDivergenceTest(d,lambda=0.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-1.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-2.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-0.5)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=2/3)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = ChisqTest(d)
m = MultinomialLRT(d)

confint(m, method = :bootstrap)
confint(m, method = :bootstrap, tail=:left)
confint(m, method = :bootstrap, tail=:right)

confint(m, method = :gold)
confint(m, method = :gold, tail=:left)
confint(m, method = :gold, tail=:right)

confint(m, method = :quesenberry_hurst)
confint(m, method = :quesenberry_hurst, tail=:left)
confint(m, method = :quesenberry_hurst, tail=:right)

confint(m, method = :sison_glaz)
confint(m, method = :sison_glaz, correct=false)
confint(m, method = :sison_glaz, tail=:left)
confint(m, method = :sison_glaz, tail=:right)

@test_throws ArgumentError confint(m, method=:FOO)
@test_throws ArgumentError confint(m, tail=:fox)

#
x=[1,2,3,1,2,3]
y=[1,1,1,2,2,3]

d = counts(x,y,3)

ChisqTest(d)
MultinomialLRT(d)

PowerDivergenceTest(x,y,3)
PowerDivergenceTest(x,y,(1:3,1:3))

ChisqTest(x,y,3)
ChisqTest(x,y,(1:3,1:3))

MultinomialLRT(x,y,3)
MultinomialLRT(x,y,(1:3,1:3))

# Test that large counts don't cause overflow (issue #43)
d = [113997 1031298
     334453 37471]
PowerDivergenceTest(d)
end


using Test, PartialSvdStoch, Serialization, Distributions

using Logging
using Base.CoreLogging

logger = ConsoleLogger(stdout, CoreLogging.Debug)
global_logger(logger)

include("testvrpca.jl")
include("testifsvd.jl")

# a data file for a matrix(11520, 22499) can be found in ./data/plimat.serialized
# mass spectrometry image

plimat = Some(Matrix{Float64})
dataf = "data/plimat.serialized"
if isfile(dataf)
    plimat = Some(Serialization.deserialize(open(dataf, "r")))
end

# This test compares singular values from exact svd with those extracted from
# incremental svd from LowRankApproxMc

@testset "ifsvd_versus_svd" begin
    @info "========================================================="
    @info "\n\n in test ifsvd_versus_svd"
    d = 500
    n = 5000
    k = 50
    mat = rand(d,n)
    for j in 1:k
    mat[:,j] = mat[:,j] * 20.
    end
    for j in 1:5
        mat[:,j+k] = mat[:,j+k] * 100.
    end
    @test testifastsvd_S(mat, 200)
end

# This test shows how in presence high variability of data
# it is possible to get a competitive approximation with incremental svd.

@testset "ifsvd_lowRankApproxEpsil" begin
    @info "========================================================="
    @info "\n\n in test ifsvd_lowRankApproxEpsil"
    d = 10000
    n = 20000
    mat = zeros(d,n)
    #
    Xlaw = Normal(0, 1000.)
    # multiply some columns
    blocsize = 100
    for bloc in 1:round(Int,n/blocsize)
        val = abs(rand(Xlaw))
        for i in 1:d
            if rand() < 0.01
                for j in 1:blocsize
                    mat[i,(bloc-1)*blocsize+j] = val * (1. - rand()/10.)
                end
            end
        end
    end
    @test testifastsvd_epsil(mat, 100, 0.05)
end

#
# test vrpca_epsil

# This tests asks for an approx at 0.01
# It extracts a rank 100 matrix at 0.005 precision within 3s (after first compilation).
# With LowRankApprox.prange alone we need to go up to rank=500 (initialized with
# opts = LowRankApprox.LRAOptions(rank=500, rtol = 0.0001) to achieve a relative
# Froebonius error of  0.005. It then runs in 1.15s.

# So we see that LowRankApprox is really fast but needs many more singular vectors
# and that VrPCA gives a clear reduction of relative error.
# Of course VrPCA initialized with LowRankApprox would give an analogous reduction
# in relative error. Cf testset "vrpca_withLoxwRankApprox"
#

@testset "vrpca_epsil" begin
    @info "========================================================="
    @info "\n\n in test vrpca_epsil"
    d = 5000
    n = 10000
    k = 100
    maxval = 1000
    mat = rand(d,n)
    # multiply some columns
    for i in 1:k
        mat[:,i] = mat[:,i] * maxval
    end
    @test testvrpca_withepsil(mat, 0.01)
end

@testset "vrpca_withLowRankApprox" begin
    @info "========================================================="
    @info "\n\n in test vrpca_withLowRankApprox"
    d = 5000
    n = 10000
    k = 100
    maxval = 1000
    mat = rand(d,n)
    # multiply some columns. but do not sample them( some will be sampled twice and will affect sketching)
    for i in 1:k
        mat[:,i] = mat[:,i] * maxval
    end
    @test testvrpca_withLowRankApprox(mat, 100)
end


module _EspEn2D
export EspEn2D
    """
        Esp2D,  = EspEn2D(Mat)

    Returns the bidimensional Espinosa entropy estimate (`Esp2D`)
    estimated for the data matrix (`Mat`) using the default parameters:
    time delay = 1, tolerance threshold = 20, percentage similarity = 0.7
    logarithm = natural, matrix template size = [floor(H/10) floor(W/10)]
    (where H and W represent the height (rows) and width (columns) of
    the data matrix `Mat`)
    ** The minimum number of rows and columns of `Mat` must be > 10.

        Esp2D = EspEn2D(Mat::AbstractArray{T,2} where T<:Real; m::Union{Int,Tuple{Int,Int}}=floor.(Int, size(Mat)./10),
                                        tau::Int=1, r::Real=20, ps::Float=.7, Logx::Real=exp(1), Lock::Bool=true)

    Returns the bidimensional Espinosa entropy (`Esp2D`) estimates for the data
    matrix (`Mat`) using the specified 'keyword' arguments:

    # Arguments:
    `m`     - Template submatrix dimensions, an integer scalar (i.e. the same
              height and width) or a two-element vector of integers [height, width] with a minimum value > 1.
              (default: [floor(H/10) floor(W/10)]) \n
    `tau`   - Time Delay, a positive integer       (default: 1) \n
    `r`     - Tolerance threshold, a positive scalar  (default: 20) \n
    `ps`    - Percentage similarity, a value in range [0 1]  (default: 0.7) \n
    `Logx`  - Logarithm base, a positive scalar    (default: natural)  \n
    `Lock`  - By default, EspEn2D only permits matrices with a maximum
              size of 128 x 128 to prevent memory errors when storing data
              on RAM. e.g. For Mat = [200 x 200], m = 3, and tau = 1,
              EspEn2D creates a vector of 753049836 elements.
              To enable matrices greater than [128 x 128] elements,
              set `Lock` to false.  (default: true)  \n
              `WARNING: unlocking the permitted matrix size may cause your Julia
              IDE to crash.`

    #  See also `SampEn2D`, `FuzzEn2D`, `DispEn2D`, `DistEn2D`, `PermEn2D`

    # References:
        [1] Ricardo Espinosa, et al.,
            "Two-dimensional EspEn: A New Approach to Analyze Image Texture
            by Irregularity."
            Entropy,
            23:1261 (2021)

    """
    function EspEn2D(Mat::AbstractArray{T,2} where T<:Real;
        m::Union{Int,Tuple{Int,Int}}=floor.(Int, size(Mat)./10),
        tau::Int=1, r::Real=20, ps::Real=0.7, Logx::Real=exp(1), Lock::Bool=true)

    NL, NW = size(Mat)
    ((NL > 128 || NW > 128) && Lock) ?
        error("To prevent memory errors, matrix width & length must have <= 128 elements.
            To estimate EspEn2D  for the current matrix ($NL,$NW) change Lock to 'false'.
            Caution: unlocking the safe matrix size may cause the Julia IDE to crash.") :
            nothing

    length(m)==1 ?  (mL = m; mW = m) : (mL = m[1];  mW = m[2])
    (NL > 10 && NW > 10) ? nothing :
        error("Number of rows and columns in Mat must be > 10")
    (minimum(m)>1) ? nothing :
        error("m:     must be an integer > 1, or a 2 element tuple of integer values > 1")
    (tau > 0) ? nothing : error("tau:   must be an integer > 0")
    (r >= 0) ?  nothing : error("r:     must be a positive value")
    ((ps >= 0) && (ps <= 1)) ?  nothing : error("ps:   must be a value in range [0 1]")
    (Logx>0) ? nothing :  error("Logx:     must be a positive number > 0")

    NL = NL - (mL-1)*tau
    NW = NW - (mW-1)*tau
    X = zeros(mL,mW,NL*NW)
    p = 0
    for k = 1:NL
        for n = 1:NW
            p += 1
            X[:,:,p] = Mat[k:tau:(mL-1)*tau+k,n:tau:(mW-1)*tau+n]
        end
    end

    p = size(X,3)
    p != NL*NW ? @warn("Potential error with submatrix division.") : nothing
    Ny = p*(p-1)/2
    Ny > 300000000 ? @warn("Number of pairwise distance calculations is $Ny") : nothing

    Cij = -ones(p-1,p-1)
    for k = 1:p-1
        Temp = abs.(X[:,:,k+1:p] .- X[:,:,k]) .<= r
        Cij[1:(end-k+1),k] = sum(Temp,dims=(1,2))
    end

    Dm = sum((Cij[:]/(mL*mW)).>=ps)/(p*(p-1)/2)
    Esp2D = -log(Logx, Dm)

    return Esp2D
    end

end

"""
Copyright 2021 Matthew W. Flood, EntropyHub

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

For Terms of Use see https://github.com/MattWillFlood/EntropyHub
"""


import Base: diff

## diff(ex, x)         f'
## what is the rest of the interface. This does:
## diff(ex, x, n)      f^(n)
## diff(ex, x, y, ...) f_{xy...}  # also diff(ex, (x,y))
## no support for diff(ex, x,n1, y,n2, ...), but can do diff(ex, (x,y), (n1, n2))

function diff(b1::SymbolicType, b2::BasicType{Val{:Symbol}})
    a = Basic()
    ret = ccall((:basic_diff, libsymengine), Int, (Ref{Basic}, Ref{Basic}, Ref{Basic}), a, b1, b2)
    return a
end

diff(b1::SymbolicType, b2::BasicType) =
    throw(ArgumentError("Second argument must be of Symbol type"))

function diff(b1::SymbolicType, b2::SymbolicType, n::Integer=1)
    n < 0 && throw(DomainError("n must be non-negative integer"))
    n==0 && return b1
    n==1 && return diff(b1, BasicType(b2))
    n > 1 && return diff(diff(b1, BasicType(b2)), BasicType(b2), n-1)
end

function diff(b1::SymbolicType, b2::SymbolicType, b3::SymbolicType)
    isa(BasicType(b3), BasicType{Val{:Integer}}) ? diff(b1, b2, N(b3)) : diff(b1, (b2, b3))
end

diff(b1::SymbolicType, b2::SymbolicType, b3::SymbolicType, b4::SymbolicType, b5...) =
    diff(b1, (b2,b3,b4,b5...))

## mixed partials
diff(ex::SymbolicType, bs::Tuple) = reduce((ex, x) -> diff(ex, x), bs, init=ex)
diff(ex::SymbolicType, bs::Tuple, ns::Tuple) =
    reduce((ex, x) -> diff(ex, x[1],x[2]), zip(bs, ns), init=ex)

diff(b1::SymbolicType, x::Union{String,Symbol}) = diff(b1, Basic(x))

"""
Series expansion to order `n` about point `x0`
"""
function series(ex::SymbolicType, x::SymbolicType, x0=0, n::Union{Integer, Basic}=6)
    (!isa(N(n), Integer) || n < 0) && throw(DomainError("n must be non-negative integer"))

    fc = subs(ex, x, x0)
    n==0 && return fc

    fp = ex
    nfact = Basic(1)
    for k in 1:n
        fp = diff(fp, x)
        nfact = k * nfact
        fc = fc + subs(fp, x, x0) * (x-x0)^k / nfact
    end

    fc
end


export proxPositive!, normPositive

"""
    proxPositive!(x::Vector{T},λ::Float64=1.0;kargs...) where T

    enforce positivity and realness of solution `x`.
"""
function proxPositive!(x::Vector{T},λ::Float64=1.0;kargs...) where T
  enfReal!(x)
  enfPos!(x)
end

"""
    returns the value of the characteristic function of real, positive numbers.
    normPositive(x) = (isreal(x)&&x>0) ? 0 : Inf
"""
function normPositive(x::Vector{T},λ::Float64=1.0;kargs...) where T
  y = copy(x)
  proxPositive!(y)
  if y != x
    return Inf
  end
  return 0.
end


const APL = AbstractPolynomialLike

function _expectation(μ::Measure, p::APL, f)
    i = 1
    s = 0
    for t in terms(p)
        while i <= length(μ.x) && monomial(t) != μ.x[i]
            i += 1
        end
        if i > length(μ.x)
            error("The polynomial $p has a nonzero term $t with monomial $(t.x) for which the expectation is not known in $μ")
        end
        s += f(μ.a[i], coefficient(t))
        i += 1
    end
    s
end

"""
    MultivariateMoments.expectation(μ::AbstractMeasureLike, p::AbstractPolynomialLike)
    MultivariateMoments.expectation(p::AbstractPolynomialLike, μ::AbstractMeasureLike)

Computes the expectation ``\\mathbb{E}_{\\mu}[p]``.
"""
function expectation end

expectation(μ::Measure, p::APL) = _expectation(μ, p, (*))
expectation(p::APL, μ::Measure) = _expectation(μ, p, (a, b) -> b * a) # a and b may be noncommutative

"""
    dot(μ::AbstractMeasureLike, p::AbstractPolynomialLike)
    dot(p::AbstractPolynomialLike, μ::AbstractMeasureLike)

See [`expectation`](@ref)
"""
LinearAlgebra.dot(μ::AbstractMeasureLike, p::APL) = expectation(μ, p)
LinearAlgebra.dot(p::APL, μ::AbstractMeasureLike) = expectation(p, μ)


using ModelingToolkit, ModelingToolkitStandardLibrary, OrdinaryDiffEq
using ModelingToolkitStandardLibrary.Blocks

@parameters t

@testset "Constant" begin
    @named src = Constant(k=2)
    @named int = Integrator()
    @named iosys = ODESystem([
        connect(src.output, int.input),
        ],
        t,
        systems=[int, src],
    )
    sys = structural_simplify(iosys)

    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))

    sol = solve(prob, Rodas4())
    @test sol[src.output.u][end] ≈ 2 atol=1e-3
end

@testset "Sine" begin
    sine(t, frequency, amplitude, phase, offset, start_time) = offset + ifelse(t < start_time, 0, amplitude* sin(2*pi*frequency*(t - start_time) + phase))

    frequency=1
    amplitude=2
    phase=0
    offset=1
    start_time=0

    @named src = Sine(frequency=frequency, amplitude=amplitude, phase=phase, offset=offset, start_time=start_time)
    @named int = Integrator()
    @named iosys = ODESystem([
        connect(src.output, int.input),
        ],
        t,
        systems=[int, src],
    )
    sys = structural_simplify(iosys)

    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))

    sol = solve(prob, Rodas4())
    @test sol[src.output.u] ≈ sine.(sol.t, frequency, amplitude, phase, offset, start_time) atol=1e-3
end

@testset "Cosine" begin
    cosine(t, frequency, amplitude, phase, offset, start_time) = offset + ifelse(t < start_time, 0, amplitude* cos(2*pi*frequency*(t - start_time) + phase))

    frequency=1
    amplitude=2
    phase=0
    offset=1
    start_time=0

    @named src = Cosine(frequency=frequency, amplitude=amplitude, phase=phase, offset=offset, start_time=start_time)
    @named int = Integrator()
    @named iosys = ODESystem([
        connect(src.output, int.input),
        ],
        t,
        systems=[int, src],
    )
    sys = structural_simplify(iosys)

    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))

    sol = solve(prob, Rodas4())
    @test sol[src.output.u] ≈ cosine.(sol.t, frequency, amplitude, phase, offset, start_time) atol=1e-3
end

@testset "ContinuousClock" begin
    cont_clock(t, offset, start_time) = offset + ifelse(t < start_time, 0, t - start_time)

    offset, start_time = 1, 0

    @named src = ContinuousClock(offset=offset, start_time=start_time)
    @named int = Integrator()
    @named iosys = ODESystem([
        connect(src.output, int.input),
        ],
        t,
        systems=[int, src],
    )
    sys = structural_simplify(iosys)

    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))

    sol = solve(prob, Rodas4())
    @test sol[src.output.u] ≈ cont_clock.(sol.t, offset, start_time) atol=1e-3
end

@testset "Ramp" begin
    ramp(t, offset, height, duration, start_time) = offset + ifelse(t < start_time, 0, ifelse(t < (start_time + duration), (t - start_time) * height / duration, height))

    offset, height, duration, start_time = 1, 2, 2, 0

    @named src = Ramp(offset=offset, height=height, duration=duration, start_time=start_time)
    @named int = Integrator()
    @named iosys = ODESystem([
        connect(src.output, int.input),
        ],
        t,
        systems=[int, src],
    )
    sys = structural_simplify(iosys)

    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))

    sol = solve(prob, Rodas4())
    @test sol[src.output.u] ≈ ramp.(sol.t, offset, height, duration, start_time) atol=1e-3
end

@testset "Step" begin
    step(t, offset, height, start_time) = offset + ifelse(t < start_time, 0, height)

    offset, height, start_time = 1, 2, 5

    @named src = Step(offset=offset, height=height, start_time=start_time)
    @named int = Integrator()
    @named iosys = ODESystem([
        connect(src.output, int.input),
        ],
        t,
        systems=[int, src],
    )
    sys = structural_simplify(iosys)

    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))

    sol = solve(prob, Rodas4())
    @test sol[src.output.u] ≈ step.(sol.t, offset, height, start_time) atol=1e-3
end

@testset "ExpSine" begin
    exp_sine(t, amplitude, frequency, damping, phase, start_time) = offset + ifelse(t < start_time, 0, amplitude * exp(-damping * (t - start_time)) * sin(2*pi*frequency*(t - start_time) + phase))

    frequency, amplitude, damping, phase, offset, start_time = 3, 2, 0.10, 0, 0, 0

    @named src = ExpSine(frequency=frequency, amplitude=amplitude, damping=damping, phase=phase, offset=offset, start_time=start_time)
    @named int = Integrator()
    @named iosys = ODESystem([
        connect(src.output, int.input),
        ],
        t,
        systems=[int, src],
    )
    sys = structural_simplify(iosys)

    prob = ODEProblem(sys, Pair[int.x=>0.0], (0.0, 10.0))

    sol = solve(prob, Rodas4())
    @test sol[src.output.u] ≈ exp_sine.(sol.t, amplitude, frequency, damping, phase, start_time) atol=1e-3
end


export kronecker

"""
    kronecker(
        scale::Int64,
        edgefactor::Int64;
        a::Float64=0.57,
        b::Float64=0.19,
        c::Float64=0.19
    )

Generates edges for a Kronecker generator graph. Returns an array of 2 rows
with 1st being the start edge and 2nd row having the end edge.
"""
function kronecker(
    scale::Int64,
    edgefactor::Int64;
    a::Float64=0.57,
    b::Float64=0.19,
    c::Float64 = 0.19
    )
#Ported from https://gitorious.org/graph500/graph500?p=graph500:graph500.git;a=blob;f=octave/kronecker_generator.m;h=064b6d4f42194cc95ec4ab375dcbee7c3243b233;hb=01a32a9c16994c3cf30bbf4caabf9be128379416.
    n = 2 ^ scale
    m = edgefactor * n
    a,b,c = 0.57, 0.19, 0.19
    ij = ones(Int64, 2, m)
    ab = a + b
    cnorm = c/(1-ab)
    anorm = a/ab

    for ib in 1:scale
        ii_bit = rand(1, m) .> ab
        jj_bit = rand(1, m) .> ( cnorm * ii_bit + anorm * .!(ii_bit) )
        ij = ij .+ 2^(ib-1) .* [ii_bit; jj_bit]
    end

    p = randperm(n)
    ij = p[ij]

    p = randperm(m)
    ij = ij[:, p]

    return ij .- 1 #Convert to zero index for Stinger
end


using MLBase, Random

@testset "cross validation" begin

datapath = joinpath(dirname(@__FILE__), "data")

data = readcsvmat(joinpath(datapath,"gamlr.gaussian.data.csv"))
y = data[:,1]
X = data[:,2:end]
offset = fill(0.001,size(y))

path = fit(LassoPath, X, y; offset=offset)
β = coef(path; select=:all)

coefsAICc = coef(path;select=:AICc)
segminAICc = minAICc(path)
@test segminAICc == 71
@test coefsAICc == β[:,segminAICc]

coefsBIC = coef(path;select=:BIC)
segminBIC = Lasso.minBIC(path)
@test segminBIC == 55
@test coefsBIC == β[:,segminBIC]

coefsAIC = coef(path;select=:AIC)
segminAIC = Lasso.minAIC(path)
@test segminAIC == 71
@test coefsAIC == β[:,segminAIC]

Random.seed!(13)
gen = Kfold(length(y),10)
segCVmin = cross_validate_path(path;gen=gen)
coefsCVmin = coef(path;select=:CVmin)
@test segCVmin == 71
@test coefsCVmin == β[:,segCVmin]

Random.seed!(13)
gen = Kfold(length(y),10)
segCVmin = cross_validate_path(path,X,y; gen=gen, offset=offset)
coefsCVmin = coef(path;select=:CVmin)
@test segCVmin == 71
@test coefsCVmin == β[:,segCVmin]

Random.seed!(13)
coefsCV1se = coef(path;select=:CV1se,nCVfolds=20)
segCV1se = cross_validate_path(path,X,y;select=:CV1se,gen=gen,offset=offset)
@test segCV1se == 42
@test coefsCV1se == β[:,segCV1se]

end


A_unwrapped = collect(range(0, stop=10π, length=100))
A_wrapped = A_unwrapped .% (2π)

@test unwrap(A_wrapped) == A_unwrapped

# test in-place version
unwrap!(A_wrapped)
@test A_wrapped == A_unwrapped

A_bf_uw = collect(range(0, stop=10*BigFloat(π), length=100))
A_bf_w = A_bf_uw .% (2*BigFloat(π))
@test unwrap(A_bf_w) ≈ A_bf_uw

A_f32_uw = collect(range(0, stop=10*Float32(π), length=100))
A_f32_w = A_bf_uw .% (2*Float32(π))
@test unwrap(A_bf_w) ≈ A_bf_uw


################################################################################
#                       Euler Angle Axis <=> Quaternions
################################################################################

for k = 1:samples
    # Sample a vector that will be aligned with the Euler angle.
    v = [randn();randn();randn()]
    v = v/norm(v)

    # Sample a angle between (0,2π).
    a = 2*pi*rand()

    # Create the Euler Angle and Axis representation.
    angleaxis = EulerAngleAxis(a,v)

    # Convert to quaternion.
    q = angleaxis_to_quat(angleaxis)

    # Convert back to Euler Angle and Axis.
    angleaxis_conv = quat_to_angleaxis(q)

    # Notice that the returned Euler angle will be always in the interval
    # [0,π].
    s = +1
    (a > π) && (a = 2π - a; s = -1)

    # Compare.
    @test  abs(angleaxis_conv.a -   a) < 1e-10
    @test norm(angleaxis_conv.v - s*v) < 1e-10

    # Rotate a vector aligned with the Euler axis using the quaternion.
    r     = v*randn()
    r_rot = vect(q\r*q)

    # The vector representation must be the same after the rotation.
    @test norm(r-r_rot) < 1e-10
end

# Test the exceptions.
@test_throws ArgumentError angleaxis_to_quat(0, [1;2;3;4])


using PolyChaos, Test, LinearAlgebra

# values taken from https://en.wikipedia.org/wiki/Hermite_polynomials#Definition
c =    [
       [-0.0] ,
       [-1.0, -0.0]   ,
       [0.0, -3.0, -0.0]       ,
       [3.0, 0.0, -6.0, -0.0]     ,
       [-0.0, 15.0, 0.0, -10.0, -0.0]  ,
       [-15.0, -0.0, 45.0, 0.0, -15.0, -0.0]    ,
       [0.0, -105.0, -0.0, 105.0, 0.0, -21.0, -0.0]  ,
       [105.0, 0.0, -420.0, -0.0, 210.0, 0.0, -28.0, -0.0] ,
       [-0.0, 945.0, 0.0, -1260.0, -0.0, 378.0, 0.0, -36.0, -0.0]  ,
       [-945.0, -0.0, 4725.0, 0.0, -3150.0, -0.0, 630.0, 0.0, -45.0, -0.0]   ,
       [0.0, -10395.0, -0.0, 17325.0, 0.0, -6930.0, -0.0, 990.0, 0.0, -55.0, -0.0]
       ]
α, β = rm_hermite_prob(length(c))
cc = rec2coeff(α,β)

@testset "showpoly" begin
    for (i, c_) in enumerate(c)
        @test isapprox(norm(c_ - cc[i]), 0)
    end
end


module EFT
export twoSum, twoDiff, twoProd, twoDiv

# * Documentation of functions

@doc raw"""
    twoSum(a, b)

Error-Free Transformation for the sum of two numbers:

(x,y) = twoSum(a,b)

  ``\Leftrightarrow``

``x = fl(a+b)``  and  ``x+y = a+b``
"""
function twoSum end

@doc raw"""
    twoProd(a, b)

Error-Free Transformation for the product of two numbers:

(x,y) = twoProd(a, b)

  ``\Leftrightarrow``

``x = fl(a\,b)``  and  ``x+y = a\,b``
"""
function twoProd end

@doc raw"""
    twoDiv(a, b)

Approximate transformation for the division. The transformation is not exact
(the error is not representable) but at least the sign of y should be correct:

(x, y) = twoDiv(a, b)

  ``\Leftrightarrow``

``x = fl(a/b)``  and  ``y ≈ a/b - x``
"""
function twoDiv end

# * Generic implementation

# Generic implementation of twoSum. Uses the algorithm by D. Knuth.
function twoSum(a, b)
    x = a + b
    z = x - a
    y = (a - (x-z)) + (b-z)
    return (x,y)
end

twoDiff(a, b) = twoSum(a,-b)

@doc raw"""
    split(a)

Error-free splitting of a FP number:

(x,y) = split(a)

  ``\Leftrightarrow``

``x+y = a``, and the representations of x and y do not overlap

This implementation uses the algorithm by Dekker.
"""
function split(a)
    # New methods should be added to compute the correct
    # constant for each FP representation
    #
    # factor = 1+2^s, where s is the mantissa length
    factor(x::Float64) = 134217729
    factor(x::Float32) = 4097

    c = factor(a) * a
    x = c - (c - a)
    y = a - x
    return (x, y)
end

# Generic implementation of TwoProd. Uses the algorithm by Veltkamp.
function twoProd(a, b)
    x = a * b
    (a1, a2) = split(a)
    (b1, b2) = split(b)
    y = a2*b2 - (((x - a1*b1) - a2*b1) - a1*b2)
    return (x,y)
end

function twoDiv(a, b)
    x = a / b
    (p1, p2) = twoProd(x, b)
    y = (a-p1-p2)/b
    return (x, y)
end

# * Specific algorithms for Float32

# Since double precision (Float64) is available to run higher-precision
# computations, it should be used to compute more efficiently transformations of
# Float32 operations.

macro float32_transformation(operator, name)
    quote
        function $(esc(name))(a::Float32, b::Float32)
            x = $operator(a, b)
            r = $operator(Float64(a), Float64(b)) - Float64(x)
            y = Float32(r)
            return (x,y)
        end
    end
end

@float32_transformation(+, twoSum)
@float32_transformation(*, twoProd)
@float32_transformation(/, twoDiv)

end


using Agents
using Random
using DataFrames
using Distributions
using DrWatson
using StatsPlots
using BenchmarkTools

function rate_to_proportion(r::Float64,t::Float64)
    1-exp(-r*t)
end;

mutable struct Person <: AbstractAgent
    id::Int64
    status::Symbol
end

function init_model(β::Float64,c::Float64,γ::Float64,N::Int64,I0::Int64)
    properties = @dict(β,c,γ)
    model = ABM(Person; properties=properties)
    for i in 1:N
        if i <= I0
            s = :I
        else
            s = :S
        end
        p = Person(i,s)
        p = add_agent!(p,model)
    end
    return model
end;

function agent_step!(agent, model)
    transmit!(agent, model)
    recover!(agent, model)
end;

function transmit!(agent, model)
    # If I'm not susceptible, I return
    agent.status != :S && return
    ncontacts = rand(Poisson(model.properties[:c]))
    for i in 1:ncontacts
        # Choose random individual
        alter = random_agent(model)
        if alter.status == :I && (rand() ≤ model.properties[:β])
            # An infection occurs
            agent.status = :I
            break
        end
    end
end;

function recover!(agent, model)
    agent.status != :I && return
    if rand() ≤ model.properties[:γ]
            agent.status = :R
    end
end;

susceptible(x) = count(i == :S for i in x)
infected(x) = count(i == :I for i in x)
recovered(x) = count(i == :R for i in x);

δt = 0.1
nsteps = 400
tf = nsteps*δt
t = 0:δt:tf;

β = 0.05
c = 10.0*δt
γ = rate_to_proportion(0.25,δt);

N = 1000
I0 = 10;

Random.seed!(1234);

abm_model = init_model(β,c,γ,N,I0)

to_collect = [(:status, f) for f in (susceptible, infected, recovered)]
abm_data, _ = run!(abm_model, agent_step!, nsteps; adata = to_collect);

abm_data[!,:t] = t;

plot(t,abm_data[:,2],label="S",xlab="Time",ylabel="Number")
plot!(t,abm_data[:,3],label="I")
plot!(t,abm_data[:,4],label="R")


@doc raw"""
    Cityblock <: Metric
    SumAbsoluteDifference <: Metric
    cityblock(x, y)
    sad(x, y)

sum of absolute difference(sad), also known as [`Cityblock`](@ref) distance, is calculated by `sum(abs(x - y))`
"""
struct SumAbsoluteDifference <: Metric end

@doc raw"""
    SqEuclidean <: Metric
    SumSquaredDifference <: Metric
    sqeuclidean(x, y)
    ssd(x, y)

sum of squared difference(ssd), also known as [`SqEuclidean`](@ref) distance, is calculated by `sum(abs2(x - y))`
"""
struct SumSquaredDifference <: Metric end

@doc raw"""
    MeanAbsoluteError <: Metric
    mae(x, y)
    sadn(x, y)

mean absolute error(mae) is calculated by [`sad`](@ref)`(x, y)/length(x)`
"""
struct MeanAbsoluteError <: Metric end

@doc raw"""
    MeanSquaredError <: Metric
    mse(x, y)
    ssdn(x, y)

mean squared error(mse) is calculated by [`ssd`](@ref)`(x, y)/length(x)`
"""
struct MeanSquaredError <: Metric end

@doc raw"""
    NCC <: Metric
    ncc(x, y)

Normalized Cross-correlation(ncc) is calculated by `dot(mean(x),mean(y))/(norm(mean(x))*norm(mean(y)))`

```math
\frac{<\bar{x}, \bar{y}>}{||\bar{x}||*||\bar{y}||}
```

!!! info

    NCC isn't a Metric because `ncc(x, x) == 1.0`
"""
struct NCC <: PreMetric end

@doc raw"""
    RootMeanSquaredError <: Metric
    rmse(x, y)

Root Mean Square Error(rmse) is calculated by `sqrt(mse(x, y))`
"""
struct RootMeanSquaredError <: Metric end

# SumAbsoluteDifference
(::SumAbsoluteDifference)(a::GenericImage, b::GenericImage) =
    Cityblock()(a, b)

@doc (@doc SumAbsoluteDifference)
sad(a::GenericImage, b::GenericImage) = SumAbsoluteDifference()(a, b)

# SumSquaredDifference
(::SumSquaredDifference)(a::GenericImage, b::GenericImage) =
    SqEuclidean()(a, b)

@doc (@doc SumSquaredDifference)
ssd(a::GenericImage, b::GenericImage) = SumSquaredDifference()(a, b)

# MeanAbsoluteError
(::MeanAbsoluteError)(a::GenericImage, b::GenericImage) =
    SumAbsoluteDifference()(a, b) / length(a)

@doc (@doc MeanAbsoluteError)
mae(a::GenericImage, b::GenericImage) = MeanAbsoluteError()(a, b)
@doc (@doc MeanAbsoluteError)
const sadn = mae

# MeanSquaredError
(::MeanSquaredError)(a::GenericImage, b::GenericImage) =
    SumSquaredDifference()(a, b) / length(a)

@doc (@doc MeanSquaredError)
mse(a::GenericImage, b::GenericImage) = MeanSquaredError()(a, b)
@doc (@doc MeanSquaredError)
const ssdn = mse

# RootMeanSquaredError
(::RootMeanSquaredError)(a::GenericImage, b::GenericImage) =
    sqrt(MeanSquaredError()(a, b))

@doc (@doc RootMeanSquaredError)
rmse(a::GenericImage, b::GenericImage) = RootMeanSquaredError()(a, b)

# NCC
function (::NCC)(a::GenericGrayImage, b::GenericGrayImage)
    A = channelview(of_eltype(floattype(eltype(a)), a))
    B = channelview(of_eltype(floattype(eltype(b)), b))
    return _ncc(A, B)
end

function (::NCC)(a::AbstractArray{<:Color3}, b::AbstractArray{<:Color3})
    A = of_eltype(floattype(eltype(a)), a)
    B = of_eltype(floattype(eltype(b)), b)
    return _ncc(A, B)
end

function _ncc(A::GenericImage, B::GenericImage)
    Am = (A.-mean(A))[:]
    Bm = (B.-mean(B))[:]
    return _dot(Am,Bm)/(norm(Am)*norm(Bm))
end

_dot(a::AbstractArray{<:Number}, b::AbstractArray{<:Number}) =
    dot(a, b)
_dot(a::GenericImage, b::GenericImage) =
    mapreduce(xy->dotc(xy...), +, zip(a, b))

@doc (@doc NCC)
ncc(a::GenericImage, b::GenericImage) = NCC()(a, b)


using StaticArrays
@testset "Spectral radius" begin
    @test_throws ArgumentError ρ(zeros(0, 0))
    # There is a specialized function for StaticArrays
    @test ρ(@SMatrix [1 2; 3 4]) ≈ ρ([1 2; 3 4])
    @test ρ(-@SMatrix [1 2; 3 4]) ≈ ρ(-[1 2; 3 4])
    @test ρ(@SMatrix [1 1; 0 1]) ≈ ρ([1 1; 0 1])
    # Matrix A1 from [Section 4, PEDJ16]
    A = @SMatrix [0.94  0.56; -0.35  0.73]
    @test ρ(A) ≈ ρ(Matrix(A))
    @test ρ(-A) ≈ ρ(Matrix(-A))
end


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

function _sphdist{T<:AbstractFloat}(long1::T, lat1::T, long2::T, lat2::T,
                                    degrees::Bool)
    # Convert both points to rectangular coordinates.
    rxy, z1 = polrec(1.0, lat1,  degrees=degrees)
    x1, y1  = polrec(rxy, long1, degrees=degrees)
    rxy, z2 = polrec(1.0, lat2,  degrees=degrees)
    x2, y2  = polrec(rxy, long2, degrees=degrees)
    # Compute vector dot product for both points.
    cs = x1*x2 + y1*y2 + z1*z2
    # Compute the vector cross product for both points.
    xc = y1*z2 - z1*y2
    yc = z1*x2 - x1*z2
    zc = x1*y2 - y1*x2
    sn = vecnorm((xc, yc, zc))
    # Convert to polar coordinates.
    radius, angle = recpol(cs, sn)
    return degrees ? rad2deg(angle) : angle
end

"""
    sphdist(long1, lat1, long2, lat2[, degrees=true]) -> angular_distance

### Purpose ###

Angular distance between points on a sphere.

### Arguments ###

* `long1`:  longitude of point 1
* `lat1`: latitude of point 1
* `long2`: longitude of point 2
* `lat2`: latitude of point 2
* `degrees` (optional boolean keyword): if `true`, all angles, including the
  output distance, are assumed to be in degrees, otherwise they are all in
  radians.  It defaults to `false`.

### Output ###

Angular distance on a sphere between points 1 and 2, as an `AbstractFloat`.  It
is expressed in radians unless `degrees` keyword is set to `true`.

### Example ###

``` julia
sphdist(120, -43, 175, +22)
# => 1.5904422616007134
```

### Notes ###

* `gcirc` function is similar to `sphdist`, but may be more suitable for
  astronomical applications.
* If `long1`, `lat1` are scalars, and `long2`, `lat2` are vectors, then the
 output is a vector giving the distance of each element of `long2`, `lat2` to
 `long1`, `lat1`.  Similarly, if `long1`,`de1` are vectors, and `long2`,` lat2`
 are scalars, then the output is a vector giving the distance of each element of
 `long1`, `lat1` to `long2`, `lat2`.  If both `long1`, `lat1` and `long2`,
 `lat2` are vectors then the output is a vector giving the distance of each
 element of `long1`, `lat1` to the corresponding element of `long2`, `lat2`.

Code of this function is based on IDL Astronomy User's Library.
"""
sphdist(long1::Real, lat1::Real,
        long2::Real, lat2::Real; degrees::Bool=false) =
            _sphdist(promote(float(long1), float(lat1),
                             float(long2), float(lat2))... , degrees)

function sphdist{LO1<:Real, LA1<:Real}(long1::AbstractArray{LO1},
                                       lat1::AbstractArray{LA1},
                                       long2::Real,
                                       lat2::Real;
                                       degrees::Bool=false)
    @assert length(long1) == length(lat1)
    dist = similar(long1, typeof(float(one(LO1))))
    for i in eachindex(long1)
        dist[i] = sphdist(long1[i], lat1[i], long2, lat2, degrees=degrees)
    end
    return dist
end

function sphdist{LO2<:Real, LA2<:Real}(long1::Real,
                                       lat1::Real,
                                       long2::AbstractArray{LO2},
                                       lat2::AbstractArray{LA2};
                                       degrees::Bool=false)
    @assert length(long2) == length(lat2)
    dist = similar(long2, typeof(float(one(LO2))))
    for i in eachindex(long2)
        dist[i] = sphdist(long1, lat1, long2[i], lat2[i], degrees=degrees)
    end
    return dist
end

function sphdist{LO1<:Real, LA1<:Real, LO2<:Real, LA2<:Real}(long1::AbstractArray{LO1},
                                                             lat1::AbstractArray{LA1},
                                                             long2::AbstractArray{LO2},
                                                             lat2::AbstractArray{LA2};
                                                             degrees::Bool=false)
    @assert length(long1) == length(lat1) == length(long2) == length(lat2)
    dist = similar(long1, typeof(float(one(LO1))))
    for i in eachindex(long1)
        dist[i] = sphdist(long1[i], lat1[i], long2[i], lat2[i], degrees=degrees)
    end
    return dist
end


function xdot(x,y,z; sigma=10)
    return sigma*(y-x)
end

function ydot(x,y,z; rho=48)
    return x*(rho-z)-y
end

function zdot(x,y,z; beta=8/3)
    return x*y - beta*z
end

function make_step(x,y,z; delta=0.01)
    xp = x + xdot(x,y,z)*delta
    yp = y + ydot(x,y,z)*delta
    zp = z + zdot(x,y,z)*delta

    xn = x + 0.5*(xdot(x,y,z)+xdot(xp,yp,zp))*delta
    yn = y + 0.5*(ydot(x,y,z)+ydot(xp,yp,zp))*delta
    zn = z + 0.5*(zdot(x,y,z)+zdot(xp,yp,zp))*delta

    return xn, yn, zn
end

function run_sim(;runs=3000, count_z = false, timing=false, thresh = 0.0)
    x, y, z = 0, 1, 0
    X, Y, Z = Float64[0,], Float64[1,], Float64[0,]
    #dist = fill(0,2)
    cnt = 0
    zless = true
    cnt_list = Int[]

    for i in 1:runs
        x,y,z = make_step(x,y,z)
        push!(X, x)
        push!(Y, y)
        push!(Z, z)

        if (z < thresh && zless) || (z > thresh && !zless)
            cnt += 1
        else
            push!(cnt_list, cnt)
            zless = !zless
            cnt = 0
        end

        # if count_z
        #     if z > thresh
        #         dist[1] += 1
        #     else
        #         dist[2] += 1
        #     end
        # end
    end
    return X, Y, Z, cnt_list #,dist
end

X, Y, Z, times = run_sim(;runs=30000, timing=true, count_z = false, thresh = 24)

using Plots
Plots.plot(X,Y)
Plots.plot(Z)
Plots.histogram(Z;bins=100)

Plots.histogram((times*0.01)[3:end],bins=20)

function get_maxima(ls; add_min=false)
    maxs = Float64[]
    if add_min
        mins = Float64[]
        if ls[1]<ls[2]
            push!(mins,ls[1])
        end
    end
    if ls[1]>ls[2]
        push!(maxs,ls[1])
    end
    for i=2:length(ls)-1
        if ls[i-1]<ls[i]>ls[i+1]
            push!(maxs,ls[i])
        end
        if add_min
            if ls[i-1]>ls[i]<ls[i+1]
                push!(mins,ls[i])
            end
        end
    end
    if ls[end]>ls[end-1]
        push!(maxs,ls[end])
    end
    if add_min
        if ls[end]<ls[end-1]
            push!(mins,ls[end])
        end
        return maxs, mins
    else
        return maxs
    end
end

# get subsequent maxima plot
z_max = get_maxima(Z)
z_plot = Tuple{Float64, Float64}[]
for i=1:length(z_max)-1
    push!(z_plot,(z_max[i], z_max[i+1]))
end
i1 = [x[1] for x in z_plot]
i2 = [x[2] for x in z_plot]
Plots.scatter(i1,i2)

# get sequential max/min plot
z_max, z_min = get_maxima(Z, add_min=true)
Plots.scatter(z_max, z_min)

#=
points = Observable(Point3f[])
colors = Observable(Int[])
set_theme!(theme_black())

fig, ax, l = lines(points, color = colors,
    colormap = :inferno, transparency = true,
    axis = (; type = Axis3, protrusions = (0, 0, 0, 0),
        viewmode = :fit, limits = (-30, 30, -30, 30, 0, 50)))

frames = 1:120
for frame in frames
    for i in 1:50
        push!(points[], step!(attractor))
        push!(colors[], frame)
    end
    ax.azimuth[] = 1.7pi + 0.3 * sin(2pi * frame / 120)
    notify.((points, colors))
    l.colorrange = (0, frame)
    sleep(0.1)
    #println(frame)
end
=#


const IntFloat = Union{Int, Float64}
const Location = Union{Vector{Int}, Int}
# 1-qubit gates
i = zeros(ComplexF64,2,2)
i[1,1] = 1
i[2,2] = 1
const I_DATA = i

x = zeros(ComplexF64,2,2)
x[2,1] = 1
x[1,2] = 1
const X_DATA = x

y = zeros(ComplexF64,2,2)
y[1,2] = -1im
y[2,1] = 1im
const Y_DATA = y

z = zeros(ComplexF64,2,2)
z[1,1] = 1
z[2,2] = -1
const Z_DATA = z

h = ones(ComplexF64,2,2)
h[2,2] = -1
const H_DATA = h/√2

s = zeros(ComplexF64,2,2)
s[1,1] = 1
s[2,2] = 1im
const S_DATA = s

sdag = zeros(ComplexF64,2,2)
sdag[1,1] = 1
sdag[2,2] = -1im
const SDAG_DATA = sdag

t = zeros(ComplexF64,2,2)
t[1,1] = 1
t[2,2] = exp(π/4im)
const T_DATA = t

tdag = zeros(ComplexF64,2,2)
tdag[1,1] = 1
tdag[2,2] = exp(-π/4im)
const TDAG_DATA = tdag

#2-qubit gates
swap = zeros(ComplexF64,2,2,2,2)
swap[1,1,1,1] = 1
swap[2,2,2,2] = 1
swap[1,2,2,1] = 1
swap[2,1,1,2] = 1
const SWAP_DATA =swap

cnot = zeros(ComplexF64, 2,2,2,2)
cnot[1,1,1,1] = 1
cnot[1,2,1,2] = 1
cnot[2,1,2,2] = 1
cnot[2,2,2,1] = 1
const CNOT_DATA = cnot

# 3-qubit gates
toffoli = zeros(ComplexF64,2,2,2,2,2,2)
toffoli[1,1,1,1,1,1] = 1
toffoli[1,1,2,1,1,2] = 1
toffoli[1,2,1,1,2,1] = 1
toffoli[2,1,1,2,1,1] = 1
toffoli[2,2,1,2,2,2] = 1
toffoli[2,1,2,2,1,2] = 1
toffoli[1,2,2,1,2,2] = 1
toffoli[2,2,2,2,2,1] = 1
const TOFFOLI_DATA = toffoli

fredkin = zeros(ComplexF64,2,2,2,2,2,2)
for i =1:2
	for j= 1:2
		fredkin[1,i,j,1,i,j] = 1
		fredkin[2,i,j,2,j,i] = 1
	end
end
const FREDKIN_DATA = fredkin
const CSWAP_DATA = fredkin

cz = zeros(ComplexF64, 2,2,2,2)
cz[1,1,1,1] = 1
cz[1,2,1,2] = 1
cz[2,1,2,1] = 1
cz[2,2,2,2] = -1
const CZ_DATA = cz


"""
    y = bunquantize(q)

Calculate the value corresponding to a bidirectionally quantized quantity.
`q` is a 2n by m matrix containing the powers of 2 and their signs for each
quantized value.
"""
function bunquantize(q)
    n = size(q, 1) ÷ 2
    y = zeros(n)
    signs = 2:2:2*n
    powers = signs .- 1
    for i = 1:size(q, 2)
        y = y + 2.0 .^ q[powers,i] .* q[signs,i]
    end
    return y
end


"""
    extract_col_feats(df, cols)

find mean, std, minimum, maximum in df[!, col]
default value of columns are all numeric columns except date
"""
function extract_col_statvals(df::DataFrame, cols::Array{Symbol, 1})
    syms = []
    types = []
    vals = []
    for col in cols
        μ, σ = mean_and_std(skipmissing(df[!, col]))
        minval = minimum(skipmissing(df[!, col]))
        maxval = maximum(skipmissing(df[!, col]))

        push!(syms, String(col))
        push!(types, "μ")
        push!(vals, μ)

        push!(syms, String(col))
        push!(types, "σ")
        push!(vals, σ)

        push!(syms, String(col))
        push!(types, "minimum")
        push!(vals, minval)

        push!(syms, String(col))
        push!(types, "maximum")
        push!(vals, maxval)
    end

    ndsparse((
        symbol = syms,
        types = types),
        (value = vals,))
end

"""
    zscore!(df, col, new_col)

Apply zscore (normalization) to dataframe `df`
No need to implement `zscore`, just implement `zscore!``.
"""
function zscore!(df::DataFrame, col::Symbol, new_col::Symbol)
    to_be_normalized = df[!, col]
    df[!, new_col] = zscore(to_be_normalized)
end

function zscore!(df::DataFrame, col::Symbol, new_col::Symbol, μ::Real, σ::Real)
    to_be_normalized = df[!, col]
    df[!, new_col] = zscore(to_be_normalized, μ, σ)
end

function zscore!(df::DataFrame, cols::Array{Symbol, 1}, new_cols::Array{Symbol, 1})
    for (col, new_col) in zip(cols, new_cols)
        zscore!(df, col, new_col)
    end
end

function zscore!(df::DataFrame, cols::Array{Symbol, 1}, new_cols::Array{Symbol, 1}, μσs::Array{Real, 1})
    for (col, new_col) in zip(cols, new_cols)
        μ, σ = μσs[String(col), "μ"].value, μσs[String(col), "σ"].value
        zscore!(df, col, new_col, μ, σ)
    end
end

"""
    unzscore(A, μ, σ)

unzscore in Array using given μ and σ
"""
unzscore(A, μ::Real, σ::Real) = A .* σ .+ μ

"""
    unzscore!(df, zcol, ocol, μ, σ)

revert zscore normalization (single column, zcol -> ocol)
"""
function unzscore!(df::DataFrame, zcol::Symbol, ocol::Symbol, μ::Real, σ::Real)
    df[!, ocol] = unzscore(df[!, zcol], μ, σ)
end

"""
    unzscore!(df, zcols, ocols, μσs)

revert zscore normalization (single column, zcol -> ocol)
"""
function unzscore!(df::DataFrame, zcols::Array{Symbol, 1}, ocols::Array{Symbol, 1}, stattb::AbstractNDSparse)
    for (zcol, ocol) in zip(zcols, ocols)
        unzscore!(df, zcol, ocol,
            stattb[String(ocol), "μ"].value, stattb[String(ocol), "σ"].value)
    end
end

minmax_scaling(X::AbstractVector, a::F, b::F) where F<:AbstractFloat =
    a .+ (b - a) .* (X .- minimum(X)) ./ (maximum(X) - minimum(X))

minmax_scaling!(df::DataFrame, ocol::Symbol, mcol::Symbol,
    a::F, b::F) where F<:AbstractFloat =
        df[!, mcol] = minmax_scaling(df[!, ocol], a, b)

"""
    minmax_scaling!(df, ocols, mcols, a, b)

min-max normalization from df[!, ocols] to df[!, mcols], new range is (a, b)
https://en.wikipedia.org/wiki/Feature_scaling#Rescaling_(min-max_normalization)

Y = a + (X - min(X)) * (b - a) / (maximum(X) - minimum(X))
"""
function minmax_scaling!(df::DataFrame, ocols::Array{Symbol, 1}, mcols::Array{Symbol, 1},
    a::F, b::F) where F<:AbstractFloat

    for (ocol, mcol) in zip(ocols, mcols)
        minmax_scaling!(df, ocol, mcol, a, b)
    end
end

unminmax_scaling(Y::AbstractVector, minY::Real, maxY::Real, a::F, b::F) where F<:AbstractFloat =
    (Y .- a) .* ((maxY - minY) / (b - a)) .+ minY

"""
    unminmax_scaling!(df, mcol, ocol,  minY, maxY, a, b)

revert min-max normalization (single column)
"""
function unminmax_scaling!(df::DataFrame,
    mcol::Symbol, ocol::Symbol, minY::Real, maxY::Real, a::F, b::F) where F<:AbstractFloat
    df[!, ocol] = unminmax_scaling(df[!, mcol], minY, maxY, a, b)
end

"""
    unminmax_scaling!(df, mcols, ocols, minY, maxY, a, b)

revert min-max normalization (multiple column)
"""
function unminmax_scaling!(df::DataFrame, zcols::Array{Symbol, 1}, ocols::Array{Symbol, 1},
    stattb::AbstractNDSparse, a::F, b::F) where F<:AbstractFloat
    for (zcol, ocol) in zip(zcols, ocols)
        unminmax_scaling!(df, zcol, ocol,
        stattb[String(ocol), "minimum"].value, stattb[String(ocol), "maximum"].value, a, b)
    end
end

"""
    exclude_elem(cols, target_col)

Exclude element and return new splited array
"""
function exclude_elem(cols, target_col)
    new_cols = copy(cols)
    deleteat!(new_cols, new_cols .== target_col)

    new_cols
end

"""
    findrow(df, col, val)

Find fist row number in df[!, `col`] as `val` by brute-force
"""
function findrow(df::DataFrame, col::Symbol, val::Union{<:Real, DateTime, ZonedDateTime})

    idx = 0
    for row in eachrow(df)
        idx += 1
        if (row[col] == val)
            return idx
        end
    end

    idx = 0

    idx
end

"""
    WHO_PM10(val::Real)
    return WHO PM10 level

1: Good
2: Normal
3: Bad
4: Very Bad
"""
function WHO_PM10(val::Real)
    if 0 <= val < 31
        return 1
    elseif 31 <= val < 81
        return 2
    elseif 81 <= val < 151
        return 3
    else
        return 4
    end
end

"""
    WHO_PM25(val::Real)
    return WHO PM25 level

1: Good
2: Normal
3: Bad
4: Very Bad
"""
function WHO_PM25(val::Real)
    if 0 <= val < 16
        return 1
    elseif 16 <= val < 36
        return 2
    elseif 36 <= val < 76
        return 3
    else
        return 4
    end
end


function isdivisible(n)
   (0 == (n % 3)) || (0 == (n % 5))
end

function solve()
   sum(filter(isdivisible, 1:999))
end

result = solve()
println(result)


"""
    gammaReg_log(topredict::String, varnames::Vector{Symbol}, train::DataFrame, test::DataFrame, RMSD::Bool=true)

Performs a gamma regression (log link) with variables from variables and data of D.

# Arguments

- `topredict::String`: The name of the column of the dataframe of interest that is to be predicted.
- `varnames::Vector{Symbol}`: The explanatory variables to include in the regression.
- `train::DataFrame`: The dataset used to train the model.
- `test::DataFrame`: The dataset used to test the model.
- `RMSD::Bool=true`: If true, returns the RMSD of the model. If false, returns the vector of predictions. Default = true.
"""
function gammaReg_log(topredict::String, varnames::Vector{Symbol}, train::DataFrame, test::DataFrame, RMSD::Bool=true)
    model = glm(Term(Symbol(topredict)) ~ sum(Term.(varnames)), train, Gamma(), LogLink())
    predictions = convert(Vector{Float64},GLM.predict(model, test))

    RMSD ? (return rmsd(predictions, test[:, Symbol(topredict)])) : return predictions
end


function clip(l, rect::Rectangle)
    A = VoronoiDelaunay.geta(l)
    B = VoronoiDelaunay.getb(l)

    clip(A, B, rect)
end

function clip(A::VoronoiDelaunay.AbstractPoint2D, B::VoronoiDelaunay.AbstractPoint2D, rect::Rectangle)
    if isinside(A, rect) && isinside(B, rect)
        VoronoiDelaunay.Line(A, B)
        # GeometryBasics.Line(A, B)
	end

	t0 = 0.0
	t1 = 1.0
	D = B - A
	# left, right, bottom, top
	# Parametrization of line segment from A to B
	p = [-getx(D) ; getx(D) ; -gety(D) ; gety(D)]
	q = [
		getx(A) - left(rect) ;
		right(rect) - getx(A) ;
		gety(A) - lower(rect) ;
		upper(rect) - gety(A)
	]

	for k in 1:4
		if p[k] == 0.0
			# Line parallel with k'th edge
			if q[k] < 0.0
				return nothing
			end
		elseif p[k] < 0.0
			# Outside to inside
			u = q[k] / p[k]
			if u > t1
				return nothing
			end
			t0 = max(u, t0)
		else # p[k] > 0.0
			# Inside to outside
			u = q[k] / p[k]
			if u < t0
				return nothing
			end
			t1 = min(u, t1)
		end
	end

	return VoronoiDelaunay.Line(A + t0*D, A + t1*D)
end


# This file is part of the IntervalArithmetic.jl package; MIT licensed

isempty(x::ValidInterval) = false
isempty(x::EmptyInterval) = true

isnai(::ValidInterval) = false
isnai(::NotAnInterval) = true

isentire(x::ValidInterval) = x == entireinterval(x)

isbounded(x::Interval) = isfinite(x.lo) && isfinite(x.hi)
isunbounded(x::Interval) = !isbounded(x)

"""
    isthin(x)

Checks if `x` is the set consisting of a single exactly
representable float. Any float which is not exactly representable
does *not* yield a thin interval. Corresponds to `isSingleton` of
the standard.
"""
isthin(x::Interval) = x.lo == x.hi

"""
    isatomic(x::Interval)

Check whether an interval `x` is *atomic*, i.e. is unable to be split.
This occurs when the interval is empty, or when the upper bound equals the lower bound or the `nextfloat` of the lower bound.
"""
isatomic(x::Interval) = (x.hi == x.lo) || (x.hi == nextfloat(x.lo))

iscommon(x::Interval) = isbounded(x)

for f in (:isentire, :isbounded, :isunbounded, :isthin, :isatomic, :iscommon)
    @eval ($f)(::EmptyInterval) = false
end

## Comparisons

"""
    ==(a,b)

Checks if the intervals `a` and `b` have equal bounds.
"""
==(a::Interval, b::Interval) = a.lo == b.lo && a.hi == b.hi
==(::EmptyInterval, ::EmptyInterval) = true
==(::ValidInterval, ::ValidInterval) = false

# Weakly less, \le, <=
<=(a::Interval, b::Interval) = (a.lo ≤ b.lo) && (a.hi ≤ b.hi)
<=(::EmptyInterval, ::EmptyInterval) = true
<=(::ValidInterval, ::ValidInterval) = false

# Auxiliary function: equivalent to </<=, but Inf <,<= Inf returning true
islessprime(a::T, b::T) where T <: Real = a < b ||
    ( a == b && (isinf(a) || isinf(b)) )

# Strict less: <
<(a::Interval, b::Interval) =
    islessprime(a.lo, b.lo) && islessprime(a.hi, b.hi)
<(::EmptyInterval, ::EmptyInterval) = true
<(::ValidInterval, ::ValidInterval) = false

function precedes end
const ≼ = precedes # \preccurlyeq
≼(a::Interval, b::Interval) = a.hi ≤ b.lo
≼(::ValidInterval, ::ValidInterval) = true

function strictprecedes end
const ≺ = strictprecedes # \prec
≺(a::Interval, b::Interval) = a.hi < b.lo
≺(::ValidInterval, ::ValidInterval) = true

"""
    in(x, a)
    ∈(x, a)

Checks if the number `x` is a member of the interval `a`, treated as a set.
Corresponds to `isMember` in the ITF-1788 Standard.
"""
in(x::Real, a::Interval) = isfinite(x) && a.lo <= x <= a.hi
in(::Real, ::EmptyInterval) = false

"""
    issubset(a,b)
    ⊆(a,b)

Checks if all the points of the interval `a` are within the interval `b`.
"""
⊆(a::Interval, b::Interval) = b.lo ≤ a.lo && a.hi ≤ b.hi
⊆(::Interval, ::EmptyInterval) = false
⊆(::EmptyInterval, ::ValidInterval) = true

zero_in(a::ValidInterval) = zero(a) ⊆ a

⊂(a::ValidInterval, b::ValidInterval) = a ⊆ b && a ≠ b

⊇(a::ValidInterval, b::ValidInterval) = b ⊆ a

⊃(a::ValidInterval, b::ValidInterval) = b ⊂ a

function isinterior end
const ⪽ = isinterior  # \subsetdot
⪽(a::Interval, b::Interval) = islessprime(b.lo, a.lo) && islessprime(a.hi, b.hi)
⪽(::Interval, ::EmptyInterval) = false
⪽(::EmptyInterval, ::ValidInterval) = true

# for zero we can avoid islessprime
contains_zero(a::Interval{T}) where T = a.lo < zero(T) < a.hi
contains_zero(::EmptyInterval) = false

# Disjoint:
isdisjoint(a::Interval, b::Interval) =
    islessprime(b.hi, a.lo) || islessprime(a.hi, b.lo)
isdisjoint(::ValidInterval, ::ValidInterval) = true


using ToeplitzMatrices
import LinearAlgebra: Symmetric, cholesky, *, mul!

import Base: transpose, adjoint, copy
import ToeplitzMatrices: Toeplitz

function copy(T::Toeplitz)
    return Toeplitz(copy(T.vc), copy(T.vr), copy(T.vcvr_dft), copy(T.tmp), T.dft)
end
function copy(T::SymmetricToeplitz)
    return SymmetricToeplitz(copy(T.vc), copy(T.vcvr_dft), copy(T.tmp), T.dft)
end

"""
    cholesky(T::SymmetricToeplitz)

Implementation adapted from "On the stability of the Bareiss and related
Toeplitz factorization algorithms", Bojanczyk et al, 1993.
"""
function cholesky!(L::AbstractMatrix, T::SymmetricToeplitz)

    # Initialize.
    L[:, 1] .= T.vc ./ sqrt(T.vc[1])
    v = copy(L[:, 1])
    N = size(T, 1)

    # Iterate.
    @inbounds for n in 1:N-1
        sinθn = v[n + 1] / L[n, n]
        cosθn = sqrt(1 - sinθn^2)

        for n′ in n+1:N
            v[n′] = (v[n′] - sinθn * L[n′ - 1, n]) / cosθn
            L[n′, n + 1] = -sinθn * v[n′] + cosθn * L[n′ - 1, n]
        end
    end
    return Cholesky(L, 'L', 0)
end
function cholesky(T::SymmetricToeplitz)
    return cholesky!(Matrix{eltype(T)}(undef, size(T, 1), size(T, 1)), T)
end

function +(T::SymmetricToeplitz, u::UniformScaling)
    Tu = copy(T)
    Tu.vc[1] += u.λ
    Tu.vcvr_dft .+= u.λ
    return Tu
end
+(u::UniformScaling, T::SymmetricToeplitz) = T + u

transpose(T::Toeplitz) = Toeplitz(T.vr, T.vc)
adjoint(T::Toeplitz) = Toeplitz(conj.(T.vr), conj.(T.vc))
transpose(T::SymmetricToeplitz) = T
adjoint(T::SymmetricToeplitz) = T

@inline LinearAlgebra.Symmetric(T::SymmetricToeplitz) = T

Toeplitz(vc::AbstractVector, vr::AbstractVector) = Toeplitz(Vector(vc), Vector(vr))

# """
#     mul!(C::Matrix, A::AbstractToeplitz, B::AbstractToeplitz)

# `O(prod(size(C)))` matrix multiplication for Toeplitz matrices. Follows from a skeleton of
# an algorithm on stackoverflow:
# https://stackoverflow.com/questions/15889521/product-of-two-toeplitz-matrices
# """
# function mul!(C::Matrix, A::AbstractToeplitz, B::AbstractToeplitz)
#     for q in 1:size(C, 2)
#         for p in 1:size(C, 1)
#             C[p, q]
#         end
#     end
#     return C
# end

# # function *(A::AbstractToeplitz, B::AbstractToeplitz)

# # end


struct Interval{T}
    a::T
    b::T
end
Interval(a::T,b::U) where {T,U} = Interval(promote(a,b)...)

import Base: ∪, ∩, \, ⊆, ∈, isempty, show, isless

isless(i₁::Interval,i₂::Interval) = i₁.a < i₂.a
adjacent(i₁::Interval,i₂::Interval) = i₁.b == i₂.a
isempty(i::Interval) = i.a >= i.b

⊂(i₁::Interval,i₂::Interval) = i₂.a < i₁.a && i₁.b < i₂.b
⊆(i₁::Interval,i₂::Interval) = i₂.a <= i₁.a && i₁.b <= i₂.b

# This imposes right-continuity, as de Boor uses.
∈(e::Number, i::Interval) = i.a < e && e ≤ i.b

function ∩(i₁::Interval,i₂::Interval)
    a = max(i₁.a,i₂.a)
    b = min(i₁.b,i₂.b)
    Interval(a,b)
end

function ∪(i₁::Interval,i₂::Interval)
    a = min(i₁.a,i₂.a)
    b = max(i₁.b,i₂.b)
    if !isempty(i₁ ∩ i₂) || adjacent(i₁,i₂) || adjacent(i₂,i₁)
        Interval(a,b)
    else
        Interval(1,0)
    end
end

function ∪(i₁::Vector{Interval{T}}, i₂::Interval{T}) where T
    sort!(vcat([i ∪ i₂ for i ∈ i₁]...))
    # Should merge possibly adjacent intervals
end

function \(i₁::Interval,i₂::Interval)
    ii = i₁ ∩ i₂
    ia = Interval(i₁.a,i₂.a)
    ib = Interval(i₂.b,i₁.b)
    if isempty(ii)
        [i₁],[i₂]
    elseif i₂ ⊆ i₁
        if isempty(ia) && isempty(ib)
            Interval[],Interval[]
        elseif isempty(ia)
            [ib],Interval[]
        elseif isempty(ib)
            [ia],Interval[]
        else
            [ia,ib],Interval[]
        end
    elseif i₁ ⊂ i₂
        Interval[],[Interval(i₂.a,i₁.a),Interval(i₁.b,i₂.b)]
    else # i₁ and i₂ partially overlap
        if i₁.a <= i₂.a
            isempty(ia) ? Interval[] : [ia],[Interval(i₁.b,i₂.b)]
        else
            isempty(ib) ? Interval[] : [ib],[Interval(i₂.a,i₁.a)]
        end
    end
end

function \(i₁::Vector{Interval}, i₂::Interval)
    is = Interval[]
    for i ∈ i₁
        i′ = i \ i₂
        !isempty(i′) && append!(is, i′)
    end
    is
end

function collapse!(is::Vector{Interval{T}}) where T
    isempty(is) && return is
    sort!(is)
    nis = Interval{T}[is[1]]
    i = 1
    for j = 2:length(is)
        ij = nis[i] ∪ is[j]
        if !isempty(ij)
            nis[i] = ij
        else
            push!(nis, is[j])
            i += 1
        end
    end
    is[1:length(nis)] = nis
    resize!(is, length(nis))
    is
end

function show(io::IO, i::Interval)
    if isempty(i)
        write(io, "Empty Interval")
    else
        write(io, "Interval[$(i.a),$(i.b)]")
    end
end

export Interval, ⊂


# module Arrays

export
    pad_matrix,
    cross_product_operator,
    one_hot,
    wrap_idx,
    wrap_get

"""
    helper to pad a matrix with some value around the edges
"""
function pad_matrix(mat::Matrix{T}, pad_size::Tuple{Int,Int},pad_val::T) where T
    new_size = [size(mat)...] + 2*[pad_size...]
    A = fill!(typeof(mat)(undef,new_size[1],new_size[2]),pad_val)
    A[
        pad_size[1]+1:pad_size[1]+size(mat,1),
        pad_size[2]+1:pad_size[2]+size(mat,2)
    ] .= mat
    A
end

cross_product_operator(x) = SMatrix{3,3}(
    [0.0    -x[3]   x[2];
     x[3]   0.0     -x[1];
     -x[2]  x[1]    0.0]
)

"""
    one_hot(::Type{T},n::Int,i::Int) where {T}

Returns a Vector{T} of length n such that `v[i] == 1` and `v[j] .== 0` for all
`j != i`.
"""
function one_hot(::Type{T},n::Int,i::Int) where {T}
    @assert (i > 0 && n > 0) "i and n must be positive integers"
    if i > n
        return one_hot(T,i,n)
    end
    v = zeros(T,n)
    v[i] = T(1)
    return v
end
one_hot(n::Int,i::Int) = one_hot(Float64,n,i)

"""
    wrap_idx(n,idx)

Wrap index to a one-dimensional array of length `n`
"""
wrap_idx(n,idx) = mod(idx-1,n)+1

"""
    wrap_get

Index into array `a` by first wrapping the indices `idx`.
"""
function wrap_get(a::A,idxs) where {R,N,A<:AbstractArray{R,N}}
    a[map(i->wrap_idx(size(a,i),idxs[i]),1:N)...]
end

# end


function test_uniform_grid_type(T, N, H)
  grid = UniformGrid(T, N, H)
  T == typeof(grid.H) && T == eltype(grid.zc) && T == eltype(grid.zf)
end

function test_uniform_grid_spacing(T, N, H)
  Δ = T(H/N)
  grid = UniformGrid(T, N, H)
  Δ == grid.Δc && Δ == grid.Δf
end

function test_uniform_grid_limits_zc_left(T, N, H)
  Δ = H/N
  grid = UniformGrid(T, N, H)
  grid.zc[1] ≈ -H+0.5Δ
end

function test_uniform_grid_limits_zc_right(T, N, H)
  Δ = H/N
  grid = UniformGrid(T, N, H)
  grid.zc[N] ≈ -0.5Δ
end

function test_uniform_grid_limits_zf_left(T, N, H)
  Δ = H/N
  grid = UniformGrid(T, N, H)
  grid.zf[1] ≈ -H
end

function test_uniform_grid_limits_zf_right(T, N, H)
  Δ = H/N
  grid = UniformGrid(T, N, H)
  grid.zf[N+1] ≈ 0
end

@testset "Grids" begin
    N, H = 3, 4.2
    for T in (Float64, Float32, Float16)
        @test test_uniform_grid_type(T, N, H)
        @test test_uniform_grid_spacing(T, N, H)
        @test test_uniform_grid_limits_zc_right(T, N, H)
        @test test_uniform_grid_limits_zf_right(T, N, H)
        @test test_uniform_grid_limits_zc_left(T, N, H)
        @test test_uniform_grid_limits_zf_left(T, N, H)
    end
end


# Snippet 09/clip-03.jl

using StatisticalRethinking

ProjDir = @__DIR__
cd(ProjDir)

# ### snippet 9.3

# Construct the logdensity problem

struct clip_9_3_model{TY <: AbstractVector, TX <: AbstractVector}
    "Observations."
    y::TY
    "Covariate"
    x::TX
end

# Make the type callable with the parameters *as a single argument*.

function (problem:: clip_9_3_model)(θ)
    @unpack y, x, = problem    # extract the data
    @unpack muy, mux = θ     # works on the named tuple too
    ll = 0.0
    ll += loglikelihood(Normal(mux, 1), x)
    ll += loglikelihood(Normal(muy, 1), y)
    ll += logpdf(Normal(0, 1), mux)
    ll += logpdf(Normal(0, 1), muy)
    ll
end

# Instantiate the model with data and inits.

Random.seed!(1234591)

N = 100
x = rand(Normal(0, 1), N)
y = rand(Normal(0, 1), N)

p = clip_9_3_model(y, x)
θ = (muy = 0.0, mux=0.0)
p(θ)

# Write a function to return properly dimensioned transformation.

problem_transformation(p::clip_9_3_model) =
    as((muy = asℝ, mux = asℝ))

# Wrap the problem with a transformation, then use Flux for the gradient.

P = TransformedLogDensity(problem_transformation(p), p)
∇P = ADgradient(:ForwardDiff, P);

# Generate and show fig 9.3

fig = StatisticalRethinking.generate_n_samples(p, ∇P);

# End of `09/clip-03d.jl`


using BayesianOptimization, GaussianProcesses, PGFPlotsX, Random
push!(PGFPlotsX.CUSTOM_PREAMBLE, "\\usepgfplotslibrary{fillbetween}")

Random.seed!(134)
f(x, noisevariance = 1) = .1*sum((x .- 2).^2) + cos(sum(π/2 * x)) + noisevariance * randn()
model = ElasticGPE(1, mean = MeanConst(0.),
                   kernel = SEArd([0.], 5.), logNoise = 0.)
modeloptimizer = MAPGPOptimizer(every = 50, noisebounds = [-2., 3],
                                kernbounds = [[-1, 0], [4, 10]], maxeval = 40)
opt = BOpt(f, model, ExpectedImprovement(),
           modeloptimizer, [-5.], [5.],
           maxiterations = 5, sense = Min, repetitions = 5,
           acquisitionoptions = (maxeval = 4000, restarts = 50),
           verbosity = Progress)
result = boptimize!(opt)

acqfunc = BayesianOptimization.acquisitionfunction(opt.acquisition, model)
xs = -5:.02:5
ms, var = predict_f(model, xs)
sig = sqrt.(var)
fmax, xmax = BayesianOptimization.acquire_max(opt.opt,
                        opt.lowerbounds, opt.upperbounds,
                        opt.acquisitionoptions.restarts)
@pgf GroupPlot({group_style = {group_size = "1 by 2", vertical_sep = "4mm"},
                height = "4cm", width = "8cm", legend_pos = "outer north east",
                legend_style = {draw = "none"}},
               {legend_columns = 1, xticklabels = ""},
             Plot({only_marks}, Coordinates(model.x[:], -model.y[:])),
             "\\addlegendentry{observations}",
             Plot({no_marks, "red!20", name_path = "A", forget_plot}, Coordinates(xs, -ms .+ sig)),
             Plot({no_marks, "red!20", name_path = "B", forget_plot}, Coordinates(xs, -ms .- sig)),
             "\\addplot[red!20] fill between [of=A and B];",
             "\\addlegendentry{model std}",
             Plot({no_marks, blue, very_thick}, Coordinates(xs, -ms)),
             "\\addlegendentry{model mean}",
             Plot({no_marks, "green!80!black", very_thick}, Coordinates(xs, f.(xs, 0))),
             "\\addlegendentry{noisefree target}",
             {height = "3cm", ytick=[0, .05, .1], yticklabels = [0, "0.05", "0.1"]},
             Plot({only_marks, red, mark="triangle*",
                   mark_size = "5pt", mark_options = {rotate = "0"}},
                  Coordinates(xmax, [fmax])),
             Plot({no_marks, very_thick}, Coordinates(xs, (x -> acqfunc([x])).(xs))),
             "\\addlegendentry{next acquisition}",
             "\\addlegendentry{acquisition function}")


n = rand(1:7)
@test matrixdepot("sampling", n) == matrixdepot("sampling", Float64, n)

A = matrixdepot("sampling", n)
v = sort(eigvals(A))
@test_approx_eq_eps v [0: n - 1;] 1e-10
println("'sampling' passed test...")


### A Pluto.jl notebook ###
# v0.12.4

using Markdown
using InteractiveUtils

# ╔═╡ ad01571d-3bc3-4d11-851d-5a00e477d6df
md"""
# Condicionales

En Julia, la sintaxis

```julia
if *condición 1*
    *opción 1*
elseif *condición 2*
    *opción 2*
else
    *opción 3*
end
```

Nos permite eventualmente evaluar una de nuestras opciones.

Por ejemplo, tal vez queremos implementar la prueba de FizzBuzz: Dado un número N, imprime "Fizz" si N es divisible entre 3, "Buzz" si N es divisible entre 5, y "FizzBuzz" si N es divisible entre ambos 3 y 5. En cualquier otro caso, imprimo el número mismo.
"""

# ╔═╡ 7dc5bb25-f3f3-4fdf-91f7-c6c5ce8c3934
N = 3

# ╔═╡ ec5720b1-de5c-4eba-aa37-9c2248eb9b01
if (N % 3 == 0) & (N % 5 == 0)
    println("FizzBuzz")
elseif N % 3 == 0
    println("Fizz")
elseif N % 5 == 0
    println("Buzz")
else
    println(N)
end

# ╔═╡ c0e38468-a793-4946-ab22-28b302c50c42
md"""
Ahora digamos que queremos regresar el mayor número de ambos. Escoge tus propios x y y
"""

# ╔═╡ 7f187a5d-561c-4010-9c8e-82fd27de0fed
x = 2

# ╔═╡ d4b84b42-193e-11eb-3a56-b5c792a3bf2e
y = 3

# ╔═╡ f3b473cf-fdbb-43ad-b5b6-54aa039f1ea0
if x > y
    x
else
    y
end

# ╔═╡ a1e393f1-fd92-43f0-b8f5-4f5a760fd690
md"""
Para el último bloque, podemos usar el operador ternario, con la sintaxis

```julia
a ? b : c
```

que equivale a

```julia
if a
    b
else
    c
end
```
"""

# ╔═╡ 9f672275-efe8-4474-b8a2-34502e3301e7
(x > y) ? x : y

# ╔═╡ 903a6dea-57ef-4141-9a93-830cb0f4cc2f
md"""
Un truco relacionado es la evaluación de corto-circuito

```julia
a && b
```
"""

# ╔═╡ 4d2656be-7402-413e-b178-145eb7d73209
(x > y) && println(x)

# ╔═╡ 768514c7-a170-4a9a-8c09-e614d86825d6
(x < y) && println(y)

# ╔═╡ ab52e37a-d650-45a9-a6a7-857d047533df
md"""
Cuando escribimos `a && b`, `b` se ejecuta sólo si `a` se evalúa a `true`.

Si `a` se evalúa a `false`, la expresión `a && b` regresa `false`
"""

# ╔═╡ e4a7bf0b-8ce3-4848-a657-382c71a196f4
md"""
### Ejercicios

5.1 Reescribe FizzBuzz sin usar `elseif`.
"""

# ╔═╡ 9555b539-88b9-434d-b60b-9593dfb76571

# ╔═╡ 27737899-7776-438b-b616-ee7bf2ed901d
md"""
5.2 Reescribe FizzBuzz usando el operador ternario.
"""

# ╔═╡ 0699b9c0-88f3-4bcd-a16c-cc1313a68a96

# ╔═╡ Cell order:
# ╟─ad01571d-3bc3-4d11-851d-5a00e477d6df
# ╠═7dc5bb25-f3f3-4fdf-91f7-c6c5ce8c3934
# ╠═ec5720b1-de5c-4eba-aa37-9c2248eb9b01
# ╟─c0e38468-a793-4946-ab22-28b302c50c42
# ╠═7f187a5d-561c-4010-9c8e-82fd27de0fed
# ╠═d4b84b42-193e-11eb-3a56-b5c792a3bf2e
# ╠═f3b473cf-fdbb-43ad-b5b6-54aa039f1ea0
# ╟─a1e393f1-fd92-43f0-b8f5-4f5a760fd690
# ╠═9f672275-efe8-4474-b8a2-34502e3301e7
# ╟─903a6dea-57ef-4141-9a93-830cb0f4cc2f
# ╠═4d2656be-7402-413e-b178-145eb7d73209
# ╠═768514c7-a170-4a9a-8c09-e614d86825d6
# ╟─ab52e37a-d650-45a9-a6a7-857d047533df
# ╟─e4a7bf0b-8ce3-4848-a657-382c71a196f4
# ╠═9555b539-88b9-434d-b60b-9593dfb76571
# ╟─27737899-7776-438b-b616-ee7bf2ed901d
# ╠═0699b9c0-88f3-4bcd-a16c-cc1313a68a96


"""
nsolsc(f,x0, fp=difffp; rtol=1.e-6, atol=1.e-12, maxit=10,
        solver="newton", sham=1, armmax=10, resdec=.1, dx=1.e-7,
        armfix=false, pdata=nothing,
        printerr=true, keepsolhist=true, stagnationok=false)

C. T. Kelley, 2021

Newton's method for scalar equations. Has most of the features a
code for systems of equations needs. This is a wrapper for a call
to nsol.jl, the real code for systems.

Input:\n
f: function\n
x0: initial iterate\n
fp: derivative. If your derivative function is fp, you give me
its name. For example fp=foobar tells me that foobar is your
function for the derivative. The default is a forward difference
Jacobian that I provide.\n

Keyword Arguments (kwargs):\n
rtol, atol: real and absolute error tolerances\n

maxit: upper bound on number of nonlinear iterations\n

solver:\n
Your choices are "newton"(default) or "chord". However,
you have sham at your disposal only if you chose newton. "chord"
will keep using the initial derivative until the iterate converges,
uses the iteration budget, or the line search fails. It is not the
same as sham=Inf, which is smarter.\n

If you use secant and your initial iterate is poor, you have made
a mistake. I will help you by driving the line search with a finite
difference derivative.\n

sham:\n
This is the Shamanskii method. If sham=1, you have Newton.
The iteration updates the derivative every sham iterations.
The convergence rate has local q-order sham+1 if you only count
iterations where you update the derivative. You need not
provide your own derivative function to use this option. sham=Inf
is chord only if chord is converging well.\n

armmax: upper bound on stepsize reductions in linesearch

resdec: target value for residual reduction. \n
The default value is .1. In the old MATLAB codes it was .5.
I only turn Shamanskii on if the residuals are decreasing
rapidly, at least a factor of resdec, and the line search is quiescent.
If you want to eliminate resdec from the method ( you don't ) then set
resdec = 1.0 and you will never hear from it again.

dx:\n
This is the increment for forward difference, default = 1.e-7.
dx should be roughly the square root of the noise in the function.

armfix:\n
The default is a parabolic line search (ie false). Set to true and
the stepsize will be fixed at .5. Don't do this unless you are doing
experiments for research.

pdata:\n
precomputed data for the function/derivative.
Things will go better if you use this rather than hide the data
in global variables within the module for your function/derivative
If you use this option your function and derivative must take pdata
as a second argument. eg f(x,pdata) and fp(x,pdata)

printerr:\n
I print a helpful message when the solver fails. To supress that
message set printerr to false.

keepsolhist:\n
Set this to true to get the history of the iteration in the output
tuple. This is on by default for scalar equations and off for systems.
Only turn it on if you have use for the data, which can get REALLY LARGE.

stagnationok:\n
Set this to true if you want to disable the line search and either
observe divergence or stagnation. This is only useful for research
or writing a book.

Output:\n
A named tuple (solution, functionval, history, stats, idid,
               errcode, solhist)
where

solution = converged result
functionval = F(solution)
history = the vector of residual norms (||F(x)||) for the iteration
stats = named tuple of the history of (ifun, ijac, iarm), the number
of functions/derivatives/steplength reductions at each iteration.

I do not count the function values for a finite-difference derivative
because they count toward a Jacobian evaluation. I do count them for
the secant method model.

idid=true if the iteration succeeded and false if not.

errcode = 0 if if the iteration succeeded
        = -1 if the initial iterate satisifies the termination criteria
        = 10 if no convergence after maxit iterations
        = 1  if the line search failed

solhist:\n
This is the entire history of the iteration if you've set
keepsolhist=true\n
nsolsc builds solhist with a function from the Tools directory. For
systems, solhist is an N x K array where N is the length of x and K
is the number of iteration + 1. So, for scalar equations (N=1), solhist
is a row vector. Hence the use of solhist' in the example below.

### Examples for nsolsc.jl

```jldoctest
julia> nsolout=nsolsc(atan,1.0;maxit=5,atol=1.e-12,rtol=1.e-12);

julia> nsolout.history
6-element Array{Float64,1}:
 7.85398e-01
 5.18669e-01
 1.16332e-01
 1.06102e-03
 7.96200e-10
 2.79173e-24
```

# If you have an analytic derivative, I will use it.

```jldoctest
julia> fs(x)=x^2-4.0; fsp(x)=2x;

julia> nsolout=nsolsc(fs,1.0,fsp; maxit=5,atol=1.e-9,rtol=1.e-9);

julia> [nsolout.solhist'.-2 nsolout.history]
6×2 Array{Float64,2}:
 -1.00000e+00  3.00000e+00
  5.00000e-01  2.25000e+00
  5.00000e-02  2.02500e-01
  6.09756e-04  2.43940e-03
  9.29223e-08  3.71689e-07
  2.22045e-15  8.88178e-15

```

"""
function nsolsc(
    f,
    x0,
    fp = difffp;
    rtol = 1.e-6,
    atol = 1.e-12,
    maxit = 10,
    solver = "newton",
    sham = 1,
    armmax = 5,
    resdec = 0.1,
    dx = 1.e-7,
    armfix = false,
    pdata = nothing,
    printerr = true,
    keepsolhist = true,
    stagnationok = false,
)
    #
    # The scalar code is a simple wrapper for the real code (nsol). The
    # wrapper puts placeholders for the memory allocations and the precomputed
    # data.
    #
    fp0 = copy(x0)
    fpp0 = copy(x0)
    newtonout = nsol(
        f,
        x0,
        fp0,
        fpp0,
        fp;
        rtol = rtol,
        atol = atol,
        maxit = maxit,
        solver = solver,
        sham = sham,
        armmax = armmax,
        resdec = resdec,
        dx = dx,
        armfix = armfix,
        pdata = pdata,
        printerr = printerr,
        keepsolhist = keepsolhist,
        stagnationok = stagnationok,
    )
    return newtonout
end


function newton_solve(f, g, H, x0; ftol=-Inf, gtol=1e-4, iter=100,
        lower=-Inf, upper=Inf, reg = 1e-3)
    x = copy(x0)
    X = [copy(x0)]
    for k = 1:iter
		fe = f(x)
		ge = g(x)
		He = H(x)
        He += reg * norm(He, Inf) * I
        ((norm(ge, Inf) < gtol) || (fe < ftol)) && break
        p = - He \ ge
        α = clamped_linesearch(f, x, p, lower=lower, upper=upper)
        x = clamp.(x + α*p, lower, upper)
        push!(X, copy(x))
        println("k:", k,
            "   f:", scn(fe, digits=3),
            "   ∇:", scn.(norm(ge, Inf)),
            "   α:", scn.(α),
            # "   g:", scn.(-g(x, rot=rot)),
            # "   p:", scn.(p),
            # "   x:", scn.(x),
            # "   H:", scn.(H),
            )
    end
    return x, X
end

function clamped_linesearch(f, x, p; iter=10,
        lower=-Inf, upper=Inf)
    α = 1.0
    fprev = f(x)
    for k = 1:iter
        xc = clamp.(x + α*p, lower, upper)
        (f(xc) <= fprev) && break
        α /= 2
    end
    return α
end

################################################################################
# Generate & Save Dataset
################################################################################
generate_hardware_dataset(N=50, sleep_ratio=0.01)

################################################################################
# Load Dataset
################################################################################
params0, trajs0, pairs0 = open_dataset(:hardwarebox; N=50)

pairs0
params0
data0 = params0[:data]

function quad_loss(x, timestep, x0, v0)
	loss = 0.0
	for k = 1:7
		t = (k-1) * timestep
		Δx = x[k][3] - (x0 + v0*t - t^2/2 * 9.81)
		loss += Δx^2
	end
	return loss
end

function pairs_loss(pairs, θ)
	loss = 0.0
	for i = 1:50
		ps = pairs[(i-1)*6 .+ (1:6)]
		x = [[ps[1][1][1:3]]; [p[2][1:3] for p in ps]]
		loss += quad_loss(x, θ[1], θ[1 + (i-1)*2 .+ (1:2)]...)
	end
	return loss / 50
end

function pairs_grad(pairs, θ)
	grad = FiniteDiff.finite_difference_gradient(θ -> pairs_loss(pairs, θ), θ)
	return grad
end

function pairs_hess(pairs, θ)
	hess = FiniteDiff.finite_difference_hessian(θ -> pairs_loss(pairs, θ), θ)
	return hess
end

θ0 = ones(101)
pairs_loss(pairs0, θ0)
pairs_grad(pairs0, θ0)
plot(Gray.(abs.(pairs_hess(pairs0, θ0))))

f00(θ) = pairs_loss(pairs0, θ)
g00(θ) = pairs_grad(pairs0, θ)
H00(θ) = pairs_hess(pairs0, θ)

θsol, θSOL = newton_solve(f00, g00, H00, θ0, gtol=1e-6, reg = 0.)
θsol


# Copyright (c) 2020 Idiap Research Institute, http://www.idiap.ch/
#  Niccolò Antonello <nantonel@idiap.ch>

export logbackward!

"""
`logbackward!(logbeta,Nt,A,y)`

In-place version of [`logbackward`](@ref).
"""
function logbackward!(beta::AbstractMatrix{T},
                      Nt::Integer,
                      A::AbstractMatrix{T},
                      y::AbstractMatrix{T}) where {T<:AbstractFloat}
  Ns = size(A,1)
  fill!(beta, -T(Inf))
  for i = 1:Ns
    beta[Nt,i] = zero(T)
  end
  for t = Nt-1:-1:1
    for j = 1:Ns
      for k = 1:Ns
        ajk = A[j,k]
        if ajk != -T(Inf)
          beta[t,j] = logadd(beta[t,j], beta[t+1,k]+ajk+y[t+1,k])
        end
      end
    end
  end
  return beta
end

# backward with constrained path
function logbackward!(beta::Matrix{T},
                      Nt::Integer,
                      A::AbstractMatrix{T},
                      t2IJ::Vector{Pair{Vector{D},Vector{D}}},
                      y::AbstractMatrix{T}) where {D<:Integer,T<:AbstractFloat}
  Ns = size(A,1)
  fill!(beta, -T(Inf))
  for i = 1:Ns
    beta[Nt,i] = zero(T)
  end

  for t = Nt-1:-1:1
    I,J = t2IJ[t]
    for k in eachindex(I)
      i,j = I[k],J[k]
      beta[t,i] = logadd(beta[t,i], beta[t+1,j]+A[i,j]+y[t+1,j])
    end
  end
  return beta
end

export logbackward

"""
`logbackward(Nt,A,y)`

Computes the backward log-probabilities (``\\hat{\\beta}``) of an Hidden Markov Model (HMM) of `Ns` states with log-transition matrix `A` (must be a `Ns` × `Ns` matrix), scale coefficients and observation log-likelihoods `y` (must be of size `Nt2` × `Ns` with `Nt2` ≥ `Nt`).
"""
function logbackward(Nt::Integer,A::AbstractMatrix{T},y::AbstractMatrix{T}) where {T <: AbstractFloat}
  Ns = size(A,1)
  beta = zeros(T,Nt,Ns)
  logbackward!(beta,Nt,A,y)
end

"""
`logbackward(Nt,A,t2tr,y)`

Computes the backward log-probabilities with constrained path indicated by the `t2tr` vector. See [`forward`](@ref) for a description of the structure and meaning of `t2tr`.
"""
function logbackward(Nt::Integer,
                     A::AbstractMatrix{T},
                     t2tr,
                     y::AbstractMatrix{T}) where {T <: AbstractFloat, D <: Integer}
  Ns = size(A,1)
  beta = zeros(T,Nt,Ns)
  logbackward!(beta,Nt,A,t2tr,y)
end


import Calculus:differentiate,jacobian,simplify,SymbolParameter,isminus

function simplify(::SymbolParameter{:-}, args)
	if length(args) == 2 && args[2]==0
		return args[1]
	end
	# Remove any 0's in a subtraction
    args = map(simplify, filter(x -> x != 0, args))
    if length(args) == 0
        return 0
    # Special Case: simplify(:(x - x)) == 0
    elseif length(args) == 2 && args[1] == args[2]
        return 0
    # Special Case: simplify(:(x - (-y))) == x + y
    elseif length(args) == 2 && isminus(args[2])
        return Expr(:call, :+, args[1], args[2].args[2])
    else
	        return Expr(:call, :-, args...)
    end
end

function differentiate(ex::Expr,wrt::Expr)
	if ex.head==:vect || ex.head ==:vcat
		return differentiate(SymbolParameter(:vect), ex.args[1:end], wrt)
	elseif ex.head == :ref
        return ex==wrt ? 1 : 0
    elseif ex.head != :call
		error("Unrecognized expression $ex")
	end
    return simplify(differentiate(SymbolParameter(ex.args[1]), ex.args[2:end], wrt))
end

function differentiate(ex::Expr, targets::Vector{Expr})
	n = length(targets)
	exprs = Array(Any, n)
	for i in 1:n
		exprs[i] = differentiate(ex, targets[i])
	end
	return exprs
end

differentiate(ex::Union{Number,Symbol},wrt::Expr) = 0

function differentiate(::SymbolParameter{:vect}, args, wrt)
	for i = 1:length(args)
		args[i] = differentiate(args[i],wrt)
	end
	return Expr(:vect,args...)
end

function jacobian(ex::Expr, targets::Vector)
    @assert ex.head==:vect || ex.head==:vcat
    exprs = Expr(:vcat)
    for i = 1:length(ex.args)
        push!(exprs.args,Expr(:row))
        for j = 1:length(targets)
            push!(exprs.args[i].args,simplify(differentiate(ex.args[i],targets[j])))
        end
    end
    return exprs
end

differentiate(::SymbolParameter{:max}, args, wrt) = Expr(:if,:($(args[1])>$(args[2])),differentiate(args[1],wrt),differentiate(args[2],wrt))
differentiate(::SymbolParameter{:min}, args, wrt) = Expr(:if,:($(args[1])<$(args[2])),differentiate(args[1],wrt),differentiate(args[2],wrt))

function simplify(ex::Expr)
	if in(ex.head,[:vcat,:row])
		for i = 1:length(ex.args)
			ex.args[i]=simplify(ex.args[i])
		end
		return ex
	end

	if ex.head != :call
        return ex
    end

    if all(Calculus.isnumber, ex.args[2:end]) && length(ex.args) > 1
        return eval(current_module(), ex)
    end
    new_ex = simplify(SymbolParameter(ex.args[1]), ex.args[2:end])
    while new_ex != ex
        new_ex, ex = simplify(new_ex), new_ex
    end
    return new_ex
end


"""
    unreal(::T) where T <: Real

For real numbers, return an empty array of type `T`.
"""
function unreal(::T) where T <: Real
    T[]
end

"""
    iszerodivisor(a::Real) -> Bool

For real numbers, return `true` if `a` is zero. Equivalent to `iszero(a)`.
"""
function iszerodivisor(a::Real)
    iszero(a)
end

"""
    cloak(a::Real)

For real numbers, equivalent to the minus operation.
"""
function cloak(a::Real)
    -(a)
end

"""
    dagger(a::Real)

For real numbers, equivalent to the identity operation.
"""
function dagger(a::Real)
    a
end

"""
    star(a::Real)

For real numbers, the identity operation.
"""
function star(a::Real)
    a
end

"""
    quadrance(a::Real)

For real numbers, the squared absolute value.
"""
function quadrance(a::Real)
    abs2(a)
end