# include("../utilities.jl")

module linalg

using LinearAlgebra # for triu/l
# using ..util

# =========================================================================== #
# Level 1 Vector Vector
# =========================================================================== #

# scalar + vector
# scalar * vector
# elementwise vector vector
# vector + scalar * vector
# norm2
# dot prod

function blas_update_add!(a, x)
    # x <- a + x
    @inbounds @simd for ii = eachindex(x)
        @fastmath x[ii] += a
    end

    return x
end

function blas_add!(y, a, x)
    # x <- a + x
    @inbounds @simd for ii = eachindex(x)
        @fastmath y[ii] = x[ii] + a
    end

    return x
end

function blas_update_scale!(a, x)
    # x <- a * x
    @inbounds @simd for ii = eachindex(x)
        @fastmath x[ii] *= a
    end

    return x
end

function blas_scale!(y, a, x)
    # y <- a * x
    @inbounds @simd for ii = eachindex(x)
        @fastmath y[ii] = x[ii] * a
    end

    return y
end

function blas_elementwise_scale!(y, x)
    # y <- yi * xi
    @inbounds @simd for ii = eachindex(y)
        @fastmath y[ii] *= x[ii]
    end
end

function blas_update_scale_add!(y, a, x)
    # y <- y + a * x
    @inbounds @simd for ii = eachindex(y) #
    # Threads.@threads for ii = eachindex(y)
        @fastmath y[ii] += x[ii] * a # @fastmath
    end

    return x
end

function blas_norm2(x)
    s = zero(Float64);

    @inbounds @simd for ii = eachindex(x)
        @fastmath s += x[ii] * x[ii]
    end

    return sqrt(s)
end

function blas_dot_prod(x, y)
    d = zero(Float64)
    @inbounds @simd for ii = eachindex(x)
        @fastmath d += x[ii] * y[ii]
    end
    return d
end

# =========================================================================== #
# Level 2 - Matrix Vector
# =========================================================================== #

# matrix vector prod
# M + a * v * v' rank one update
# M + a * u * v' rank two update

function blas_matrixview_update_scale_add!(y, a, mat, col)
    # y <- y + a * mat[:, col]
    @inbounds @simd for ii = 1:size(mat,1)
        @fastmath y[ii] += mat[ii, col] * a
    end

    return y
end

function blas_matrix_vector_prod(mat, x)
    # y <- A * x

    (n_row, n_col) = size(mat);

    y = zeros(n_row)
    @inbounds for cc = 1:n_col
		y = blas_matrixview_update_scale_add!(y, x[cc], mat, cc)
    end

	return y;

end

# =========================================================================== #
# Level 3 - Matrix Matrix
# =========================================================================== #

# matrix matrix prod

function blas_matrix_matrix_prod(A, B)
    # C <- A * B

    (n_row, n_inner) = size(A);
    n_col = size(B,2);

    C = zeros(n_row, n_col)
    @inbounds for cc = 1:n_col
        # y = zeros(n_row)
        @inbounds for icc = 1:n_inner
            # C[:, cc] = blas_matrixview_update_scale_add!(C[:, cc], B[icc, cc], A, icc)
            # y = blas_matrixview_update_scale_add!(y, B[icc, cc], A, icc)
            blas_matrixview_update_scale_add!( @view(C[:, cc]) , B[icc, cc], A, icc)
        end
        # C[:,cc] = y
    end

	return C;

end

#=
function saxpy_matrixview(scalar, mat, col, y)
    # y <- y + scalar * mat[:, col]
	# unsafe, dimensions are not checked
	@inbounds @simd for ii = 1:size(y, 1) #
		@fastmath y[ii] += scalar * mat[ii, col];
        # y[ii] += scalar * mat[ii, col];
    end
    return y
end

function matrix_vector_prod(mat, vec)

	n_row = size(mat, 1);
	n_col = size(mat, 2);

    #=
	res = Vector{eltype(vec)}(undef, n_row);
	@inbounds @simd for rr = 1:n_row
		@fastmath res[rr] = mat[rr, 1] * vec[1];
    end

	# @inbounds Threads.@threads for cc = 1:n_col
    @inbounds for cc = 2:n_col
		res = saxpy_matrixview(vec[cc], mat, cc, res);
    end

    =#

    res = zeros(n_row)
    @inbounds for cc = 1:n_col
		res = saxpy_matrixview(vec[cc], mat, cc, res);
    end

	return res;
end
=#

end # linalg

module ltest

using ..linalg
using LinearAlgebra
using BenchmarkTools

function test_blas_update_scale_add!()
    n = 10
    y = zeros(n)
    x = collect(1:n)
    a = 10

    n = 100_000_000 # 1_000_000_000 -> doesnt fit into the memory!!!
    y = rand(n)
    x = rand(n)
    a = rand()

    # y_theo = x .* a .+ y

    @time linalg.blas_update_scale_add!(y, a, x)

    # println( all( (y_theo - y) .<= 1e-16 ) )

end

function benchmark_blas_update_scale_add!(y, a, x)

    b_ = @benchmark linalg.blas_update_scale_add!($y, $a, $x)

    println("=== blas_update_scale_add! ===")
    show(
            stdout,
            MIME("text/plain"),
            b_ )
    println("\n\n")
end

function benchmark_blas_update_scale_add_default(y, a, x)
    # b_ = @benchmark $y = $y .+ $a .* $x;
    b_ = @benchmark $y .+= $a .* $x;

    println("=== blas_update_scale_add_default ===")
    show(
            stdout,
            MIME("text/plain"),
            b_ )
    println("\n\n")
end

function benchmark_blas_update_scale_add_all()
    n = 1_000_000 # for 10_000_000 ~10ms
    y = rand(n)
    x = rand(n)
    a = rand()

    benchmark_blas_update_scale_add!(y, a, x)
    benchmark_blas_update_scale_add_default(y, a, x)
end

function test_matrix_vector_prod()

    mat = reshape( collect(0:14), (5,3) )
    vec = collect(1:3)

    x_theo = mat * vec

    x_mine = linalg.blas_matrix_vector_prod(mat, vec)

    println("norm dx: $(norm(x_theo - x_mine))" )

end

function benchmark_matrix_vector_prod()

    n_row = 1000 # 100000;
	n_col = 100000 # 1000;

    mat = rand(n_row, n_col)
    vec = rand(n_col)

    b_ = @benchmark linalg.blas_matrix_vector_prod($mat, $vec);

    println("=== matrix_vector_prod ===")
    show(
            stdout,
            MIME("text/plain"),
            b_ )
    println("\n\n")

    b_ = @benchmark $mat * $vec;

    println("=== matrix_vector_prod_default ===")
    show(
            stdout,
            MIME("text/plain"),
            b_ )
    println("\n\n")
end

function test_blas_matrix_matrix_prod()

    n_row = 100
    n_inner = 150000
    n_col = 500
    A = rand(n_row, n_inner)
    B = rand(n_inner, n_col)

    @time C = linalg.blas_matrix_matrix_prod(A, B)

    @time C_theo = A * B;

    dC = C_theo - C

    println( all( dC .<= 1e-10 )  )

end

function benchmark_blas_matrix_matrix_prod()

    linalg.blas_matrix_matrix_prod(A, B)

end

#=

function benchmark_linsolve_all()
    N = 6_00
    A = rand(N, N)
    A = diagm( diag(A) ) .+ A
    A = A .+ A'
    b = rand(N)

    benchmark_linsolve_lu(A, b)
    benchmark_linsolve_ldl(A, b)
    benchmark_linsolve_default(A, b)

end

function benchmark_linsolve_default(A, b)
    b_ = @benchmark $A \ $b

    println("=== benchmark_linsolve_default ===")
    show(
            stdout,
            MIME("text/plain"),
            b_ )
    println("\n\n")
end
=#

end # ltest


#!/usr/bin/env julia
# -*- coding: utf-8 -*-
cd(dirname(@__FILE__))
include("../HANTS.jl")
using .HANTS, Plots

y = [
    0.055952, 0.037081, 0.062657, 0.120110, 0.178219, 0.244443, 0.213190, 0.494648,
    0.328767, 0.561776, 0.380300, 0.641233, 0.532486, 0.827757, 0.654052, 0.816695,
    0.611424, 0.661557, 0.398067, 0.339881, 0.113957, 0.086790, 0.058600
]

nbase = 365
nfreq = 3
tseries = 1:16:365
validrange = (-1, 1)
fet = 0.02
dod = 1
δ = 0.1

A, φ, yrec = hants(
    y, fet, dod, δ; nbase=nbase, nfreq=nfreq, validrange=validrange, tseries=tseries, outlier=nothing
)
y_reconstruct = reconstruct(A, φ, nbase)

A_Hi, φ_Hi, yrec_Hi = hants(
    y, fet, dod, δ; nbase=nbase, nfreq=nfreq, validrange=validrange, tseries=tseries, outlier=:Hi
)
y_reconstruct_Hi = reconstruct(A_Hi, φ_Hi, nbase)

A_Lo, φ_Lo, yrec_Lo = hants(
    y, fet, dod, δ; nbase=nbase, nfreq=nfreq, validrange=validrange, tseries=tseries, outlier=:Lo
)
y_reconstruct_Lo = reconstruct(A_Lo, φ_Lo, nbase)

fig1 = plot(tseries, y, lc=:black, label="Original Data", shape=:circle, mc=:black)
plot!(tseries, yrec, lc=:blue, label="HANTS - None", shape=:circle, mc=:blue)
plot!(tseries, yrec_Hi, lc=:red, label="HANTS - Hi", shape=:circle, mc=:red)
plot!(tseries, yrec_Lo, lc=:green, label="HANTS - Lo", shape=:circle, mc=:green)
title!("Testing HANTS Algorithm")

fig2 = plot(tseries, y, lc=:black, label="Original Data", shape=:circle, mc=:black)
plot!(1:365, y_reconstruct, lc=:blue, label="HANTS - None")
plot!(1:365, y_reconstruct_Hi, lc=:red, label="HANTS - Hi")
plot!(1:365, y_reconstruct_Lo, lc=:green, label="HANTS - Lo")
title!("Testing HANTS Reconstruct")

plot(fig1, fig2; layout=(1, 2), size=(800,300))


struct extended_function
    fmin::Function
    spline::Function
    fmax::Function
    xmin::Float64
    xmax::Float64
end

function extend_func(k, Pk, kmin, kmax) # Needs serious modification, Log/Lin??, slope eval != kmin/kmax
    filt = (k .>= kmin) .&  (k .<= kmax) .& (Pk .> 0)
    spl = Spline1D(log.(k[filt]), log.(Pk[filt]), k=2)
    splf = x -> exp(spl(log(x)))

    smin = derivative(spl, log(kmin))
    smax = derivative(spl, log(kmax))

    fmin = x -> splf(kmin) * (x/kmin)^smin
    fmax = x -> splf(kmax) * (x/kmax)^smax

    return extended_function(fmin, splf, fmax, kmin, kmax)
end

function eval_extend_func(Pkfunc, x)
    if x < Pkfunc.xmin
        Pkfunc.fmin(x)
    elseif x < Pkfunc.xmax
        Pkfunc.spline(x)
    else
        Pkfunc.fmax(x)
    end
end

(pk::extended_function)(x) = eval_extend_func(pk,x)


# Visualization of the interference of two point sources in 2D

using PyPlot

const L = 6
const sep = 2
const N = 500
const k = 5

V = linspace(-L, L, N)
X = V
Y = X'
#X, Y = meshgrid(V, V)
R1 = sqrt( (X.-sep).^2 .+ Y.^2)
R2 = sqrt( (X.+sep).^2 .+ Y.^2)

# Sum of Green's functions for two point sources
Z = exp(1im*k*R1)./R1 + exp(1im*k*R2)./R2

M = 10
T = linspace(0.0, 2*pi, M)
T = T[1:(M-1)]
cut = 0.8
scale = 55/(2*cut)

for p=1:1
  for iter=1:length(T)

    figure(1)
    clf()
    hold()

    W = real(Z*exp(-1im * T[iter]))
    W = max(W, -cut)
    W = min(W, cut)

    imshow(scale*(W+cut), vmin=nothing, vmax=nothing, aspect="equal")

    filename = "Frame$(1000+iter)"
    print("Saving to $(filename)")
    savefig(filename)
  end
 end

 # convert to gif
 run(`convert -density 100 -loop 1000 -delay 10 Frame100* interference_two_sources.gif`)


"""
Banzhaf (1965) value
"""
function fᵦⁱ(G::Game, i::Int)
    N = length(G.N)

    banzhaf = ∑(
        G.v(S) - G.v(setdiff(S, i))
        for S in subsets(G.N)
            if i ∈ S
    )

    return banzhaf ./ (2^N - 1)
end


julia> expr="2 * (3 -1) + 2 * 5"
"2 * (3 -1) + 2 * 5"

julia> parsed = parse(expr) #Julia provides low-level access to language parser for AST/Expr creation
:(+(*(2,-(3,1)),*(2,5)))

julia> t = typeof(parsed)
Expr

julia> names(t) #shows type fields
(:head,:args,:typ)

julia> parsed.args #Inspect our 'Expr' type innards
3-element Any Array:
 :+
 :(*(2,-(3,1)))
 :(*(2,5))

julia> typeof(parsed.args[2]) #'Expr' types can nest
Expr

julia> parsed.args[2].args
3-element Any Array:
  :*
 2
  :(-(3,1))

julia> parsed.args[2].args[3].args #Will nest until lowest level of AST
3-element Any Array:
  :-
 3
 1

julia> eval(parsed)
14

julia> eval(parse("1 - 5 * 2 / 20 + 1"))
1.5

julia> eval(parse("2 * (3 + ((5) / (7 - 11)))"))
3.5


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .jl
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.5.0
#   kernelspec:
#     display_name: Julia 1.4.2
#     language: julia
#     name: julia-1.4
# ---

using Plots
using Distributions
using LinearAlgebra
using ForwardDiff

# +
struct Point{T<:Real}
    x::T
    y::T
    z::T
end

Point(xyz::Vector{T}) where {T} = Point{T}(T.(xyz)...)

# +
struct Tp
    zfunc::Function
    gx::Float64
    gy::Float64
    p::Point{Float64}
end

function Tp(zfunc, p_xy)
    zfuncwrap(xy) = zfunc(xy...)
    g = xy -> ForwardDiff.gradient(zfuncwrap, xy)
    ∇g_x, ∇g_y = g(p_xy)
    Tp(zfunc, ∇g_x, ∇g_y, Point([p_xy..., zfuncwrap(p_xy)]))
end

function tangent_plane(tp::Tp, x, y)
    z = tp.gx * (x-tp.p.x) + tp.gy * (y-tp.p.y)+ tp.p.z
    return z
end
# -

zfunc(x, y) = -x^2-y^2+1.
tp = Tp(zfunc, [3.0, 1.0])

function visualize(tp::Tp; camera = (0, 30))
    tp_offset=0.5
    offset=1.0
    tp_xs = range(tp.p.x - tp_offset, tp.p.x + tp_offset, length = 10)
    tp_ys = range(tp.p.y - tp_offset, tp.p.y + tp_offset, length = 10)
    xs = range(tp.p.x - offset, tp.p.x + offset, length = 20)
    ys = range(tp.p.y - offset, tp.p.y + offset, length = 20)
    p = plot(xlabel="x", ylabel="y")
    wireframe!(p, xs, ys, (x, y) -> tp.zfunc(x, y), camera = camera)
    surface!(p, tp_xs, tp_ys, (x, y) -> tangent_plane(tp, x, y), camera = camera)
    scatter3d!(p, [tp.p.x],[tp.p.y],[tp.p.z], label="p")
    return p
end

ps=[]
for θ in 1:90
    IJulia.clear_output(true)
    p = visualize(tp, camera = (θ, 30))
    p |> display
    push!(ps, p)
end

@gif for p in ps
    plot(p)
end


# Baumgartner-Weis-Schindler statistic and hypothesis testing.
# Copyright (c) Robert Feldt 2013 (robert.feldt@gmail.com)
#
# For license details see the LICENSE.md file in the top dir of the repository
# where you found this file.
#
# The algorithms in this file are based on the definitions of the BWS in the paper:
#
#  Markus Neuhäuser, "Exact tests based on the Baumgartner-WeiB-Schindler statistic - A survey",
#  Statistical Papers, vol. 46, pp. 1-30, Springer Verlag, 2005.
#

export bws_statistic, bws_test_sampled

#############################################################################
## Part I. Calculating the BWS.
#############################################################################

# Calculate the BWS statistic from two samples of values.
function bws_statistic(x, y)
  # Get the ranks. The first 1:n are ranks for x, (n+1):end for y.
  rs = ranks(vcat(x, y))

  FeldtLib.bws_statistic_from_ranks(rs, length(x), length(y))
end

# Test if according to the Baumgartner-Weis-Schindler (BWS) statistic we
# can reject the hypothesis that the two set of samples x and y are likely to have
# the same location (mean).
function bws_test_sampled(x, y, numsamples = 10000)
  # Get the ranks. The first 1:n are ranks for x, (n+1):(n+m) for y.
  rs = ranks(vcat(x, y))

  FeldtLib.bws_test_sampled_from_ranks(rs, length(x), length(y), numsamples)
end

# Calculate the BWS statistic from a set of ranks of n values from one sample
# and m values from another sample. The first 1:n rank values in ranks are for
# the first sample and the rest are from the other.
function bws_statistic_from_ranks(ranks, n, m)
  # Pre-calc some values needed in both b calcs.
  nm = n+m
  np1 = n + 1
  mp1 = m + 1

  # Now calc the two b values
  b_x = (1/n) * (n / (m*nm)) * calc_bws_sum(ranks,   1,  n, n, np1, nm, 0)
  b_y = (1/m) * (m / (n*nm)) * calc_bws_sum(ranks, np1, nm, m, mp1, nm, n)

  (b_x + b_y) / 2
end

function calc_bws_sum(ranks, mini, maxi, n, np1, nm, subi = 0)
  sum = 0.0
  for(i in mini:maxi)
    k = i - subi
    nom = ( ranks[i] - (nm/n*k) )^2
    denom = (k/np1) * (1 - (k/np1))
    sum += (nom / denom)
  end
  sum
end

function bws_test_sampled_from_ranks(ranks, n, m, numsamples = 10000)
  rs = copy(ranks) # Copy since we will shuffle it below.

  actual_b = FeldtLib.bws_statistic_from_ranks(rs, n, m)

  # Array to save the b statistics
  samples = Array(Float64, numsamples)
  numlarger = 0

  # Set up for inline sorts. Not sure if this really speeds things up...
  ns = Array(Int64, n)
  ms = Array(Int64, m)
  np1 = n+1

  for(i in 1:numsamples)
    # Now shuffle the ranks and then ensure they are sorted for each group.
    # Not sure this is actually faster than letting julia decide on the copying
    # and gc of intermediate vectors/arrays... ;)
    shuffle!(rs)
    ns[1:n] = rs[1:n]
    ms[1:m] = rs[np1:end]
    sort!(ns)
    sort!(ms)
    rs[1:n] = ns
    rs[np1:end] = ms

    # Now we can calc the bws again and note if it was larger than before.
    samples[i] = FeldtLib.bws_statistic_from_ranks(rs, n, m)
    if samples[i] >= actual_b
      numlarger += 1
    end
  end

  # A larger (or equal) BWS statistic value indicates samples from distributions
  # that are "more apart" and thus that there is a difference between the samples.
  pvalue = numlarger / numsamples

  (actual_b, pvalue, numlarger, numsamples) # , mean(samples), std(samples))
end

#############################################################################
## Part II. Adding the same interface as used by hypothesis tests in
##          HypothesisTests.jl
#############################################################################
using HypothesisTests

type BWSTest <: HypothesisTests.HypothesisTest
  # Ranks of all values where first 1:n are from one sample and (n+1):end are
  # from the other.
  ranks::Array{Int64,1}
  n::Integer
  m::Integer

  b::Float64 # The BWS statistic itself

  # We save the numlarger and numsamples taken instead of the p-value since
  # we can then add additional iterations later if needed.
  numlarger::Integer
  numsamples::Integer
end
function BWSTest{S <: Number}(x::Vector{S}, y::Vector{S}, numsamples = 10000)
  rs = ranks(vcat(x, y))
  n = length(x)
  m = length(y)
  b, pvalue, numlarger, numsamples = bws_test_sampled_from_ranks(rs, n, m)
  BWSTest(rs, n, m, b, numlarger, numsamples)
end

function pvalue(bwst::BWSTest)
  bwst.numlarger / bwst.numsamples
end


using GridArrays
using Test, LinearAlgebra, DomainSets, Plots
using GridArrays: MaskedGrid, IndexSubGrid, randompoint, element, elements, cartesianproduct, iscomposite

function delimit(s::AbstractString)
    println()
    println("## ",s)
end
interval_grids = (EquispacedGrid, PeriodicEquispacedGrid, MidpointEquispacedGrid, ChebyshevNodes, ChebyshevExtremae, FourierGrid,
                    ChebyshevUNodes, LegendreNodes)
types = (Float64,BigFloat)

function test_grids(T)
    ## Equispaced grids
    len = 21
    a = -T(1.2)
    b = T(3.5)
    g1 = EquispacedGrid(len, Interval(a, b))
    g2 = PeriodicEquispacedGrid(len, Interval(a-1, b+1))
    g3 = g1 × g2
    g4 = g1 × g3

    test_generic_grid(g1)
    test_generic_grid(g2)
    test_generic_grid(g3)
    test_generic_grid(g4)

    # Test a subgrid
    g5 = g1[10:20]
    @test g5[1] == g1[10]
    @test g5[11] == g1[20]
    @test length(g5) == 20-10+1
    test_generic_grid(g5)
    g6 = g1[10:2:20]
    @test g6[2] == g1[12]
    @test length(g6) == 6

    g = EquispacedGrid(len, a, b)
    idx = 5
    @test g[idx] ≈ a + (idx-1) * (b-a)/(len-1)
    @test g[len] ≈ b
    @test_throws BoundsError g[len+1] == b

    # Test iterations
    (l,s) = grid_iterator1(g)
    @assert s ≈ len * (a+b)/2

    ## Periodic equispaced grids
    len = 120
    a = -T(1.2)
    b = T(3.5)
    g = PeriodicEquispacedGrid(len, a, b)

    idx = 5
    @test g[idx] ≈ a + (idx-1) * (b-a)/len
    @test g[len] ≈ b - step(g)
    @test_throws BoundsError g[len+1] == b

    (l,s) = grid_iterator1(g)
    @test s ≈ (len-1)*(a+b)/2 + a

    (l,s) = grid_iterator2(g)
    @test s ≈ (len-1)*(a+b)/2 + a

    ## Tensor product grids
    len = 11
    g1 = PeriodicEquispacedGrid(len, -one(T), one(T))
    g = g1^2
    @test isperiodic(g1)
    @test isperiodic(g)
    @test size(g) == (length(g1),length(g1))
    @test cartesianproduct(g1) ==g1

    g2 = EquispacedGrid(len, -one(T), one(T))
    @test !isperiodic(g2)
    g = g1 × g2
    @test !isperiodic(g)
    @test length(g) == length(g1) * length(g2)
    @test size(g) == (length(g1),length(g2))
    @test size(g,1) == length(g1)

    @test element(g, 1) == g1
    @test element(g, 2) == g2
    @test element(g,1:2) == g
    @test !iscomposite(g1)&& !iscomposite(g1)
    @test iscomposite(g)
    @test support(g) ≈ support(g1)×support(g2)

    idx1 = 5
    idx2 = 9
    x1 = g1[idx1]
    x2 = g2[idx2]
    x = g[idx1,idx2]
    @test x[1] ≈ x1
    @test x[2] ≈ x2

    (l,s) = grid_iterator1(g)
    @test s ≈ -len

    (l,s) = grid_iterator2(g)
    @test s ≈ -len

    # Test a tensor of a tensor
    g3 = g × g2
    idx1 = 5
    idx2 = 7
    idx3 = 4
    x = g3[idx1,idx2,idx3]
    x1 = g1[idx1]
    x2 = g2[idx2]
    x3 = g2[idx3]
    @test x[1] ≈ x1
    @test x[2] ≈ x2
    @test x[3] ≈ x3

    # Test a mapped grid
    m = interval_map(T(0), T(1), T(2), T(3))
    # Make a MappedGrid by hand because mapped_grid would simplify
    mg1 = MappedGrid(PeriodicEquispacedGrid(30, T(0), T(1)), m)
    test_generic_grid(mg1)
    # Does mapped_grid simplify?
    mg2 = mapped_grid(PeriodicEquispacedGrid(30, T(0), T(1)), m)
    @test typeof(mg2) <: PeriodicEquispacedGrid
    @test infimum(support(mg2)) ≈ T(2)
    @test supremum(support(mg2)) ≈ T(3)

    # Apply a second map and check whether everything simplified
    m2 = interval_map(T(2), T(3), T(4), T(5))
    mg3 = mapped_grid(mg1, m2)
    @test infimum(support(mg3)) ≈ T(4)
    @test supremum(support(mg3)) ≈ T(5)
    @test typeof(supergrid(mg3)) <: PeriodicEquispacedGrid

    # Scattered grid
    pts = rand(T, 10)
    sg = ScatteredGrid(pts)
    test_generic_grid(sg)
end

function test_laguerre(T)
    grid = LaguerreNodes(10,rand(T))
    @test infimum(support(LaguerreNodes(0.,rand(10)))) == 0
    @test supremum(support(LaguerreNodes(0.,rand(10)))) == Inf
    test_generic_grid(grid)
end

function test_hermite(T)
    grid = HermiteNodes(10)
    @test DomainSets.GeometricSpace{T}()== support(HermiteNodes(rand(T,10)))
    test_generic_grid(grid)
end

function test_jacobi(T)
    grid = JacobiNodes(10,rand(T),rand(T))
    @test support(JacobiNodes(T(0),T(0),rand(T,10))) == ChebyshevInterval{T}()
    test_generic_grid(grid)
    @test JacobiNodes(10,zero(T),zero(T)) ≈ LegendreNodes(10)
    @test JacobiNodes(10,T(1//2),T(1//2)) ≈ ChebyshevUNodes(10)
    @test JacobiNodes(10,T(-1//2),T(-1//2)) ≈ ChebyshevTNodes(10)
end

for T in types
    delimit(string(T))
    for GRID in interval_grids
        @testset "$(rpad(string(GRID),80))" begin
            g = instantiate(GRID,10,T)
            test_interval_grid(g)
        end
    end

    for grid in (JacobiNodes(10,rand(T),rand(T)),)
        @testset "$(rpad(string(typeof(grid)),80))" begin
            test_interval_grid(grid)
        end
    end

    @testset "HermiteNodes" begin
        test_hermite(T)
    end

    @testset "Laguerre" begin
        test_laguerre(T)
    end

    @testset "Jacobi" begin
        test_jacobi(T)
    end

    @testset "$(rpad("Specific grid tests",80))" begin
        test_grids(T)
    end
end

function test_generic_subgrid(grid, s)
    @test support(grid) ≈ s
    for x in grid
        @test x ∈ support(grid)
    end
    for x in subindices(grid)
        @test issubindex(x, grid)
    end

    cnt = 0
    for i in eachindex(supergrid(grid))
        if issubindex(i, grid)
            cnt += 1
        end
    end
    @test cnt == length(grid)
end

function test_subgrids()
    delimit("Grid functionality")
    @testset  "SubGrids" begin
    n = 20
    grid1 = EquispacedGrid(n, -1.0, 1.0)
    subgrid1 = MaskedGrid(grid1, -0.5..0.7)
    test_generic_subgrid(subgrid1, -0.5..0.7)

    subgrid2 = IndexSubGrid(grid1, 4:12)
    subgrid3 = subgrid(grid1, -0.5..0.7)
    test_generic_subgrid(subgrid3, -0.5..0.7)

    @test subgrid1 == subgrid3
    @test mask(subgrid1) == mask(subgrid3)
    @test subindices(subgrid1) == subindices(subgrid3)

    @test support(subgrid1) ∈ support(grid1)
    @test support(subgrid2) ∈ support(grid1)

    G1 = EquispacedGrid(n, -1.0, 1.0)
    G2 = EquispacedGrid(n, -1.0, 1.0)
    ProductG = G1 × G2

    C = UnitDisk()
    refgrid = MaskedGrid(ProductG, C)
    circle_grid = subgrid(ProductG, C)
    @test circle_grid isa MaskedGrid
    @test refgrid ≈ circle_grid
    test_generic_subgrid(circle_grid, C)
    @testset "Generic MaskedGrid" begin

        @test (length(circle_grid)/length(ProductG)-pi*0.25) < 0.01

        G1s = IndexSubGrid(G1,2:4)
        G2s = IndexSubGrid(G2,3:5)
        ProductGs = G1s × G2s
        @test G1s[1] == G1[2]
        @test G2s[1] == G2[3]
        @test ProductGs[1,1] == [G1[2],G2[3]]
    end

    # subgrid of a subgrid
    g1 = EquispacedGrid(10,-1,1)^2
    @test mask(subgrid(subgrid(g1,(0.0..1.0)^2),UnitSimplex{2}()))==[i+j<6 for i in 1:5 , j in 1:5]

    g1 = EquispacedGrid(10,0,1)^2
    for x in g1[BitArray([i+j<12 for i in 1:10 , j in 1:10])]
        @test x ∈ UnitSimplex{2}()
    end

    C = UnitInterval()^2
    productgrid = subgrid(ProductG, C)
    refgrid = MaskedGrid(ProductG, C)

    @test supergrid(productgrid) == ProductG
    @test productgrid isa TensorSubGrid
    test_generic_subgrid(productgrid, C)
    refgrid = MaskedGrid(ProductG, C)
    @test reshape(refgrid,10,10) == productgrid
    @test subindices(refgrid) == subindices(productgrid)

    # Generic tests for the subgrids
    @testset "result" for (grid,subgrid) in ( (grid1,subgrid1), (grid1,subgrid2), (ProductG, circle_grid))
        # print("Subgrid is ")
        # println(typeof(subgrid))
        # Count the number of elements in the subgrid
        cnt = 0
        for i in 1:length(grid)
            if issubindex(i, subgrid)
                cnt += 1
            end
        end
        @test cnt == length(subgrid)
    end

    g = subgrid(ScatteredGrid(rand(10)), Interval(0,.5))
    @test support(g) ≈ Interval(0,.5)
    for x in g
        @test x ∈ support(g)
    end

    for x in boundary(EquispacedGrid(100,-1,1)^2,UnitDisk())
        @test norm(x)≈1
    end

    for x in  boundary(subgrid(EquispacedGrid(100,-1,1)^2,UnitDisk()),UnitDisk())
        @test norm(x)≈1
    end
end
end

function test_randomgrids()
    println("Random grids:")
    @testset begin
        d = UnitDisk()
        g = randomgrid(d, 10)
        @test length(g) == 10
        @test length(eltype(g)) == dimension(d)
        @test reduce(&, [x ∈ d for x in g])

        g2 = randomgrid(UnitDisk{BigFloat}(), 10)
        @test eltype(g2[1]) == BigFloat

        g3 = randomgrid(0.0..1.0, 10)
        @test length(g3) == 10
        # 1D is a special case where we don't use vectors of length 1
        @test eltype(g3) == Float64

        box1 = UnitInterval()^2
        p1 = randompoint(box1)
        @test length(p1) == dimension(box1)
        @test p1 ∈ box1
        box2 = 0.0..1.0
        p2 = randompoint(box2)
        @test typeof(p2) == Float64
        @test p2 ∈ box2

        g3 = randompoint(UnionDomain(0.0..1.5,1.5..2.0))
        @test typeof(g3) == Float64
        @test g3 ∈ 0.0..2.0

        g3 = randompoint(IntersectionDomain(0.0..1.5,1.0..2.0))
        @test typeof(g3) == Float64
        @test g3 ∈ 1.0..1.5

        g3 = randompoint(DifferenceDomain(0.0..1.5,1.0..2.0))
        @test typeof(g3) == Float64
        @test g3 ∈ 0.0..1.0

    end
end
using StaticArrays

include("test_modcartesianindices.jl")

include("test_boundingbox.jl")
include("test_broadcast.jl")
include("test_gauss.jl")
test_subgrids()
test_randomgrids()

@testset "Plots" begin
    plot(FourierGrid(4))
    plot(FourierGrid(4)× FourierGrid(4) )
    plot(FourierGrid(4)× FourierGrid(4) × FourierGrid(4) )
    plot(FourierGrid(4)× FourierGrid(4) ,rand(4,4))
    plot(FourierGrid(4) ,rand(4))
end


"""
```
M = imfiltermtx(imgsize, kernel, [border, value])
```

creates a sparse matrix such that matrix-vector multiplication is identical to
calling `Images.imfilter` with the same options:

`imfiltermtx(size(img),kernel,border,value) * vec(img)  ≈  vec(imfilter(img,kernel,border,value))`

See `Images.imfilter` for an explanation of the available options.
Note that boundary condition `"value"` does not support any `value`s other than 0.
"""

function imfiltermtx{T,N}(szimg::NTuple{N,Int}, k::AbstractArray{T,N}, border::AbstractString="replicate", value=zero(T))
  border != "value" || convert(T,value) == zero(T) || throw(ArgumentError("Only value '0' supported for border condition 'value'."))
  npixels = prod(szimg)
  idximg = reshape(1:npixels,szimg)
  idximg = padforfilter(idximg, size(k), border, value)

  cols::Matrix{Int} = im2col(idximg,size(k),"sliding")
  nelems,ncols  = size(cols)
  I::Vector{Int} = vec(repmat(reshape(1:ncols,1,ncols),nelems))
  J::Vector{Int} = vec(cols)
  V::Vector{T}   = repmat(vec(k),ncols)
  if border == "value"
    valid = find(J)
    I,J,V = map(x->x[valid],(I,J,V))
  end

  sparse(I,J,V,ncols,npixels)
end

"""
```
otf = psf2otf(psf, [outsize])
```

creates the optical transfer function (`otf`) from the provided point-spread function (`psf`).
The tuple `outsize` can be used to specify the size of the `otf` array; by default, `outsize = size(psf)`.

The convolution theorem relates convolution with the `psf` in the spatial domain to
multiplication with the `otf` in the frequency domain:

`Images.imfilter(x,Images.reflect(psf),"circular") ≈ real(ifft( fft(x) .* psf2otf(psf,size(x)) ))`

This function is similar to Matlab's `psf2otf`.
See http://www.mathworks.com/help/images/ref/psf2otf.html
"""

function psf2otf{T,N}(psf::AbstractArray{T,N}, outsz::NTuple{N,Int}=size(psf))
  psfsz = size(psf)
  if psfsz != outsz
    pad = map(-,outsz,psfsz)
    all(x -> x >= 0, pad) || throw(DimensionMismatch("psf too large for outsz."))
    psf = padarray(psf,tuple(zeros(Int,N)...),pad,"value",T(0))
  end
  shift = map(x -> -floor(Int,x/2), psfsz)
  fft(circshift(psf,shift))
end

"""
```
imgpad = padforfilter(img, kernelsize, [border, value])
```

returns a padded copy of array `img` such that the following holds true:

`imfilter(padforfilter(img,size(kernel),border,value),kernel,"inner") == imfilter(img,kernel,border,value)`

See `Images.imfilter` for an explanation of options `border` and `value`.
"""

function padforfilter{T,N}(A::AbstractArray{T,N}, sz::NTuple{N,Int}, border::AbstractString="replicate", value=zero(T))
  if border in ("replicate", "circular", "reflect", "symmetric", "value")
    prepad  = ntuple(d->div(sz[d]-1, 2), N)
    postpad = ntuple(d->div(sz[d],   2), N)
    A = padarray(A, prepad, postpad, border, convert(T,value))
  elseif border == "inner" # nothing to do
  else
    throw(ArgumentError("Border condition '$border' not supported."))
  end
  A
end


#=

# 226: Terminal flux calculation via test functions, nD, boundary reaction
([source code](SOURCE_URL))

=#

module Example226_BoundaryIntegral

using VoronoiFVM, GridVisualize,ExtendableGrids

function main(;n=10,Plotter=nothing,verbose=false, unknown_storage=:sparse,dim=2)
    n=[101,21,5]
    X=collect(range(0.0,1,length=n[dim]))
    if dim==1
        grid=simplexgrid(X)
        Γ_where_T_equal_1=[2]
        Γ_where_T_equal_0=[1]
    elseif dim==2
        grid=simplexgrid(X,X)
        Γ_where_T_equal_1=[2]
        Γ_where_T_equal_0=[4]
    elseif dim==3
        grid=simplexgrid(X,X,X)
        Γ_where_T_equal_1=[2]
        Γ_where_T_equal_0=[4]
    end

    function storage(f,u,node)
        f.=u
    end

    function flux(f,u,edge)
	f[1]=u[1,1]-u[1,2]
    end

    function breaction(f,u,node)
        if node.region==Γ_where_T_equal_1[1]
            f[1]= u[1]^2
        end
    end

    physics=VoronoiFVM.Physics(flux=flux,
	                       storage=storage,
	                       breaction=breaction)

    system=VoronoiFVM.System(grid,physics)
    enable_species!(system,1,[1])
    boundary_dirichlet!(system,1,Γ_where_T_equal_0[1],1.0);
    inival=unknowns(system,inival=0.0)

    U=solve(inival,system)

    tf=VoronoiFVM.TestFunctionFactory(system)
    T=testfunction(tf,Γ_where_T_equal_0,Γ_where_T_equal_1)

    scalarplot(grid,U[1,:],Plotter=Plotter,zplane=0.50001)
    I=integrate(system,T,U)
    B=integrate(system,breaction,U; boundary=true)
    isapprox(-I[1], B[Γ_where_T_equal_1[1]],rtol=1.0e-12)
end

function test()
    main(dim=1, unknown_storage=:sparse )  ? true : return false
    main(dim=1, unknown_storage=:dense  )  ? true : return false
    main(dim=2, unknown_storage=:sparse )  ? true : return false
    main(dim=2, unknown_storage=:dense  )  ? true : return false
    main(dim=3, unknown_storage=:sparse )  ? true : return false
    main(dim=3, unknown_storage=:dense  )  ? true : return false
end

end


# nanstd.jl
#
#   Standard deviation, ignoring NaNs
#
# Jeremy Rüffer
# Thünen Institut
# Institut für Agrarklimaschutz
# Junior Research Group NITROSPHERE
# Julia 1.4.0
# 09.04.2020
# Last Edit: 09.04.2020

nanstd(x::Array{T}) where T <: Number = std(filter(!isnan,x))

nanstd(x::Array{Union{T,Missing}}) where T <: Number = std(filter(!isnan,filter(!ismissing,x)))


#= Load these if running these tests in terminal
import Pkg; Pkg.activate("../")  # CytofRepFAM
import CytofRepFAM.MCMC
=#

@testset "Test logit / sigmoid" begin
  @test MCMC.logit(.6) ≈ log(.6 / .4)
  @test MCMC.sigmoid(3.) ≈ 1 / (1 + exp(-3.))
end

@testset "Test logsumexp" begin
  x = randn(10)
  @test MCMC.logsumexp(x) ≈ log(sum(exp.(x)))
  @test MCMC.logsumexp(x[1:3]) ≈ MCMC.logsumexp(x[1], x[2], x[3])
end


using ModelingToolkit, OrdinaryDiffEq
using AnalogMTK
using AnalogMTK.EasyPlot #set, cons
import Printf: @sprintf
j=im

#==Circuit parameters
===============================================================================#
R = 1000
C = 10e-6 #10e-6, 10e-9
V = 1.0
𝑓 = 60 #Hz

#==Basic calculations
===============================================================================#
𝜔 = 2π*𝑓
ZC = 1/(j*𝜔*C)

println()
@info("Simulation parameters:")
@show V, 𝑓, 𝜔
@show R, C, abs(ZC)

𝜏 = R*C
@show 𝜏

#==Circuit definition
===============================================================================#
@named resistor = Resistor(R=R)
@named capacitor = Capacitor(C=C)
#@named source = VSource(V)
@named source = VSource(Sine(A=V, f=𝑓))
@named ground = Ground()

rc_eqs = [
          connect(source.p, resistor.p)
          connect(resistor.n, capacitor.p)
          connect(capacitor.n, source.n, ground.g)
         ]

@named rc_model = ODESystem(rc_eqs, AnalogMTK.t, systems=[resistor, capacitor, source, ground])

println()
@info("Initial ODESystem (model definition):"); flush(stdout); flush(stderr)
display(rc_model)

println()
@info("Computing simplified model (ODESystem)..."); flush(stdout); flush(stderr)
sys = structural_simplify(rc_model)
display(sys)
println("\tdone.")

#==Circuit simulation
===============================================================================#
println()
@info("Running simulation..."); flush(stdout); flush(stderr)
u0 = [
      capacitor.v => 0.0
      capacitor.p.i => 0.0
      resistor.v => 0.0
     ]
prob = ODEProblem(sys, u0, (0, 10.0))
sol = solve(prob, Trapezoid())
	#Trapezoid(), ImplicitEuler()
	#Rodas4(), RadauIIA5(), Tsit5()
@show length(sol.t)
println("\tdone.")

#==Plot results
===============================================================================#
println()
@info("Plotting results..."); flush(stdout); flush(stderr)
ylabels = ["Voltage [V]", "Current [A]"]
plot = plot_transient(sol.t, sol[source.p.v], id="Vsrc"; ylabels)
plot_transient!(plot, sol.t, sol[capacitor.p.v], id="Vout")
plot_transient!(plot, sol.t, sol[capacitor.p.i], id="Iout", strip=2)
displaygui(plot)
println("\tdone.")

:TEST_DONE


#=

  From the WebPPL model (written months ago):
  """
  Today I changed the last of two bulbs which were supposed to hold for
  2000h (they are Halogen bulbs). The first went out some day ago,
  say 25h ago.

  What is the probability of this to happen, i.e. that both bulbs went out
  in a 25h period, if we assume that the time for light bulbs are an
  Exponential distribution?
  """

  Mathematica code:
    Probability[Abs[a - b] <= c, {a, b} \[Distributed]
      ProductDistribution[ExponentialDistribution[p],
      ExponentialDistribution[p]]]

  1 - E^(-c p) (if c > 0)

  -> 0.0124222

  Here's the summary of the three models: exponential, gaussian, and poisson,
  where 1 is the probability that both bulbs went out in the range of 25h.
  I guess that the exponential model is the most realistic of these.

  Model 1: Exponential
  Distributions of variable p (num:0)
  0.00000 =>    9881  (0.988100)
  1.00000 =>     119  (0.011900)

  Model 2: Gaussian
  Distributions of variable p (num:0)
  0.00000 =>    8639  (0.863900)
  1.00000 =>    1361  (0.136100)

  Model 3: Poisson
  Distributions of variable p (num:0)
  0.00000 =>    6851  (0.685100)
  1.00000 =>    3149  (0.314900)

  Model 4: Weibull (experimental)
  Distributions of variable p (num:0)
  0.00000 =>    9880  (0.988000)
  1.00000 =>     120  (0.012000)

  Note: 2000h is 83.33 days.
  1/0.011900 (from the exponential model) ~ 84.03: This happens once each 84'th day...

  Cf ~/webppl/light_bulbs.jl

=#

using Turing
include("jl_utils.jl")

# Exponential
@model function light_bulbs_exp()
    lambda = 2000
    b1 ~ Exponential(lambda)
    b2 ~ Exponential(lambda)
    # The differences in failures
    diff ~ Dirac(abs(b1-b2))
    # Probability that the difference between the events is <= 25
    p ~ Dirac(diff <= 25)
end

println("Model 1: Exponential")
model = light_bulbs_exp()

# chns = sample(model, Prior(), 100_000)
# chns = sample(model, MH(), 100_000)
# chns = sample(model, PG(15), 10_000)
chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 100_000)

display(chns)
show_var_dist_pct(chns, :p)

# Gaussian distribution: 2000h, standard deviation 100h
@model function light_bulbs_gauss()
    mu = 2000;
    stdev = 100;
    b1 ~ Normal(mu,stdev)
    b2 ~ Normal(mu,stdev)
    diff ~ Dirac(abs(b1-b2))
    p ~ Dirac(diff <= 25)
end

println("Model 2: Gaussian")
model = light_bulbs_gauss()

# chns = sample(model, Prior(), 100_000)
# chns = sample(model, MH(), 100_000)
# chns = sample(model, PG(15), 10_000)
chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 100_000)

display(chns)
show_var_dist_pct(chns, :p)

# Poisson distribution: 1/2000h
@model function light_bulbs_poisson()
    lambda = 2000
    b1 ~ Poisson(lambda)
    b2 ~ Poisson(lambda)
    diff ~ Dirac(abs(b1-b2))
    p ~ Dirac(diff <= 25)
end

println("Model 3: Poisson")
model = light_bulbs_poisson()

# chns = sample(model, Prior(), 100_000)
# chns = sample(model, MH(), 100_000)
# chns = sample(model, PG(15), 10_000)
chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 100_000)

display(chns)
show_var_dist_pct(chns, :p)

# Weibul distribution
@model function light_bulbs_weibul()
    shape = 1
    scale = 2000
    b1 ~ Weibull(shape,scale)
    b2 ~ Weibull(shape,scale)
    diff ~ Dirac(abs(b1-b2))
    p ~ Dirac(diff <= 25)
end

println("Model 4: Weibull (experimental)")
model = light_bulbs_weibul()

# chns = sample(model, Prior(), 100_000)
# chns = sample(model, MH(), 100_000)
# chns = sample(model, PG(15), 10_000)
chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 100_000)

display(chns)
show_var_dist_pct(chns, :p)


using Test
using DataDrivenControl
using LinearAlgebra

@testset "utils" begin
    @testset "convert" begin
        # convert_to_matrix
        n = 2
        w = rand(Int(n*(n+1)/2))
        P = DataDrivenControl.convert_to_matrix(w)
        @test P == [w[1] 0.5*w[2]; 0.5*w[2] w[3]]
        # convert_quadratic_to_linear_basis
        x = rand(n)
        w' * DataDrivenControl.convert_quadratic_to_linear_basis(x) == x' * P * x
    end
    @testset "cost" begin
        x = ones(3)
        u = ones(3)
        Q = Matrix(I, 3, 3)
        R = 2*Matrix(I, 3, 3)

        quadratic_cost = DataDrivenControl.QuadraticCost(Q, R)
        r = quadratic_cost(x, u)
        @test r == x'*Q*x + u'*R*u  # analytically obtained
        Q_func(x) = x'*Q*x
        R_func(x) = R
        quadratic_in_input_cost = DataDrivenControl.QuadraticInInputCost(Q_func, R_func)
        @test r == quadratic_in_input_cost(x, u)
    end
end


function descent(p::Param, η::Real)
  function ()
    @. p.x -= η * p.Δ
    @. p.Δ = 0
  end
end

function momentum(p::Param, ρ, η)
  v = zeros(p.x)
  function ()
    @. v = ρ * v - η * p.Δ
    @. p.Δ = -v
  end
end

# Ref. https://arxiv.org/pdf/1212.0901.pdf
function nesterov(p::Param, ρ, η)
  v = zeros(p.x)
  function ()
    d = @. ρ^2 * v - (1+ρ) * η * p.Δ
    @. v = ρ*v - η*p.Δ
    @. p.Δ = -d
  end
end

function rmsprop(p::Param; η::Real = 0.001, ρ::Real = 0.9, ϵ::Real = 1e-8)
  acc  = zeros(p.x)
  function ()
    @. acc = ρ * acc + (1 - ρ) * p.Δ^2
    @. p.Δ *= η / √(acc + ϵ)
  end
end

function adagrad(p::Param; η::Real = 0.01, ϵ::Real = 1e-8)
  acc = zeros(p.x) .+ ϵ
  function ()
    @. acc += p.Δ^2
    @. p.Δ *= η / √(acc + ϵ)
  end
end

function adadelta(p::Param; ρ::Real = 0.9, ϵ::Real = 1e-8)
  acc = zeros(p.x)
  Δacc = zeros(p.x)
  function ()
    @. acc = ρ * acc + (1 - ρ) * p.Δ^2
    @. p.Δ *= √(Δacc + ϵ) / √(acc + ϵ)
    @. Δacc = ρ * Δacc + (1 - ρ) * p.Δ^2
   end
end

function adam(p::Param; η::Real = 0.001, β1::Real = 0.9, β2::Real = 0.999, ϵ::Real = 1e-8)
  mt = zeros(p.x)
  vt = zeros(p.x)
  β1p, β2p = β1, β2
  function ()
    @. mt = β1 * mt + (1 - β1) * p.Δ
    @. vt = β2 * vt + (1 - β2) * p.Δ^2
    @. p.Δ =  mt / (1 - β1p) / √(vt / (1 - β2p) + ϵ) * η
    β1p *= β1
    β2p *= β2
  end
end

function adamax(p::Param; η::Real = 0.002, β1::Real = 0.9, β2::Real = 0.999, ϵ::Real = 1e-8)
  mt = zeros(p.x)
  ut = zeros(p.x)
  β1p = β1
  function ()
    @. mt = β1 * mt + (1 - β1) * p.Δ
    @. ut = max(β2 * ut, abs(p.Δ))
    @. p.Δ = (η/(1 - β1p)) * mt/(ut + ϵ)
    β1p *= β1
  end
end

function amsgrad(p::Param; η::Real = 0.001, β1::Real = 0.9, β2::Real = 0.999, ϵ::Real = 1e-8)
  mt = zeros(p.x)
  vt = zeros(p.x) .+ ϵ
  v̂t = zeros(p.x) .+ ϵ
  function ()
    @. mt = β1 * mt + (1 - β1) * p.Δ
    @. vt = β2 * vt + (1 - β2) * p.Δ ^ 2
    @. v̂t = max.(v̂t, vt)
    @. p.Δ = η * mt / √v̂t
  end
end

function nadam(p::Param; η::Real = 0.001, β1::Real = 0.9, β2::Real = 0.999, ϵ::Real = 1e-8)
  mt = zeros(p.x)
  vt = zeros(p.x)
  β1p, β2p = β1, β2
  function ()
    @. mt = β1 * mt + (1 - β1) * p.Δ
    @. vt = β2 * vt + (1 - β2) * p.Δ^2
    @. p.Δ = (β1 * mt / (1 - β1 * β1p) + (1 - β1) * p.Δ / (1 - β1p)) / √(vt * β2 / (1 - β2p) + ϵ) * η
    β1p *= β1
    β2p *= β2
  end
end

clip(p::Param, thresh::Real) = () -> clamp!(p.Δ, -thresh, thresh)

function expdecay(p::Param, γ::Real)
  if γ != 0
    return () -> p.Δ .+= γ .* p.x
  else
    return () -> nothing
  end
end

function invdecay(p::Param, γ::Real)
  if γ != 0
    n = 0
    return () -> begin
      p.Δ .*= 1 / (1 + γ * n)
      n += 1
    end
  else
    return () -> nothing
  end
end


module TestLEM
	using ManifoldLearning
	using Base.Test

	k = 12
	d = 2
	X, L = swiss_roll()
	Y = transform(LEM, X, k=k, d=d)

	@test outdim(Y) == d
	@test size(projection(Y), 2) == size(X, 2)
	@test neighbors(Y) == k
	@test length(eigvals(Y)) == d
	@test length(ccomponent(Y)) > 1
end


using NumericFuns
using Base.Test
using Compat

## auxiliary function

function check_rtype{T<:Number}(f::Functor{1}, ::Type{T})
    pt = result_type(f, T)
    rt = typeof(evaluate(f, one(T)))
    if !(pt == rt)
        error("result_type($f, $T) should be $rt (but predicted $pt)")
    end
end

function check_rtype{T1<:Number,T2<:Number}(f::Functor{2}, ::Type{T1}, ::Type{T2})
    pt = result_type(f, T1, T2)
    rt = typeof(evaluate(f, one(T1), one(T2)))
    if !(pt == rt)
        error("result_type($f, $T1, $T2) should be $rt (but predicted $pt)")
    end
end

## type categories

const inttypes = [Int8,
                  Int16,
                  Int32,
                  Int64,
                  Int128,
                  UInt8,
                  UInt16,
                  UInt32,
                  UInt64,
                  UInt128]

const integraltypes = [Bool, inttypes...]
const fptypes = [Float32, Float64]
const realtypes = [integraltypes..., fptypes...]

const complextypes = [Complex{Bool},
                      Complex{Int16},
                      Complex{Int32},
                      Complex{Int64},
                      Complex64,
                      Complex128]

const numerictypes_r = [realtypes..., Complex64, Complex128]
const numerictypes = [realtypes..., complextypes...]

## test fptype

println("    fptype")

for t in realtypes
    ft = typeof(convert(AbstractFloat, zero(t)))
    @test fptype(t) == ft
end

@test fptype(Complex{Int}) == Complex128
@test fptype(Complex64) == Complex64
@test fptype(Complex128) == Complex128

## arithmetics

println("    arithmetics (unary)")

for F in [Negate, AbsFun, Abs2Fun, RealFun, ImagFun, SqrFun, RcpFun]
    for T in numerictypes
        check_rtype(F(), T)
    end
end

for F in [SignFun, SignbitFun]
    for T in realtypes
        check_rtype(F(), T)
    end
end

println("    arithmetics (binary)")

for F in [Add, Subtract, Multiply, Divide, RDivide, Pow]
    println("        $F")
    for T1 in numerictypes, T2 in numerictypes
        check_rtype(F(), T1, T2)
    end
end

println("    arithmetics (quotient & modulo)")

for F in [DivFun, FldFun, RemFun, ModFun, MaxFun, MinFun]
    println("        $F")
    for T1 in realtypes, T2 in realtypes
        check_rtype(F(), T1, T2)
    end
end

## comparison

println("    comparison operators")

for F in [LT, GT, LE, GE, EQ, NE]
    println("        $F")
    for T1 in realtypes, T2 in realtypes
        check_rtype(F(), T1, T2)
    end
end

## logical & bitwise

println("    logical operators")

check_rtype(Not(), Bool)
check_rtype(And(), Bool, Bool)
check_rtype(Or(),  Bool, Bool)

println("    bitwise operators")

for T in integraltypes
    check_rtype(BitwiseNot(), T)
end

for F in [BitwiseAnd, BitwiseOr, BitwiseXor]
    for T1 in integraltypes, T2 in integraltypes
        check_rtype(F(), T1, T2)
    end
end

# rounding functions

println("    rounding functions")

for F in [FloorFun, CeilFun, TruncFun, RoundFun,
          IfloorFun, IceilFun, ItruncFun, IroundFun]
    for T in realtypes
        check_rtype(F(), T)
    end
end

# number classification functions

println("    number classification")

for F in [IsnanFun, IsinfFun, IsfiniteFun]
    for T in realtypes
        check_rtype(F(), T)
    end
end

# algebraic functions

println("    algebraic functions")

for F in [SqrtFun, RsqrtFun]
    for T in numerictypes
        check_rtype(F(), T)
    end
end

for F in [CbrtFun, RcbrtFun]
    for T in realtypes
        check_rtype(F(), T)
    end
end

for T1 in realtypes, T2 in realtypes
    check_rtype(HypotFun(), T1, T2)
end

println("    exponential & logarithm")

for F in [ExpFun, Exp2Fun, Exp10Fun,
          LogFun, Log2Fun, Log10Fun]
    for T in numerictypes_r
        check_rtype(F(), T)
    end
end

for F in [Expm1Fun, Log1pFun, XlogxFun,
          LogisticFun, LogitFun, SoftplusFun, InvsoftplusFun]
    for T in realtypes
        check_rtype(F(), T)
    end
end

for F in [XlogyFun, LogsumexpFun]
    for T1 in realtypes, T2 in realtypes
        check_rtype(F(), T1, T2)
    end
end

println("    trigonometric functions")

for F in [SinFun, CosFun, TanFun, CotFun, SecFun, CscFun,
          AsinFun, AcosFun, AtanFun, AcotFun, AsecFun, AcscFun,
          SincFun, CoscFun, SinpiFun, CospiFun]
    for T in realtypes
        check_rtype(F(), T)
    end
end

for T1 in realtypes, T2 in realtypes
    check_rtype(Atan2Fun(), T1, T2)
end

println("    hyperbolic functions")

for F in [SinhFun, CoshFun, TanhFun, CothFun, SechFun, CschFun,
          AsinhFun, AcoshFun, AtanhFun, AcothFun, AsechFun, AcschFun]
    for T in numerictypes_r
        check_rtype(F(), T)
    end
end

println("    error functions & friends")

for F in [ErfFun,ErfcFun,ErfiFun,ErfcxFun]
    for T in numerictypes_r
        check_rtype(F(), T)
    end
end

for F in [ErfinvFun, ErfcinvFun]
    for T in realtypes
        check_rtype(F(), T)
    end
end

println("    gamma & friends")

for F in [GammaFun,LgammaFun,EtaFun,ZetaFun]
    for T in numerictypes_r
        check_rtype(F(), T)
    end
end

for T in realtypes
    check_rtype(DigammaFun(), T)
end

println("    beta & lbeta")

for F in [BetaFun, LbetaFun]
    for T1 in realtypes, T2 in realtypes
        check_rtype(F(), T1, T2)
    end
end

println("    airy & friends")

for F in [AiryFun,AiryprimeFun,AiryaiFun,AiryaiprimeFun,AirybiFun,AirybiprimeFun]
    for T in numerictypes_r
        check_rtype(F(), T)
    end
end

println("    bessel & friends")

for F in [Besselj0Fun, Besselj1Fun, Bessely0Fun, Bessely1Fun]
    for T in numerictypes_r
        check_rtype(F(), T)
    end
end


function Ematrix(e, v, cond)
    if cond == "stress"
        E = (e / (1 - v * v)) * [
            1 v 0
            v 1 0
            0 0 (1 - v) / 2]
    elseif cond == "strain"
        E =
            (e * (1 - v) / ((1 + v) * (1 - 2 * v))) * [
                1 v / (v - 1) 0
                v / (v - 1) 1 0
                0 0 (1 - 2 * v) / (2 - 2 * v)]
    else
        println("no planestress or planestrain condition")
        exit()
    end
    return E
end
function quadra(uv,xs,ys)
    (u,v) = uv
    N1 = u*v*(u-1)*(v-1)/4; N5 = v*(1-u*u)*(v-1)/2;
    N2 = u*v*(u+1)*(v-1)/4; N6 = u*(u+1)*(1-v*v)/2;
    N3 = u*v*(u+1)*(v+1)/4; N7 = v*(1-u*u)*(v+1)/2;
    N4 = u*v*(u-1)*(v+1)/4; N8 = u*(u-1)*(1-v*v)/2;
            N9 = (1-u*u)*(1-v*v);
    x = dot(xs,[N1 N2 N3 N4 N5 N6 N7 N8 N9])
    y = dot(ys,[N1 N2 N3 N4 N5 N6 N7 N8 N9])
    return x,y
end
function Nuv(uv)
    (u,v) = uv
    Nu1 = (2*u-1)*(v-1)*v/4;  Nv1 = (u-1)*u*(2*v-1)/4;
    Nu2 = (2*u+1)*(v-1)*v/4;  Nv2 = (u+1)*u*(2*v-1)/4;
    Nu3 = (2*u+1)*(v+1)*v/4;  Nv3 = (u+1)*u*(2*v+1)/4;
    Nu4 = (2*u-1)*(v+1)*v/4;  Nv4 = (u-1)*u*(2*v+1)/4;
    Nu5 = -u*(v-1)*v;         Nv5 = -(u*u-1)*(2*v-1)/2;
    Nu6 = -(v*v-1)*(2*u+1)/2; Nv6 = -u*(u+1)*v;
    Nu7 = -u*(v+1)*v;         Nv7 = -(u*u-1)*(2*v+1)/2;
    Nu8 = -(v*v-1)*(2*u-1)/2; Nv8 = -u*(u-1)*v;
    Nu9 = 2*u*(v*v-1);        Nv9 = 2*v*(u*u-1);

    Nu = [Nu1 Nu2 Nu3 Nu4 Nu5 Nu6 Nu7 Nu8 Nu9]
    Nv = [Nv1 Nv2 Nv3 Nv4 Nv5 Nv6 Nv7 Nv8 Nv9]

    return [Nu; Nv]
end
function Jacobian(uv,xs,ys)
    J = Nuv(uv)*[xs' ys']
    return J
end
function Nxy(uv,xs,ys)
    J = Jacobian(uv,xs,ys)
    nuv = Nuv(uv)
    nxy = J\nuv
    return nxy
end
function N(uv)
    (u,v) = uv
    N1 = u*v*(u-1)*(v-1)/4; N5 = v*(1-u*u)*(v-1)/2;
    N2 = u*v*(u+1)*(v-1)/4; N6 = u*(u+1)*(1-v*v)/2;
    N3 = u*v*(u+1)*(v+1)/4; N7 = v*(1-u*u)*(v+1)/2;
    N4 = u*v*(u-1)*(v+1)/4; N8 = u*(u-1)*(1-v*v)/2;
            N9 = (1-u*u)*(1-v*v);

    N = [N1 N2 N3 N4 N5 N6 N7 N8 N9 zeros(1,9);
              zeros(1,9)           N1 N2 N3 N4 N5 N6 N7 N8 N9]
    return N
end
function LN(uv,xs,ys)
    nxy = Nxy(uv,xs,ys)
    nx = nxy[1,:]'
    ny = nxy[2,:]'
    nx = [nx zeros(1,9);
          zeros(1,9) nx]
    ny = [ny zeros(1,9);
          zeros(1,9) ny]

    LN = [1 0; 0 0; 0 1]*nx + [0 0; 0 1; 1 0]*ny

    return LN
end
function ke(E,xs,ys)
    g = [-0.949108 -0.741531 -0.405845 0.405845 0.741531 0.949108 0.0]
    w = [0.129485 0.279705 0.38183 0.38183 0.279705 0.129485 0.417959]
    kre = zeros(18,18)
    for i=1:7,j=1:7
        uv = [g[i] g[j]]
        ln = LN(uv,xs,ys)
        J = Jacobian(uv,xs,ys)
        kre = kre + w[i]*w[j]*ln'*E*ln*abs(det(J))
    end
    return kre
end
function IDs(bc)
    #note: dangerously RAMMY
    #returns
    #nbc: number of boundary conditions
    #ix: ids of edges for each condition
    #ni: contains number of edges for each condition
    nbc = count(x->x==-1,bc)+1
    ix = findall(x->x==-1,bc)
    ix = [ix[i][1] for i=1:nbc-1]
    prepend!(ix,0)
    ix = [collect(ix[i-1]+1:ix[i]-1) for i=2:nbc]

    ni = [length(i) for i in ix]
    ix = [ [i for i in bc[ix[j]]] for j=1:nbc-1]
    ix = [edge[ix[i],:] for i=1:nbc-1]
    return Int64(nbc-1),ni,ix
end
function BC(f,xedge,yedge,map)
    uv = [0. 0.]
    if map[1]==1 & map[2]==1
        ind = 1
        uv[2] = -1
    elseif map[2]==1 & map[3]==1
        ind = 2
        uv[1] = 1
    elseif map[3]==1 & map[4]==1
        ind = 1
        uv[2] = 1
    else
        ind = 2
        uv[1] = -1
    end
    g = [-0.949108 -0.741531 -0.405845 0.405845 0.741531 0.949108 0.0]
    w = [0.129485 0.279705 0.38183 0.38183 0.279705 0.129485 0.417959]
    rhs = zeros(18,1)
    for i=1:7
        uv[ind] = g[i]
        (x,y) = quadra(uv,xedge,yedge)
        nuv = Nuv(uv)
        xdot = nuv[ind,:]'*xedge'
        ydot = nuv[ind,:]'*yedge'
        road = (xdot[1]^2+ydot[1]^2)^(0.5)
        rhs = rhs + w[i]*N(uv)'*f([x y])*road
    end
    return rhs
end


using AlphaShapes
using Test

@testset "Test Volume" begin
   p = [0. 1.;1. 0.;1. 1.]
   @test AlphaShapes.SimplexVolume(p) == 0.5
end


"""
Creates a sampler from distribution(s)
"""
function make_sample_prior(distr::D) where D <: Distributions.Distribution
    return function sample_prior()
        return rand(distr)
    end
end

function make_sample_prior(distr::Array{D, 1}) where D <: Distributions.Distribution
    return function sample_prior()
        return rand.(distr)
    end
end

function make_sample_prior(param::T, distr::D) where {T <: Tuple, D <: UnionAll}
    return function sample_prior()
        return rand(distr(param...))
    end
end

function make_sample_prior(param::Array{T, 1}, distr::D) where {T <: Tuple, D <: UnionAll}
    d = [distr(p...) for p in param]
    return function sample_prior()
        return rand.(d)
    end
end

function make_sample_prior(param::Array{T, 1}, distr::Array{D, 1}) where {T <: Tuple, D <: UnionAll}
    dA = [d(p...) for (p, d) in zip(param, distr)]
    return function sample_prior()
        return rand.(dA)
    end
end

"""
Creates a density from distribution or set of distributions
"""
function make_density_prior(distr::D) where D <: Distributions.Distribution
    return function density_prior(x::Float64)
        return Distributions.pdf(distr, x)
    end
end

function make_density_prior(distr::Array{D, 1}) where D <: Distributions.Distribution
    return function density_prior(x::Array{Float64, 1})
        tot = 0.0
        for (d, x) in zip(distr, x)
            tot += Distributions.logpdf(d, x)
        end
        return exp(tot)
    end
end

"""
Simultaneously generate sampler and density function for distribution and parameters
"""
function make_model_prior(d::D) where D <: Distributions.Distribution
    return make_sample_prior(d), make_density_prior(d)
end

function make_model_prior(d::Array{D, 1}) where D <: Distributions.Distribution
    return make_sample_prior(d), make_density_prior(d)
end

function make_model_prior(param::T, distr::D) where {T <: Tuple, D <: UnionAll}
    d = distr(param...)
    return make_sample_prior(d), make_density_prior(d)
end

function make_model_prior(param::Array{T, 1}, distr::D) where {T <: Tuple, D <: UnionAll}
    dA = [distr(p...) for p in param]
    return make_sample_prior(dA), make_density_prior(dA)
end

function make_model_prior(param::Array{T, 1}, d::Array{D, 1}) where {T <: Tuple, D <: UnionAll}
    dA = [d(p...) for (p, d) in zip(param, distr)]
    return make_sample_prior(dA), make_density_prior(dA)
end


@testsection "Isotropic tensors" begin
    for T ∈ (Sym, Float64), dim ∈ (2, 3)
        @testsection "type $T, dim $dim" begin
            𝟏 = tensId2(Val(dim), Val(T))
            @test opequal(𝟏, I)
            @test opequal(tr(𝟏), dim)

            if T == Sym
                α = symbols("α", real = true)
            else
                α = rand()
                while opequal(α, one(T)) || opequal(α, zero(T))
                    α = rand()
                end
            end
            @test α * 𝟏 isa TensISO{2}
            t = α * 𝟏 + (1 - α) * 𝟏
            @test opequal(t, 𝟏)
            @test t isa TensISO{2}
            @test opequal(inv(α * 𝟏), inv(α) * 𝟏)
            @test isISO(t)

            𝕀, 𝕁, 𝕂 = ISO(Val(dim), Val(T))
            @test opequal(𝕁 + 𝕂, 𝕀)
            @test opequal((𝟏 ⊗ 𝟏) / dim, 𝕁)
            if T == Sym
                α, β = symbols("α β", real = true)
            else
                α = rand()
                β = rand()
                while opequal(α + β, one(T)) || opequal(α, zero(T)) || opequal(β, zero(T))
                    α = rand()
                    β = rand()
                end
            end
            𝕋 = α * 𝕁 + β * 𝕂
            @test 𝕋 isa TensISO{4}
            @test opequal(inv(𝕋), inv(α) * 𝕁 + inv(β) * 𝕂)
            @test isISO(𝕋)

            if T == Sym && dim == 3
                E, ν = symbols("E ν", real = true)
                k = E / 3(1 - 2ν)
                μ = E / 2(1 + ν)
                λ = E * ν / ((1 + ν) * (1 - 2ν))
                ℂ = simplify(3k * 𝕁 + 2μ * 𝕂)
                @test ℂ == simplify(TensISO{dim}(3k, 2μ))
                𝕊 = simplify(inv(ℂ))
                @test simplify.(KM(𝕊)) == [
                    1/E -ν/E -ν/E 0 0 0
                    -ν/E 1/E -ν/E 0 0 0
                    -ν/E -ν/E 1/E 0 0 0
                    0 0 0 (1+ν)/E 0 0
                    0 0 0 0 (1+ν)/E 0
                    0 0 0 0 0 (1+ν)/E
                ]
                @test simplify(ℂ ⊡ 𝕊) == 𝕀

                n = 𝐞(3)
                Eᵒᵉᵈᵒ = E * (1 - ν) / ((1 + ν) * (1 - 2ν))
                Kref = simplify.([μ 0 0; 0 μ 0; 0 0 Eᵒᵉᵈᵒ])
                @test factor(n ⋅ ℂ ⋅ n) == factor(dotdot(n, ℂ, n)) == Kref
                # Hooke law
                for i ∈ 1:3, j ∈ 1:3
                    @eval $(Symbol("ε$i$j")) = symbols($"ε$i$j", real = true)
                end
                𝛆 = Tens(SymmetricTensor{2,3}((i, j) -> eval(Symbol("ε$i$j"))))
                𝛔 = ℂ ⊡ 𝛆
                @test factor(𝛔) == factor(λ * tr(𝛆) * 𝟏 + 2μ * 𝛆)
                @test factor(simplify(𝛔 ⊡ 𝛆)) == factor(simplify(λ * tr(𝛆)^2 + 2μ * 𝛆 ⊡ 𝛆))

                @test 𝕀 == 𝟏 ⊠ˢ 𝟏
                @test 3𝕁 == 𝟏 ⊗ 𝟏
                @test 𝕀 ⊙ 𝕀 == 6
                @test 𝕁 ⊙ 𝕀 == 𝕁 ⊙ 𝕁 == 1
                @test 𝕂 ⊙ 𝕀 == 𝕂 ⊙ 𝕂 == 5
                @test 𝕂 ⊙ 𝕁 == 𝕁 ⊙ 𝕂 == 0
                @test simplify(ℂ ⊙ 𝕁) == simplify(3k)
                @test simplify(ℂ ⊙ 𝕂) == simplify(10μ)

            end

        end
    end
end


""" Arrow Category as a Double Category of Squares

In every category C, you can construct Sq(C) as a
[double category](https://ncatlab.org/nlab/show/double+category)
of commutative squares. This module uses SquareDiagrams to implement
this construction.
"""
module Squares

using AutoHashEquals

using ...GAT
using ...Theories: DoubleCategory
import ...Theories: ob, hom, dom, codom, compose, ⋅, ⋆, HomV, HomH,
                    composeH, composeV, id, idH, idV, id2, id2H, id2V
using ..FreeDiagrams, ..FinSets

@instance DoubleCategory{FinSet, FinFunction, FinFunction, SquareDiagram} begin
  @import dom, codom, top, bottom, left, right, ⋅
  idH(A::FinSet) = FinFunction(identity, A, A)
  idV(A::FinSet) = FinFunction(identity, A, A)

  composeH(f::FinFunction, g::FinFunction) = compose(f,g)
  composeV(f::FinFunction, g::FinFunction) = compose(f,g)

  id2(A::FinSet) = SquareDiagram(idH(A), idH(A), idV(A), idV(A))
  # FIXME: how do you distinguish between vertical and horizontal if they are the same type?
  id2V(f::FinFunction) = SquareDiagram(f, f, idV(A), idV(A))
  id2H(f::FinFunction) = SquareDiagram(idH(A), idH(A), f, f)

  """    composeH(s₁, s₂)

  compose two squares horizontally as shown below:
      1   -f->   3  -g->   5
      |          |         |
      |          |         |
      v          v         v
      2  -f'->   4  -g'->  6
   """
  function composeH(s₁::SquareDiagram, s₂::SquareDiagram)
    @assert ob(s₁)[3] == ob(s₂)[1]
    @assert ob(s₁)[4] == ob(s₂)[2]
    @assert right(s₁) == left(s₂)

    f = top(s₁)
    f′= bottom(s₁)
    g = top(s₂)
    g′= bottom(s₂)
    return SquareDiagram(f⋅g, f′⋅g′, left(s₁), right(s₂))
  end

  """    composeV(s₁, s₂)

  compose two squares vertically as shown below:
      1   -->  3
      |        |
      |        |
      v        v
      2  -->   4
      |        |
      |        |
      v        v
      5  -->   6
  """
  function composeV(s₁::SquareDiagram, s₂::SquareDiagram)
    @assert ob(s₁)[2] == ob(s₂)[1]
    @assert ob(s₁)[4] == ob(s₂)[3]
    @assert bottom(s₁) == top(s₂)

    f = left(s₁)
    f′= right(s₁)
    g = left(s₂)
    g′= right(s₂)
    return SquareDiagram(top(s₁), bottom(s₂), f⋅g, f′⋅g′)
  end
end

end


println("Computing the amount of wrapping paper the elves need...")
# Open input
cd(raw"/Users/wapisani/Documents/Programming/Julia/Advent_of_Code/")
open("AoC_2015_2.input","r") do f
    global wrapping_dimensions_list = split(read(f,String),"\n")
end

surface_area = 0
length_ribbon = 0
for dimensions in wrapping_dimensions_list
    l = parse(Int64,split(dimensions,'x')[1])
    w = parse(Int64,split(dimensions,'x')[2])
    h = parse(Int64,split(dimensions,'x')[3])

    # Wrapping paper calculations
    side_areas = [l*w,l*h,w*h]
    SA = 2 * l * w + 2 * w * h + 2 * h * l
    SA += minimum(side_areas)
    surface_area += SA

    # Ribbon calculations
    side_1_perimeter = l + l + h + h
    side_2_perimeter = l + l + w + w
    side_3_perimeter = h + h + w + w
    volume = l * w * h
    min_perimeter = minimum([side_1_perimeter,side_2_perimeter,side_3_perimeter])
    length_ribbon += volume
    length_ribbon += min_perimeter
end

println("The elves should order $surface_area ft^2 of wrapping paper and $length_ribbon ft of ribbon")


#==
Functions related to calculating physical parameters of the ionospheric plasma
(susceptibility) relevant to the propagation of electromagnetic waves.
==#

@doc raw"""
    susceptibility(altitude, frequency, bfield, species; params=LMPParams())
    susceptibility(altitude, frequency, w::HomogeneousWaveguide; params=LMPParams())
    susceptibility(altitude, me::ModeEquation; params=LMPParams())

Compute the ionosphere susceptibility tensor `M` as a `SMatrix{3,3}` using
`species.numberdensity` and `species.collisionfrequency` at `altitude`.

Multiple species can be passed as an iterable. Use a `tuple` of `Species`, rather than a
`Vector`, for better performance.

If `params.earthcurvature == true`, `M` includes a first order correction for earth
curvature by means of a fictitious refractive index [Pappert1967].

The susceptibility matrix is calculated from the constitutive relations presented in
[Ratcliffe1959]. This includes the effect of earth's magnetic field vector and
collisional damping on electron motion.

The tensor is:
```math
M = -\frac{X}{U(U²-Y²)}
        \begin{pmatrix}
            U² - x²Y²    & -izUY - xyY² & iyUY - xzY² \\
            izUY - xyY²  & U² - y²Y²    & -ixUY - yzY² \\
            -iyUY - xzY² & ixUY - yzY²  & U² - z²Y²
        \end{pmatrix}
```
where ``X = ωₚ²/ω²``, ``Y = |ωₕ/ω|``, ``Z = ν/ω``, and ``U = 1 - iZ``. The earth curvature
correction subtracts ``2/Rₑ*(H - altitude)`` from the diagonal of ``M`` where ``H`` is
`params.curvatureheight`.

# References

[Pappert1967]: R. A. Pappert, E. E. Gossard, and I. J. Rothmuller, “A numerical
    investigation of classical approximations used in VLF propagation,” Radio Science,
    vol. 2, no. 4, pp. 387–400, Apr. 1967, doi: 10.1002/rds196724387.

[Ratcliffe1959]: J. A. Ratcliffe, "The magneto-ionic theory & its applications to the
    ionosphere," Cambridge University Press, 1959.
"""
function susceptibility(altitude, frequency, bfield, species; params=LMPParams())
    @unpack earthradius, earthcurvature, curvatureheight = params

    B, x, y, z = bfield.B, bfield.dcl, bfield.dcm, bfield.dcn
    ω = frequency.ω

    # Precompute constants (if multiple species)
    invω = inv(ω)
    invE0ω = invω/E0

    #== TODO:
    The zero type should be inferred instead of hard coded, but because we species N and nu
    are FunctionWrappers, we know the types will be ComplexF64.
    ==#
    U²D = zero(ComplexF64)
    Y²D = zero(ComplexF64)
    UYD = zero(ComplexF64)
    @inbounds for i in eachindex(species)
        X, Y, Z = _magnetoionicparameters(altitude, invω, invE0ω, bfield, species[i])

        U = 1 - 1im*Z
        U² = U^2
        Y² = Y^2

        D = -X/(U*(U² - Y²))

        U²D += U²*D
        Y²D += Y²*D
        UYD += U*Y*D
    end

    # Leverage partial symmetry of M to reduce computations
    izUYD = 1im*z*UYD
    xyY²D = x*y*Y²D
    iyUYD = 1im*y*UYD
    xzY²D = x*z*Y²D
    ixUYD = 1im*x*UYD
    yzY²D = y*z*Y²D

    # Elements of `M`
    M11 = U²D - x^2*Y²D
    M21 = izUYD - xyY²D
    M31 = -iyUYD - xzY²D
    M12 = -izUYD - xyY²D
    M22 = U²D - y^2*Y²D
    M32 = ixUYD - yzY²D
    M13 = iyUYD - xzY²D
    M23 = -ixUYD - yzY²D
    M33 = U²D - z^2*Y²D

    if earthcurvature
        curvaturecorrection = 2/earthradius*(curvatureheight - altitude)

        M11 -= curvaturecorrection
        M22 -= curvaturecorrection
        M33 -= curvaturecorrection
    end

    M = SMatrix{3,3}(M11, M21, M31, M12, M22, M32, M13, M23, M33)

    return M
end

susceptibility(altitude, me::ModeEquation; params=LMPParams()) =
    susceptibility(altitude, me.frequency, me.waveguide; params=params)

susceptibility(altitude, frequency, w::HomogeneousWaveguide; params=LMPParams()) =
    susceptibility(altitude, frequency, w.bfield, w.species; params=params)

"""
    susceptibilityspline(frequency, bfield, species; params=LMPParams())
    susceptibilityspline(frequency, w::HomogeneousWaveguide; params=LMPParams())
    susceptibilityspline(me::ModeEquation; params=LMPParams())

Construct a cubic interpolating spline of [`susceptibility`](@ref) and return the callable
`Interpolations` type.

`params.susceptibilitysplinestep` is the altitude step in meters used to construct the
spline between `BOTTOMHEIGHT` and `params.topheight`.
"""
function susceptibilityspline(frequency, bfield, species; params=LMPParams())
    zs = BOTTOMHEIGHT:params.susceptibilitysplinestep:params.topheight
    Ms = susceptibility.(zs, (frequency,), (bfield,), (species,))

    itp = CubicSplineInterpolation(zs, Ms)

    return itp
end

susceptibilityspline(me::ModeEquation; params=LMPParams()) =
    susceptibilityspline(me.frequency, me.waveguide; params=params)

susceptibilityspline(frequency, w::HomogeneousWaveguide; params=LMPParams()) =
    susceptibilityspline(frequency, w.bfield, w.species; params=params)


using Test

using InvariantSets
using LinearAlgebra: I
using Polyhedra
using CDDLib

@testset "Scaling with `*ᶜ` operator" begin
    P = Hyperrectangle(low=[-.6, -2.0],
                       high=[1.0, 2.0])
    P1 = 2.0*I *ᶜ P
    P2 = 2.0 *ᶜ P
    @test P1 ⊆ P2 && P2 ⊆ P1
    @test any(2*canonical_length(P) .== canonical_length(P2))
    P3 = convert(HPolytope, P)
    P4 = 2.0 *ᶜ P3
    @test any(2*canonical_length(P3) .== canonical_length(P4))
end

@testset "Concrete Addition operator +ᶜ" begin
    P = Hyperrectangle(low=[-.6, -2.0],
                       high=[1.0, 2.0])
    P1 = convert(HPolytope, P)
    vec =  [1.0,-10]
    P3 = P1 +ᶜ vec
    P4 = vec +ᶜ P1
    @test any(canonical_length(P3) .≈ vec .+ canonical_length(P1))
    @test any(canonical_length(P3) .== canonical_length(P4))
    P5 = P1 +ᶜ P1
    @test P5 ⊆ 2 *ᶜ P1 && 2 *ᶜ P1 ⊆ P5
end

@testset "Concrete Subtraction operator -ᶜ" begin
    P = Hyperrectangle(low=[-.6, -2.0],
                       high=[1.0, 2.0])
    P1 = convert(HPolytope, P)
    vec =  [1.0, -0.5]
    P3 = P1 -ᶜ vec
    @test any(canonical_length(P3) .≈ canonical_length(P1) .- vec )
    P4 = vec -ᶜ P1
    @test any(canonical_length(P4) .≈ vec .+ canonical_length(reflect(P1)))
    P5 = P1 -ᶜ P1
    @test P5 ⊆ BallInf(zeros(2), 0.0) &&  BallInf(zeros(2), 0.0) ⊆ P5
end


using GenericLinearAlgebra
using LinearAlgebra
using SpecialFunctions
using Dates
using MAT
using Roots
using FFTW
using PyPlot
using PolynomialRoots
using Printf

function Remez(targetf, parity, degree, xapp, lef = -1, rig = 1, maxiter = 10, sample_size = 20, eps = 1e-12)
    #--------------------------------------------------------------------------------------------------------------
    # Input:
    #     targetf: target funciton
    #     parity: the parity of approximation function, 0 -- even, 1 -- odd, 2 -- no constraint
    #     degree: the degree of approximation polynomial
    #     xapp: initial points to approximate, if provide a single zero, will choose roots of Chebyshev polynomials
    #     lef, rig: left and right endpoints of approximation interval
    #     maxiter: Max iteration
    #     sample_size: sample size used in Remez algorithm
    #     eps: accuracy of approximation
    #
    # Output:
    #     Expansion of approximation polynomial under Chebyshev basis
    #
    # We note that our implementation does not strictly follow the reference. The algorithm may crush when
    # the degree is very large or the problem is ill-conditioned.
    #
    #------------------------------------------------------------------------------------------------------------
    #
    # Reference:
    #     E. W. Cheney.
    #     Introduction to approximation theory
    #
    # Author: X. Meng
    # Version 1.0 .... 02/2020
    #
    #------------------------------------------------------------------------------------------------------------

    if(xapp==0)
        xapp = big.(cos.(collect(range(pi/2,stop=0,length=degree+1))))
    end
    iter = 0
    eps2 = 1e-18
    comp = ones(degree+1,1)
    comp[2:2:end] .= -1
    while(true)
        iter += 1

        # Step1: Find the best approximation on n+1 given points by solving a linear equation

        A = zeros(BigFloat,degree+1,degree+1)
        b = zeros(BigFloat,degree+1,1)
        for i=1:degree
            if(parity==0)
                deg = 2*(i-1)
            elseif(parity==1)
                deg = 2*i-1
            else
                deg = i-1
            end
            for j=1:degree+1
                A[j,i] = chebyshev(xapp[j],deg)
            end
        end
        A[:,end] = comp
        for i=1:degree+1
            b[i] = targetf(xapp[i])
        end
        sol = A\b

        # Step2: Find roots of residual function

        eps3 = findmin([eps2,1e-4*abs(sol[end])])[1]
        xroot = zeros(BigFloat,degree,1)
        for i=1:degree
            rootl = xapp[i]
            rootr = xapp[i+1]
            pm = (-1)^(i)*sign(sol[end])
            froot(x) = chebyshevfunc(x,sol[1:end-1],parity,-targetf(x))
            rootiter = 0
            xroot[i] =  brent(rootl,rootr,froot,eps3,eps3)
            if(xroot[i]==Inf)
                return
            end
        end

        # Step3: In each pair of adjacent roots, find a point x such that the absolute value of
        #        residual function is maximized. In addition, values of residual function alternate
        #        in sign.

        xappnew = copy(xapp)
        maxtot = abs(sol[end])
        r_max = -Inf
        y_max = -Inf

        for i=1:degree+1
            maxapp = abs(sol[end])
            pm = (-1)^(i)*sign(sol[end])
            if(i==1)
                lend = lef
            else
                lend = xroot[i-1]
            end
            if(i==degree+1)
                rend = rig
            else
                rend = xroot[i]
            end
            exh = collect(range(lend,stop=rend,length=sample_size))
            for j=1:length(exh)
                fval = chebyshevfunc(exh[j],sol[1:end-1],parity,-targetf(exh[j]))
                if(abs(fval)>r_max)
                    r_max = abs(fval)
                end
                if(fval*pm>y_max)
                    y_max = fval*pm
                end
                if(fval*pm>maxapp)
                    maxapp = fval*pm
                    maxtot = maximum([maxtot,maxapp])
                    xappnew[i] = exh[j]
                end
            end
        end
        xapp = copy(xappnew)

        if(abs(r_max-y_max)>1e-12)
            @printf("Warning: the interpolation points maybe incorrect\n")
        end
        @printf("The %3d-th itertion: previous error is %5.4e, L_inf approximation error is %5.4e\n",iter,sol[end],maxtot)
        if(maxtot<eps||iter>=maxiter||abs((sol[end]-maxtot)/sol[end])<1e-4)
            return sol[1:end-1]
        end
    end
    end

function chebyshev(x,n) # T_n(x)
    if(abs(x)<=1)
        return cos(n*acos(x))
    elseif(x>1)
        return cosh(n*acosh(x))
    else
        return (-1)^n*cosh(n*acosh(-x))
    end
end

function chebyshevfunc(x,sol,parity,init)
    # Compute the value of a summation of Chebyshev polynomials at x
    y = init
    for i=1:length(sol)
        if(parity==0)
            deg = 2*(i-1)
        elseif(parity==1)
            deg = 2*i-1
        else
            deg = i-1
        end
        y += sol[i]*chebyshev(x,deg)
    end
    return y
end

function brent(a,b,f,tol1,tol2)
    # brent method for finding roots on a given interval [a,b]
    # f(a)f(b) should be less than 0, tol1, tol2 are stopping criteria
    fa = f(a)
    fb = f(b)
    if(f(a)*f(b)>0)
        println("Error, f(a)f(b)>0")
        return Inf
    end
    if(abs(f(a))<abs(f(b)))
        tmp = a
        tmp2 = fa
        a = b
        fa = fb
        b = tmp
        fb = tmp2
    end
    c = a
    fc = fa
    s = b
    d = 1e-10
    iter = 0
    mflag = true
    while(true)
        iter += 1
        if(iter>1000)
            println("Brent method: reaches max iteration.")
            return Inf
        end
        if(abs(b-a)<tol2||abs(f(s))<tol1)
            return s
        end
        if(fa!=fc&&fb!=fc)
            s = a*fb*fc/((fa-fb)*(fa-fc))+b*fa*fc/((fb-fa)*(fb-fc))+c*fa*fb/((fc-fa)*(fc-fb))
        else
            s = b-fb*(b-a)/(fb-fa)
        end
        if((s>=b)||(s<=(3*a+b)/4)||((abs(s-b)*2>abs(b-c))&&mflag)||((abs(s-b)*2>abs(d-c))&&(!mflag))||(tol1>abs(b-c)&&mflag)||((tol1>abs(d-c))&&(!mflag)))
            s = (a+b)/2
            mflag = true
        else
            mflag = false
        end
        fs = f(s)
        d = c
        c = b
        fc = fb
        if(fa*fs<0)
            b = s
            fb = fs
        else
            a = s
            fa = fs
        end
        if(abs(f(a))<abs(f(b)))
            tmp = a
            tmp2 = fa
            a = b
            fa = fb
            b = tmp
            fb = tmp2
        end
    end
end

d = Remez(x->1/x, 1, 10, 0, 1, 10)

using DelimitedFiles

# open file in append mode
file = open("example_file.txt", "w")

# writing to a file using write() method
writedlm(file, d)

# We need to close the file in order to write the content from the disk to file
close(file)


# Compute sum of squared errors between a learner's estimation and the truth

function computesquareddiff(true_history::Array{Array{Float64,1},1},
                            estimated_history::Array{Array{Float64,1},1})
    [sum((true_history[j] .- estimated_history[j]).^ 2) for j in 1:length(true_history)]
end

function computesquareddiff(true_history::Array{Float64,1}, estimated_history::Array{Float64,1})
    [(true_history[j] - estimated_history[j])^ 2 for j in 1:length(true_history)]
end


gauss_nodes_weights(p::Type{P}, n) where {P <: Chebyshev} =
    FastGaussQuadrature.gausschebyshev(n)

gauss_nodes_weights(p::Type{P}, n) where {α, β, P <: Jacobi{α, β}} =
    FastGaussQuadrature.gaussjacobi(n, α, β)

include("Orthogonal/FastLegendre.jl")

eval_basis(::Type{P}, n, x::Float64) where {P <: Legendre} = FastLegendre.fastlegendre(n,x)
eval_basis(::Type{P}, n, x::Union{Int, Int32, Float16, Float32}) where {P <: Legendre} =
    FastLegendre.fastlegendre(n,float(x))
gauss_nodes_weights(p::Type{P}, n) where {P <: Legendre} =
    FastGaussQuadrature.gausslegendre(n)


# test1.jl
# based on ex1.jl
using Distributions

function test1()
println("# Starting Test 1")
num_data_default = 200
gen_data_normal(theta::Array, n::Integer = num_data_default) = rand(Normal(theta[1],theta[2]),num_data_default)

normalize_theta2_pos(theta::Array) =  theta[2] = abs(theta[2])
is_valid_theta2_pos(theta::Array) =  theta[2]>0.0 ? true : false

theta_true = [0.0, 1.0]
param_prior = Distributions.MvNormal(theta_true,ones(length(theta_true)))

abc_plan = abc_pmc_plan_type(gen_data_normal,ABC.calc_summary_stats_mean_var,ABC.calc_dist_max, param_prior;
                             target_epsilon=0.01,is_valid=is_valid_theta2_pos,num_max_attempt=10000);

num_param = 2
# Generate "observed" data
data_obs = abc_plan.gen_data(theta_true)
ss_obs = abc_plan.calc_summary_stats(data_obs)
println("# Setting 'true' parameters of theta= ",theta_true)
println("# Generated 'observed' data with summary statistics = ",vec(ss_obs))
println("# Starting ABC-PMC run")
pop_out = run_abc(abc_plan,ss_obs;verbose=true);

println("# Results: ",mean(pop_out.theta[1,:]), " ",mean(pop_out.theta[2,:]))
if abs(mean(pop_out.theta[1,:]))>0.05
  return false
end
if abs(mean(pop_out.theta[2,:])-1)>0.05
  return false
end
return true
end


alpha = rand()+im*rand()
beta = rand()+im*rand()
#beta = 0.0+im*rand()

# Compute covariance matrix:
nt = 5
t = linspace(0,nt-1,nt)

cov = zeros(Complex{Float64},nt,nt)
#cov = zeros(Float64,nt,nt)
for i=1:nt
  for j=i:nt
    cov[i,j]=alpha*exp(-beta*(t[j]-t[i]))
    cov[j,i]=conj(cov[i,j])
  end
  cov[i,i]=real(alpha)
end

ccov=chol(cov)

#covinv=inv(cov)
#
#for i=1:nt
#  for j=1:nt
#    if abs(covinv[i,j]) < eps()
#       covinv[i,j]=0.0+im*0.0
##       covinv[i,j]=0.0
#    end
#  end
#end
#
#covinvchol=chol(covinv)

# Check that cholesky decomposition is correct:
*(ctranspose(ccov),ccov)-cov

# Now try constructing:
a=zeros(Complex64,nt)
b=zeros(Complex64,nt)
v=zeros(Complex64,nt)
matrix = zeros(Complex64,nt,nt)
for i=1:nt
  a[i]=sqrt(alpha)*exp(beta*t[i])
  b[i]=sqrt(alpha)*exp(-beta*t[i])
  v[i]=b[i]/a[i]
end
d = Diagonal(a)
l = zeros(Complex64,nt,nt)
lam=zeros(Complex64,nt,nt)
for i=1:nt
  for j=1:i
    l[i,j]=1.0+im*0.0
  end
  if i < nt
    lam[i,i] = sqrt(v[i]-v[i+1])
  else
    lam[i,i] = sqrt(v[i])
  end
  for j=1:nt
    if j >= i
      matrix[i,j] = a[i]*b[j]
    else
      matrix[i,j] = a[j]*b[i]
    end
  end
end
cholm = *(d,*(l,lam))


"""
    hmcr(Mvir; <keyword arguments>)

Compute the expected halo concentration from halo mass using the relation of
Dutton & Maccio 2014 (equations 7, 10-13).

# Arguments
- `mdef::AbstractString = default_mdef`: halo mass definition (e.g., "200c", "vir")
- `cosmo::AbstractCosmology = default_cosmo`: cosmology under which to evaluate the overdensity
- `z::Real = 0.0`: redshift at which to evaluate the overdensity
"""
function hmcr(Mvir; mdef = default_mdef, cosmo = default_cosmo, z = 0.0)
    mdef == lowercase(strip(mdef))
    if mdef == "200c"
        a = 0.520 + (0.905 - 0.520) * exp(-0.617 * z ^ 1.21)
        b = -0.101 + 0.026 * z
    elseif mdef == "vir"
        a = 0.537 + (1.025 - 0.537) * exp(-0.718 * z ^ 1.08)
        b = -0.097 + 0.024 * z
    else
        throw("only implemented for 200c and vir halo mass definitions")
    end
    # convert Mvir to h-scaled units
    return @. exp10(a + b * (log10(Mvir * cosmo.h) - 12.0))
end

"""
    hmcr_prior(Mvir, cvir; <keyword arguments>)

Compute the probability of drawing a halo with virial parameters (Mvir, cvir),
using the halo mass-concentration relation of Dutton & Maccio 2014.

# Arguments
- `mdef::AbstractString = default_mdef`: halo mass definition (e.g., "200c", "vir")
- `cosmo::AbstractCosmology = default_cosmo`: cosmology under which to evaluate the overdensity
- `z::Real = 0.0`: redshift at which to evaluate the overdensity
- `sigma_logc::Real = 0.16`: scatter in log_concentration at fixed Mvir (in dex)
"""
function hmcr_prior(Mvir, cvir;
                    mdef = default_mdef, cosmo = default_cosmo, z = 0.0,
                    sigma_logc = 0.16)
    logc_expected = log10(hmcr(Mvir; mdef = mdef, cosmo = cosmo, z = z))
    logc = log10(cvir)
    return log_gauss(logc, logc_expected, sigma_logc)
end

function hmcr_prior(halo::HaloModel;
                    mdef = default_mdef, cosmo = default_cosmo, z = 0.0,
                    sigma_logc = 0.16)
    Mvir = virial_mass(halo; mdef = mdef, z = z, cosmo = cosmo)
    logc_expected = log10(hmcr(Mvir; mdef = mdef, cosmo = cosmo, z =z))
    logc_expected = log10(cvir_expected)
    logc = log10(concentration(halo; mdef = mdef, z = z, cosmo = cosmo))
    return log_gauss(logc, logc_expected, sigma_logc)
end

"""
    shmr(Mvir; <keyword arguments>)

Compute the expected stellar mass from halo mass using the relation of
Rodriguez-Puebla et al. 2017.

See equations 25-33 for the functional form and 49 - 55 for the parameters.

# Arguments
- `mdef::AbstractString = default_mdef`: halo mass definition (e.g., "200c", "vir")
- `cosmo::AbstractCosmology = default_cosmo`: cosmology under which to evaluate the overdensity
- `z::Real = 0.0`: redshift at which to evaluate the overdensity
"""
function shmr(Mvir; z = 0.0, cosmo = default_cosmo, mdef = default_mdef)

    if mdef != "vir"
        @warn("converting $mdef to vir halo mass definition", maxlog=1)
        @warn("assuming an NFW profile with standard concentration", maxlog=1)
        cvir = hmcr(Mvir; z = z, cosmo = cosmo, mdef = mdef)
        halo = NFW_from_virial(Mvir, cvir; z = z, cosmo = cosmo, mdef = mdef)
        Mvir = virial_mass(halo; z = z, cosmo = cosmo, mdef = "vir")
    end

    P(x, y, z) = y * z - x * z / (1.0 + z)
    Q(z) = exp(-4.0 / (1.0 + z)^2)

    ϵ0  =  -1.758
    ϵ1  =  +0.110
    ϵ2  =  -0.061
    ϵ3  =  -0.023
    M00 = +11.548
    M01 =  -1.297
    M02 =  -0.026
    α0  =  +1.975
    α1  =  +0.714
    α2  =  +0.042
    δ0  =  +3.390
    δ1  =  -0.472
    δ2  =  -0.931
    γ0  =  +0.498
    γ1  =  -0.157

    logϵ = ϵ0 + P(ϵ1, ϵ2, z) * Q(z) + P(ϵ3, 0.0, z)
    logM0 = M00 + P(M01, M02, z) * Q(z)
    α = α0 + P(α1, α2, z) * Q(z)
    δ = δ0 + P(δ1, δ2, z) * Q(z)
    γ = γ0 + P(γ1, 0.0, z) * Q(z)

    x = log10(Mvir) - logM0
    g(x) = δ * log10(1.0 + exp(x))^γ / (1.0 + exp(10.0^-x)) - log10(10^(-α * x) + 1.0)
    logMstar = logϵ + logM0 + g(x) - g(0)

    return exp10(logMstar)
end

function shmr(halo::HaloModel; mdef = default_mdef, z = 0.0, cosmo = default_cosmo)
    Mvir = virial_mass(halo; mdef = "vir", z = z, cosmo = cosmo)
    return shmr(Mvir; z = z, cosmo = cosmo, mdef = "vir")
end

"""
    shmr_prior(Mvir, Mstar; <keyword arguments>)

Compute the probability of drawing a halo mass and stellar mass pair (Mvir, Mstar).

# Arguments
- `mdef::AbstractString = default_mdef`: halo mass definition (e.g., "200c", "vir")
- `cosmo::AbstractCosmology = default_cosmo`: cosmology under which to evaluate the overdensity
- `z::Real = 0.0`: redshift at which to evaluate the overdensity
- `sigma_logMstar::Real = 0.15` scatter in log_Mstar (in dex)
"""
function shmr_prior(Mvir, Mstar;
                    mdef = default_mdef, cosmo = default_cosmo, z = 0.0,
                    sigma_logMstar = 0.15)
    logMstar_expected = log10(shmr(Mvir; mdef = mdef, z = z, cosmo = cosmo))
    logMstar = log10(Mstar)
    return log_gauss(logMstar, logMstar_expected, sigma_logMstar)
end

function shmr_prior(halo::HaloModel, Mstar;
                    mdef = default_mdef, cosmo = default_cosmo, z = 0.0,
                    sigma_logMstar = 0.15)
    logMstar_expected = log10(shmr(halo; z = z, cosmo = cosmo))
    logMstar = log10(Mstar)
    return log_gauss(logMstar, logMstar_expected, sigma_logMstar)
end

"""
    peak_height(Mvir; <keyword arguments>)

Computes the dimensionless peak height,

```math
\\nu = \\delta_\\mathrm{crit}(z) / \\sigma(M, z)
```

using the approximation of Dutton & Maccio 2014 for a Planck 2013 cosmology.

# Arguments
- `mdef::AbstractString = default_mdef`: halo mass definition (e.g., "200c", "vir")
- `cosmo::AbstractCosmology = default_cosmo`: cosmology under which to evaluate the overdensity
- `z::Real = 0.0`: redshift at which to evaluate the overdensity
"""
function peak_height(Mvir; cosmo = default_cosmo, mdef="200c", z = 0.0)
    if mdef != "200c"
        throw("only implemented for 200c definition")
    end
    m = @. log10(M / 1e12 * cosmo.h)
    ν0 = @. exp10(-0.11 + 0.146 * m + 0.0138 * m^2 + 0.00123 * m^3)
    return @. ν0 * (0.033 + 0.79 * (1.0 + z) + 0.176 * exp(-1.356 * z))
end

"""
    alpha_peak(Mvir; <keyword arguments>)

Compute the Einasto shape parameter from peak height using the relation of Gao+2008.

# Arguments
- `mdef::AbstractString = default_mdef`: halo mass definition (e.g., "200c", "vir")
- `cosmo::AbstractCosmology = default_cosmo`: cosmology under which to evaluate the overdensity
- `z::Real = 0.0`: redshift at which to evaluate the overdensity
"""
function alpha_peak(Mvir; z = 0.0, cosmo = default_cosmo, mdef = "200c")
    if mdef != "200c"
        throw("only implemented for 200c definition")
    end
    ν = peak_height(Mvir, z; cosmo = cosmo, mdef = mdef)
    return @. 0.0095 * ν^2 + 0.155
end

"""
    alpha_peak_prior(Mvir, alpha; <keyword arguments>)

Compute the probability of drawing a halo with the given virial mass and Einasto shape
parameter using the alpha-peak height relation of Gao et al. 2008.

# Arguments
- `mdef::AbstractString = default_mdef`: halo mass definition (e.g., "200c", "vir")
- `cosmo::AbstractCosmology = default_cosmo`: cosmology under which to evaluate the overdensity
- `z::Real = 0.0`: redshift at which to evaluate the overdensity
- `sigma0::Real = 0.16`: scatter (in dex) of Mvir at redshift z = 0
- `sigmaz::Real = 0.03`: slope with redshift in scatter (in dex) of Mvir
"""
function alpha_peak_prior(Mvir, alpha;
                          mdef = default_mdef, cosmo = default_cosmo, z = 0.0,
                          sigma0 = 0.16, sigmaz = 0.03)
    sigma_logalpha = sigma0 + sigmaz * z
    logalpha_expected = log10(alpha_peak(Mvir; z = z, cosmo = cosmo, mdef = mdef))
    logalpha = log10(alpha)
    return log_gauss(logalpha, logalpha_expected, sigma_logalpha)
end

"""
    abg_from_logshm(logshm)

αβγ scalings with log10(Mstar / Mhalo) from DiCintio+2014a.
"""
function abg_from_logshm(logshm)
    x = logshm
    if x < -4.1 || x > -1.3
        return 1, 3, 1
    end
    α = 2.94 - log10(exp10(-1.08 * (x + 2.33)) + exp10(2.29 * (x + 2.33)))
    β = 4.23 + 1.34 * x + 0.26 * x^2
    γ = -0.06 + log10(exp10(-0.68 * (x + 2.56)) + exp10(x + 2.56))
    return α, β, γ
end


@testset "Dynamics" begin
    T = 3
    num_state = 2
    num_action = 1
    num_parameter = 0
    w_dim = [num_parameter for t = 1:T]

    function pendulum(z, u, w)
        mass = 1.0
        lc = 1.0
        gravity = 9.81
        damping = 0.1
        [z[2], (u[1] / ((mass * lc * lc)) - gravity * sin(z[1]) / lc - damping * z[2] / (mass * lc * lc))]
    end

    function euler_explicit(x, u, w)
        h = 0.1
        x + h * pendulum(x, u, w)
    end

    dynamics = Dynamics(euler_explicit, num_state, num_action, num_parameter=num_parameter)
    model = [dynamics for t = 1:T-1]

    x1 = ones(num_state)
    u1 = ones(num_action)
    w1 = zeros(num_parameter)
    X = [x1 for t = 1:T]
    U = [u1 for t = 1:T]
    W = [w1 for t = 1:T]

    dynamics.evaluate(dynamics.evaluate_cache, x1, u1, w1)
    @test norm(dynamics.evaluate_cache - euler_explicit(x1, u1, w1)) < 1.0e-8
    dynamics.jacobian_state(dynamics.jacobian_state_cache, x1, u1, w1)
    dynamics.jacobian_action(dynamics.jacobian_action_cache, x1, u1, w1)

    jac_fd = ForwardDiff.jacobian(a -> euler_explicit(a[1:num_state], a[num_state .+ (1:num_action)], w1), [x1; u1])
    @test norm([dynamics.jacobian_state_cache dynamics.jacobian_action_cache] - jac_fd) < 1.0e-8

    d = [zeros(num_state) for t = 1:T-1]
    for (t, dynamics) in enumerate(model)
        d[t] .= dynamics!(dynamics, X[t], U[t], W[t])
    end
    @test norm(vcat(d...) - vcat([euler_explicit(X[t], U[t], W[t]) for t = 1:T-1]...)) < 1.0e-8

    jx = [zeros(num_state, num_state) for t = 1:T-1]
    ju = [zeros(num_state, num_action) for t = 1:T-1]
    IterativeLQR.jacobian!(jx, ju, model, X, U, W)
    jac_dense = [jx[1] ju[1] zeros(num_state, num_state + num_action);
                 zeros(num_state, num_state + num_action) jx[2] ju[2]]
    @test norm(jac_dense - [jac_fd zeros(model[2].num_state, model[2].num_action + model[2].num_next_state); zeros(model[2].num_next_state, model[1].num_state + model[1].num_action) jac_fd]) < 1.0e-8
end


# Update_CovHypers.jl

function llik_ScInvChiSq(y::Vector{T}, ν::T, s0::T) where T <: Real
    sum1divy = sum(1.0 ./ y)
    sumlogy = sum(log.(y))
    n = length(y)

    halfnu = 0.5 * ν
    a1 = halfnu * n * (log(ν) + log(s0) - log(2.0))
    a2 = n * SpecialFunctions.loggamma(halfnu)
    a3 = halfnu * sumlogy
    a4 = halfnu * s0 * sum1divy

    return a1 - a2 - a3 - a4
end

function rpost_ScInvChiSq_s0(y::Vector{T}, ν::T, prior::GammaParams) where T <: Real
    sum1divy = sum(1.0 ./ y)
    n = length(y)

    halfnu = 0.5 * ν
    a1 = prior.shape + halfnu * n
    b1 = prior.rate + halfnu * sum1divy

    return rand(Distributions.Gamma(a1, 1.0/b1))
end

function rpost_ScInvChiSq_ν(y::Vector{T}, s0::T, prior::Vector{T}) where T <: Real
    ## Here the prior is a vector of values representing a discrete uniform over them.
    lp = [ llik_ScInvChiSq(y, nu, s0) for nu in prior ]
    w = exp.( lp .- maximum(lp) )
    return StatsBase.sample(prior, Weights(w))
end

function update_cov_hypers!(state::State_GPMTD, prior::PriorCovHyper_Matern) where T <: Real

    L = length(state.mixcomps)

    nζ = StatsBase.counts(state.ζ, 1:L) # intercept does not involve these parameters
    use_indx = findall( nζ .> 0 )

    if length(use_indx) > 0

        κ_dat = [ state.mixcomps[j].κ for j in use_indx ]
        lenscale_dat = [ state.mixcomps[j].corParams.lenscale for j in use_indx ]

        κ_ν_out = rpost_ScInvChiSq_ν(κ_dat, state.mixcomps[1].κ_hypers.κ0, prior.κ_ν)
        κ0_out = rpost_ScInvChiSq_s0(κ_dat, κ_ν_out, prior.κ0)

        lenscale_ν_out = rpost_ScInvChiSq_ν(lenscale_dat, state.mixcomps[1].corHypers.lenscale0, prior.lenscale_ν)
        lenscale0_out = rpost_ScInvChiSq_s0(lenscale_dat, lenscale_ν_out, prior.lenscale0)

    else

        κ_ν_out = StatsBase.sample(prior.κ_ν)
        κ0_out = rand(Distributions.Gamma(prior.κ0.shape, 1.0/prior.κ0.rate))

        lenscale_ν_out = StatsBase.sample(prior.lenscale_ν)
        lenscale0_out = rand(Distributions.Gamma(prior.lenscale0.shape, 1.0/prior.lenscale0.rate))

    end

    # update these parameters in all mixcomps
    for ℓ = 1:L
        state.mixcomps[ℓ].κ_hypers.κ_ν = κ_ν_out
        state.mixcomps[ℓ].κ_hypers.κ0 = κ0_out
        state.mixcomps[ℓ].corHypers.lenscale_ν = lenscale_ν_out
        state.mixcomps[ℓ].corHypers.lenscale0 = lenscale0_out
    end

    return nothing
end

### tests passed 4/12/19
# ν = 7.5
# s0 = 10.0
# using Distributions
# using StatsBase
# using SpecialFunctions
# n = 1000
#
# xx = rand(InverseGamma(0.5*ν, 0.5*ν*s0), n)
#
# nsamp = 1000
# samp = ones(nsamp, 2)
# for i = 2:nsamp
#     samp[i,1] = rpost_ScInvChiSq_ν(xx, samp[i-1,2], [5.0, 7.5, 10.0, 25.0, 50.0])
#     samp[i,2] = rpost_ScInvChiSq_s0(xx, samp[i,1], GammaParams(1.0, 1.0))
# end
#
# using Plotly
# plot(samp[:,1])
# plot(samp[:,2])


square(x) = x^2


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,jl:hydrogen
#     text_representation:
#       extension: .jl
#       format_name: hydrogen
#       format_version: '1.3'
#       jupytext_version: 1.10.3
#   kernelspec:
#     display_name: Julia 1.7.0
#     language: julia
#     name: julia-1.7
# ---

# %%
using LinearAlgebra
using Distributions
using Plots
using Random
using Optim
using QuadGK

"""最小二乗法による多項式フィッティング"""
function polynomialfit(x, y, d)
    n = length(y)   # データサイズ
    X = x .^ (0:d)' # design matrix
    β̂ = X \ y       # Juliaでは X \ y の一発で最小二乗法が可能
    f(x) = evalpoly(x, β̂)
    ŷ = f.(x)
    r = ŷ - y
    σ̂ = norm(r)/√n  # モデルの標準偏差パラメータの推定
    dist = product_distribution([Normal(μ, σ̂) for μ in ŷ])
    loglik = logpdf(dist, y)  # 最大化された対数尤度
    AIC = -2loglik + 2(d + 2) # 所謂赤池情報量規準
    (; x, y, d, β̂, σ̂, f, loglik, AIC)
end

"""optimize函数を使った最尤法による多項式フィッティング"""
function polynomialfit_optim(x, y, d)
    negloglik(w) = begin
        β = @view w[1:end-1]
        σ = exp(w[end])
        n = length(y)
        dist = product_distribution([Normal(evalpoly(x[i], β), σ) for i in 1:n])
        -loglikelihood(dist, y)
    end
    o = optimize(negloglik, zeros(d+2), Optim.Options(store_trace=true, extended_trace = true, iterations = 10^4))
    ŵ = o.minimizer
    β̂ = ŵ[1:end-1]
    σ̂ = exp(ŵ[end])
    loglik = -o.minimum
    AIC = 2o.minimum + 2(d + 2)
    f(x) = evalpoly(x, β̂)
    (; x, y, d, β̂, σ̂, f, loglik, AIC, negloglik, o)
end

function plotpolyfit(result; legend=:topleft, titlefontsize=10, kwargs...)
    (; x, y, d, f, loglik, AIC) = result
    xs = range(extrema(x)...; length=1000)
    plot(; legend, titlefontsize)
    scatter!(x, y; label="data")
    plot!(xs, f.(xs); label="degree-$d fitting")
    title!("degree = $d,  log likelihood = $(round(loglik; digits=2)),  AIC = $(round(AIC; digits=2))")
    plot!(; kwargs...)
end

# %%
# データの生成

rng = MersenneTwister(1234567890)

n = 21
x = range(-1, 1; length=n)
σ₀ = 0.3
f₀(x) = sinpi(x)
noise = σ₀*randn(rng, length(x))
y = f₀.(x) + noise

xs = range(extrema(x)...; length=1000)
P = plot(; legend=:topleft)
scatter!(x, y; label="data")
plot!(xs, sinpi.(xs); label="sin(πx)")
title!("data and true law"; titlefontsize=10)
plot!(; size=(400, 300))

# %%
β₀ = [isodd(k) ? (-1)^(k÷2)*π^k/factorial(k) : 0.0 for k in 0:3]
@show β₀ σ₀;

# %%
(; β̂, σ̂, loglik, AIC) = polynomialfit(x, y, 3)
@show β̂ σ̂ loglik AIC;

# %%
(; β̂, σ̂, loglik, AIC) = polynomialfit_optim(x, y, 3)
@show β̂ σ̂ loglik AIC;

# %%
Q = plotpolyfit(polynomialfit(x, y, 3))
R = plotpolyfit(polynomialfit(x, y, 9))
S = plotpolyfit(polynomialfit(x, y, 15))

plot(P, Q, R, S; size=(800, 600), layout=(2, 2), titlefontsize=9)

# %% [markdown]
# 対数尤度(log likelihood)の大きさは「もっともらしさ」の度合いではない！
#
# 対数尤度の大きさはモデルのデータへのフィッティングの良さの指標の1つに過ぎない。
#
# 上の結果を見れば分かるように、フィッティングで使う多項式の次数を上げた方がフィッティングの度合いが上がって、対数尤度も大きくなるが、次数が大きな場合の結果は全然「もっともらしい」とは言えない。
#
# 尤度を最大化することによってモデルのパラメータ値を調節する最尤法は「最も尤もらしいパラメータ値」を探すのではなく、単に「モデルをデータに最もフィットさせるようなパラメータ値」を探しているだけである。
#
# 最尤法は「最も尤もらしいパラメータ値」を探す方法ではない！

# %%
function plot_loglikAIC(x, y)
    result = polynomialfit.(Ref(x), Ref(y), 0:18)
    d = getproperty.(result, :d)
    loglik = getproperty.(result, :loglik)
    AIC = getproperty.(result, :AIC)
    dbest = d[argmin(AIC)]

    T = plot(d, loglik; label="log likelihood", legend=:topleft)
    plot!(; xtick=d, xlabel="degree", ylabel="log likelihood")
    title!("log likelihoods of polynomial fitting"; titlefontsize=10)

    U = plot(d, AIC; label="AIC", legend=:topright)
    plot!(; xtick=d, xlabel="degree", ylabel="AIC")
    title!("AICs of polynomial fitting (minimized at degree $dbest)"; titlefontsize=10)

    plot(T, U; size=(800, 300), titlefontsize=9, leftmargin=3Plots.mm, bottommargin=3Plots.mm)
end

# %%
plot_loglikAIC(x, y)

# %%
function animate_polyfit(x, y, d, f₀, σ₀; gifname="polyfit$d.gif", fps=20, ylim=(-10, 45), ytick=-100:5:100)
    (; x, y, d, negloglik, o) = polynomialfit_optim(x, y, d)
    a, b = extrema(x)
    n = length(y)
    value = -getproperty.(o.trace, :value)
    centroid = getindex.(getproperty.(o.trace, :metadata), "centroid")

    ngenerr = Vector{Float64}(undef, length(centroid))
    Threads.@threads for t in keys(centroid)
        β = centroid[t][1:end-1]
        f(x) = evalpoly(x, β)
        σ = exp(centroid[t][end])
        GE = (1/2)*(log(2π*σ^2) + σ₀^2/σ^2 + (1/σ^2)*1/(b-a)*quadgk(x -> (f(x) - f₀(x))^2, a, b)[1])
        ngenerr[t] = n*GE
    end

    xs = range(extrema(x)...; length=1000)
    ylim = isnothing(ylim) ? extrema([-value; ngenerr]) .+ (-1, 1) : ylim

    L = length(value)
    tstep = max(1, L ÷ 400)

    anim = @animate for t in [fill(1, fps); 1:tstep:L; fill(L, 3fps)]
        β = centroid[t][1:end-1]
        f(x) = evalpoly(x, β)
        P = plot(; legend=false)
        scatter!(x, y)
        plot!(xs, f.(xs); ylim=(-1.5, 1.5))
        title!("degree-$d fitting of size-$n data"; titlefontsize=10)

        Q = plot(; legend=:topleft)
        plot!(-value[1:t]; label="(-1)×(log likelihood)")
        plot!(ngenerr[1:t]; label="n×(generalization error)")
        plot!(;xlim=extrema(axes(value, 1)).+(-0.05L, 0.05L), ylim, ytick)
        title!("t = $t"; titlefontsize=10)

        plot(P, Q; size=(800, 400))    end
    gif(anim, gifname; fps)
end

# %%
animate_polyfit(x, y, 1, f₀, σ₀)

# %%
animate_polyfit(x, y, 3, f₀, σ₀)

# %%
animate_polyfit(x, y, 9, f₀, σ₀)

# %%
animate_polyfit(x, y, 15, f₀, σ₀)

# %%
GE₀ = (1/2)*(log(2π*σ₀^2) + σ₀^2/σ₀^2)
n * GE₀

# %%
rng = MersenneTwister(1234567891)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3a.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5a.gif")

# %%
rng = MersenneTwister(1234567892)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3b.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5b.gif")

# %%
rng = MersenneTwister(1234567893)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3c.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5c.gif")

# %%
animate_polyfit(x, y, 9, f₀, σ₀; gifname="polyfit9c.gif")

# %%
rng = MersenneTwister(1234567894)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3d.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5d.gif")

# %%
rng = MersenneTwister(1234567895)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3e.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5e.gif")

# %%
rng = MersenneTwister(1234567896)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3f.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5f.gif")

# %%
rng = MersenneTwister(1234567897)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3g.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5g.gif")

# %%
rng = MersenneTwister(1234567898)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3h.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5h.gif")

# %%
rng = MersenneTwister(1234567899)
y = f₀.(x) + σ₀*randn(rng, length(x))
plot_loglikAIC(x, y)

# %%
animate_polyfit(x, y, 3, f₀, σ₀; gifname="polyfit3i.gif")

# %%
animate_polyfit(x, y, 5, f₀, σ₀; gifname="polyfit5i.gif")

# %%