# - Julia: High Performance Programming
# - Module 1 Chapter 2

using Allnix: add, power, fpower, fplus
using Base

@testset "add" begin
# int comparison
@test add(1,2) == 3

# - Floating point comparison - #

# type \approx and use tab complete to get ≈
@test add(1.0, 2.0) ≈ 3.0
# or use isapprox() function
@test isapprox(3.0, add(1.0,2.0))

# Exact floating point in some cases
@test add(1.0, 2.0) == 3.0
end

@testset "power" begin
@test power(2,2) == 4
@test power(2.0, 2.0) == 4.0
@test fpower(2.0, 2.0) == 4.0
end

@testset "fplus" begin
@test fplus(1,2,3) == 6
end

# ---------------------- #
# - Variable arguments - #
# ---------------------- #
@testset "varargs" begin
args = [1,2]
# Notice the ellipses
@test add(args...) == 3
end

# --------------------- #
# - Ranges and arrays - #
# --------------------- #
@testset "ones" begin
length = 100
v = ones(Float64, length)
@test Base.length(v) == length
for i in v
    @test i == 1.0
end
end

@testset "zeros" begin
length = 100
v = zeros(Float64, length)
@test Base.length(v) == length
for i in v
    @test i == 0.0
end
end

@testset "linspace" begin
start = 0
stop = 100
# IMPORTANT: n is the number of nodes not intervals
n = 201

@test linspace(start,stop,n) == 0.:0.5:100.
end

@testset "fill" begin
v = Vector{Float64}()
Base.resize!(v, 100)
### - fill! - ###
fill!(v, 13.0)
for i in v
    @test i == 13.0
end
end

# ------------------------------------------ #
# - Deep copy of array with primitive type - #
# ------------------------------------------ #
@testset "copy" begin
count = 100
v = Vector{Float64}(linspace(1,100,count))
w = copy(v)
# - Expect deep copy - #
@test pointer_from_objref(v) != pointer_from_objref(w)
@test length(w) == count
for i in 1:count
    @test w[i] == v[i]
end
end

@testset "Common Array Operators" begin
count = 100
# source = [1, 2, ... 100]
source = Vector{Float64}(linspace(1,100.,count))

####  append!  ####
v = copy(source)
append!(v, 13.0)
@test length(v) == 1 + count
@test v[1+count] == 13.0

v = copy(source)
b = copy(source)
append!(v, b)
@test length(v) == 2*count
@test v[2] == 2.0
@test v[101] == 1.0
@test v[200] == 100

### push! ###
v = copy(source)
w = push!(v, 101) # return v reference
@test length(v) == count + 1
# w and v point to the same array
@test pointer_from_objref(w) == pointer_from_objref(v)

#### pop! ####
# - Remove the last item and returns it
v = copy(source)
@test pop!(v) == 100.0
@test length(v) == count - 1

### shift! ###
# - Remove the first item and returns it
v = copy(source)
@test shift!(v) == 1.0
# v = 2,...10
@test length(v) == count - 1

### splice! ###
v = copy(source)
splice!(v, 55)
@test v[55] == 56

### in ###
v = copy(source)
@test in(55, v)

### sort ###
v = Vector{Float64}(linspace(100.,1.,count))
@test v[1] == 100.0
sort!(v) # mutable version
@test v[1] == 1.0

v = Vector{Float64}(linspace(100.,1.,count))
@test v[1] == 100.0
t = sort(v) # immutable version
@test v[1] == 100.0
@test t[1] == 1.0
end

@testset "dot operator" begin
v = Vector{Float64}()
Base.resize!(v, 100)

# - Assign value to every element - #
v .= 13.0

for i in v
    @test i == 13.0
end

@testset "pointer" begin
a::Vector{Float64} = 1:10.0::Float64
b = a
@test b == a
### pointer_from_objref ###
a_ptr = pointer_from_objref(a)
b_ptr = pointer_from_objref(b)
@test b_ptr == a_ptr

c::Vector{Float64} = 1:10.0::Float64
# compare values
@test b == c
c_ptr = pointer_from_objref(c)
@test b_ptr != c_ptr

@test b[10] == 10.0
a[10] = 13.0
@test b[10] == 13.0
@test b == a

end

end


struct ActiveState
    weighted_label::Float64
    weight::Float64
    lower::Int64
    upper::Int64

end

function merge_state(l::ActiveState, r::ActiveState)
    return ActiveState(l.weighted_label + r.weighted_label,
                       l.weight + r.weight,
                       l.lower,
                       r.upper)
end

function below(l::ActiveState, r::ActiveState)
    return l.weighted_label * r.weight <= l.weight * r.weighted_label
end

function active_set_isotonic_regression!(y::Vector{Float64}, weights::Vector{Float64})

    n = length(y)
    if n <= 1
        return y
    end
    if n != length(weights)
        throw(DimensionMismatch("Lengths of values and weights mismatch"))
    end

    @inbounds begin
        active_set = [ActiveState(weights[i] * y[i], weights[i], i, i) for i in 1 : n]
        current = 1
        while current < size(active_set, 1)
            while current < size(active_set, 1) && below(active_set[current], active_set[current+1])
                current += 1
            end
            if current == size(active_set, 1)
                break
            end

            merged = merge_state(active_set[current], active_set[current+1])
            splice!(active_set, current:current+1, [merged])
            while current > 1 && !below(active_set[current-1], active_set[current])
                current -= 1
                merged = merge_state(active_set[current], active_set[current+1])
                splice!(active_set, current:current+1, [merged])
            end
        end

        for as in active_set
            y[as.lower:as.upper] .= as.weighted_label / as.weight
        end
    end
    return y
end

active_set_isotonic_regression!(y::Vector{Float64}) = active_set_isotonic_regression!(y, fill!(similar(y), one(eltype(y))))

# non-mutating versions
active_set_isotonic_regression(y::Vector{Float64}, weights::Vector{Float64}) = active_set_isotonic_regression!(copy(y), weights)
active_set_isotonic_regression(y::Vector{Float64}) = active_set_isotonic_regression(y, fill!(similar(y), one(eltype(y))))


using Soss

multiple_poisson_regression = @model (traps, live_in_super, log_sq_foot) begin
    beta ∼ Normal(-0.25, 1)
    beta_super ∼ Normal(-0.5, 1)
    alpha ∼ Normal(log(4), 1)

    complaints ∼ For(eachindex(traps)) do i
        # TODO: Numerically safe version
        λ = exp(alpha + beta * traps[i] + beta_super * live_in_super[i] + log_sq_foot[i])
        Poisson(λ)
    end
end


using JuMP, Cbc, CSV

file = "campaignData.csv"

data_import = CSV.read(file, header=false)
campData = Matrix{Int}(data_import)

totalsColumn = size(campData,2)
numOptions = size(campData,2)-1

customersData = campData[1,1:numOptions]
resourcesData = campData[2:end,1:numOptions]
totalsData = campData[2:end,totalsColumn]

adModel = Model(Cbc.Optimizer)

@variable(adModel, options[1:numOptions], binary=true)
@objective(adModel, Max, sum(options[:].*customersData[:]))

for r = 1:size(resourcesData,1)
    @constraint(adModel, sum(resourcesData[r,:].*options[:])<=totalsData[r])
end
@constraints(adModel, begin
    options[6]-(options[4]+options[5])<=0
    options[2]+options[4]<=1
end)

stats = JuMP.optimize!(adModel)
result = JuMP.value.(options)


# A bright soliton in 1D.

push!(LOAD_PATH, "../src")

using Fluxion

t = UnknownDimension("t")
x = BasicDimension("x", RegularGrid(0, 20, 100))
psi = UnknownField("psi", [x, t])

eq = equal(diff(psi, t), 0.5im * diff(psi, x, x) + (1im * abs(psi)^2 - 0.5im) * psi)

function xdensity(psi, t)
    abs(psi)^2
end

k = fourier_space(x)
function kdensity(psi, t)
    psi_k = fourier(psi, x => k)
    abs(psi_k)^2
end

psi0 = 1 / cosh(10 - x)

results = integrate(
    eq, psi0, 0,
    samplers=Dict(
        :xdensity => (xdensity, linspace(0, 5, 101)),
        :kdensity => (kdensity, linspace(0, 5, 101))
        ))

plot_results(results)


using Plots
using ImplicitEquations

f(x,y) = y^2 - x^3 + x

p1 = plot(Eq(f,0))

xs = -1:0.01:4
ys = -4:0.01:4
zs = f.(xs',ys)

@show size(zs)
p1 = plot(Eq(f,0))
p2 = plot(xs,ys,zs,st=:surface,camera=(10,30))

lay = @layout [a b]
plot(p1,p2,layout=lay,xlabel="x",ylabel="y")


function LineArc(line::Geometry, arc::Geometry)
    if line.GType != "line" || arc.GType != "arc"
        error("input type incorrect: line and arc needed")
    end
    Center = ArcCenter(arc)
    SymPy.@vars x y t
    x = line.Sp.x + t*(line.Ep.x - line.Sp.x)
    y = line.Sp.y + t*(line.Ep.y - line.Sp.y)
    Eq = (x - Center.x)^2 + (y - Center.y)^2 - (1/arc.Sp.cur)^2
    Roots = SymPy.solve(Eq)
    #Ips = Array{Tuple{Point2D{Float64},Float64, Float64}, 1}()
    #if !isa(SymPy.N(Roots[1]), Real)  # line and circle do not Intersect
        #println(typeof(Roots[1]))
    #    return (intersect=false, type="none")
    #else
    if isa(SymPy.N(Roots[1]), Real)
        for ti in Roots
            if 0.0 <= ti <= 1.0 + 1.0e-6  # check if intersection point is in line segment
                Ipc = Point2D{Float64}(SymPy.subs(x, t, ti), SymPy.subs(y, t, ti))
                dist1 = SymPy.N(ti*line.Length)
                Result = PointInSector(Ipc, arc)
                if Result[1] # check if intersection point is in arc
                    dist2 = Result[2]
                    return (intersect=true, type="cross", Ip=Ipc, Dist1=dist1, Dist2=dist2) # only the first junction point is returned
                    #push!(Ips, (Ipc, Dist1, Dist2))
                end
            end
        end
    end

    return (intersect=false, type="none")
end


using Plots, LaTeXStrings, ProgressBars
function LogisticMap(x::Matrix{<:Number}, r::Vector{<:Number})
    return x = 4 .* r .* x .* (1 .- x)
end
rcount= 2001
r_s = collect(range(0.0, 1.0, length=rcount))
x_s = rand(rcount, 10)
anim = @animate for n in ProgressBar(1:70)
    global x_s = LogisticMap(x_s, r_s)
    scatter(r_s, x_s, ms=0.5, msw=0, xlims=(0.0, 1.0), ylims = (0, 1), marker=:rect,
        legend=false, color=:black, alpha=0.25, xlabel=L"r", ylabel=L"x_n",
        title="\$\\textrm{Logistic\\ Map},\\quad n=$n\$",
        dpi=120)
end
gif(anim, "bifurcation.gif", fps=10)


@doc raw"""
    AsymptoticSKCETest(kernel::Kernel, predictions, targets)

Calibration hypothesis test based on the unbiased estimator of the squared kernel
calibration error (SKCE) with quadratic sample complexity.

# Details

Let ``\mathcal{D} = (P_{X_i}, Y_i)_{i=1,\ldots,n}`` be a data set of predictions and
corresponding targets. Denote the null hypothesis "the predictive probabilistic model is
calibrated" with ``H_0``.

The hypothesis test approximates the p-value ``ℙ(\mathrm{SKCE}_{uq} > c \,|\, H_0)``, where
``\mathrm{SKCE}_{uq}`` is the unbiased estimator of the SKCE, defined as
```math
\frac{2}{n(n-1)} \sum_{1 \leq i < j \leq n} h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big),
```
where
```math
\begin{aligned}
h_k\big((μ, y), (μ', y')\big) ={}&   k\big((μ, y), (μ', y')\big)
                                   - 𝔼_{Z ∼ μ} k\big((μ, Z), (μ', y')\big) \\
                                 & - 𝔼_{Z' ∼ μ'} k\big((μ, y), (μ', Z')\big)
                                   + 𝔼_{Z ∼ μ, Z' ∼ μ'} k\big((μ, Z), (μ', Z')\big).
\end{aligned}
```

The p-value is estimated based on the asymptotically valid approximation
```math
ℙ(n\mathrm{SKCE}_{uq} > c \,|\, H_0) \approx ℙ(T > c \,|\, \mathcal{D}),
```
where ``T`` is the bootstrap statistic
```math
T = \frac{2}{n} \sum_{1 \leq i < j \leq n} \bigg(h_k\big((P^*_{X_i}, Y^*_i), (P^*_{X_j}, Y^*_j)\big)
- \frac{1}{n} \sum_{r = 1}^n h_k\big((P^*_{X_i}, Y^*_i), (P_{X_r}, Y_r)\big)
- \frac{1}{n} \sum_{r = 1}^n h_k\big((P_{X_r}, Y_r), (P^*_{X_j}, Y^*_j)\big)
+ \frac{1}{n^2} \sum_{r, s = 1}^n h_k\big((P_{X_r}, Y_r), (P_{X_s}, Y_s)\big)\bigg)
```
for bootstrap samples ``(P^*_{X_i}, Y^*_i)_{i=1,\ldots,n}`` of ``\mathcal{D}``.
This can be reformulated to the approximation
```math
ℙ(n\mathrm{SKCE}_{uq}/(n - 1) - \mathrm{SKCE}_b > c \,|\, H_0) \approx ℙ(T' > c \,|\, \mathcal{D}),
```
where
```math
\mathrm{SKCE}_b = \frac{1}{n^2} \sum_{i, j = 1}^n h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big)
```
and
```math
T' = \frac{2}{n(n - 1)} \sum_{1 \leq i < j \leq n} h_k\big((P^*_{X_i}, Y^*_i), (P^*_{X_j}, Y^*_j)\big)
- \frac{2}{n^2} \sum_{i, r=1}^n h_k\big((P^*_{X_i}, Y^*_i), (P_{X_r}, Y_r)\big).
```

# References

Widmann, D., Lindsten, F., & Zachariah, D. (2019). [Calibration tests in multi-class
classification: A unifying framework](https://proceedings.neurips.cc/paper/2019/hash/1c336b8080f82bcc2cd2499b4c57261d-Abstract.html).
In: Advances in Neural Information Processing Systems (NeurIPS 2019) (pp. 12257–12267).

Widmann, D., Lindsten, F., & Zachariah, D. (2021). [Calibration tests beyond
classification](https://openreview.net/forum?id=-bxf89v3Nx).
"""
struct AsymptoticSKCETest{K<:Kernel,E,V,M} <: HypothesisTests.HypothesisTest
    """Kernel."""
    kernel::K
    """Calibration error estimate."""
    estimate::E
    """Test statistic."""
    statistic::V
    """Symmetric kernel matrix, consisting of pairwise evaluations of ``h_{ij}``."""
    kernelmatrix::M
end

function AsymptoticSKCETest(
    kernel::Kernel, predictions::AbstractVector, targets::AbstractVector
)
    # compute the calibration error estimate, the test statistic, and the kernel matrix
    estimate, statistic, kernelmatrix = estimate_statistic_kernelmatrix(
        kernel, predictions, targets
    )

    return AsymptoticSKCETest(kernel, estimate, statistic, kernelmatrix)
end

# HypothesisTests interface

HypothesisTests.default_tail(::AsymptoticSKCETest) = :right

function HypothesisTests.pvalue(test::AsymptoticSKCETest; kwargs...)
    return pvalue(Random.GLOBAL_RNG, test; kwargs...)
end
function HypothesisTests.pvalue(
    rng::Random.AbstractRNG, test::AsymptoticSKCETest; bootstrap_iters::Int=1_000
)
    return bootstrap_ccdf(rng, test.statistic, test.kernelmatrix, bootstrap_iters)
end

HypothesisTests.testname(::AsymptoticSKCETest) = "Asymptotic SKCE test"

# parameter of interest: name, value under H0, point estimate
function HypothesisTests.population_param_of_interest(test::AsymptoticSKCETest)
    return "SKCE", zero(test.estimate), test.estimate
end

function HypothesisTests.show_params(io::IO, test::AsymptoticSKCETest, ident="")
    return println(io, ident, "test statistic: $(test.statistic)")
end

@doc raw"""
    estimate_statistic_kernelmatrix(kernel, predictions, targets)

Compute the estimate of the SKCE, the test statistic, and the matrix of the evaluations of
the kernel function.

# Details

Let ``\mathcal{D} = (P_{X_i}, Y_i)_{i=1,\ldots,n}`` be a data set of predictions and
corresponding targets.

The unbiased estimator ``\mathrm{SKCE}_{uq}`` of the SKCE is defined as
```math
\frac{2}{n(n-1)} \sum_{1 \leq i < j \leq n} h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big),
```
where
```math
\begin{aligned}
h_k\big((μ, y), (μ', y')\big) ={}&   k\big((μ, y), (μ', y')\big)
                                   - 𝔼_{Z ∼ μ} k\big((μ, Z), (μ', y')\big) \\
                                 & - 𝔼_{Z' ∼ μ'} k\big((μ, y), (μ', Z')\big)
                                   + 𝔼_{Z ∼ μ, Z' ∼ μ'} k\big((μ, Z), (μ', Z')\big).
\end{aligned}
```

The test statistic is defined as
```math
\frac{n}{n-1} \mathrm{SKCE}_{uq} - \mathrm{SKCE}_b,
```
where
```math
\mathrm{SKCE}_b = \frac{1}{n^2} \sum_{i, j = 1}^n h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big)
```
(see [`AsymptoticSKCETest`](@ref)). This is equivalent to
```math
\frac{1}{n^2} \sum_{i, j = 1} h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big) \bigg(\frac{n^2}{(n - 1)^2} 1(i \neq j) - \bigg).
```

The kernelmatrix ``K \in \mathbb{R}^{n \times n}`` is defined as
```math
K_{ij} = h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big)
```
for ``i, j \in \{1, \ldots, n\}``.
"""
function estimate_statistic_kernelmatrix(kernel, predictions, targets)
    # obtain number of samples
    nsamples = length(predictions)
    nsamples > 1 || error("there must be at least two samples")

    # pre-computations
    α = (2 * nsamples - 1) / (nsamples - 1)^2

    @inbounds begin
        # evaluate the kernel function for the first pair of samples
        prediction = predictions[1]
        target = targets[1]

        # initialize the kernel matrix
        hij = unsafe_skce_eval(kernel, prediction, target, prediction, target)
        kernelmatrix = Matrix{typeof(hij)}(undef, nsamples, nsamples)
        kernelmatrix[1, 1] = hij

        # initialize the test statistic and the unbiased estimate of the SKCE
        statistic = -hij / 1
        estimate = zero(statistic)

        # add evaluations of all other pairs of samples
        nstatistic = 1
        nestimate = 0
        for i in 2:nsamples
            predictioni = predictions[i]
            targeti = targets[i]

            for j in 1:(i - 1)
                predictionj = predictions[j]
                targetj = targets[j]

                # evaluate the kernel function
                hij = unsafe_skce_eval(kernel, predictioni, targeti, predictionj, targetj)

                # update the kernel matrix
                kernelmatrix[j, i] = hij

                # update the estimate and the test statistic
                nstatistic += 2
                statistic += 2 * (α * hij - statistic) / nstatistic
                nestimate += 1
                estimate += (hij - estimate) / nestimate
            end

            # evaluate the kernel function for the `i`th sample
            hij = unsafe_skce_eval(kernel, predictioni, targeti, predictioni, targeti)

            # update the kernel matrix
            kernelmatrix[i, i] = hij

            # update the test statistic
            nstatistic += 1
            statistic -= (statistic + hij) / nstatistic
        end
    end

    # add lower triangle of the kernel matrix
    LinearAlgebra.copytri!(kernelmatrix, 'U')

    return estimate, statistic, kernelmatrix
end

@doc raw"""
    bootstrap_ccdf(rng::AbstractRNG, statistic, kernelmatrix, bootstrap_iters::Int)

Estimate the value of the inverse CDF of the test statistic under the calibration null
hypothesis by bootstrapping.

# Details

Let ``\mathcal{D} = (P_{X_i}, Y_i)_{i=1,\ldots,n}`` be a data set of predictions and
corresponding targets. Denote the null hypothesis "the predictive probabilistic model is
calibrated" with ``H_0``, and the test statistic with ``T``.

The value of the inverse CDF under the null hypothesis is estimated based on the
asymptotically valid approximation
```math
ℙ(T > c \,|\, H_0) \approx ℙ(T' > c \,|\, \mathcal{D}),
```
where the bootstrap statistic ``T'`` is defined as
```math
T' = \frac{2}{n(n - 1)} \sum_{1 \leq i < j \leq n} h_k\big((P^*_{X_i}, Y^*_i), (P^*_{X_j}, Y^*_j)\big)
- \frac{2}{n^2} \sum_{i, r=1}^n h_k\big((P^*_{X_i}, Y^*_i), (P_{X_r}, Y_r)\big)
```
for bootstrap samples ``(P^*_{X_i}, Y^*_i)_{i=1,\ldots,n}`` of ``\mathcal{D}``
(see [`AsymptoticSKCETest`](@ref)).

Let ``C_i`` be the number of times that data pair ``(P_{X_i}, Y_i)`` was resampled.
Then we obtain
```math
T' = \frac{1}{n^2} \sum_{i=1}^n C_i \sum_{j=1}^n \bigg(\frac{n}{n-1} (C_j - \delta_{i,j}) - 2\bigg) h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big).
```
"""
function bootstrap_ccdf(
    rng::Random.AbstractRNG, statistic, kernelmatrix, bootstrap_iters::Int
)
    # initialize array of counts of resampled indices
    nsamples = LinearAlgebra.checksquare(kernelmatrix)
    resampling_counts = Vector{Int}(undef, nsamples)

    # for each bootstrap sample
    α = nsamples / (nsamples - 1)
    extreme_count = 0
    sampler = Random.Sampler(rng, 1:nsamples)
    for _ in 1:bootstrap_iters
        # resample data set
        fill!(resampling_counts, 0)
        for _ in 1:nsamples
            idx = rand(rng, sampler)
            @inbounds resampling_counts[idx] += 1
        end

        # evaluate the bootstrap statistic
        z = zero(statistic)
        n = 0
        for i in 1:nsamples
            # check if the `i`th data pair was sampled
            ci = resampling_counts[i]
            iszero(ci) && continue

            zi = Statistics.mean(enumerate(resampling_counts)) do (j, cj)
                # obtain evaluation of the kernel function
                @inbounds hij = kernelmatrix[j, i]
                return ((cj - (i == j)) * α - 2) * hij
            end

            # update bootstrap statistic
            n += ci
            z += ci * (zi - z) / n
        end

        # check if the bootstrap statistic is ≥ the original statistic
        if z ≥ statistic
            extreme_count += 1
        end
    end

    return extreme_count / bootstrap_iters
end


# Exponential map via Rodriguez formula
# =====================================
function expmap(O)
  O = vec(O);
  dp = dot(O,O);
  ln = sqrt(dp);
  W = skew(O);
  if ln < 1E-15
    R = Matrix{Float64}(I, 3, 3);
  else
    R = I + (sin(ln)/ln)*W + ((1-cos(ln))/dp)*W*W;
  end
  return R;
end

# Skew-symmetric matrix out of 3-vector
# =====================================
function skew(O)
  [0.0 -O[3] O[2];
   O[3] 0.0 -O[1];
  -O[2] O[1] 0.0];
end


using UncertaintyQuantification

σx = [1, 1.1, 0.9, 1.2, 0.8]
σω = [0.7, 1.3, 1.4, 0.6, 0.95]

X = RandomVariable.(Normal.(0, σx), [:X1, :X2, :X3, :X4, :X5])
ω = RandomVariable.(Normal.(0, σω), [:ω1, :ω2, :ω3, :ω4, :ω5])

B = Model(
    df ->
        df.X1 .* df.ω1 + df.X2 .* df.ω2 + df.X3 .* df.ω3 + df.X4 .* df.ω4 + df.X5 .* df.ω5,
    :B,
)

mc = MonteCarlo(10000)

s_mc = sobolindices([B], [X; ω], :B, mc)

# Compare with analytical solution
VB = sum(σx .^ 2 .* σω .^ 2)
analytical = (σx .^ 2 .* σω .^ 2) / VB

# Monte Carlo
@assert all(isapprox.(s_mc.FirstOrder, 0.0, atol=0.1))
@assert all(isapprox.(s_mc.TotalEffect[1:5], analytical, rtol=0.1))
@assert all(isapprox.(s_mc.TotalEffect[6:end], analytical, rtol=0.1))


using Nemo
using Hecke
using ToyFHE
using Primes
using Test

const n_plaintext_slots = 3
const plaintext_modulus = 256

p_fact = Primes.factor(plaintext_modulus)
@assert length(p_fact) == 1
pbase = first(keys(p_fact))

const ℤ = Nemo.ZZ
const ℤx, x = PolynomialRing(ℤ, "x")
const ℤp = ResidueRing(ℤ, pbase)
const ℤpx, xp = PolynomialRing(ℤp, "xp")

# Find a prime cyclotomic that supports at least `n_plaintext_slots` slots
function find_cyclotomic(nslots)
    # TODO: What are the security considerations on ring dimensions
    for m in primes(2^20)
        poly = cyclotomic(m, x)
        fact = Nemo.factor(ℤpx(poly))
        if length(fact) > nslots
            return m
        end
    end
end

m = find_cyclotomic(n_plaintext_slots)
poly = cyclotomic(m, x)

# TODO: What are the security considerations for choosing the ciphertext modulus
q = nextprime(big(2)^51)
ℛ = ResidueRing(PolynomialRing(ResidueRing(ℤ, q), "x")[1], poly)
ℛbig = ResidueRing(PolynomialRing(ResidueRing(ℤ, nextprime(big(2)^111)), "x")[1], poly)

params = BFVParams(
    ℛ,
    ℛbig,
    plaintext_space(ℛ, plaintext_modulus),
    1,
    8/√(2π),
    div(q, plaintext_modulus)
)

kp = keygen(params)

plain = PolyCRTEncoding(zero(plaintext_space(params)))
plain[:] .= [1, 2, 3, 4, 5, 6]

c1 = encrypt(kp, plain)
let dec = PolyCRTEncoding(decrypt(kp, c1*c1))
    @test dec == [1, 4, 9, 16, 25, 36]
end


using Distributions
using NHST

srand(1)

x = randn(20)
y = 1.0 + randn(20)

@assert isapprox(test_statistic(TTest, x, y), -2.8204)
@assert isapprox(p_value(TTest, x, y), 0.00758227)
@assert isapprox(ci(TTest, x, y)[1], -1.5176904)
@assert isapprox(ci(TTest, x, y)[2], -0.2493534)

@assert isapprox(test_statistic(run(TTest, x, y)), -2.8204)
@assert isapprox(p_value(run(TTest, x, y)), 0.00758227)
@assert isapprox(ci(run(TTest, x, y))[1], -1.5176904)
@assert isapprox(ci(run(TTest, x, y))[2], -0.2493534)

run(TTest, x, y)


@testset "Linear Spaces" begin
    @testset "LinearSubspace" begin
        A = LinearSubspace([1 0 3; 2 1 3], [5, -2])
        @test dim(A) == dim(intrinsic(A)) == dim(extrinsic(A)) == 1
        @test codim(A) == codim(intrinsic(A)) == codim(extrinsic(A)) == 2
        @test ambient_dim(A) == 3
        @test startswith(sprint(show, A), "1-dim. affine linear subspace")
        @test identity.(A) == A

        @test intrinsic(A) isa IntrinsicDescription
        @test startswith(sprint(show, intrinsic(A)), "IntrinsicDescription")
        @test extrinsic(A) isa ExtrinsicDescription
        @test startswith(sprint(show, extrinsic(A)), "ExtrinsicDescription")

        @test identity.(extrinsic(A)) == extrinsic(A)
        @test identity.(intrinsic(A)) == intrinsic(A)
        @test identity.(Intrinsic) == Intrinsic

        u = rand(1)
        x = A(u, Intrinsic)
        @test coord_change(A, Extrinsic, Intrinsic, x) ≈ u rtol = 1e-12
        @test coord_change(A, Intrinsic, Extrinsic, u) ≈ x rtol = 1e-12
        @test norm(A(x, Extrinsic)) ≈ 0 atol = 1e-14
        @test norm(ExtrinsicDescription(intrinsic(A))(x)) ≈ 0.0 atol = 1e-12

        B = rand_subspace(3; codim = 2)
        @test B != A
        copy!(B, A)
        @test extrinsic(B) == extrinsic(A)
        @test intrinsic(B) == intrinsic(A)
        @test B == A
        @test copy(A) == A
        @test copy(A) !== A

        C = rand_subspace(3, dim = 1, real = true)
        @test geodesic_distance(A, C) > 0
        γ = geodesic(A, C)
        @test size(γ(1)) == size(intrinsic(A).Y)

        A2 = translate(A, [1, 1], Extrinsic)
        A3 = LinearSubspace(extrinsic(A).A, extrinsic(A).b + [1, 1])
        @test A2.intrinsic.Y ≈ A3.intrinsic.Y

        # rand subspace through a point
        x = randn(ComplexF64, 5)
        L = rand_subspace(x; dim = 2)
        @test norm(L(x)) ≈ 0 atol = 1e-8
        L2 = rand_subspace(x; dim = 2, affine = false)
        @test is_linear(L2)
        @test norm(L2(x)) ≈ 0 atol = 1e-8
    end

    @testset "Intersect subspaces" begin
        L₁ = rand_subspace(7; codim = 2)
        L₂ = rand_subspace(7; codim = 3)
        L₃ = L₁ ∩ L₂
        @test codim(L₃) == 5
        @test dim(L₃) == 2
        @test ambient_dim(L₃) == 7
        E₃ = extrinsic(L₃)
        @test norm(L₁(E₃.A \ E₃.b, Extrinsic)) ≈ 0 atol = 1e-12
        @test norm(L₂(E₃.A \ E₃.b, Extrinsic)) ≈ 0 atol = 1e-12
        @test norm(L₃(E₃.A \ E₃.b, Extrinsic)) ≈ 0 atol = 1e-12
    end

    @testset "IntrinsicSubspaceHomotopy" begin
        @var x[1:4]
        f1 = rand_poly(x, 2)
        f2 = rand_poly(x, 2)
        F = System([f1, f2], x)

        A = rand_subspace(4; codim = 2)
        # Commpute witness set
        L = A(x, Extrinsic)
        res = track.(total_degree(F ∩ L)...)
        W = solution.(res)

        B = rand_subspace(4, codim = 2)

        graff_tracker = Tracker(
            IntrinsicSubspaceHomotopy(F, A, B),
            options = (automatic_differentiation = 3,),
        )
        graff_result = track.(graff_tracker, W)
        @test all(is_success, graff_result)

        C = rand_subspace(4, codim = 2)
        copy_A = copy(A)
        set_subspaces!(graff_tracker.homotopy, B, C)
        @test A == copy_A
        @test all(is_success, track.(graff_tracker, solution.(graff_result)))

        graff_path_tracker = EndgameTracker(IntrinsicSubspaceHomotopy(F, A, B))
        graff_path_result = track.(graff_path_tracker, W)
        @test all(is_success, graff_path_result)

        @test all(
            isapprox.(solution.(graff_path_result), solution.(graff_result), rtol = 1e-12),
        )
    end
end


using Flux, Flux.Data.MNIST, LinearAlgebra
using Flux: onehotbatch

imgs   = Flux.Data.MNIST.images()
labels = Flux.Data.MNIST.labels()
nTrain = length(imgs)
trainData = vcat([hcat(float.(imgs[i])...) for i in 1:nTrain]...)
trainLabels = labels[1:nTrain]
testImgs = Flux.Data.MNIST.images(:test)
testLabels = Flux.Data.MNIST.labels(:test)
nTest = length(testImgs)
testData = vcat([hcat(float.(testImgs[i])...) for i in 1:nTest]...)

A = [ones(nTrain) trainData]
Adag = pinv(A)
tfPM(x) = x ? +1 : -1
yDat(k) = tfPM.(onehotbatch(trainLabels,0:9)'[:,k+1])
bets = [Adag*yDat(k) for k in 0:9]

classify(input) = findmax([([1 ; input])'*bets[k] for k in 1:10])[2]-1

predictions = [classify(testData[k,:]) for k in 1:nTest]
confusionMatrix = [sum((predictions .== i) .& (testLabels .== j))
				for i in 0:9, j in 0:9]
accuracy = sum(diag(confusionMatrix))/nTest

println("Accuracy: ", accuracy, "\nConfusion Matrix:")
show(stdout, "text/plain", confusionMatrix)


using Plots,ApproxFun

x=Fun()
D=Derivative()

L=D^10 + cosh(x)*D^8 + x^3*D^6 + x^4*D^4 + cos(x)*D^2 + x^2
d=Interval()
B=[dirichlet(d);neumann(d);[Evaluation(Interval(),false,k) for k=2:4];
           [Evaluation(Interval(),true,k) for k=2:4]]

u=[B;L]\[0.;0.;1.;1.;zeros(6);exp(x)]

ApproxFun.plot(u)


function get_indecies(k, n)
    a = collect(1:k-1)
    append!(a, collect(k+1:n))
    return a
end

function update_W!(W, A, B, r, m)
    for k = 1:r
        indecies = get_indecies(k, r)
        ΣWB = sum(i -> W[:, i] * B[i, k], indecies)
        W[:, k] = maximum(hcat(zeros(m, 1), (A[:, k] - ΣWB) / B[k, k]), 2)
    end
end

function update_H!(H, C, D, r, n)
    for k = 1:r
        indecies = get_indecies(k, r)
        ΣDH = sum(i -> D[k, i] * H[i, :], indecies)
        H[k, :] = maximum(hcat(zeros(n, 1), (C[k, :] - ΣDH) / D[k, k]), 2)'
    end
end

function hierarchical_als(X, W, H, r)
    (m, n) = size(X)
    while stopping_criteria
        A = X * H'
        B = H * H'
        update_W!(W, A, B, r, m)
        C = W' * X
        D = W' * W
        update_H!(H, C, D, r, n)
    end
end


## Primitives: Functions to be implemented for a newsvendor problems

"""
    underage_cost(anp::AbstractNewsvendorProblem)

Get the cost of underage of a newsvendor problem.
"""
underage_cost(anp::AbstractNewsvendorProblem) = error("Underage cost is not
 defined in the concrete type")

"""
    overage_cost(anp::AbstractNewsvendorProblem)

Get the cost of overage of a newsvendor problem.
"""
overage_cost(anp::AbstractNewsvendorProblem) = error("Overage cost is not defined
in the concrete type")

"""
	distr(anp::AbstractNewsvendorProblem)

Get the demand distribution of a newsvendor problem.
"""
distr(anp::AbstractNewsvendorProblem) = error("Demand distribution is not
defined in the concrete type")
##

## Optional primitive

"""
profit_shift(anp::AbstractNewsvendorProblem)

Define how profit is shifted because of fixed cost, penalty, etc.;
defautls to 0.0.
"""
profit_shift(anp::AbstractNewsvendorProblem) = zero(underage_cost(anp))

"""
	q_min(anp::AbstractNewsvendorProblem)

Define minimal feasible quanitity, e.g., due to contractual limitations;
defautls to 0.0.
"""
q_min(anp::AbstractNewsvendorProblem) = zero(underage_cost(anp))

"""
	q_max(anp::AbstractNewsvendorProblem)

Define maximal feasible quanitity, e.g., due to contractual limitations;
defautls to `Inf`.
"""
q_max(anp::AbstractNewsvendorProblem) = Inf
##

## Maximizing order quanitity

"""
	critical_fractile(anp::AbstractNewsvendorProblem)

Compute the critical fractile for a newsvendor problem:
```math
\\textrm{critical fractile} = \\frac{\\textrm{underage cost}}{\\textrm{underage cost} + \\textrm{overage cost}}
```
"""
function critical_fractile(anp::AbstractNewsvendorProblem)
    underage_cost(anp) / (underage_cost(anp) + overage_cost(anp))
end

"""
	q_opt(anp::AbstractNewsvendorProblem; rounded = true)

Compute the quanitity that maximizes expected profit for a newsvendor problem
(i.e., where critical fractile equals in-stock probability). Attempts to solve
```math
F(q_{opt}) = \\textrm{critical fractile}
```
where F is the c.d.f. of the demand distribution.
Returns closest next integer unless `rounded=false`; then, it returns exact real.
Clamps at q_min and q_max.
"""
function q_opt(anp::AbstractNewsvendorProblem; rounded = true)

    if critical_fractile(anp) <= 0
        return q_min(anp)
    end

    if critical_fractile(anp) >= 1
        return min(maximum(distr(anp)), q_max(anp))
    end

    q = clamp(quantile(distr(anp), critical_fractile(anp)), q_min(anp), q_max(anp))

    if rounded == false  return q end

    q = round(Int, q)

    if q_min(anp) < q < q_max(anp)
        return q
    else
        return clamp(q, q_min(anp), q_max(anp))
    end

end

@doc raw"""
	q_scarf(anp::AbstractNewsvendorProblem)

Compute the quanitity that maximizes the minimal expected profit among all
distributions with the same mean and variance. Worst-case solution (Scarf, 1958)
```math
q_{scarf} = \mu + \sigma/2 * (\sqrt{r} - \sqrt{1/r})
```
where
```math
r = \frac{\textrm{underage cost}}{ \textrm{overage cost}}
```
"""
function q_scarf(anp::AbstractNewsvendorProblem)
    μ = mean(distr(anp))
    σ = std(distr(anp))
    r = underage_cost(anp) / overage_cost(anp)
    return μ + σ / 2 * (√r - √(1 / r))
end

##

## Metrics: Expected values depending on order quanitity

@doc raw"""
	leftover(anp::AbstractNewsvendorProblem, q)

Compute expected leftover inventory when stocking quantity q.

```math
E[\textrm{leftover}] = \int_{-\infty}^q (q - x)f(x)dx
```
"""
function leftover(anp::AbstractNewsvendorProblem, q)
    if isinf(q) return Inf end
    return leftover(distr(anp), q)
end

"""
    sales(anp::AbstractNewsvendorProblem, q)

Compute expected sales when stocking quantity q.
"""
sales(anp::AbstractNewsvendorProblem, q) = q - leftover(anp, q)

"""
	lost_sales(anp::AbstractNewsvendorProblem, q)

Compute expected lost sales when stocking quantity q.
"""
lost_sales(anp::AbstractNewsvendorProblem, q) = mean(distr(anp)) - sales(anp, q)

"""
	mismatch_cost(anp::AbstractNewsvendorProblem, q)

Compute expected mismatch cost when stocking quantity q. It is given by
```math
E[\\textrm{mismatch cost}] = \\textrm{underage cost} \\times  E[\\textrm{lost sales}] + \\textrm{overage cost} \\times  E[\\textrm{leftover}]
```
"""
function mismatch_cost(anp::AbstractNewsvendorProblem, q)
    if isinf(q) return 0 end
    return (underage_cost(anp) * lost_sales(anp, q)
            +
            overage_cost(anp) * leftover(anp, q))
end

"""
    profit(anp::AbstractNewsvendorProblem)

Compute expected profit when stocking quantity q_opt.

	profit(anp::AbstractNewsvendorProblem, q)

Compute expected profit when stocking quantity q. It is given by
```math
E[\\textrm{profit}] = \\textrm{underage cost} \\times  \\mu - E[\\textrm{mismatch cost}] +  \\textrm{profit shift}
```
"""
profit(anp::AbstractNewsvendorProblem, q) = (underage_cost(anp) * mean(distr(anp))
                                             -
                                             mismatch_cost(anp, q) + profit_shift(anp))

profit(anp::AbstractNewsvendorProblem) = profit(anp, q_opt(anp))


function bisection_search(f::Function, x0::Vector, x1::Vector, t::Vector;
        tol::Real = 1e-5,
        maxiters::Int = 1000,
        verbose = true
    )

    @assert all(x0 .<= x1)
    @assert all(length.([x0, x1, t]) .== length(x0))

    x0, x1 = (x0, x1) .|> deepcopy

    # finding sense
    r0 = f(x0)
    r1 = f(x1)
    @assert length(r0) == length(r1) == length(x0)
    sense = map((x) -> x[1] <= x[2] ? 1.0 : -1.0, zip(r0, r1))
    for (i, (tc, sc, rc0, rc1)) in enumerate(zip(t, sense, r0, r1))
        !(sc*rc0 <= tc <= sc*rc1) && error("target[$i] = $tc not in range: [$(sc*rc0), $(sc*rc1)]")
    end

    verbose && (prog = ProgressThresh(tol, "searching:  "))
    xi = x1
    last_xi = x1
    for it in 1:maxiters
        xi = (x1 - x0)/2 + x0
        ri = f(xi)

        # Assume a constant monotony TODO: rethink this
        foreach(eachindex(x0)) do c
            sense[c] > 0.0 ?
                ri[c] < t[c] ? (x0[c] = xi[c]) : (x1[c] = xi[c]) :
                ri[c] < t[c] ? (x1[c] = xi[c]) : (x0[c] = xi[c])
        end

        error = maximum(abs.(ri - t))
        verbose && update!(prog, error)
        if all(error < tol)
            verbose && finish!(prog)
            return xi
        end
        last_xi = xi
    end
    verbose && finish!(prog)
    return xi
end


module PeriodicDataToolbox
    using NCDatasets
    using SparseArrays
    mutable struct Bundle
        interpolate_mtx :: AbstractArray{Float64, 2}

        fine_time       :: AbstractArray{Float64, 1}
        coarse_time     :: AbstractArray{Float64, 1}

        var_names       :: AbstractArray

        fine_data       :: Dict
        coarse_data     :: Dict
    end

    function readCoarseNCFile(filename, varnames, fine_time, )
        ds = Dataset(filename, "r")

        coarse_data = Dict()
        fine_data   = Dict()

        for varname in varnames
            coarse_data[varname] = nomissing(ds[varname][:], NaN)
            coarse_data[varname]
        end

        close(ds)
    end

    function interpolate!(
        fine_data       :: AbstractArray{Float64, 1},
        coarse_data     :: AbstractArray{Float64, 1},
        interpolate_mtx :: AbstractArray{Float64, 2},
    )

        for i = 1:length(fine_data)
            fine_data[i] = interpolate_mtx[i, 1] * coarse_data[1]
            for j = 2:length(coarse_data)
                fine_data[i] += interpolate_mtx[i, j] * coarse_data[j]
            end
        end

    end

    function genInterpolateMatrix(
        time1 :: AbstractArray{Float64, 1},  # coarse
        time2 :: AbstractArray{Float64, 1},  # fine
        period :: Float64,
    )

        N_ext = length(time1)+2
        time1_ext = zeros(Float64, N_ext)

        time1_ext[1]       = time1[end] - period

        if length(time1) > 1
            time1_ext[2:end-1] = time1[1:end]
        end

        time1_ext[end]     = time1[1] + period

        mtx = spzeros(Float64, length(time2), length(time1))

        if length(time1) == 1
            mtx .= 1.0
            return mtx
        end

        # The algorithm below assume length(time1) > 1
        i_2 = 1
        done = false
        for i = 1:N_ext-1

            ta = time1_ext[i]
            tb = time1_ext[i+1]
            while ta <= time2[i_2] && time2[i_2] < tb
                Δt = tb - ta
                ia = (i   ==     1  ) ? length(time1) : i-1
                ib = (i   == N_ext-1) ? 1             : i

                mtx[i_2, ia] = (tb - time2[i_2]) / Δt
                mtx[i_2, ib] = (time2[i_2] - ta) / Δt

                #println(tb - time2[i_2], "; ", time2[i_2] - ta, "; ", Δt, "; ", ia, " , ", ib)

                if i_2 < length(time2)
                    i_2 += 1
                else
                    done = true
                    break
                end
            end

            if done
                break
            end
        end

        if !done
            throw(ErrorException("Not all element of time2 is within [0, period]"))
        end

        return mtx

    end

end


# Simple Adaptive Metropolis Proposal
# The proposal at each step is equal to a scalar multiple
# of the empirical posterior covariance plus a fixed, small covariance
# matrix epsilon which is also used for initial exploration.
#
# Reference:
#   H. Haario, E. Saksman, and J. Tamminen, "An adaptive Metropolis algorithm",
#   Bernoulli 7(2): 223-242 (2001)
mutable struct AMProposal{FT <: Real, CT <: AbstractMatrix{FT},
                          MNT <: AbstractMvNormal} <: Proposal{MNT}
    epsilon::CT
    scalefactor::FT
    proposal::MNT
    μ::Vector{FT}
    M::Matrix{FT}
    δ::Vector{FT}
    N::Int
end

function AMProposal(
    epsilon::AbstractMatrix{FT}, scalefactor=FT(2.38^2 / size(epsilon, 1))
) where {FT <: Real}
    sym = PDMat(Symmetric(epsilon))
    proposal = MvNormal(zeros(size(sym, 1)), sym)
    AMProposal(sym, scalefactor, proposal, zeros(size(sym,1)), zeros(size(sym)...),
               zeros(size(sym,1)), 0)
end

logratio_proposal_density(::AMProposal, params_prev, params) = 0

# When the proposal is initialised the empirical posterior covariance is zero
function trackstep!(proposal::AMProposal, params)
    proposal.μ .= params
    proposal.M .= 0
    proposal.proposal = MvNormal(zeros(size(proposal.μ)), proposal.epsilon)
    proposal.N = 1
end

# Recompute the empirical posterior covariance matrix
function trackstep!(proposal::AMProposal, params, ::Union{Val{true}, Val{false}})
    proposal.N += 1
    proposal.δ .= params .- proposal.μ
    proposal.μ .+= proposal.δ ./ proposal.N

    mul!(proposal.M, params .- proposal.μ, proposal.δ', true, true)

    prop_cov = proposal.M .* proposal.scalefactor ./ proposal.N .+ proposal.epsilon
    proposal.proposal = MvNormal(mean(proposal.proposal), Symmetric(prop_cov))
end

function propose(rng::Random.AbstractRNG, p::AMProposal, ::DensityModel)
    return rand(rng, p.proposal)
end

function propose(rng::Random.AbstractRNG, p::AMProposal, ::DensityModel, t)
    return t + rand(rng, p.proposal)
end


@testset "bayesianeva.jl" begin

    n = 100

    x = Variable("x",randn(n))

    θ = [1; 1; -.5; .3; .1]

    μ = θ[1] .+ θ[2]*x.value
    ϕ = θ[3] .+ θ[4]*x.value
    σ = exp.(ϕ)
    ξ = θ[5]

    pd = GeneralizedExtremeValue.(μ,σ, ξ)

    y = rand.(pd)

    model = BlockMaxima(Variable("y", y), locationcov = [x] ,logscalecov = [x])

    fm = Extremes.BayesianEVA(model, Mamba.Chains(collect(θ')))

    rl = returnlevel(fm, 100)

    ci = cint(rl)

    @testset "getdistribution(fittedmodel)" begin
        @test all(vec(Extremes.getdistribution(fm)) .== pd)
    end

    @testset "quantile(fm, p)" begin
        # p not in [0, 1] throws
        @test_throws AssertionError Extremes.quantile(fm, -1)

        # Test with known values
        @test vec(quantile(fm, .95)) ≈ quantile.(pd, .95)
    end

    @testset "returnlevel(fm, returnPeriod)" begin
        # returnPeriod < 0 throws
        @test_throws AssertionError returnlevel(fm, -100)

        # Test with known values
        @test vec(rl.value) ≈ quantile.(pd, 1-1/100)
    end

    @testset "cint(fm, returnPeriod, confidencelevel)" begin
        # confidencelevel not in [0, 1]
        @test_throws AssertionError cint(rl, -1)

        # Test with known values
        @test ci[1][1] ≈ rl.value[1,1]
        @test ci[5][1] ≈ rl.value[1,5]

    end

    n = 1000
    nobservation = 1000
    nobsperblock = 1

    threshold = 10.0

    x = Variable("x",randn(n))

    θ = [-.5; 1; 0.1]

    ϕ = θ[1] .+ θ[2]*x.value
    σ = exp.(ϕ)
    ξ = θ[3]

    pd = GeneralizedPareto.(σ, ξ)

    y = rand.(pd)

    model = ThresholdExceedance(Variable("y", y), logscalecov = [x])

    fm = Extremes.BayesianEVA(model, Mamba.Chains(collect(θ')))

    rl = returnlevel(fm, 0, nobservation, nobsperblock, 100)

    ci = cint(rl)

    @testset "getdistribution(fittedmodel)" begin
        @test all(vec(Extremes.getdistribution(fm)) .== pd)
    end

    @testset "quantile(fm, p)" begin
        # p not in [0, 1] throws
        @test_throws AssertionError Extremes.quantile(fm, -1)

        # Test with known values
        @test vec(quantile(fm, .95)) ≈ quantile.(pd, .95)
    end

    @testset "returnlevel(fm, returnPeriod)" begin
        # returnPeriod < 0 throws
        @test_throws AssertionError returnlevel(fm, 0, nobservation, nobsperblock, -100)

        # Test with known values
        p = 1-nobservation/(100 * nobsperblock * n)
        @test vec(rl.value) ≈ quantile.(pd, p)
    end

    @testset "cint(fm, returnPeriod, confidencelevel)" begin
        # confidencelevel not in [0, 1]
        @test_throws AssertionError cint(returnlevel(fm, 0, nobservation, nobsperblock, 100), -1)

        # Test with known values
        @test ci[1][1] ≈ rl.value[1,1]
        @test ci[5][1] ≈ rl.value[1,5]

    end

    @testset "showfittedEVA(io, obj, prefix)" begin
        # print does not throw
        buffer = IOBuffer()
        @test_logs Extremes.showfittedEVA(buffer, fm, prefix = "\t")
    end

    @testset "showChain(io, chain, prefix)" begin
        # print does not throw
        buffer = IOBuffer()
        @test_logs Extremes.showChain(buffer, fm.sim)
    end

    @testset "findposteriormode(fm::BayesianEVA)" begin

        x = Variable("x", randn(10))
        μ = 10 .+ x.value
        σ = 1.0
        ξ = .1
        pd = GeneralizedExtremeValue.(μ, σ, ξ)
        y = rand.(pd)
        fm = Extremes.BayesianEVA(Extremes.BlockMaxima(Variable("y", y), locationcov=[x]),
            Mamba.Chains([10.0 1.0 0.0 .1; -10.0 1.0 0.0 .1; 20.0 1.0 0.0 .1]))

        θ̂ = Extremes.findposteriormode(fm)
        @test θ̂ ≈ [10.0; 1.0; 0.0; .1]

    end

end


using ConstructiveSolidGeometry
using Base.Test

# Unit tests for ray <-> plane intersection

# Parallel
test_ray = Ray(Coord(0.0, 0.0, 0.0), Coord(0.0, 1.0, 0.0) )
test_plane = Plane(Coord(1.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0))
@test raytrace(test_ray, test_plane)[1] == false

# Hit
test_ray = Ray(Coord(0.0, 0.0, 0.0), unitize(Coord(1.0, 1.0, 0.0)) )
test_plane = Plane(Coord(1.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0))
@test raytrace(test_ray, test_plane) == (true, sqrt(2))

# Miss
test_ray = Ray(Coord(0.0, 0.0, 0.0), unitize(Coord(-1.0, -1.0, 0.0)) )
test_plane = Plane(Coord(1.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0))
@test raytrace(test_ray, test_plane)[1] == false

# Inside Plane
test_ray = Ray(Coord(0.0, 0.0, 0.0), Coord(0.0, 1.0, 0.0) )
test_plane = Plane(Coord(0.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0))
@test raytrace(test_ray, test_plane)[1] == true

# Unit tests for ray <-> sphere intersection

# Hit from outside
test_ray = Ray(Coord(-2.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0) )
test_sphere = Sphere(Coord(0.0, 0.0, 0.0), 1.0)
@test raytrace(test_ray, test_sphere) == (true, 1.0)

# Hit from inside (backwards is closer)
test_ray = Ray(Coord(-0.9, 0.0, 0.0), Coord(1.0, 0.0, 0.0) )
test_sphere = Sphere(Coord(0.0, 0.0, 0.0), 1.0)
@test raytrace(test_ray, test_sphere) == (true, 1.9)

# Hit from inside (backwards is farther)
test_ray = Ray(Coord(-0.5, 0.0, 0.0), Coord(-1.0, 0.0, 0.0) )
test_sphere = Sphere(Coord(0.0, 0.0, 0.0), 1.0)
@test raytrace(test_ray, test_sphere) == (true, 0.5)

# Miss from outside
test_ray = Ray(Coord(-2.0, 0.0, 0.0), Coord(-1.0, 0.0, 0.0) )
test_sphere = Sphere(Coord(0.0, 0.0, 0.0), 1.0)
@test raytrace(test_ray, test_sphere)[1] == false

# Ray origin is on sphere
test_ray = Ray(Coord(-1.0, 0.0, 0.0), Coord(0.0, 1.0, 0.0) )
test_sphere = Sphere(Coord(0.0, 0.0, 0.0), 1.0)
@test raytrace(test_ray, test_sphere) == (true, 0.0)

# Unit tests for ray <-> Cone intersection

# Hit from inside
test_ray = Ray(Coord(0.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0) )
test_cone = Cone(Coord(0.0, 0.0, 1.0), Coord(0.0, 0.0, -1.0), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == true
@test raytrace(test_ray, test_cone)[2] ≈ 1.0

# Hit from inside
test_ray = Ray(Coord(-0.5, 0.0, 0.0), Coord(1.0, 0.0, 0.0) )
test_cone = Cone(Coord(0.0, 0.0, 1.0), Coord(0.0, 0.0, -1.0), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == true
@test raytrace(test_ray, test_cone)[2] ≈ 1.5

# Hit from outside
test_ray = Ray(Coord(-5.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0) )
test_cone = Cone(Coord(0.0, 0.0, 1.0), Coord(0.0, 0.0, -1.0), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == true
@test raytrace(test_ray, test_cone)[2] ≈ 4.0

# Hit at angle from outside
test_ray = Ray(Coord(-5.0, 1.0, 1.0), unitize(Coord(1.0, 0.0, -1.0)) )
test_cone = Cone(Coord(0.0, 0.0, 1.0), unitize(Coord(0.0, 0.1, -1.0)), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == true
@test raytrace(test_ray, test_cone)[2] ≈ 3.5511721769144815

# Hit at 45 at angle from outside, for rotated cylinder
test_ray = Ray(Coord(1.0, 1.0, 2.0), unitize(Coord(1.0, 1.0, 0.0)) )
test_cone = Cone(Coord(1.0, 1.0, 1.0), unitize(Coord(1.0, 1.0, 0.0)), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == true
@test raytrace(test_ray, test_cone)[2] ≈ 1.0

# Hit of shadow of cone
test_ray = Ray(Coord(0.0, 0.0, 2.0), Coord(1.0, 0.0, 0.0) )
test_cone = Cone(Coord(0.0, 0.0, 1.0), Coord(0.0, 0.0, -1.0), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == false

# Valid hit after shadow of cone
test_ray = Ray(Coord(0.5, 0.0, 2.0), Coord(0.0, 0.0, -1.0) )
test_cone = Cone(Coord(0.0, 0.0, 1.0), Coord(0.0, 0.0, -1.0), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == true
@test raytrace(test_ray, test_cone)[2] ≈ 1.5

# Valid hit before shadow of cone
test_ray = Ray(Coord(0.5, 0.0, 0.0), Coord(0.0, 0.0, 1.0) )
test_cone = Cone(Coord(0.0, 0.0, 1.0), Coord(0.0, 0.0, -1.0), pi/4.0 )
@test raytrace(test_ray, test_cone)[1] == true
@test raytrace(test_ray, test_cone)[2] ≈ 0.5

# Unit tests for ray <-> Infinite Cylinder intersection
# Z Cylinder Tests

test_cylinder = InfCylinder(Coord(-2.0, 0.0, 0.0), Coord(0.0, 0.0, 1.0), 1.0)

# Hit from outside
test_ray = Ray(Coord(0, 0.0, 0.0), Coord(-1.0, 0.0, 0.0) )
@test raytrace(test_ray, test_cylinder) == (true, 1.0)

# Hit from inside (backwards is closer)
test_ray = Ray(Coord(-1.5, 0.0, 0.0), Coord(-1.0, 0.0, 0.0) )
@test raytrace(test_ray, test_cylinder) == (true, 1.5)

# Hit from inside (forwards is closer)
test_ray = Ray(Coord(-2.5, 0.0, 0.0), Coord(-1.0, 0.0, 0.0) )
@test raytrace(test_ray, test_cylinder) == (true, 0.5)

# Miss from outside
test_ray = Ray(Coord(0.0, 2.5, 0.0), Coord(-1.0, 0.0, 0.0) )
@test raytrace(test_ray, test_cylinder)[1] == false

# Tilted Cylinder tests
test_cylinder = InfCylinder(Coord(0.0, 0.0, 0.0), unitize(Coord(1.0, 0.0, 1.0)), 1.0)

# Hit from outside
test_ray = Ray(Coord(-2.0, 0.0, 2.0), unitize(Coord(1.0, 0.0, -1.0)) )
@test raytrace(test_ray, test_cylinder)[1] == true
@test raytrace(test_ray, test_cylinder)[2] ≈ sqrt(8)-1.0

# Hit from inside (backwards is closer)
test_ray = Ray(Coord(-.5, 0.0, 0.5), unitize(Coord(1.0, 0.0, -1.0)) )
@test raytrace(test_ray, test_cylinder)[1] == true
@test raytrace(test_ray, test_cylinder)[2] ≈ sqrt(0.5^2+0.5^2) + 1

# Hit from inside (forwards is closer)
test_ray = Ray(Coord(0.5, 0.0, -.5), unitize(Coord(1.0, 0.0, -1.0)) )
@test raytrace(test_ray, test_cylinder)[2] ≈ 1 - sqrt(0.5^2+0.5^2)

# Miss from outside
test_ray = Ray(Coord(0.0, 2.5, 0.0), Coord(-1.0, 0.0, 0.0) )
@test raytrace(test_ray, test_cylinder)[1] == false

# Reflection testing
test_plane = Plane(Coord(1, 0, 0), Coord(-1, 0, 0), "reflective")

# Straight on
test_ray = Ray(Coord(0, 0, 0), Coord(1, 0, 0))
@test reflect(test_ray, test_plane).direction.x ≈ -1.0
@test reflect(test_ray, test_plane).direction.y ≈ 0.0
@test reflect(test_ray, test_plane).direction.z ≈ 0.0

# 45 degree angle
test_ray = Ray(Coord(0, 0, 0), unitize(Coord(1, 1, 0)))
@test reflect(test_ray, test_plane).direction.x ≈ -1.0/sqrt(2.0)
@test reflect(test_ray, test_plane).direction.y ≈ 1.0/sqrt(2.0)
@test reflect(test_ray, test_plane).direction.z ≈ 0.0

# 45 degree plane
test_plane = Plane(Coord(1, 1, 0), unitize(Coord(-1, -1, 0)), "reflective")

# Straight on
test_ray = Ray(Coord(0, 0, 0), unitize(Coord(1, 1, 0)))
@test reflect(test_ray, test_plane).direction.x ≈ -1.0/sqrt(2.0)
@test reflect(test_ray, test_plane).direction.y ≈ -1.0/sqrt(2.0)
@test reflect(test_ray, test_plane).direction.z ≈ 0.0

# At an angle
test_ray = Ray(Coord(0, 0, 0), Coord(1, 0, 0))
@test reflect(test_ray, test_plane).direction.x <= 1e-10
@test reflect(test_ray, test_plane).direction.y ≈ -1.0
@test reflect(test_ray, test_plane).direction.z ≈ 0.0

# Test plane halfspace

# Point is below a z-plane with normal pointing up
test_coord = Coord(0.0, 0.0, 0.0)
test_plane = Plane(Coord(0.0, 0.0, 1.0), unitize(Coord(0.0, 0.0, 1.0)))
@test halfspace(test_coord, test_plane) == -1

# Point is below a z-plane with normal pointing down
test_coord = Coord(0.0, 0.0, 0.0)
test_plane = Plane(Coord(0.0, 0.0, 1.0), unitize(Coord(0.0, 0.0, -1.0)))
@test halfspace(test_coord, test_plane) == 1

# Point is above a z-plane with normal pointing up
test_coord = Coord(0.0, 0.0, 2.0)
test_plane = Plane(Coord(0.0, 0.0, 1.0), unitize(Coord(0.0, 0.0, 1.0)))
@test halfspace(test_coord, test_plane) == 1

# Point is above a z-plane with normal pointing down
test_coord = Coord(0.0, 0.0, 2.0)
test_plane = Plane(Coord(0.0, 0.0, 1.0), unitize(Coord(0.0, 0.0, -1.0)))
@test halfspace(test_coord, test_plane) == -1

# Point is on a z-plane
test_coord = Coord(0.0, 0.0, 1.0)
test_plane = Plane(Coord(0.0, 0.0, 1.0), unitize(Coord(0.0, 0.0, -1.0)))
@test halfspace(test_coord, test_plane) == -1

# Test sphere halfspace

# Point is inside a sphere
test_coord = Coord(-1.9, -1.9, -2.5)
test_sphere = Sphere(Coord(-2.0, -2.0, -2.0), 1.0)
@test halfspace(test_coord, test_sphere) == -1

# Point is outside a sphere
test_coord = Coord(-1.9, -1.9, -2.5)
test_sphere = Sphere(Coord(-2.0, -2.0, -2.0), 0.1)
@test halfspace(test_coord, test_sphere) == 1

# Point on a sphere
test_coord = Coord(1.0, 0.0, 0.0)
test_sphere = Sphere(Coord(0.0, 0.0, 0.0), 1.0)
@test halfspace(test_coord, test_sphere) == -1

# Test Cone halfspace

# Inside cone
test_coord = Coord(0.0, 0.0, 0.0)
test_cone = Cone(Coord(0.0, 0.0, 1.0), Coord(0.0, 0.0, -1.0),  pi/8.0)
@test halfspace(test_coord, test_cone) == 1

# outside cone
test_coord = Coord(5.0, 0.0, 0.0)
@test halfspace(test_coord, test_cone) == -1

# outside cone (but inside shadow)
#test_coord = Coord(0.0, 0.0, 2.0)
@test halfspace(test_coord, test_cone) == -1

# Inside rotated cylinder
test_coord = Coord(1.5, 1.5, 1.1)
test_cone = Cone(Coord(1.0, 1.0, 1.0), unitize(Coord(1.0, 1.0, 0.0)), pi/4.0 )
@test halfspace(test_coord, test_cone) == 1

# Outside rotated cylinder
test_coord = Coord(1.5, 1.5, 2.1)
test_cone = Cone(Coord(1.0, 1.0, 1.0), unitize(Coord(1.0, 1.0, 0.0)), pi/4.0 )
@test halfspace(test_coord, test_cone) == -1

# Test InfCylinder Halfspace

# Inside cylinder
test_coord = Coord(2.0, 2.0, 2.0)
test_cylinder = InfCylinder(Coord(1.9, 1.9, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 1.0)
@test halfspace(test_coord, test_cylinder) == -1

# Outside cylinder
test_coord = Coord(-2.0, 2.0, 2.0)
test_cylinder = InfCylinder(Coord(1.9, 1.9, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 1.0)
@test halfspace(test_coord, test_cylinder) == 1

# On cylinder
test_coord = Coord(2.9, 1.9, 10.0)
test_cylinder = InfCylinder(Coord(1.9, 1.9, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 1.0)
@test halfspace(test_coord, test_cylinder) == -1

# Bounding Box for the Problem
bounding_box = Box(Coord(-.63, -.63, -150), Coord(.63, .63, 150))

# Containers to store cells as we make them
cells = Array{Cell}(0)

# Define all the surfaces we will use up front
# Box
top = Plane(Coord(0.0, 0.0, 150.0), unitize(Coord(0.0, 0.0, 1.0)),   "reflective")
bot = Plane(Coord(0.0, 0.0, -150.0), unitize(Coord(0.0, 0.0, -1.0)), "reflective")
left = Plane(Coord(-.63, 0.0, 0.0), unitize(Coord(-1.0, 0.0, 0.0)),  "reflective")
right = Plane(Coord(0.63, 0.0, 0.0), unitize(Coord(1.0, 0.0, 0.0)),  "reflective")
up = Plane(Coord(0.0, 0.63, 0.0), unitize(Coord(0.0, 1.0, 0.0)),     "reflective")
down = Plane(Coord(0.0, -0.63, 0.0), unitize(Coord(0.0, -1.0, 0.0)), "reflective")
clad_outer = InfCylinder(Coord(0.0, 0.0, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 0.4750)
clad_inner = InfCylinder(Coord(0.0, 0.0, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 0.4180)
fuel = InfCylinder(Coord(0.0, 0.0, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 0.4096)

# Make the Water Region
regions = Array{Region}(0)

push!(regions, Region(top, -1))
push!(regions, Region(bot, -1))
push!(regions, Region(left, -1))
push!(regions, Region(right, -1))
push!(regions, Region(up, -1))
push!(regions, Region(down, -1))
push!(regions, Region(clad_outer, 1))

ex = :(1 ^ 2 ^ 3 ^ 4 ^ 5 ^ 6 ^ 7)

push!(cells, Cell(regions, ex))

# Make the Cladding
regions = Array{Region}(0)

push!(regions, Region(top, -1))
push!(regions, Region(bot, -1))
push!(regions, Region(clad_outer, -1))
push!(regions, Region(clad_inner, 1))

ex = :(1 ^ 2 ^ 3 ^ 4)

push!(cells, Cell(regions, ex))

# Make the Gap
regions = Array{Region}(0)

push!(regions, Region(top, -1))
push!(regions, Region(bot, -1))
push!(regions, Region(clad_inner, -1))
push!(regions, Region(fuel, 1))

ex = :(1 ^ 2 ^ 3 ^ 4)

push!(cells, Cell(regions, ex))

# Make the Fuel
regions = Array{Region}(0)

push!(regions, Region(top, -1))
push!(regions, Region(bot, -1))
push!(regions, Region(fuel, -1))

ex = :(1 ^ 2 ^ 3)

push!(cells, Cell(regions, ex))

geometry = Geometry(cells, bounding_box)

# Some tests for our test geometry
@test find_cell_id(Coord(0.0,0.1,0), geometry) == 4
@test find_cell_id(Coord(0.0,0.41,0), geometry) == 3
@test find_cell_id(Coord(0.0,0.42,0), geometry) == 2
@test find_cell_id(Coord(0.0,0.55,0), geometry) == 1

# Some tests for weirder definitions
# Bounding Box for the Problem
bounding_box = Box(Coord(-.63, -.63, -150), Coord(.63, .63, 150))

# Containers to store cells as we make them
cells = Array{Cell}(0)

# Define all the surfaces we will use up front
# Box
top = Plane(Coord(0.0, 0.0, 150.0), unitize(Coord(0.0, 0.0, 1.0)), "reflective")
bot = Plane(Coord(0.0, 0.0, -150.0), unitize(Coord(0.0, 0.0, -1.0)), "reflective")
left = Plane(Coord(-.63, 0.0, 0.0), unitize(Coord(-1.0, 0.0, 0.0)), "reflective")
right = Plane(Coord(0.63, 0.0, 0.0), unitize(Coord(1.0, 0.0, 0.0)), "reflective")
up = Plane(Coord(0.0, 0.63, 0.0), unitize(Coord(0.0, 1.0, 0.0)), "reflective")
down = Plane(Coord(0.0, -0.63, 0.0), unitize(Coord(0.0, -1.0, 0.0)), "reflective")
left_cyl = InfCylinder(Coord(-0.175, 0.0, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 0.3 )
right_cyl = InfCylinder(Coord(0.1, 0.0, 0.0), unitize(Coord(0.0, 0.0, 1.0)), 0.2)

# Make the outer Region
regions = Array{Region}(0)

push!(regions, Region(top, -1))
push!(regions, Region(bot, -1))
push!(regions, Region(left, -1))
push!(regions, Region(right, -1))
push!(regions, Region(up, -1))
push!(regions, Region(down, -1))
push!(regions, Region(left_cyl, -1))
push!(regions, Region(right_cyl, -1))

ex = :(1 ^ 2 ^ 3 ^ 4 ^ 5 ^ (~(7 | 8)) ^ 6 )

push!(cells, Cell(regions, ex))

# Make the inner Region
regions = Array{Region}(0)

push!(regions, Region(top, -1))
push!(regions, Region(bot, -1))
push!(regions, Region(left_cyl, -1))
push!(regions, Region(right_cyl, -1))

ex = :(1 ^ 2 ^ (3 | 4))

push!(cells, Cell(regions, ex))

geometry = Geometry(cells, bounding_box)

@test find_cell_id(Coord(0.41,0.0,0), geometry) == 1
@test find_cell_id(Coord(0.05,0.0,0), geometry) == 2

# Test random ray generation
ray = generate_random_ray(geometry.bounding_box)
@test ray.origin.x >= geometry.bounding_box.lower_left.x
@test ray.origin.y >= geometry.bounding_box.lower_left.y
@test ray.origin.z >= geometry.bounding_box.lower_left.z
@test ray.origin.x <= geometry.bounding_box.upper_right.x
@test ray.origin.y <= geometry.bounding_box.upper_right.y
@test ray.origin.z <= geometry.bounding_box.upper_right.z
ray2 = generate_random_ray(geometry.bounding_box)
@test ray.origin != ray2.origin
@test ray.direction != ray2.direction

# Test intersection routines
ray = Ray(Coord(0.0, 0.0, 0.0), Coord(1.0, 0.0, 0.0) )
new_ray, id, bc_type = find_intersection(ray, geometry)
@test new_ray.origin.x ≈ 0.125
@test new_ray.origin.y ≈ 0.0
@test new_ray.origin.z ≈ 0.0
@test new_ray.direction.x ≈ 1.0
@test new_ray.direction.y ≈ 0.0
@test new_ray.direction.z ≈ 0.0
@test id == 3
@test bc_type == "transmission"

println("Tests complete")


struct topic_modeling_hyper <: distribution_hyper_params
    α::AbstractArray{Float64,1}
end

mutable struct topic_modeling_sufficient_statistics <: sufficient_statistics
    N::Float64
    points_sum::AbstractArray{Int64,1}
end

function calc_posterior(prior:: topic_modeling_hyper, suff_statistics::topic_modeling_sufficient_statistics)
    if suff_statistics.N == 0
        return prior
    end
    return topic_modeling_hyper(prior.α + suff_statistics.points_sum)
end

function sample_distribution(hyperparams::topic_modeling_hyper)
    cat_dist = rand(Dirichlet(hyperparams.α))
    return topic_modeling_dist(log.(cat_dist))
end

function create_sufficient_statistics(hyper::topic_modeling_hyper,posterior::topic_modeling_hyper,points::AbstractArray{Float64,2}, pts_to_group = 0)
    if length(points) == 0
        return topic_modeling_sufficient_statistics(size(points,2),zeros(Int64,length(hyper.α)))
    end
    pt_count = counts(Int.(points),length(hyper.α))
    return topic_modeling_sufficient_statistics(size(points,2),pt_count)
end

function log_marginal_likelihood(hyper::topic_modeling_hyper, posterior_hyper::topic_modeling_hyper, suff_stats::topic_modeling_sufficient_statistics)
    D = length(suff_stats.points_sum)
    N = Int(ceil(sum(posterior_hyper.α - hyper.α)))
    logpi = log(pi)
    val = sum((x-> logabsgamma(x)[1]).(posterior_hyper.α) - suff_stats.points_sum .* (x-> logabsgamma(x)[1]).(hyper.α)) + logabsgamma(sum(hyper.α))[1] -logabsgamma(N + sum(hyper.α))[1]
    return val
end

function aggregate_suff_stats(suff_l::topic_modeling_sufficient_statistics, suff_r::topic_modeling_sufficient_statistics)
    return topic_modeling_sufficient_statistics(suff_l.N+suff_r.N, suff_l.points_sum + suff_r.points_sum)
end


using KitBase, FluxReconstruction, OrdinaryDiffEq, LinearAlgebra, Plots

begin
    x0 = -1
    x1 = 1
    ncell = 16
    dx = (x1 - x0) / ncell
    deg = 3
    nsp = deg + 1
    cfl = 0.1
    dt = cfl * dx
    t = 0.0
    a = 1.0
    tspan = (0.0, 2.0)
    nt = tspan[2] / dt |> Int
    bc = :period
end
ps = FRPSpace1D(x0, x1, ncell, deg)

u = zeros(ncell, deg + 1)
for i = 1:ncell, j = 1:deg+1
    u[i, j] = sin(π * ps.xpg[i, j])
end

prob = FR.FRAdvectionProblem(u, tspan, ps, a, bc)
itg = init(prob, Tsit5(), saveat = tspan[2], adaptive = false, dt = dt)

for iter = 1:nt
    step!(itg)
end

begin
    x = zeros(ncell * nsp)
    sol = zeros(ncell * nsp)
    for i = 1:ncell
        idx0 = (i - 1) * nsp
        idx = idx0+1:idx0+nsp

        for j = 1:nsp
            idx = idx0 + j
            x[idx] = ps.xpg[i, j]
            sol[idx] = itg.u[i, j]
        end
    end
    ref = @. sin(π * x)
end

plot(x, sol, label = "t=1")
plot!(x, ref, line = :dash, label = "t=0")

begin
    dx |> println
    FR.L1_error(sol, ref, dx) |> println
    FR.L2_error(sol, ref, dx) |> println
    FR.L∞_error(sol, ref, dx) |> println
end


abstract type AbstractABC end
abstract type AbstractPointABC end
weights(x::AbstractABC) = x.weights
particles(x::AbstractABC) = x.particles
Distributions.mean(x::AbstractABC) = sum(weights(x) .* particles(x))
function Distributions.quantile(x::AbstractABC, q)
    [quantile([p[i] for p in particles(x)], Weights(weights(x)), q)
     for i in eachindex(first(particles(x)))]
end
Distributions.median(x::AbstractABC) = quantile(x, .5)

weights(log_weights) = weights!(similar(log_weights), log_weights)
weights!(weights, log_weights) = @. weights = exp(log_weights)

function logsumexp(x::Vector{T}) where T
    res = zero(T)
    for i in eachindex(x)
        res += exp(x[i])
    end
    log(res)
end

log_ess(logw) = -logsumexp(2 .* logw)
ess(w) = 1/sum(w.^2)

euclidean(x, y) = √sum(abs2, x .- y)
euclidean(x, y, b::Number) = euclidean(x, y)/b
euclidean(x, y, b::AbstractVector) = √sum(abs2, (x .- y) ./ b)
pairwise_euclidean(x::AbstractVector{<:AbstractVector{<:AbstractVector}}) = pairwise_euclidean(vcat(x...))
function pairwise_euclidean(x, b = 1)
    n = length(x)
    result = zeros(n * (n - 1) ÷ 2)
    k = 0
    for i in 1:n
        for j in i+1:n
            k += 1
            result[k] = euclidean(x[i], x[j], b)
        end
    end
    result
end
function energydistance(x, y, b)
    c1, c2, c3, nx, ny = _inner(x, y, (i, j) -> euclidean(i, j, b))
    2*(c1/(nx*ny) - c2/nx^2 - c3/ny^2)
end
function energydistance(x, y)
    c1, c2, c3, nx, ny = _inner(x, y, euclidean)
    2*(c1/(nx*ny) - c2/nx^2 - c3/ny^2)
end
struct StandardMMD end
struct LinearMMD end
struct RandomFourierMMD end
function _inner(x, y, k)
    c1 = 0.; c2 = 0.; c3 = 0.
    nx = length(x)
    ny = length(y)
    @simd for i in 1:nx
        for j in 1:ny
            @inbounds c1 += k(x[i], y[j])
        end
        for j in i+1:nx
            @inbounds c2 += k(x[i], x[j])
        end
    end
    @simd for i in 1:ny
        for j in i+1:ny
            @inbounds c3 += k(y[i], y[j])
        end
    end
    c1, c2, c3, nx, ny
end
mmd(x, y, k) = mmd(StandardMMD, x, y, k)
function mmd(::Type{StandardMMD},
             x::AbstractVector{<:AbstractVector},
             y::AbstractVector{<:AbstractVector}, k)
    c1, c2, c3, nx, ny = _inner(x, y, k)
    2*((nx == 1 ? 0. : c2/(nx * (nx - 1))) +
       (ny == 1 ? 0. : c3/(ny * (ny - 1))) -
       c1/(nx * ny))
end
function mmd(::Type{StandardMMD},
             x::AbstractVector{<:Number},
             y::AbstractVector{<:Number}, k)
    -2 * k(x, y)
end
function kldistance(x, y)
    res = 0.
    for xi in x
        res += log(minimum(euclidean(xi, yj) for yj in y))
    end
    res
end

"""
    defaultproposal(prior)

Returns a function that takes a list of `particles` and returns a proposal
distribution. This function is called during initialization of
[`PMC`](@ref) and [`AdaptiveSMC`](@ref).
"""
defaultproposal(d::Any) = particles -> MultiParticleNormal(deepcopy(particles),
                                                           fill(1/length(particles),
                                                           length(particles)))
function defaultproposal(d::Union{<:TruncatedMultivariateNormal, <:MultivariateUniform})
    particles -> MultiParticleNormal(deepcopy(particles),
                                     fill(1/length(particles), length(particles)),
                                     diagonal = length(d.lower) > 10,
                                     lower = d.lower, upper = d.upper)
end
defaultproposal(d::Uniform) = particles -> MultiParticleNormal(deepcopy(particles),
                                                               fill(1/length(particles),
                                                                    length(particles)),
                                                               lower = [d.a],
                                                               upper = [d.b])

init_particles(prior, K) = [randn(eltype(prior), length(prior)) for _ in 1:K]

"""
    EpsilonExponentialDecay(init, last, decay)

Exponentially decreasing sequence starting at `init`, decaying with factor `decay`,
ending as soon as it equals (or drops below) `last`.
# Example
```julia-repl
julia> collect(EpsilonExponentialDecay(1, .2, .5))
4-element Array{Any,1}:
 1.0
 0.5
 0.25
 0.125
```
"""
struct EpsilonExponentialDecay
    init::Float64
    last::Float64
    decay::Float64
end
Base.length(e::EpsilonExponentialDecay) = ceil(Int, -(log(e.init) - log(e.last))/log(e.decay)) + 1
function Base.show(io::IO, ::MIME"text/plain", d::EpsilonExponentialDecay)
    println(io, "EpsilonExponentialDecay(init = $(d.init), last = $(d.last), decay factor = $(d.decay))")
end
function Base.iterate(e::EpsilonExponentialDecay, s = e.init)
    s/e.decay < e.last + 2eps() && return nothing
    (s, s * e.decay)
end

"""
    EpsilonLinearDecay(init, last, decay)

Linearly decreasing sequence starting at `init`, decaying with step size `decay`,
ending as soon as it equals (or drops below) `last`.
# Example
```julia-repl
julia> collect(EpsilonLinearDecay(10, 5, 2))
4-element Array{Any,1}:
 10.0
  8.0
  6.0
  4.0
```
"""
struct EpsilonLinearDecay
    init::Float64
    last::Float64
    decay::Float64
end
Base.length(e::EpsilonLinearDecay) = ceil(Int, (e.init - e.last)/e.decay) + 1
function Base.show(io::IO, ::MIME"text/plain", d::EpsilonLinearDecay)
    println(io, "EpsilonLinearDecay(init = $(d.init), last = $(d.last), delta decay = $(d.decay))")
end
function Base.iterate(e::EpsilonLinearDecay, s = e.init)
    s + e.decay <= e.last && return nothing
    (s, s - e.decay)
end


using Random

include("get_div_grad.jl")

function test_cg()
  cg_tol = 1.0e-6;

  # Cubic spline matrix.
  n = 10;
  A = spdiagm(-1 => ones(n-1), 0 => 4*ones(n), 1 => ones(n-1))
  b = A * [1:n;];

  (x, stats) = cg(A, b, itmax=10);
  r = b - A * x;
  resid = norm(r) / norm(b)
  @printf("CG: Relative residual: %8.1e\n", resid);
  @test(resid <= cg_tol);
  @test(stats.solved);

  # Code coverage.
  (x, stats) = cg(Matrix(A), b);
  show(stats);

  radius = 0.75 * norm(x);
  (x, stats) = cg(A, b, radius=radius, itmax=10);
  show(stats)
  @test(stats.solved);
  @test(abs(radius - norm(x)) <= cg_tol * radius);

  # Sparse Laplacian.
  A = get_div_grad(16, 16, 16);
  b = randn(size(A, 1));
  (x, stats) = cg(A, b);
  r = b - A * x;
  resid = norm(r) / norm(b);
  @printf("CG: Relative residual: %8.1e\n", resid);
  @test(resid <= cg_tol);
  @test(stats.solved);

  radius = 0.75 * norm(x);
  (x, stats) = cg(A, b, radius=radius, itmax=10);
  show(stats)
  @test(stats.solved);
  @test(abs(radius - norm(x)) <= cg_tol * radius);

  opA = LinearOperator(A)
  (xop, statsop) = cg(opA, b, radius=radius, itmax=10)
  @test(abs(radius - norm(xop)) <= cg_tol * radius)

  n = 100
  B = LBFGSOperator(n)
  Random.seed!(0)
  for i = 1:5
    push!(B, rand(n), rand(n))
  end
  b = B * ones(n)
  (x, stats) = cg(B, b, itmax=2n)
  @test norm(x - ones(n)) ≤ cg_tol * norm(x)
  @test stats.solved

  # Test b == 0
  (x, stats) = cg(A, zeros(size(A,1)))
  @test x == zeros(size(A,1))
  @test stats.status == "x = 0 is a zero-residual solution"

  # Test integer values
  A = [4 -1 0; -1 4 -1; 0 -1 4]
  b = [7; 2; -1]
  (x, stats) = cg(A, b)
  @test stats.solved

  # Test with Jacobi (or diagonal) preconditioner
  A = ones(10,10) + 9 * I
  b = 10 * [1:10;]
  M = 1/10 * opEye(10)
  (x, stats) = cg(A, b, M=M, itmax=10);
  show(stats)
  r = b - A * x;
  resid = norm(r) / norm(b);
  @printf("CG: Relative residual: %8.1e\n", resid);
  @test(resid <= cg_tol);
  @test(stats.solved);
end

test_cg()


f(x::Vector) = (1.0 - x[1])^2
function g!(x::Vector, storage::Vector)
	storage[1] = -2.0 * (1.0 - x[1])
end

x = [0.0]
gradient = [0.0]
ls_x = [0.0]
ls_gradient = [0.0]
g!(x, gradient)
dx = -gradient
d = DifferentiableFunction(f, g!)

t, f_up, g_up = Optim.backtracking_line_search!(d, x, dx, ls_x, ls_gradient)
@assert f(x + t * dx) < f(x) + 0.9 * t * -dot(gradient, dx)


module MatrixNetworks
	using LinearAlgebra
	using DelimitedFiles

	export random_adjmat, symmetrize, neighbours, degree, load_matrix_network

	"""
		symmetrize(M::Matrix) -> SymM::Symmetric

	Convert rectangular (`n_L` × `n_T`) matrix into a symmetric (`n_L` + `n_T`) order
	matrix.

	# Examples
	```julia-repl
	julia> A = [ 1 0 1 ; 0 0 1 ]
	2×3 Array{Int64,2}:
	1  0  1
	0  0  1
	julia> symmetrize(A)
	5×5 Array{Int64,2}:
	0  0  1  0  1
	0  0  0  0  1
	1  0  0  0  0
	0  0  0  0  0
	1  1  0  0  0
	```

	# Arguments
	- `M::Matrix`: rectangular (`n_L` × `n_T`) matrix.

	"""
	function symmetrize(M::Matrix)::Symmetric
		# Get dimensions of non-symmetric matrix
		R, C = size(M)

		# Create empty matrices to fill spaces between our matrix
		Rzeros = zeros(Int64, R, R)
		Czeros = zeros(Int64, C, C)

		# Create symmetric matrix
		M	 = [ Rzeros M ; M' Czeros ]
		SymM = Symmetric(M)

		return SymM
	end

	"""
		neighbours(M::Matrix, node::Int, orientation::String = "row") -> neighbours_array::Array{Int}

	Get neighbours of node in adjacency matrix.

	# Examples
	```julia-repl
	julia> A = [ 0 0 1 0 1 ; 0 0 0 0 1 ; 1 0 0 0 0 ; 0 0 0 0 0 ; 1 1 0 0 0 ]
	5×5 Array{Int64,2}:
	0  0  1  0  1
	0  0  0  0  1
	1  0  0  0  0
	0  0  0  0  0
	1  1  0  0  0
	julia> neighbours(A, 1)
	2-element Array{Int64,1}:
	3
	5
	```

	```julia-repl
	julia> B = [ 0 1 0 0 ; 1 0 1 0 ]
	2×4 Array{Int64,2}:
	0  1  0  0
	1  0  1  0
	julia> neighbours(B, 2, "row")
	2-element Array{Int64,1}:
	1
	3
	julia> neighbours(B, 2, "column")
	1-element Array{Int64,1}:
	2
	```

	# Arguments
	- `M::Matrix`: adjacency matrix.
	- `node::Int`: idx of node.
	- `orientation::String="row"` : orientation of search (column or row-wise).

	"""
	function neighbours(M::Matrix, node::Int, orientation::String = "row")::Array{Int}
		if orientation == "row"
			node_adjacency	 = M[node,:]
		elseif orientation == "column"
			node_adjacency	 = M[:, node]
		else
			error("Orientation value must be either \"row\" of \"column\"")
		end

		neighbours_array = [idx for (idx,neighbour) in enumerate(node_adjacency)
							if neighbour != 0]

		return neighbours_array
	end

	"""
		degree(M::Matrix, node::Int, orientation::String = "row") -> node_degree::Int

	Get degree of node in symmetric adjacency matrix.

	# Examples
	```julia-repl
	julia> A = [ 0 0 1 0 1 ; 0 0 0 0 1 ; 1 0 0 0 0 ; 0 0 0 0 0 ; 1 1 0 0 0 ]
	5×5 Array{Int64,2}:
	0  0  1  0  1
	0  0  0  0  1
	1  0  0  0  0
	0  0  0  0  0
	1  1  0  0  0
	julia> degree(A, 1)
	2
	```

	```julia-repl
	julia> B = [ 0 1 0 0 ; 1 0 1 0 ]
	2×4 Array{Int64,2}:
	0  1  0  0
	1  0  1  0
	julia> degree(B, 2, "row")
	2
	julia> degree(B, 2, "column")
	1
	```

	# Arguments
	- `M::Matrix`: adjacency matrix.
	- `node::Int`: idx of node.
	- `orientation::String="row"` : orientation of search (column or row-wise).

	"""
	function degree(M::Matrix, node::Int, orientation::String = "row")::Int
		if orientation == "row"
			node_neighbours = neighbours(M, node, "row")
		elseif orientation == "column"
			node_neighbours = neighbours(M, node, "column")
		else
			error("Orientation value must be either \"row\" of \"column\"")
		end

		node_degree = length(node_neighbours)

		return node_degree
	end

	"""
		load_matrix_network(filepath::String, separator::Char = ' ') -> A::Matrix{Int64}
	"""
	function load_matrix_network(filepath::String, separator::Char = ' ')::Matrix{Int64}
		A = readdlm(filepath, separator, Int64, '\n')
		return A
	end

end


using LinearAlgebra
using CVChannel
using Test

"""
This verifies the multiplicativity of the [dephrasure channel](https://arxiv.org/abs/1806.08327)

The dephrasure channel is parameterized by two parameters. We scan over these parameters
and check that it is multiplicative at each point.
...
"""

println("\nVerifying the multiplicativity of channel value for dephrasure channel...")

@testset "Verifying Multiplicativity of Dephrasure Channel" begin
    println("We are going to check the multiplicativity when p,q in {0,0.1,...0.9,1}.")
    p_vals = [0.0:0.1:1;]; q_vals = [0.0:0.1:1;];
    pCtr = 0; qCtr = 0;
    max_diff = 0;
    for q_id in q_vals
        print("\nStarting q= ", q_id,". \n")
        orig_cv = 2 - q_id
        for p_id in p_vals
            #Get Data
            dephrasurepq(ρ) = dephrasureChannel(ρ,p_id,q_id)
            orig_choi = choi(dephrasurepq,2,3);
            par_choi = permuteSubsystems(kron(orig_choi,orig_choi),[1,3,2,4],[2,3,2,3]);

            #This guarantees upper bound which is important since the claim
            #is (effectively) cv = cv_ppt
            par_cv, opt1, opt2 = pptCVDual(par_choi,4,9);

            diff = par_cv - orig_cv^2
            abs(diff) > max_diff ? max_diff = abs(diff) : nothing

            #Iterate
            pCtr = pCtr + 1
        end
        qCtr += 1
        pCtr = 0
    end

    println("The difference between single copy channel value and parallel is upper bounded by:", max_diff)
    @test max_diff <= 5e-6
end


# TODO: optimize sigmoid and etc. with threshold

"""
    __BASIC_ACTIVATIONS__

a collection of activations with no parameter and only one input.
"""
__BASIC_ACTIVATIONS__ = [
    (:Sin, x->:(sin.($x)), x->:(cos.($x))),
    (:Cos, x->:(cos.($x)), x->:(-sin.($x))),
    (:Sinh, x->:(sinh.($x)), x->:(cosh.($x))),
    (:Cosh, x->:(cosh.($x)), x->:(-sinh.($x))),
    (:Tan, x->:(tan.($x)), x->:(sec.($x) .* sec.($x))),
    (:Tanh, x->:(tanh.($x)), x->:(sec.($x) .* tan.($x))),
    (:Csc, x->:(csc.($x)), x->:(-csc.($x) .* cot.($x))),
    (:Cot, x->:(cot.($x)), x->:(-csc.($x) .* csc.($x))),
    (:ArcSin, x->:(asin.($x)), x->:(1 ./ sqrt(1 .- ($x .* $x)))),
    (:ArcCos, x->:(acos.($x)), x->:(-1 ./ sqrt(1 .- ($x .* $x)))),
    (:ArcTan, x->:(atan.($x)), x->:(1 ./ (1 .+ ($x .* $x)))),
    (:Exp, x->:(exp.($x)), x->:(exp.($x))),
    (:Log, x->:(log.($x)), x->:(1 ./ $x)),
]

for (OP, FUNC, DFUNC) in __BASIC_ACTIVATIONS__
    @eval begin
        export $OP

        mutable struct $OP{T, O} <: AbstractBlock
            input::O
            $OP{T, O}() where {T, O} = new{T, O}()
        end

        $OP(::Type{T}, ::Type{O}) where {T, O} = $OP{T, O}()
        $OP(::Type{T};nbatch=1) where T = $OP(T, nbatch > 1 ? Matrix{T} : Vector{T})
        $OP(;nbatch=1) = $OP(Float64, nbatch=nbatch)

        function forward(op::$OP{T, O}, input::O) where {T, O}
            op.input = input
            return $(FUNC(:(input)))
        end

        function backward(op::$OP{T, O}, grad::O) where {T, O}
            grad .* $(DFUNC(:(op.input)))
        end
    end
end

include("Complex.jl")
include("Sigmoid.jl")
include("PReLU.jl")
include("ReLU.jl")
include("Gaussian.jl")


using ReachabilityAnalysis, SparseArrays

LazySets.set_rtol(Float64, 1e-13)
LazySets.set_ztol(Float64, 1e-13)

const eₓ = sparsevec([2], [1.], 4)
const x0 = 0.05

# model without parameter variation
function embrake_no_pv(; Tsample=1.E-4, ζ=1e-6, x0=0.05)
    # model's constants
    L = 1.e-3
    KP = 10000.
    KI = 1000.
    R = 0.5
    K = 0.02
    drot = 0.1
    i = 113.1167

    # state variables: [I, x, xe, xc]
    A = Matrix([-(R+K^2/drot)/L 0      KP/L  KI/L;
                          K/i/drot      0      0      0;
                          0             0      0      0;
                          0             0      0      0])

    EMbrake = @system(x' = Ax)

    # initial conditions
    I₀  = Singleton([0.0])
    x₀  = Singleton([0.0])
    xe₀ = Singleton([0.0])
    xc₀ = Singleton([0.0])
    X₀ = I₀ × x₀ × xe₀ × xc₀

    # reset map
    Ar = sparse([1, 2, 3, 4, 4], [1, 2, 2, 2, 4], [1., 1., -1., -Tsample, 1.], 4, 4)
    br = sparsevec([3, 4], [x0, Tsample*x0], 4)
    reset_map(X) = Ar * X + br

    # hybrid system with clocked linear dynamics
    ha = HACLD1(EMbrake, reset_map, Tsample, ζ)
    return IVP(ha, X₀)
end

# model with parameter variation changing only 1 coefficient, corresponds to the Flow* settings in [SO15]
function embrake_pv_1(; Tsample=1.E-4, ζ=1e-6, Δ=3.0, x0=0.05)
    # model's constants
    L = 1.e-3
    KP = 10000.
    KI = 1000.
    R = 0.5
    K = 0.02
    drot = 0.1
    i = 113.1167
    p = 504. + (-Δ .. Δ)

    # state variables: [I, x, xe, xc]
    A = IntervalMatrix([-p            0      KP/L   KI/L;
                        K/i/drot      0      0      0;
                        0             0      0      0;
                        0             0      0      0])

    EMbrake = @system(x' = Ax)

    # initial conditions
    I₀  = Singleton([0.0])
    x₀  = Singleton([0.0])
    xe₀ = Singleton([0.0])
    xc₀ = Singleton([0.0])
    X₀ = I₀ × x₀ × xe₀ × xc₀

    # reset map
    Ar = sparse([1, 2, 3, 4, 4], [1, 2, 2, 2, 4], [1., 1., -1., -Tsample, 1.], 4, 4)
    br = sparsevec([3, 4], [x0, Tsample*x0], 4)
    reset_map(X) = Ar * X + br

    # hybrid system with clocked linear dynamics
    ha = HACLD1(EMbrake, reset_map, Tsample, ζ)
    return IVP(ha, X₀)
end

# maximum time by which the solution is below the x0 threshold
function max_time(sol)
    sfpos = []
    times = []

    for fp in sol
        for (j, X) in enumerate(fp)
            push!(sfpos, ρ(eₓ, X))
            push!(times, tspan(fp, j))
        end
    end
    idx = findfirst(x -> x > x0, sfpos)

    if isnothing(idx)
        return tend(sol)
    else
        return sup(times[idx-1])
    end
end


function SimpleSyntheticHypergraph(n,m,K,pvals,rmin,rmax,cluster_sizes=ones(K),r_sizes=ones(rmax-rmin),cluster_prefs=ones(K))
    """
    Generate a synthetic hypergraph, store in incidence matrix
    Output several other things about hypergraph like edge sizes and degrees
    and edge list.

    This can be viewed as a simplified approximation to the HSBM, that is
    easier and faster to generate in practice.

    Procedure:
    * Set a fixed number of hyperedges m and clusters K (you can randomize this outside this function)
    * Set a distribution determining the relative proportion of hyperedges of each size from r_min to r_max
    * For each hyperedge, with probability p, assign it to one community at random,
        selecting a r nodes uniformly at random for a hyperedge of size r
    * Otherwise, with probability 1-p, select r nodes uniformly at random from across the entire hypergraph
    """

    # Normalize to get proportions/probabilities
    r_sizes = Weights(r_sizes/sum(r_sizes))
    cluster_sizes = Weights(cluster_sizes/sum(cluster_sizes))         # proportion of nodes in each cluster
    cluster_prefs = Weights(cluster_prefs/sum(cluster_prefs))        # relative proportion of interior hyperedge in each cluster

    # Generate the ground truth clustering of nodes
    # ground_truth = GenerateTruth(n,cluster_sizes)
    Clusters = Vector{Vector{Int64}}()
    for j = 1:K
        push!(Clusters,Vector{Int64}())
        @assert(n*cluster_sizes[j] > 2*rmax)
    end

    ground_truth = zeros(Int64,n)
    for i = 1:n
        c = sample(1:K,cluster_sizes)
        ground_truth[i] = c
        push!(Clusters[c],i)
    end

    U = Vector{Int64}()
    E = Vector{Int64}()
    EdgeList = Vector{Vector{Int64}}()
    E_lengths = zeros(Int64,m)
    for enum = 1:m
        e = SampleEdge(pvals,n,rmin,rmax,cluster_prefs,r_sizes,Clusters)
        push!(EdgeList,e)
        E_lengths[enum] = length(e)
        for node in e
            push!(U,node)
            push!(E,enum)
        end
    end
    He2n = SparseArrays.sparse(E,U,ones(length(U)),m,n)
    deg = vec(sum(He2n,dims = 1))
    return He2n, EdgeList, E_lengths, deg, ground_truth

end

function SampleEdge(pvals,n,rmin,rmax,cluster_prefs,r_sizes,Clusters)
    replace = false
    K = length(Clusters)
    r = sample(rmin:rmax,r_sizes)       # select a hyperedge size
    if rand(1)[1] < pvals[r-rmin+1]

        c = sample(1:K,cluster_prefs)   # select a random cluster to put the edge in
        return sort(sample(Clusters[c],r,replace = replace))
    else
        return sort(sample(1:n,r,replace = replace))
    end
end


using ThompsonSampling, Base.Test

x = linspace(0,2pi,100)
y = linspace(0,2pi,100)

g(x,y) = sin(x)*sin(y)

gη(x) = g(x[1],x[2]) + 0.05*randn()

x0 = [π 3π/2;π π+1.0]
y0 = [gη(x0[:,i]) for i in 1:size(x0,2)]

t = ThompsonSampler(gη,x0,y0,100,[0.0,0.0],[2pi,2pi])

collect(Iterators.take(t,2))
@time collect(Iterators.take(t,50));


# 10001st prime

using Primes

function solve()
    return prime(10001)
end

print(Int(solve()))


# -----------------------------------------------------------------------
# Total Radiative Forcing From Methane (based on FUND model equations).
# -----------------------------------------------------------------------

@defcomp rf_ch4_total_fund begin

    CH₄_0           = Parameter()             # Initial (pre-industrial) methane concentration (ppb).
    ϕ               = Parameter()             # Scaling term to represent indirect radiative forcing from methane (25% increase from tropospheric ozone, 15% increase from stratospheric water vapor).
    CH₄             = Parameter(index=[time]) # Atmospheric methane concetration for current period (ppb).
    rf_ch4_direct   = Parameter(index=[time]) # Direct forcing from atmospheric methane concentrations (Wm⁻²).

    rf_ch4_indirect = Variable(index=[time])  # Indirect radiative forcing from atmospheric methane concentrations (Wm⁻²).
    rf_ch4_total    = Variable(index=[time])  # Total radiative forcing (direct and indirect effects) from methane (Wm⁻²).

    function run_timestep(p, v, d, t)

        if is_first(t)
            # Set initial radiative forcing to 0.
            v.rf_ch4_indirect[t] = 0.0
            v.rf_ch4_total[t] = 0.0
        else
            # Indirect CH₄ forcing from FUND (following FUND equations, this is 40% of direct forcing without accounting for CH₄-N₂O absorption overlap).
            v.rf_ch4_indirect[t] = 0.036 * p.ϕ * (sqrt(p.CH₄[t]) - sqrt(p.CH₄_0))

            # Calculate total CH₄ radiative forcing.
            v.rf_ch4_total[t] = p.rf_ch4_direct[t] + v.rf_ch4_indirect[t]
        end
    end
end


"""
Samples f : R^d -> R at num points in box specified by lx and ux

INPUTS:
f: function being sampled
lx: length d array of lower bounds
ux: length d array of upper bounds

OUTPUTS:
res: length num array of sampled values
loc: num x d matrix of sample locations
"""
function sample_points(f, lx, ux; num = 10)
    @assert length(lx) == length(ux)
    d = length(lx)
    loc = rand(num, d)
    for j = 1:num
        for i = 1:d
            loc[j, i] = loc[j, i]*(ux[i] - lx[i]) + lx[i]
        end
    end
    res = zeros(num)
    for j  = 1:num
        res[j] = f(loc[j, :])
    end
    return res, loc
end


"""
Sun, W., Qu, J., Chen, Y., Di, Y. and Gao, F., 2016.
Heuristic sample reduction method for support vector data description.
Turkish Journal of Electrical Engineering & Computer Sciences, 24(1), pp.298-312.

Heuristic Sampling Reduction (HSR)
Parameter:
k
eps: 0.1 * num_dimensions
"""
mutable struct HSR <: Sampler
    k::Int
    eps::Real
    function HSR(k::Int, eps::Real=0.1)
        k > 0 || throw(ArgumentError("Invalid number of clusters $(k)."))
        eps >= 0 || throw(ArgumentError("Negative threshold parameter $(eps)."))
        return new(k, eps)
    end
end

function sample(sampler::HSR, data::DataSet, labels::Vector{Symbol})::BitArray{1}
    sampler.k = min(size(data, 2), sampler.k)
    threshold = sampler.eps * size(data, 1)
    clustering = kmeans(data, sampler.k)
    dist_center_sorted_idx = sortperm(clustering.costs)
    tree = KDTree(data)
    sample_mask = trues(size(data, 2))
    processed = falses(size(data, 2))
    for i in dist_center_sorted_idx
        if !processed[i]
            # inrange includes i itself
            # -> query sorted result and drop first element
            # Special case: another observation is equal to i
            # -> one of the two remains in sample_mask
            to_remove = inrange(tree, data[:, i], threshold, true)[2:end]
            sample_mask[to_remove] .= false
            processed[to_remove] .= true
            processed[i] = true
        end
    end
    @assert all(processed)
    return sample_mask
end


using GLM, DataFrames, Gadfly, Distributions
include("misc.jl")

function avPlot(dfrm::RegressionModel, variable::Symbol)
    varNames = names(dfrm.mf.df)
    # get e_yx
    lhs = dfrm.mf.terms.eterms[1]
    rhs = extractrhs(dfrm)
    rhsnew = replace(rhs.rhsstring, "+"*string(variable), "")
    rhsnew = rhsnew[2:end]
    rhsnew = parse(rhsnew)
    fnew = Formula(lhs, rhsnew)
    newfit = fit(LinearModel, fnew, dfrm.mf.df)
    e_yx = residuals(newfit)
    # get e_zx
    X = dfrm.mf.df[2:end]
    Z = X[variable]
    X = X[~[(name in [variable]) for name in names(X)]]
    intercept = ones(Int, nrow(X), 1)
    X = convert(Array, X)
    X = hcat(intercept, X)
    H = *(*(X, inv(*(transpose(X), X))), transpose(X))
    e_zx = *((eye(size(H)[1])-H), Z)
    df = DataFrame(e_yx=e_yx, e_zx=e_zx)
    # get alpha
    avlm = fit(LinearModel, @formula(e_yx~0+e_zx), df)
    alpha = coef(avlm)
    # plot
    xs_range = abs(maximum(df[:e_zx]) - minimum(df[:e_zx]))
    xs = linspace(minimum(df[:e_zx])-xs_range/10, maximum(df[:e_zx])+xs_range/10)
    ys = alpha .* xs
    xy = DataFrame(xs=xs, ys=ys)
    plot(layer(df, x="e_zx", y="e_yx", Geom.point), Guide.xlabel(string(variable)*" | others"),
         Guide.ylabel(string(varNames[1])*" | others"),
         Guide.title("Added-Variable Plot"),
         layer(xy, x="xs", y="ys", Theme(default_color=color("red")), Geom.line))
end

function vif(dfrm::RegressionModel)
    X = dfrm.mf.df[2:end]
    rhs = extractrhs(dfrm)
    result = DataFrame(variable=rhs.rhsarray, vif=0.0)
    i = 1
    for var in rhs.rhsarray
        lhs = parse(var)
        rhsnew = replace(rhs.rhsstring, "+"*var, "")
        rhsnew = rhsnew[2:end]
        rhsnew = parse(rhsnew)
        fnew = Formula(lhs, rhsnew)
        newfit = fit(LinearModel, fnew, X)
        r2 = rsquared(newfit)
        result[:vif][i] = 1 / (1-r2)
        i = i + 1
    end
    result
end

function rsquared(dfrm::RegressionModel)
    SStot = sum((dfrm.model.rr.y - mean(dfrm.model.rr.y)).^2)
    SSres = sum((dfrm.model.rr.y - dfrm.model.rr.mu).^2)
    return (1-(SSres/SStot))
end

function adjrsquared(dfrm::RegressionModel)
    SStot = sum((dfrm.model.rr.y - mean(dfrm.model.rr.y)).^2)
    SSres = sum((dfrm.model.rr.y - dfrm.model.rr.mu).^2)
    n = size(dfrm.model.rr.y, 1)  #number of samples
    if dfrm.mf.terms.intercept
        p = size(dfrm.mm.m, 2) - 1
    else
        p = size(dfrm.mm.m, 2)
    end
    return 1- ( (SSres/(n-p-1)) / (SStot/(n-1)) )
end

function rstudent(dfrm::RegressionModel)
    SSres = sum((dfrm.model.rr.y - dfrm.model.rr.mu).^2)
    n = size(dfrm.model.rr.y, 1)  #number of samples
    if dfrm.mf.terms.intercept
        p = size(dfrm.mm.m, 2) - 1
    else
        p = size(dfrm.mm.m, 2)
    end
    sigma2 = SSres / (n-p)
    X = dfrm.mm.m
    H = *(*(X, inv(*(transpose(X), X))), transpose(X))
    h = diag(H)
    r = residuals(dfrm) ./ (sqrt(sigma2 .* (1 - h)))
    return r
end

function jackknife(dfrm::RegressionModel)
    r = rstudent(dfrm)
    n = size(dfrm.model.rr.y, 1)  #number of samples
    if dfrm.mf.terms.intercept
        p = size(dfrm.mm.m, 2) - 1
    else
        p = size(dfrm.mm.m, 2)
    end
    t = r .* sqrt((n-p-1) ./ (n-p-r.^2))
    return t
end

function halfnorm(dfrm::RegressionModel)
    N = size(dfrm.model.rr.y, 1)
    n = 1:N
    X = dfrm.mm.m
    H = *(*(X, inv(*(transpose(X), X))), transpose(X))
    h = diag(H)
    labels = sortperm(h)
    h = h[labels]
    U = (N+n) / (2*N+1)
    d = Normal()
    u = quantile(d, U)
    # prepare labels for potential outliers
    df = DataFrame(u=u, h=h, label="")
    outlierlabel = labels[u.>2]
    no_outliers = length(outlierlabel)
    for i in (length(labels)-no_outliers+1):length(labels)
        df[:label][i] = string(labels[i])
    end
    plot(df, x="u", y="h", label="label", Geom.point, Geom.label, Guide.xlabel("Half-normal quantiles"), Guide.ylabel("Leverages"))
end

function cooksdistance(dfrm::RegressionModel; plotit::Bool=false)
    r = rstudent(dfrm)
    X = dfrm.mm.m
    H = *(*(X, inv(*(transpose(X), X))), transpose(X))
    h = diag(H)
    p = size(dfrm.mm.m, 2)
    d = (r.^2 .* h) ./ (p*(1-h))
    if !plotit
        return d
    else
        # identify outliers
        n = length(d)
        cutoff = 4/(n-p)
        labels = 1:n
        outlierlabel = labels[d.>cutoff]
        df = DataFrame(n=labels, d=d, label="")
        for i in outlierlabel
            df[:label][i] = string(i)
        end
        plot(df, x="n", y="d", label="label", Guide.xlabel("Case number"), Guide.ylabel("Cook's distance"), Geom.bar, Geom.label)
    end
end


export AffineMap, square_check, visualize

"""
An `AffineMap` represents a mapping of the plane to itself.
"""
immutable AffineMap
  A::Array{Float64,2}
  b::Array{Float64,1}
end

AffineMap(M::Matrix) = AffineMap(M,[0,0])
AffineMap() = AffineMap(eye(2))

"""
`f(x)` where `f` is an affine map and `x` is a vector
is standard funtion application taking `x` to `Ax+b`.
Equivalent form: `f*x`.
"""
function (f::AffineMap)(x::Vector)
  return f.A * x + f.b
end

(*)(f::AffineMap, x::Vector) = f(x)

function (*)(f::AffineMap, g::AffineMap)
  A = f.A * g.A
  b = f.A*g.b + f.b
  return AffineMap(A,b)
end

"""
`square_check(x)` checks if the vector `x` lies in the
unit square.
"""
square_check(x::Vector) = (0 <= x[1] <= 1) && (0 <= x[2] <= 1)

"""
`square_check(f,x)` checks if `f(x)` is in the unit square.
"""
square_check(f::AffineMap,x::Vector) = square_check(f(x))

"""
`square_check(f)` checks if `f(x)` is in the unit square
for four values of `x` corresponding
to the corners of the unit square.
"""
function square_check(f::AffineMap)
  corners = ([0,0], [0,1], [1,0], [1,1])
  return all(square_check(f,x) for x in corners)
end

function quad_draw(pts, marker::String="-")
  x = [v[1] for v in pts]
  y = [v[2] for v in pts]
  plot(x,y,marker,color="k")
  nothing
end

function visualize(f::AffineMap)
  corners = [ [0,0], [0,1], [1,1], [1,0], [0,0] ]
  quad_draw(corners,":")
  poly = map(f,corners)
  quad_draw(poly)
  axis("equal")
  axis("off")
  axis([-0.25,1.25,-0.25,1.25])
  nothing
end


#Demo 7: Bode Plots
#-------------------------------------------------------------------------------
using InspectDR
using Colors

#==Input
===============================================================================#

#Constants
#-------------------------------------------------------------------------------
black = RGB24(0, 0, 0)
white = RGB24(1, 1, 1)
red = RGB24(1, 0, 0)
green = RGB24(0, 1, 0)
blue = RGB24(0, 0, 1)
j = im

#Input data
#-------------------------------------------------------------------------------
fmax = 10e9
fstep = 5e6
f = collect(0:fstep:fmax)
fmin_disp = 10e6
G = 20.0
f_p1 = 50e6
f_p2 = .9e9
f_z = Inf #.8e9
f1 = .8e9

#==Helper functions
===============================================================================#
phase(x) = rad2deg(angle(x))
function roots(a,b,c)
	r = sqrt(b*b-4*a*c)
	result = (-b .+ [r, -r]) / (2a)
	return (result[1], result[2])
end
xf_o2(f, G, f_z, f_p1, f_p2) = #2nd order transfer function
	G*(1+j*(f./f_z))./( (1+j*(f./f_p1)).*(1+j*(f./f_p2)) )
function f_unitygain_f(f, G, f_z, f_p1, f_p2) #Compute unity gain frequency
	invsq(x) = (result = 1/x; return result*result)
	_z² = invsq(f_z); G² = G*G
	_p1² = invsq(f_p1); _p2² = invsq(f_p2)
	r = roots(_p1²*_p2², _p1²+_p2²-G²*_z², 1-G²)
	return sqrt(max(r...)) #Select sqrt(positive root)
end
f_3dB(f, G, f_z, f_p1, f_p2) = #Compute 3dB frequency
	f_unitygain_f(f, sqrt(2), f_z, f_p1, f_p2)

#==Calculations
===============================================================================#
X = xf_o2(f, G, f_z, f_p1, f_p2)
f0 = f_unitygain_f(f, G, f_z, f_p1, f_p2)
fBW = f_3dB(f, G, f_z, f_p1, f_p2)
phase0 = phase(xf_o2(f0, G, f_z, f_p1, f_p2))
pmargin = 180+phase0
@show pmargin

#==Generate plot
===============================================================================#
mplot = InspectDR.Multiplot(title="Bode Plot")
mplot.hplot*=.6 #Bode plot looks better with wider aspect ratio
mplot.ncolumns = 1

splot = plot_mag = add(mplot, InspectDR.Plot2D)
	splot.axes = InspectDR.axes(:log10, :dB20)
	splot.ext_full = InspectDR.PExtents2D(xmin=fmin_disp, ymin=-10)
	wfrm = add(splot, f, X)
		wfrm.line = line(color=blue, width=3)
	a = splot.annotation
#		a.title = ""
		a.xlabel = "Frequency (Hz)"; a.ylabel = "Magnitude (dB)"
	splot.layout.htitle = 8

splot = plot_phase = add(mplot, InspectDR.Plot2D)
	splot.axes = InspectDR.axes(:log10, :lin)
	splot.ext_full = InspectDR.PExtents2D(xmin=fmin_disp, ymin=-180)
	wfrm = add(splot, f, phase(X))
		wfrm.line = line(color=blue, width=3)
	a = splot.annotation
#		a.title = ""
		a.xlabel = "Frequency (Hz)"; a.ylabel = "Phase (°)"
	splot.layout.htitle = 8

#Add specialized annotation
#-------------------------------------------------------------------------------
afont = InspectDR.Font(12) #Font for annotation
markerline = line(style=:dash, width=2.5)
markerline_light = line(style=:dash, width=2.5, color=RGB24(.4,.4,.4))

#Add vertical markers to both plots:
for splot in [plot_mag, plot_phase]
	splot.layout.grid = grid(vmajor=true, vminor=true, hmajor=false)
	add(splot, vmarker(f0, markerline))
	add(splot, vmarker(fBW, markerline))
end

#Add annotation to Magnitude plot:
add(plot_mag, atext("0dB", y=1, xoffset=0.5, yoffset=2/100, font=afont, align=:bc))
add(plot_mag, hmarker(1, markerline_light))

fstr = "f3dB=" * @sprintf("%.1f MHz", fBW/1e6)
add(plot_mag, atext(fstr, x=fBW, xoffset=-1/100, yoffset=.5, font=afont, angle=-90, align=:bc))

fstr = "f0=" * @sprintf("%.1f MHz", f0/1e6)
add(plot_mag, atext(fstr, x=f0, xoffset=-1/100, yoffset=.5, font=afont, angle=-90, align=:bc))

#Add annotation to Phase plot:
fstr = "PM=" * @sprintf("%.1f°", pmargin)
add(plot_phase, atext(fstr, y=(phase0-180)/2, xoffset=0.5, font=afont, align=:cc))
add(plot_phase, hmarker(phase0, markerline))

#==Render plot
===============================================================================#
gplot = display(InspectDR.GtkDisplay(), mplot)

InspectDR.write_png("export_bode.png", mplot)
#InspectDR.write_svg("export_bode.svg", mplot)
#InspectDR.write_eps("export_bode.eps", mplot)
#InspectDR.write_pdf("export_bode.pdf", mplot)

:DONE


type LUSolver{T} <: LinearSolver{T}
    n::Int
    A::Matrix{T}
    b::Vector{T}
    x::Vector{T}
    pivots::Vector{Int}
    info::Int

    LUSolver(n) = new(n, zeros(T, n, n), zeros(T, n), zeros(T, n), zeros(Int, n), 0)
end

function LUSolver{T}(A::Matrix{T})
    n = checksquare(A)
    lu = LUSolver{eltype(A)}(n)
    lu.A .= A
    lu
end

function LUSolver{T}(A::Matrix{T}, b::Vector{T})
    n = checksquare(A)
    @assert n == length(b)
    lu = LUSolver{eltype(A)}(n)
    lu.A .= A
    lu.b .= b
    lu
end

# function factorize!{T}(lu::LUSolver{T}, pivot=true)
#     @inbounds begin
#         for k = 1:lu.n
#             # find index max
#             kp = k
#             if pivot
#                 amax = real(zero(T))
#                 for i = k:lu.n
#                     absi = abs(lu.A[i,k])
#                     if absi > amax
#                         kp = i
#                         amax = absi
#                     end
#                 end
#             end
#             lu.pivots[k] = kp
#             if lu.A[kp,k] != 0
#                 if k != kp
#                     # Interchange
#                     for i = 1:lu.n
#                         tmp = lu.A[k,i]
#                         lu.A[k,i] = lu.A[kp,i]
#                         lu.A[kp,i] = tmp
#                     end
#                 end
#                 # Scale first column
#                 Akkinv = one(T)/lu.A[k,k]
#                 for i = k+1:lu.n
#                     lu.A[i,k] *= Akkinv
#                 end
#             elseif lu.info == 0
#                 lu.info = k
#             end
#             # Update the rest
#             for j = k+1:lu.n
#                 for i = k+1:lu.n
#                     lu.A[i,j] -= lu.A[i,k] * lu.A[k,j]
#                 end
#             end
#         end
#     end
#     nothing
# end

function factorize!{T}(lu::LUSolver{T})
    @inbounds for k = 1:lu.n
        lu.pivots[k] = k
        Ainv = one(T)/lu.A[k,k]
        @simd for i = k+1:lu.n
            lu.A[i,k] *= Ainv
            @simd for j = k+1:lu.n
                lu.A[i,j] -= lu.A[i,k] * lu.A[k,j]
            end
        end
    end
end

function solve!{T}(lu::LUSolver{T})
    for i = 1:lu.n
        lu.x[i] = lu.b[lu.pivots[i]]
    end
    for i = lu.n:-1:1
        for j = i+1:lu.n
            lu.x[i] -= lu.A[i,j] * lu.x[j]
        end
        lu.x[i] /= lu.A[i,i]
    end
    for i = 1:lu.n
        lu.b[i] = lu.x[i]
    end
    nothing
end


module TestSBL
using Test
using LinearAlgebra
using CompressedSensing: sparse_data, sbl, SBL, fsbl, rmps, update!, rmp, foba, perturb
using SparseArrays

n, m, k = 32, 48, 3
A, x, b = sparse_data(n = n, m = m, k = k)
σ = 1e-2
y = perturb(b, σ/2)

@testset "Sparse Bayesian Learning" begin
    xsbl = sbl(A, y, σ^2) # need to pass variance or covariance of noise
    tol = σ
    # xsbl = droptol!(sparse(xsbl), tol)
    @test findall(abs.(xsbl) .> tol) == x.nzind
    @test isapprox(A*xsbl, b, atol = σ)

    # fast sparse bayesian learning
    xfsbl = fsbl(A, y, σ^2)
    @test findall(abs.(xfsbl) .> tol) == x.nzind
    @test isapprox(A*xfsbl, b, atol = σ)

    # relevance matching pursuit
    xrmps = rmps(A, y, σ^2)
    @test findall(abs.(xrmps) .> tol) == x.nzind
    @test isapprox(A*xrmps, b, atol = σ)

    # rmp with noise variance optimization
    σ_init = σ
    xrmps, σ²_opt = rmps(A, y, Val(true), σ_init^2)
    @test xrmps isa SparseVector
    @test σ²_opt isa Real
    @test norm(A*xrmps-y) < 5sqrt(σ²_opt) * size(A, 1)

    # rmp with noise variance optimization with prior over σ
    σ_init = σ
    xrmps, σ²_opt = rmps(A, y, Val(true), σ_init^2, 1, σ_init^2)
    @test norm(A*xrmps-y) < 5sqrt(σ²_opt) * size(A, 1)
    @test isapprox(σ²_opt, σ^2, rtol = 2) # noise variance is approximately recovered

    # zero noise limit of relevance matching pursuit
    xrmp = rmp(A, y, σ)
    @test xrmp.nzind == x.nzind
    @test isapprox(A*xrmp, b, atol = σ)

end

# bench = false
# using BenchmarkTools
# if bench
#     @btime sbl($A, $b, $(σ^2))
#     @btime rmps($A, $b, $(σ^2))
#     @btime fsbl($A, $b, $(σ^2))
#     @btime rmp($A, $y, $(σ^2))
#     @btime foba($A, $y, $(σ^2))
# end

end # TestSBL