include("beta_marginal.jl")

## -----------  Generic

isbounded(u :: AbstractMarginal{T1}) where T1 = true;
lb(u :: AbstractMarginal{T1}) where T1  =
    isbounded(u)  ?  u.lb  :  -Inf;
ub(u :: AbstractMarginal{T1}) where T1  =
    isbounded(u)  ?  u.ub  :  Inf;

function Base.show(io :: IO, u :: AbstractMarginal{T1}) where T1
    print(io, typeof(u));
    if isbounded(u)
        print(io,  " with bounds ",  round.([lb(u), ub(u)], digits = 2));
    end
end

"""
	$(SIGNATURES)

Quantiles for a marginal distribution. Inputs are percentiles in [0, 1].
Not defined for all distributions. Fallback returns nothing.
"""
function marginal_quantile(u :: AbstractMarginal{T1}, pct) where T1
    return nothing
end

"""
	$(SIGNATURES)

Validate draws from a given marginal distribution. Mainly bounds checking and avoiding Inf and NaN.
"""
function validate_draws end

function validate_bounded_draws(u :: AbstractMarginal{T1}, x :: AbstractArray{T1}) where T1 <: AbstractFloat

    isValid = true;
    isValid = isValid  &&  check_float_array(x, lb(u), ub(u));
    return isValid
end

function validate_bounded_draws(u :: AbstractMarginal{T1}, x :: T1) where T1 <: AbstractFloat

    isValid = true;
    isValid = isValid  &&  check_float(x, lb(u), ub(u));
    return isValid
end

function validate_unbounded_draws(nm :: AbstractMarginal{T1}, x) where T1
    isValid = true
    isValid = isValid && !any(isinf.(x));
    return isValid
end

## ------  Uniform

draw_test_endowments(m :: UniformMarginal{T1}, n, rng :: AbstractRNG) where T1 =
    lb(m) .+ (ub(m) - lb(m)) .* rand(rng, T1, n);

validate_draws(u :: UniformMarginal{T1}, x :: AbstractVector{T1})  where T1 =
    validate_bounded_draws(u, x);

function marginal_quantile(u :: UniformMarginal{T1}, pct) where T1
    return lb(u) .+ (ub(u) .- lb(u)) .* pct;
end

mean(m :: UniformMarginal{T1}) where T1 = (ub(m) + lb(m)) / T1(2);
var(m :: UniformMarginal{T1}) where T1 = ((ub(m) - lb(m)) ^ T1(2)) / T1(12);
std(m :: UniformMarginal{T1}) where T1 = sqrt(var(m));

## ----------  Normal

Base.show(io :: IO, u :: NormalMarginal{T1}) where T1 =
    print(io, typeof(u),
        " with mean ",  round(mean(u), digits = 2),
        " and std ",  round(std(u), digits = 2));

mean(n :: NormalMarginal{T1}) where T1  =  n.mean;
std(n :: NormalMarginal{T1}) where T1  =  n.std;
isbounded(u :: NormalMarginal{T1}) where T1 = false;

draw_test_endowments(nm :: NormalMarginal{T1}, n, rng :: AbstractRNG) where T1 =
    mean(nm) .+ std(nm) .* randn(rng, T1, n);

validate_draws(nm :: NormalMarginal{T1}, x :: Vector{T1}) where T1 =
    validate_unbounded_draws(nm, x);

function marginal_quantile(u :: NormalMarginal{T1}, pct) where T1
    return quantile.(Distributions.Normal(u.mean, u.std), pct)
end

## ----------  LogNormal

Base.show(io :: IO, u :: LogNormalMarginal{T1}) where T1 =
    print(io, typeof(u),
        " with mean ",  round(mean(u), digits = 2),
        " and std ",  round(std(u), digits = 2));

# https://en.wikipedia.org/wiki/Log-normal_distribution
mean(n :: LogNormalMarginal{T1}) where T1  =
    n.lb + exp(n.mean + T1(0.5) * (n.std) ^ T1(2));
var(n :: LogNormalMarginal{T1}) where T1  =
    exp(T1(2) * n.mean + n.std ^ 2) * (exp(n.std ^ 2) - one(T1));
std(n :: LogNormalMarginal{T1}) where T1 = sqrt(var(n));
isbounded(u :: LogNormalMarginal{T1}) where T1 = false;

draw_test_endowments(nm :: LogNormalMarginal{T1}, n, rng :: AbstractRNG) where T1 =
    # Do not use mean(nm) and std(nm). They mean something different.
    nm.lb .+ exp.(nm.mean .+ nm.std .* randn(rng, T1, n));

validate_draws(nm :: LogNormalMarginal{T1}, x :: Vector{T1}) where T1 =
    validate_unbounded_draws(nm, x);

function marginal_quantile(u :: LogNormalMarginal{T1}, pct) where T1
    return u.lb .+ exp.(quantile.(Distributions.Normal(u.mean, u.std), pct))
end

## -----------  Percentile

lb(u :: PercentileMarginal{T1}) where T1  =  zero(T1);
ub(u :: PercentileMarginal{T1}) where T1  =  one(T1);

# Same as uniform
mean(m :: PercentileMarginal{T1}) where T1 = (ub(m) + lb(m)) / T1(2);
var(m :: PercentileMarginal{T1}) where T1 = ((ub(m) - lb(m)) ^ T1(2)) / T1(12);
std(m :: PercentileMarginal{T1}) where T1 = sqrt(var(m));

draw_test_endowments(m :: PercentileMarginal{T1}, n, rng :: AbstractRNG) where T1 =
    rand(rng, T1, n);

validate_draws(u :: PercentileMarginal{T1}, x) where T1 =
    validate_bounded_draws(u, x);

marginal_quantile(m :: PercentileMarginal{T1}, pct) where T1 = pct;

## -----------  Bounded

draw_test_endowments(m :: BoundedMarginal{T1}, n, rng :: AbstractRNG) where T1 =
    lb(m) .+ (ub(m) - lb(m)) .* rand(rng, T1, n);

validate_draws(u :: BoundedMarginal{T1}, x) where T1 =
    validate_bounded_draws(u, x);

## -----------  Unbounded

isbounded(u :: UnboundedMarginal{T1}) where T1 = false;
validate_draws(u :: UnboundedMarginal{T1}, x) where T1 =
    validate_unbounded_draws(u, x);

draw_test_endowments(m :: UnboundedMarginal{T1}, n, rng :: AbstractRNG) where T1 =
    randn(rng, T1, n);

## ------------  Unknown

isbounded(u :: UnknownMarginal{T1}) where T1 = false;
validate_draws(u :: UnknownMarginal{T1}, x) where T1 = true;

draw_test_endowments(m :: UnknownMarginal{T1}, n, rng :: AbstractRNG) where T1 =
    [draw_test_endowment(T1, rng)  for j = 1 : n];

function draw_test_endowment(T1, rng :: AbstractRNG)
    if T1 == Vector{Int}
        return rand(rng, 1 : 100, 3);
    else
        error("Not implemented")
    end
end

# -------------


#Constant mean function

"""
    MeanVec <: Mean

Mean function for a 2D GP,
with constant mean along one dimension
```math
m(x) = [β_1, ..., β_n]
```
with constants ``β_1``, ..., ``β_n``.
"""
mutable struct MeanVec <: Mean
    "Constant vector"
    β::Array{Float64,1}
    "x points corresponding to `β`"
    x0::Array{Float64,1}
    "Priors for mean parameters"
    priors::Array
    """
        MeanVec(β::Array{Float64,1})

    Create `MeanVec` with constant vector `β`.
    """
    MeanVec(β::Array{Float64,1}, x0::Array{Float64,1}) = new(β, x0, [])
end

mean(mVec::MeanVec, x::AbstractVector) = mVec.β[ [findfirst(isequal(_x[2]),mVec.x0) for _x in x] ]
function mean(mVec::MeanVec, X::AbstractMatrix)
    mean_vector = zeros(size(X,2))
    for i in 1:size(X,2)
        mean_vector[i] = mVec.β[findfirst(isequal(X[2,i]),mVec.x0)]
    end
    return mean_vector
end

get_params(mVec::MeanVec) = mVec.β
get_param_names(::MeanVec) = [:β]
num_params(mVec::MeanVec) = length(mVec.β)
function set_params!(mVec::MeanVec, hyp::AbstractVector)
    length(hyp) == length(mVec.β) || throw(ArgumentError("ConstVec mean function only has $(mVec.dim) parameters"))
    copyto!(mVec.β, hyp)
    copyto!(mVec.x0, mVec.x0)
end
function grad_mean(mVec::MeanVec, x::AbstractVector)
    idx = findfirst(isequal(x[2]),mVec.x0)
    dM_theta = zeros(length(mVec.β))
    dM_theta[idx] = 1.0
    return dM_theta
end


"""
    RationalKernel(; α::Real=2.0, metric=Euclidean())

Rational kernel with shape parameter `α` and given `metric`.

# Definition

For inputs ``x, x'`` and metric ``d(\\cdot, \\cdot)``, the rational kernel with shape parameter ``\\alpha > 0`` is defined as
```math
k(x, x'; \\alpha) = \\bigg(1 + \\frac{d(x, x')}{\\alpha}\\bigg)^{-\\alpha}.
```
By default, ``d`` is the Euclidean metric ``d(x, x') = \\|x - x'\\|_2``.

The [`ExponentialKernel`](@ref) is recovered in the limit as ``\\alpha \\to \\infty``.

See also: [`GammaRationalKernel`](@ref)
"""
struct RationalKernel{Tα<:Real,M} <: SimpleKernel
    α::Vector{Tα}
    metric::M

    function RationalKernel(α::Real, metric)
        @check_args(RationalKernel, α, α > zero(α), "α > 0")
        return new{typeof(α),typeof(metric)}([α], metric)
    end
end

function RationalKernel(; alpha::Real=2.0, α::Real=alpha, metric=Euclidean())
    return RationalKernel(α, metric)
end

@functor RationalKernel

function kappa(κ::RationalKernel, d::Real)
    return (one(d) + d / only(κ.α))^(-only(κ.α))
end

metric(k::RationalKernel) = k.metric

function Base.show(io::IO, κ::RationalKernel)
    return print(io, "Rational Kernel (α = ", only(κ.α), ", metric = ", κ.metric, ")")
end

"""
    RationalQuadraticKernel(; α::Real=2.0, metric=Euclidean())

Rational-quadratic kernel with respect to the `metric` and with shape parameter `α`.

# Definition

For inputs ``x, x'`` and metric ``d(\\cdot, \\cdot)``, the rational-quadratic kernel with
shape parameter ``\\alpha > 0`` is defined as
```math
k(x, x'; \\alpha) = \\bigg(1 + \\frac{d(x, x')^2}{2\\alpha}\\bigg)^{-\\alpha}.
```
By default, ``d`` is the Euclidean metric ``d(x, x') = \\|x - x'\\|_2``.

The [`SqExponentialKernel`](@ref) is recovered in the limit as ``\\alpha \\to \\infty``.

See also: [`GammaRationalKernel`](@ref)
"""
struct RationalQuadraticKernel{Tα<:Real,M} <: SimpleKernel
    α::Vector{Tα}
    metric::M

    function RationalQuadraticKernel(; alpha::Real=2.0, α::Real=alpha, metric=Euclidean())
        @check_args(RationalQuadraticKernel, α, α > zero(α), "α > 0")
        return new{typeof(α),typeof(metric)}([α], metric)
    end
end

@functor RationalQuadraticKernel

function kappa(κ::RationalQuadraticKernel, d::Real)
    return (one(d) + d^2 / (2 * only(κ.α)))^(-only(κ.α))
end
function kappa(κ::RationalQuadraticKernel{<:Real,<:Euclidean}, d²::Real)
    return (one(d²) + d² / (2 * only(κ.α)))^(-only(κ.α))
end

metric(k::RationalQuadraticKernel) = k.metric
metric(::RationalQuadraticKernel{<:Real,<:Euclidean}) = SqEuclidean()

function Base.show(io::IO, κ::RationalQuadraticKernel)
    return print(
        io, "Rational Quadratic Kernel (α = ", only(κ.α), ", metric = ", κ.metric, ")"
    )
end

"""
    GammaRationalKernel(; α::Real=2.0, γ::Real=1.0, metric=Euclidean())

γ-rational kernel with respect to the `metric` with shape parameters `α` and `γ`.

# Definition

For inputs ``x, x'`` and metric ``d(\\cdot, \\cdot)``, the γ-rational kernel with shape
parameters ``\\alpha > 0`` and ``\\gamma \\in (0, 2]`` is defined as
```math
k(x, x'; \\alpha, \\gamma) = \\bigg(1 + \\frac{d(x, x')^{\\gamma}}{\\alpha}\\bigg)^{-\\alpha}.
```
By default, ``d`` is the Euclidean metric ``d(x, x') = \\|x - x'\\|_2``.

The [`GammaExponentialKernel`](@ref) is recovered in the limit as ``\\alpha \\to \\infty``.

See also: [`RationalKernel`](@ref), [`RationalQuadraticKernel`](@ref)
"""
struct GammaRationalKernel{Tα<:Real,Tγ<:Real,M} <: SimpleKernel
    α::Vector{Tα}
    γ::Vector{Tγ}
    metric::M

    function GammaRationalKernel(;
        alpha::Real=2.0, gamma::Real=1.0, α::Real=alpha, γ::Real=gamma, metric=Euclidean()
    )
        @check_args(GammaRationalKernel, α, α > zero(α), "α > 0")
        @check_args(GammaRationalKernel, γ, zero(γ) < γ ≤ 2, "γ ∈ (0, 2]")
        return new{typeof(α),typeof(γ),typeof(metric)}([α], [γ], metric)
    end
end

@functor GammaRationalKernel

function kappa(κ::GammaRationalKernel, d::Real)
    return (one(d) + d^only(κ.γ) / only(κ.α))^(-only(κ.α))
end

metric(k::GammaRationalKernel) = k.metric

function Base.show(io::IO, κ::GammaRationalKernel)
    return print(
        io,
        "Gamma Rational Kernel (α = ",
        only(κ.α),
        ", γ = ",
        only(κ.γ),
        ", metric = ",
        κ.metric,
        ")",
    )
end


# ---
# title: 546. Remove Boxes
# id: problem546
# author: Tian Jun
# date: 2020-10-31
# difficulty: Hard
# categories: Dynamic Programming, Depth-first Search
# link: <https://leetcode.com/problems/remove-boxes/description/>
# hidden: true
# ---
#
# Given several boxes with different colors represented by different positive
# numbers.
# You may experience several rounds to remove boxes until there is no box left.
# Each time you can choose some continuous boxes with the same color (composed
# of k boxes, k >= 1), remove them and get `k*k` points.
# Find the maximum points you can get.
#
#
#
# **Example 1:**
#
#
#
#     Input: boxes = [1,3,2,2,2,3,4,3,1]
#     Output: 23
#     Explanation:
#     [1, 3, 2, 2, 2, 3, 4, 3, 1]
#     ----> [1, 3, 3, 4, 3, 1] (3*3=9 points)
#     ----> [1, 3, 3, 3, 1] (1*1=1 points)
#     ----> [1, 1] (3*3=9 points)
#     ----> [] (2*2=4 points)
#
#
#
#
# **Constraints:**
#
#   * `1 <= boxes.length <= 100`
#   * `1 <= boxes[i] <= 100`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


abstract type GaussianItem end

mutable struct Gaussian <: GaussianItem
    pi::Float64
    tau::Float64
end

function Gaussian(; mu::Float64=0.0, sigma::Float64=0.0)
    if sigma == 0
        return Gaussian(0.0, 0.0)
    end
    pi = sigma^(-2)
    tau = pi * mu
    Gaussian(pi, tau)
end

@inline function mu(g::Gaussian)
    if g.pi != 0
        return g.tau / g.pi
    else
        return 0
    end
end

@inline function sigma(g::Gaussian)
    if g.pi != 0
        return(sqrt(1/g.pi))
    else
        return Inf
    end
end

@inline function isless(a::Gaussian, b::Gaussian)
    isless(mu(a), mu(b))
end

@inline (*)(a::Gaussian, b::Gaussian) = Gaussian(a.pi + b.pi, a.tau + b.tau)
@inline (/)(a::Gaussian, b::Gaussian) = Gaussian(a.pi - b.pi, a.tau - b.tau)

function show(io::IO, g::Gaussian)
    print(io, "Gaussian(mu=$(mu(g)), sigma=$(sigma(g)))")
end


using Manopt
using Test
@testset "Manifold Tests     " begin
    include("manifolds/testGr.jl")
    include("manifolds/testGraph.jl")
    include("manifolds/testHn.jl")
    include("manifolds/testRn.jl")
    include("manifolds/testS1.jl")
    include("manifolds/testSn.jl")
    include("manifolds/testSO.jl")
    include("manifolds/testSPD.jl")
    include("manifolds/testStiefel.jl")
    include("manifolds/testSym.jl")
    include("manifolds/testGraphConstruction.jl")
    #
    include("manifolds/testManifold.jl")
    include("manifolds/testExtended.jl")
    include("manifolds/testCombined.jl")
end
@testset "Plan Tests         " begin
    include("plans/testDebug.jl")
    include("plans/testRecord.jl")
    include("plans/testGradientPlan.jl")
    include("plans/testSubGradientPlan.jl")
end
@testset "Algorithm Tests    " begin
    include("algorithms/testBasicsAlgs.jl")
end
@testset "Function Tests     " begin
    include("functions/testAdjointDifferentials.jl")
    include("functions/testDifferentials.jl")
    include("functions/testCosts.jl")
    include("functions/testGradients.jl")
    include("functions/testProximalMaps.jl")
end
@testset "Helper & Data Tests" begin
    include("helpers/testErrorMeasures.jl")
    include("helpers/testData.jl")
end
@testset "Solver Tests       " begin
    include("solvers/testDR.jl")
    include("solvers/testCPP.jl")
    include("solvers/testGradDesc.jl")
end


#=
Computes a sequence of marginal densities for a continuous state space
Markov chain :math:`X_t` where the transition probabilities can be represented
as densities. The estimate of the marginal density of X_t is

    1/n sum_{i=0}^n p(X_{t-1}^i, y)

This is a density in y.

@author : Spencer Lyon <spencer.lyon@nyu.edu>

@date: 2014-08-01

References
----------

http://quant-econ.net/jl/stationary_densities.html
=#

"""
A look ahead estimator associated with a given stochastic kernel p and a vector
of observations X.

##### Fields

- `p::Function`: The stochastic kernel. Signature is `p(x, y)` and it should be
vectorized in both inputs
- `X::Matrix`: A vector containing observations. Note that this can be passed as
any kind of `AbstractArray` and will be coerced into an `n x 1` vector.

"""
type LAE
    p::Function
    X::Matrix

    function LAE(p::Function, X::AbstractArray)
        n = length(X)
        new(p, reshape(X, n, 1))
    end
end

"""
A vectorized function that returns the value of the look ahead estimate at the
values in the array y.

##### Arguments

- `l::LAE`: Instance of `LAE` type
- `y::Array`: Array that becomes the `y` in `l.p(l.x, y)`

##### Returns

- `psi_vals::Vector`: Density at `(x, y)`

"""
function lae_est{T}(l::LAE, y::AbstractArray{T})
    k = length(y)
    v = l.p(l.X, reshape(y, 1, k))
    psi_vals = mean(v, 1)
    return squeeze(psi_vals, 1)
end


"""
    residreg(pred, Y)
Compute regression prediction errors.
* `pred` : Predictions.
* `Y` : Observed data.
"""
residreg(pred, Y) = Y - pred

"""
    residcla(pred, y)
Compute classification prediction error (0 = no error, 1 = error).
* `pred` : Predictions.
* `Y` : Observed data.
"""
residcla(pred, Y) = pred .!= Y

"""
    ssr(pred, Y)
Compute the sum of squared prediction errors (SSR).
* `pred` : Predictions.
* `Y` : Observed data.
"""
function ssr(pred, Y)
    r = residreg(pred, Y)
    reshape(sum(r.^2, dims = 1), 1, :)
end

"""
    msep(pred, Y)
Compute the mean of the squared prediction errors (MSEP).
* `pred` : Predictions.
* `Y` : Observed data.
"""
function msep(pred, Y)
    r = residreg(pred, Y)
    reshape(colmeans(r.^2), 1, :)
end

"""
    rmsep(pred, Y)
Compute the square root of the mean of the squared prediction errors (RMSEP).
* `pred` : Predictions.
* `Y` : Observed data.
"""
rmsep(pred, Y) = sqrt.(msep(pred, Y))

"""
    bias(pred, Y)
Compute the prediction bias, i.e. the opposite of the mean prediction error.
* `pred` : Predictions.
* `Y` : Observed data.
"""
function bias(pred, Y)
    r = residreg(pred, Y)
    reshape(-colmeans(r), 1, :)
end

"""
    sep(pred, Y)
Compute the corrected SEP ("SEP_c"), i.e. the standard deviation of the prediction errors.
* `pred` : Predictions.
* `Y` : Observed data.

## References

Bellon-Maurel, V., Fernandez-Ahumada, E., Palagos, B., Roger, J.-M., McBratney, A., 2010.
Critical review of chemometric indicators commonly used for assessing the quality of
the prediction of soil attributes by NIR spectroscopy.
TrAC Trends in Analytical Chemistry 29, 1073–1081. https://doi.org/10.1016/j.trac.2010.05.006
"""
sep(pred, Y) = sqrt.(msep(pred, Y) .- bias(pred, Y).^2)

"""
    r2(pred, Y)
Compute the R2 coefficient.
* `pred` : Predictions.
* `Y` : Observed data.

The rate R2 is calculated by R2 = 1 - MSEP(current model) / MSEP(null model),
where the "null model" is the overall mean.
For predictions over CV or test sets, and/or for non linear models,
it can be different from the square of the correlation coefficient (`cor2`)
between the true data and the predictions.
"""
function r2(pred, Y)
    m = size(Y, 1)
    mu = colmeans(Y)
    zmu = reduce(hcat, fill(mu, m, 1))'
    1 .- msep(pred, Y) ./ msep(zmu, Y)
end

"""
    cor2(pred, Y)
Compute the squared linear correlation between data and predictions.
* `pred` : Predictions.
* `Y` : Observed data.
"""
function cor2(pred, Y)
    q = size(Y, 2)
    res = cor(pred, Y).^2
    q == 1 ? res = [res; ] : res = diag(res)
    reshape(res, 1, :)
end

"""
    rpd(pred, Y)
Compute the ratio "deviation to model performance" (RPD).
* `pred` : Predictions.
* `Y` : Observed data.

This is the ratio of the deviation to the model performance to the deviation,
defined by RPD = Std(Y) / RMSEP, where Std(Y) is the standard deviation.

Since Std(Y) = RMSEP(null model) where the null model is the simple average,
this also gives RPD = RMSEP(null model) / RMSEP.
"""
rpd(pred, Y) = std(Y, dims = 1, corrected = false) ./ rmsep(pred, Y)

"""
    rpdr(pred, Y)
Compute a robustified RPD.
* `pred` : Predictions.
* `Y` : Observed data.
"""
function rpdr(pred, Y)
    #Y = ensure_mat(Y)
    u = mapslices(Jchemo.mad, Y; dims = 1) / 1.4826
    r = residreg(pred, Y)
    v = mapslices(median, abs.(r); dims = 1)
    res = u ./ v
    reshape(res, 1, :)
end

# Robust RPD = MAD(Y) / Median(Abs(prediction errors))
# MAD = Median(Abs(prediction errors)) where the median is the null model
#rpdr <- function(pred, Y) {
#    u <- apply(.mat(Y), FUN = mad, MARGIN = 2) / 1.4826
#    r <- residreg(pred, Y)
#    v <- apply(.mat(abs(r)), FUN = median, MARGIN = 2)
#    u / v
#}

"""
    mse(pred, Y)
Summary of model performance for regression.
* `pred` : Predictions.
* `Y` : Observed data.
"""
# Summary for regression
function mse(pred, Y)
    q = size(Y, 2)
    zmsep = msep(pred, Y)
    zrmsep = sqrt.(zmsep)
    zsep = sep(pred, Y)
    zbias = bias(pred, Y)
    zcor2 = cor2(pred, Y)
    zr2 = r2(pred, Y)
    zrpd = rpd(pred, Y)
    zrpdr = rpdr(pred, Y)
    zmean = reshape(colmeans(Y), 1, :)
    nam = map(string, repeat(["y"], q), 1:q)
    nam = reshape(nam, 1, :)
    res = (nam = nam, msep = zmsep, rmsep = zrmsep, sep = zsep, bias = zbias,
        cor2 = zcor2, r2 = zr2, rpd = zrpd, rpdr = zrpdr, mean = zmean)
    DataFrame(res)
end

"""
    err(pred, y)
Compute the classification error rate (ERR).
* `pred` : Predictions.
* `y` : Observed data.
"""
function err(pred, y)
    r = residcla(pred, y)
    res = [sum(r)] / size(y, 1)
    reshape(res, 1, :)
end

# scores:
# - Must return a matrix
# - In particular for err, create multiple dispatches Y::AbstractVector, Y::AbstractMatrix


# Implements standard manipulation operation with Decision Diagrams

export
    ¬,
    ∧,
    ∨,
    apply,
    reduce,
    restrict,
    marginalize

# OPERATIONS
"Returns the complement of the function."
@inline (¬)(α::Node) = Node(index(α), ¬low(α), ¬high(α))
@inline (¬)(α::Terminal{Bool}) = value(α) ? Terminal(false) : Terminal(true)
@inline (¬)(α::Terminal{T}) where T = Terminal(one(T) - value(α))
"Returns the conjunction of the Boolean functions."
@inline (∧)(α::DecisionDiagram{Bool}, β::DecisionDiagram{Bool}) = apply(&, α, β)
"Returns the disjunction of the Boolean functions."
@inline (∨)(α::DecisionDiagram{Bool}, β::DecisionDiagram{Bool}) = apply(|, α, β)
@inline Base.:+(α::DecisionDiagram{T}, β::DecisionDiagram{T}) where T = apply(+, α, β)
"Returns the subtraction of the respective functions."
@inline Base.:-(α::DecisionDiagram{T}, β::DecisionDiagram{T}) where T = apply(-, α, β)
"Returns the product of the respective functions."
@inline Base.:*(α::DecisionDiagram{T}, β::DecisionDiagram{T}) where T = apply(*, α, β)
@inline Base.:*(c::Number,α::Terminal) = Terminal(c * value(α))
@inline Base.:*(c::Number,α::DecisionDiagram) = Node(index(α),c * low(α),c * high(α))
"Returns a Diagram canonical representation of α OP β, where OP is some binary operator."
@inline apply(OP::Function,α::DecisionDiagram{T}, β::DecisionDiagram{T}) where T = apply(OP, α, β, Dict{Tuple{Int, Int}, DecisionDiagram{T}}(), Dict{T,DecisionDiagram{T}}(),Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}())
"""
    apply(OP,α,β,opcache,vcache)

Recursively computes α OP β and cache results.

# Parameters
- `OP`: operation to apply to leaves
- `α`,`β`: decision diagrams
- `opcache`: cache of applied operations
- `vcache`: cache of terminal values
"""
function apply(OP::Function, α::DecisionDiagram{T}, β::DecisionDiagram{T}, opcache::Dict{Tuple{Int, Int}, DecisionDiagram{T}}, vcache::Dict{T,DecisionDiagram{T}}, ncache::Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}) where T
    key = (id(α), id(β))
    if haskey(opcache, key) return opcache[key] end
    #TODO: check identity and anihilator cases
    local γ::DecisionDiagram{T}
    if isa(α,Terminal) && isa(β,Terminal)
        constant = OP(value(α),value(β))
        γ = get!(vcache, constant, Terminal(constant))
        # if haskey(vcache, constant) # avoid duplicate terminals
        #     γ = vcache[constant]
        # else
        #     γ = Terminal(constant)
        #     vcache[constant] = γ
        # end
    else # decompose on least variable (terminals are associated with upper variables)
        if index(α) < index(β) # decompose on variable index(α)
            l = apply(OP, low(α), β, opcache, vcache, ncache)
            h = apply(OP, high(α), β, opcache, vcache, ncache)
        elseif index(α) > index(β) # decompose on variable index(β)
            l = apply(OP, α, low(β), opcache, vcache, ncache)
            h = apply(OP, α, high(β), opcache, vcache, ncache)
        else # both DDs are rooted at the same variable
            l = apply(OP, low(α), low(β), opcache, vcache, ncache)
            h = apply(OP, high(α), high(β), opcache, vcache, ncache)
        end
        if id(l) == id(h) # redundant node
            γ = l
            #γ = index(α) <= index(β) ? l : h
        elseif haskey(ncache,(min(index(α),index(β)),id(l),id(h))) # duplicate or already visited node
            γ = ncache[min(index(α),index(β)),id(l),id(h)]
        else
            γ = ncache[min(index(α),index(β)),id(l),id(h)] = Node(min(index(α),index(β)), l, h)
        end
    end
    opcache[key] = γ
end
"""
    apply(f::Function, α::DecisionDiagram)

Returns the application of function f to α.
"""
function apply(f::Function,α::DecisionDiagram{T}) where T
    apply(f,α,Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}(),Dict{T,DecisionDiagram{T}}())
end
function apply(f::Function,α::Terminal{T}, cache::Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}, vcache::Dict{T,DecisionDiagram{T}}) where T
    v = f(value(α))
    get!(vcache,v,Terminal(v))
end
function apply(f::Function,α::DecisionDiagram{T}, cache::Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}, vcache::Dict{T,DecisionDiagram{T}}) where T
    left, right = apply(f,low(α),cache,vcache), apply(f,high(α),cache,vcache)
    if id(left) == id(right) # redundant node
        return left
    end
    # if there is a node with the same variable and children, return it
    # otherwise, create it
    get!(cache, (index(α),id(left),id(right)), Node{T}(id(α),index(α),left,right))
end

"""
    reduce(α::DecisionDiagram)

Returns the canonical form of the decision diagram α.
A diagram is canonical iff:
- no two terminals are assigned to the same value
- no two nodes have are labeled by the same variable and have the same two reduced children low and high with identical ids

"""
function reduce(α::DecisionDiagram{T}) where T
    reduce(α,Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}(),Dict{T,DecisionDiagram{T}}())
end
"""
reduce(α::DecisionDiagram{T}, cache::Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}, vcache::Dict{T,DecisionDiagram{T}})

Returns the canonical form of decision diagram α using node and value caches.
A diagram is canonical iff:
    - no two terminals are assigned to the same value
    - no two nodes have are labeled by the same variable and have the same two reduced children low and high with identical ids

# Arguments
- `α`: decision diagram to reduce.
- `cache`: cache of inner nodes (identified by ids of variable, low and high children).
- `vcache::Dict{T,DecisionDiagram{T}}`: cache of terminal nodes identified by the associated value.
"""
reduce(α::Terminal{T}, cache::Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}, vcache::Dict{T,DecisionDiagram{T}}) where T = get!(vcache,value(α),α)
function reduce(α::DecisionDiagram{T}, cache::Dict{Tuple{Int,Int,Int},DecisionDiagram{T}}, vcache::Dict{T,DecisionDiagram{T}}) where T
    left, right = reduce(low(α),cache,vcache), reduce(high(α),cache,vcache)
    if id(left) == id(right) # redundant node
        return left
    end
    # if there is a node with the same variable and children, return it
    # otherwise, create it
    get!(cache, (index(α),id(left),id(right)), Node{T}(id(α),index(α),left,right))
    # elseif haskey(cache,(index(α),id(left),id(right))) # duplicate or already visited node
    #     return cache[index(α),id(left),id(right)]
    # end
    # cache[index(α),id(left),id(right)] = Node{T}(id(α),index(α),left,right)
end

"""
    restrict(α::DecisionDiagram, var::Int, value::Bool=true)

Returns the canonical form of the function α constrained at variable var=value.

# Example
```julia
x, y = variable(1), variable(2)
f = (one(x)-x)*(one(y)-1) + x*y
g = Terminal(4)*(one(x)-x) + Terminal(2)*x
h = f + g
r1 = restrict(h,1,false)
r2 = h | (2 => true) # shorthand for restrict(h,2,true)
r3 = h | (y => true) # same as above
```
"""
function restrict(α::DecisionDiagram, var::Int, value::Bool=true)
    if isa(α,Terminal) return α end
    if index(α) != var
        l = restrict(low(α),var,value)
        h = restrict(high(α),var,value)
        return Node(index(α), l, h)
    end
    if value return restrict(high(α),var,value) end
    restrict(low(α),var,value)
end
@inline Base.:|(α::DecisionDiagram,a::Pair{Int,Bool}) = restrict(α,a[1],a[2])

"Marginalize a variable var from function α."
@inline marginalize(α::Terminal, var::Int) = α + α
#marginalize(α::Node{T}, var::Int) = reduce(marginalize_step(α, var))
marginalize(α::DecisionDiagram, var::Int) = restrict(α,var,true) + restrict(α,var,false)
# function marginalize(α::DecisionDiagram, var::Int)
#   # produces non-canonical form (perhaps more efficient?)
#   if index(α) == var return low(α) + high(α) end
#   Node(index(α), marginalize(low(α), var), marginalize(high(α), var))
# end


using OrdinaryDiffEq, DiffEqDevTools, Test
using DiffEqProblemLibrary.ODEProblemLibrary: importodeproblems; importodeproblems()
import DiffEqProblemLibrary.ODEProblemLibrary: prob_ode_linear, prob_ode_2Dlinear

prob = prob_ode_linear
sol =solve(prob,Rosenbrock32())
dt₀ = sol.t[2]

prob = prob_ode_2Dlinear
sol =solve(prob,ExplicitRK(tableau=constructBogakiShampine3()))
dt₀ = sol.t[2]

@test  1e-7 < dt₀ < .1
@test_throws ErrorException sol = solve(prob,Euler())
#dt₀ = sol.t[2]

sol3 =solve(prob,ExplicitRK(tableau=constructDormandPrince8_64bit()))
dt₀ = sol3.t[2]

@test 1e-7 < dt₀ < .3

T = Float32
u0 = T.([1.0;0.0;0.0])

tspan = T.((0,100))
prob = remake(prob, u0=u0, tspan=tspan)
@test_nowarn solve(prob, Euler(); dt=T(0.0001))

tspan = T.((2000,2100))
prob = remake(prob, tspan=tspan)
@test_throws ArgumentError solve(prob, Euler(); dt=T(0.0001)) # Loops forever

function rober(du,u,p,t)
  y₁,y₂,y₃ = u
  k₁,k₂,k₃ = p
  du[1] = -k₁*y₁+k₃*y₂*y₃
  du[2] =  k₁*y₁-k₂*y₂^2-k₃*y₂*y₃
  du[3] =  k₂*y₂^2
  nothing
end
u0 = Float32[1.0,0.0,0.0]
tspan = (0f0, 1f5)
params = (4f-2,3f7,1f4)
prob = ODEProblem(rober,u0,tspan,params)
sol = solve(prob, Rosenbrock23())


function multinom_rand!(n::Int, p::AbstractVector{Float64},
                        x::AbstractVector{T}) where T<:Real
    k = length(p)
    length(x) == k || throw(DimensionMismatch("Invalid argument dimension."))

    rp = 1.0  # remaining total probability
    i = 0
    km1 = k - 1

    while i < km1 && n > 0
        i += 1
        @inbounds pi = p[i]
        if pi < rp
            xi = rand(Binomial(n, pi / rp))
            @inbounds x[i] = xi
            n -= xi
            rp -= pi
        else
            # In this case, we don't even have to sample
            # from Binomial. Just assign remaining counts
            # to xi.

            @inbounds x[i] = n
            n = 0
            # rp = 0.0 (no need for this, as rp is no longer needed)
        end
    end

    if i == km1
        @inbounds x[k] = n
    else  # n must have been zero
        z = zero(T)
        for j = i+1 : k
            @inbounds x[j] = z
        end
    end

    return x
end

function multinom_rand!(rng::AbstractRNG, n::Int, p::AbstractVector{Float64},
                        x::AbstractVector{T}) where T<:Real
    k = length(p)
    length(x) == k || throw(DimensionMismatch("Invalid argument dimension."))

    rp = 1.0  # remaining total probability
    i = 0
    km1 = k - 1

    while i < km1 && n > 0
        i += 1
        @inbounds pi = p[i]
        if pi < rp
            xi = rand(rng, Binomial(n, pi / rp))
            @inbounds x[i] = xi
            n -= xi
            rp -= pi
        else
            # In this case, we don't even have to sample
            # from Binomial. Just assign remaining counts
            # to xi.

            @inbounds x[i] = n
            n = 0
            # rp = 0.0 (no need for this, as rp is no longer needed)
        end
    end

    if i == km1
        @inbounds x[k] = n
    else  # n must have been zero
        z = zero(T)
        for j = i+1 : k
            @inbounds x[j] = z
        end
    end

    return x
end

struct MultinomialSampler{T<:Real} <: Sampleable{Multivariate,Discrete}
    n::Int
    prob::Vector{T}
    alias::AliasTable
end

MultinomialSampler(n::Int, prob::Vector{T}) where T<:Real =
    MultinomialSampler{T}(n, prob, AliasTable(prob))

_rand!(s::MultinomialSampler, x::AbstractVector{T}) where T<:Real =
    _rand!(GLOBAL_RNG, s, x)
function _rand!(rng::AbstractRNG, s::MultinomialSampler,
                x::AbstractVector{T}) where T<:Real
    n = s.n
    k = length(s)
    if n^2 > k
        multinom_rand!(rng, n, s.prob, x)
    else
        # Use an alias table
        fill!(x, zero(T))
        a = s.alias
        for i = 1:n
            x[rand(rng, a)] += 1
        end
    end
    return x
end

length(s::MultinomialSampler) = length(s.prob)


### A Pluto.jl notebook ###
# v0.14.7

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 230fdd70-c5ef-11eb-01d9-371bb366387d
begin
	using CSV
	using DataFrames
	using Plots
	using GLM
	using PlutoUI
end

# ╔═╡ 2d3fad52-f1ac-4383-ad78-782c5265148d
data = CSV.read("D:\\Workspace\\Machine Learning\\World GDP Analysis and Prediction\\Data\\World_GDP.csv",DataFrame)

# ╔═╡ 1a4f31a3-5b58-4f4f-8513-a416fd795ad5
scatter(data.year,data.GDP)

# ╔═╡ 79e34de2-59b5-4bdb-b16e-93c64d2010a2
reg = fit(LinearModel,@formula(GDP~year),data)

# ╔═╡ 35d8c82f-d32c-4c2c-8934-280b39761320
linear_regression = predict(reg)

# ╔═╡ 3d4580cb-eb49-4979-9063-60a03c4072b2
begin
	scatter(data.year,data.GDP)
	plot!(data.year,linear_regression)
end

# ╔═╡ 7b0a2af1-148f-42f7-87f6-3a237c00f8cf
md"Enter an year to predict : $@bind y TextField()"

# ╔═╡ 342439a5-e524-40a3-85a7-43a78cf07596
pyear = parse(Int64,y)

# ╔═╡ 75394533-5d15-47c2-9a34-5d98ffd25900
pGDP = coef(reg)[2] * pyear + coef(reg)[1]

# ╔═╡ Cell order:
# ╠═230fdd70-c5ef-11eb-01d9-371bb366387d
# ╠═2d3fad52-f1ac-4383-ad78-782c5265148d
# ╠═1a4f31a3-5b58-4f4f-8513-a416fd795ad5
# ╠═79e34de2-59b5-4bdb-b16e-93c64d2010a2
# ╠═35d8c82f-d32c-4c2c-8934-280b39761320
# ╠═3d4580cb-eb49-4979-9063-60a03c4072b2
# ╠═7b0a2af1-148f-42f7-87f6-3a237c00f8cf
# ╠═342439a5-e524-40a3-85a7-43a78cf07596
# ╠═75394533-5d15-47c2-9a34-5d98ffd25900

World GDP Analysis and Prediction/main.jl


using LazySets, MathematicalPredicates, SparseArrays

function heat_specification()
    # initial set: xᵢ ∈ [0.6, 0.625] for i ≤ 2 and xᵢ = 0 for i > 2
    X0 = Hyperrectangle(low=[fill(0.6, 2); zeros(198)],
                        high=[fill(0.625, 2); zeros(198)])

    # safety property: x133 ≤ 0.1
    property = is_contained_in(HalfSpace(sparsevec([133], [1.0], 200), 0.1))

    # time horizon: 20 time units
    time_horizon = 20.0

    # specification
    O = Dict(:T => time_horizon, :property => property)

    return X0, O
end


function floydtriangle(rows)
    r = collect(1:div(rows *(rows + 1), 2))
    spacing = Int(ceil(log10(r[end] + 1))) + 1
    for i in 1:rows
        for _ in 1:i
            print(lpad(popfirst!(r), spacing))
        end
        println()
    end
end

floydtriangle(5); println(); floydtriangle(14)


# ---
# title: 150. Evaluate Reverse Polish Notation
# id: problem150
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Stack
# link: <https://leetcode.com/problems/evaluate-reverse-polish-notation/description/>
# hidden: true
# ---
#
# Evaluate the value of an arithmetic expression in [Reverse Polish
# Notation](http://en.wikipedia.org/wiki/Reverse_Polish_notation).
#
# Valid operators are `+`, `-`, `*`, `/`. Each operand may be an integer or
# another expression.
#
# **Note:**
#
#   * Division between two integers should truncate toward zero.
#   * The given RPN expression is always valid. That means the expression would always evaluate to a result and there won't be any divide by zero operation.
#
# **Example 1:**
#
#
#
#     Input: ["2", "1", "+", "3", "*"]
#     Output: 9
#     Explanation: ((2 + 1) * 3) = 9
#
#
# **Example 2:**
#
#
#
#     Input: ["4", "13", "5", "/", "+"]
#     Output: 6
#     Explanation: (4 + (13 / 5)) = 6
#
#
# **Example 3:**
#
#
#
#     Input: ["10", "6", "9", "3", "+", "-11", "*", "/", "*", "17", "+", "5", "+"]
#     Output: 22
#     Explanation:
#       ((10 * (6 / ((9 + 3) * -11))) + 17) + 5
#     = ((10 * (6 / (12 * -11))) + 17) + 5
#     = ((10 * (6 / -132)) + 17) + 5
#     = ((10 * 0) + 17) + 5
#     = (0 + 17) + 5
#     = 17 + 5
#     = 22
#
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


# Load Julia packages (libraries) needed

# ### snippet 0.5 is replaced by below `using StatisticalRethinking`.

using StatisticalRethinking, GLM
#gr(size=(600, 600));

# ### snippet 0.4

howell1 = CSV.read(rel_path("..", "data", "Howell1.csv"), delim=';')
df = convert(DataFrame, howell1);
df2 = filter(row -> row[:age] >= 18, df);
first(df2, 5)

# Fit a linear regression of distance on speed

m = lm(@formula(height ~ weight), df2)

# estimated coefficients from the model

coef(m)

# Plot residuals against speed

scatter( df2[!, :height], residuals(m), xlab="Height",
ylab="Model residual values", lab="Model residuals")

# End of `00/clip-04-05.jl`


"""
    linearsearch(a::Union{AbstractVector{T}, T}, item::T)

Linear search in array, find first occurance. Time complexity O(n).
"""
function linearsearch(a::Union{AbstractVector{T}, T}, item::T)::Int where T
    if length(a) == 0 return 0 end

    for i in 1:length(a)
        if a[i] == first(item)
            return i
        end
    end

    return 0
end

"""
    binarysearch(a::Union{AbstractVector{T}, T}, item::T)

Binary search in array, find any occurance. Time complexity O(logn).
"""
function binarysearch(a::Union{AbstractVector{T}, T}, item::T) where T
    lo::Int = 1
    hi::Int = length(a)
    if hi == 0 return nothing end

    mid::Int = 0
    item = first(item)
    while lo <= hi
        mid = lo + (hi - lo) ÷ 2

        if item < a[mid]
            hi = mid - 1
        elseif item > a[mid]
            lo = mid + 1
        else
            break
        end
    end

    return item == a[mid] ? mid : nothing
end


#############################################################################
# JuMP
# An algebraic modelling langauge for Julia
# See http://github.com/JuliaOpt/JuMP.jl
#############################################################################
# maxcut_sdp.jl
#
# Solves the SDP relaxation of the classic MAXCUT problem:
# max   L•X
# s.t.  diag(X) == e
#       X ≽ 0
# where
#  L = 1/4(Diag(W e) - W)
#  W = edge-weight matrix
#  e = vector of 1s
#
# then applies the Goemans-Williamson algorithm
#############################################################################
using JuMP
using Mosek

function solve_maxcut_sdp(n, W)
    L = 0.25 * (diagm(W*ones(n)) - W)

    # Solve the SDP relaxation
    m = Model()
    @variable(m, X[1:n,1:n], SDP)
    @objective(m, Max, vecdot(L,X))
    @constraint(m, diag(X) .== 1)
    solve(m)

    # Cholesky the result
    F = cholfact(getvalue(X)[:,:], :U, Val{true})
    V = (F[:P]*F[:L])'

    # Normalize columns
    for i = 1:n
        V[:,i] ./= norm(V[:,i])
    end

    # Generate "random" vector
    # - seeded on problem size for repeatability
    # - for all the problems in this file, the
    #   solutions are "integral" anyway so there
    #   isn't really a need for this
    r = rand(n)
    cut = ones(n)
    for i = 1:n
        if sum(r' * V[:,i]) <= 0
            cut[i] = -1
        end
    end

    return cut, sum(L.*(cut*cut'))
end

function test0()
    #   [1] --- 5 --- [2]
    #
    # Solution:
    #  S  = {1}
    #  S' = {2}
    n = 2
    W = [0.0 5.0;
         5.0 0.0]
    cut, cutval = solve_maxcut_sdp(n, W)

    @assert cut[1] != cut[2]

    println("Solution for Graph 0 = $cutval")
    println(cut)
end

function test1()
    #   [1] --- 5 --- [2]
    #    |  \          |
    #    |    \        |
    #    7      6      1
    #    |        \    |
    #    |          \  |
    #   [3] --- 1 --- [4]
    #
    # Solution:
    #  S  = {1}
    #  S' = {2,3,4}
    n = 4
    W = [0.0 5.0 7.0 6.0;
         5.0 0.0 0.0 1.0;
         7.0 0.0 0.0 1.0;
         6.0 1.0 1.0 0.0]
    cut, cutval = solve_maxcut_sdp(n, W)
    @assert (v = cut[2]) == cut[3] == cut[4]
    @assert cut[1] != v

    println("Solution for Graph 1 = $cutval")
    println(cut)
end

function test2()
    #   [1] --- 1 --- [2]
    #    |             |
    #    |             |
    #    9             9
    #    |             |
    #    |             |
    #   [3] --- 1 --- [4]
    #
    # Solution:
    #  S  = {2,3}
    #  S' = {1,4}
    n = 4
    W = [0.0 1.0 9.0 0.0;
         1.0 0.0 0.0 9.0;
         9.0 0.0 0.0 1.0;
         0.0 9.0 1.0 0.0]
    cut, cutval = solve_maxcut_sdp(n, W)
    @assert (v = cut[1]) == cut[4]
    @assert (w = cut[2]) == cut[3]
    @assert v != w

    println("Solution for Graph 2 = $cutval")
    println(cut)
end

test0()
test1()
test2()


# It turns out that this works OK for starting points 10*x_init and
# 100*x_init.  Interesting, the Fortran code does _not_ run the case
# 1*x_init.   It looks like it converges to a different minimum from
# that starting point.
const expmax = 1.999 * log(sqrt(0.999*floatmax()))
const expmin = 1.999 * log(sqrt(1.001*floatmin()))

const box = let res_init=zeros(10), jac_init=zeros(10,3), x_init=[0.0, 1.0, 2.0]

    function res(x, r)
        -expmax > minimum(x) && throw("Error in box residual")
        for i = 1:10
            ti = -0.1 * Float64(i)
            t1 = ti * x[1]
            e1 = t1 > expmin ? exp(t1) : 0.0
            t2 = ti*x[2]
            e2 = t2 > expmin ? exp(t2) : 0.0
            r[i] = (e1 - e2) - x[3]*(exp(ti) - exp(10.0 * ti))
        end
        return r
    end

    function jac(x, jac)
        for i = 1:10
            ti = -0.1 * Float64(i)
            t = x[1] * ti
            e = t > expmin ? exp(t) : 0.0
            jac[i, 1] = ti * e
            t = x[2] * ti
            e = t > expmin ? exp(t) : 0.0
            jac[i, 2] = -ti * e
            jac[i, 3] = exp(10.0 * ti) - exp(ti)
        end
        return jac
    end

    f(;scale=1, verbose=false, print_steps=false) = testone("box", res, jac,
                                                   res_init, jac_init, x_init;
                                                   scale=scale, verbose=verbose,
                                                   print_steps=print_steps)
end


function emp_cdf(rnb::Vector{T}) where {T}
    x = sort(rnb)
    x_eps = x .+ eps.(x)

    L = length(x)
    y1 = collect( 0:(L-1) ) ./ L
    y2 = collect( 1:L ) ./ L

    cdf_x = reshape( hcat( x, x_eps )', 2*L, 1)
    cdf_y = reshape( hcat( y1, y2 )', 2*L, 1)

    return (cdf_x, cdf_y)
end

function bin_size_scott(sigma, n)
    h = sigma * 3.49 / cbrt(n)
    return h
end

function histogram(rnb::Vector{T}, n_bin::Int) where {T}

    (x_min, x_max)             = extrema(rnb)
    bin_edges                  = collect( range(x_min, x_max, n_bin+1) )
    # (bin_edges, freq, bin_idx) = histogram(rnb, bin_edges)

    # return (bin_edges, freq, bin_idx)

    (bin_edges, freq) = histogram(rnb, bin_edges)

    return (bin_edges, freq)

end

function histogram(rnb::Vector{T}, bin_edges::Vector{T}) where {T}
    # it is assumed that the bin edges are sorted
    Lr      = length(rnb)
    Le      = length(bin_edges)
    freq    = zeros(Int, Le-1)
    # bin_idx = zeros(Int, Lr) # zero means that it is outside of the edges
    srnb    = sort(rnb)

    start_idx = 1 + util.binary_search( srnb, bin_edges[1] )
    end_idx   = util.binary_search( srnb, bin_edges[end] )
    end_idx   = min(Lr, end_idx)

    ee = 2
    for rr = start_idx:end_idx

        if srnb[rr] <= bin_edges[ee]
            # increase the frequency in the current bin
            freq[ee-1] += 1
            # bin_idx[rr] = ee-1
        else
            # move to the next bin
            ee += 1
            freq[ee-1] += 1
            if ee > Le
                break
            end
        end

    end

    # return (bin_edges, freq, bin_idx)
    return (bin_edges, freq)
end

function histogram_2d(rnb::Matrix{T}, n_bin_x::Int, n_bin_y::Int) where {T}

    (x_min, x_max)             = extrema(  view(rnb, 1, :)  )
    (y_min, y_max)             = extrema(  view(rnb, 2, :)  )

    x_bin_edges   = collect( range(x_min, x_max, n_bin_x + 1) )
    y_bin_edges   = collect( range(y_min, y_max, n_bin_y + 1) )

    (x_bin_edges, y_bin_edges, freq) = histogram_2d(rnb, x_bin_edges, y_bin_edges)

    return (x_bin_edges, y_bin_edges, freq)

end

function is_inside_box_2d(x, y, x_min, x_max, y_min, y_max)
    return x_min <= x <= x_max && y_min <= y <= y_max
end

function histogram_2d(rnb::Matrix{T}, x_bin_edges::Vector{T}, y_bin_edges::Vector{T}) where {T}
    # it is assumed that the bin edges are sorted
    (n_dim, Lr)      = size(rnb)

    Lxe      = length(x_bin_edges)
    Lye      = length(y_bin_edges)

    freq    = zeros(Int, Lxe-1, Lye-1)

    x_min = x_bin_edges[1]
    x_max = x_bin_edges[end]
    y_min = y_bin_edges[1]
    y_max = y_bin_edges[end]

    for rr = 1:Lr
        x = rnb[1, rr]
        y = rnb[2, rr]

        if is_inside_box_2d(x, y, x_min, x_max, y_min, y_max)
            x_left_idx = util.binary_search(x_bin_edges, x)
            y_left_idx = util.binary_search(y_bin_edges, y)

            freq[x_left_idx, y_left_idx] += 1

        end
    end

    return (x_bin_edges, y_bin_edges, freq)

end

#=
function histogram_2d_OLD(rnb::Matrix{T}, x_bin_edges::Vector{T}, y_bin_edges::Vector{T}) where {T}
    # it is assumed that the bin edges are sorted
    (n_dim, Lr)      = size(rnb)

    Lxe      = length(x_bin_edges)
    Lye      = length(y_bin_edges)

    freq    = zeros(Int, Lxe-1, Lye-1)
    # bin_idx = zeros(Int, Lr) # zero means that it is outside of the edges

    srnb  = copy(rnb)
    p = sortperm( view( rnb, 1, : )  ) # sort by x
    permute!(rnb, p)

    x_start_idx = 1 + util.binary_search( view(srnb, 1, : ), x_bin_edges[1] )
    x_end_idx   = util.binary_search( view(srnb, 1, : ), x_bin_edges[end] )
    x_end_idx   = min(Lr, x_end_idx)

    x_edge_idx = 2
    x_left_idx = x_start_idx
    while x_edge_idx <= Lxe

        x_right_idx = util.binary_search( view(srnb, 1, : ), x_bin_edges[x_edge_idx] )
        if x_right_idx > Lr
            break
        end
        # sort the sub array by y
        sort!( view(srnb, 2, x_left_idx:x_right_idx ) )

        # find the first y index which is inside the box
        y_start_idx = 1 + util.binary_search( view(srnb, 2, x_left_idx:x_right_idx), y_bin_edges[1] )
        y_start_idx += x_left_idx
        y_end_idx   = util.binary_search( view(srnb, 2, x_left_idx:x_right_idx), y_bin_edges[end] )
        y_end_idx   += x_left_idx
        y_end_idx   = min(Lr, y_end_idx)

        # loop over the sub array and put them into the right bin

        ee = 2
        for rr = y_start_idx:y_end_idx

            if srnb[rr] <= y_bin_edges[ee]
                # increase the frequency in the current bin
                freq[x_edge_idx, ee-1] += 1
                bin_idx[rr] = ee-1
            else
                # move to the next bin
                ee += 1
                freq[x_edge_idx, ee-1] += 1
                if ee > Lye
                    break
                end
            end

        end

    end

    # return (bin_edges, freq, bin_idx)
    return (x_bin_edges, y_bin_edges, freq)
end
=#


#=
  From Hugin's BN heart_disease_regression.net

  Risk of dying from heart disease in the next 10 years

  Cf ~/webppl/hear_disease_regression.wppl
  In the WebPPL model, I did several alternative, but here I just settle with one.
  For two of four models I got very high risk factor and commented:
  """
  Darn, according to this model, there's a probability of 0.9999999983124701
  that I will die from a heart disease in the next 10 years...
  """

  But this Turing.jl model give a much better fate for me: a risk of only 31% of
  dying from heart disease in the next 10 years.

  Summary Statistics
         parameters       mean       std   naive_se      mcse         ess      rhat   ess_per_sec
             Symbol    Float64   Float64    Float64   Float64     Float64   Float64       Float64

                sex     1.4957    0.5000     0.0050    0.0048   9718.4386    0.9999     2893.2535
        cholesterol     5.8248    9.8761     0.0988    0.0952   9897.9782    0.9999     2946.7038
                age    39.9645   24.8574     0.2486    0.2697   9714.7104    1.0002     2892.1436
  score_cholesterol     0.9898   11.8513     0.1185    0.1143   9897.9782    0.9999     2946.7038
          score_sex    -1.4957    0.5000     0.0050    0.0048   9718.4386    0.9999     2893.2535
          score_age   -20.0711   49.7148     0.4971    0.5395   9714.7104    1.0002     2892.1436
              score   -25.5770   50.9925     0.5099    0.5488   9756.6159    1.0001     2904.6192
               risk     0.3088    0.4545     0.0045    0.0048   9709.6351    0.9999     2890.6327
       oneMinusRisk     0.6912    0.4545     0.0045    0.0048   9709.6351    0.9999     2890.6327

=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function heart_disease_regression(sexVal=0,ageVal=64)
    male = 0
    female = 1

    #  Priors
    sex ~ Categorical([0.5,0.5]) # [male,female]
    cholesterol ~ Normal(6,10.0) #  In the Hugin model the variance is 100.0.
    age ~ Normal(40,25) #  In the Hugin model the variance is 625.

    intercept = -5.0

    score_cholesterol ~ Dirac(1.2 * (cholesterol - 5.0))
    score_sex ~ Dirac(-1 * sex)
    score_age ~ Dirac(2.0 * (age - 50.0))

    score ~ Dirac(intercept + score_age + score_sex + score_cholesterol)

    #  Risk of dying from heart disease in the next 10 years
    risk ~ Dirac(1.0/(1.0 + exp(-score)))

    true ~ Dirac(sex==male)
    # true ~ Dirac(age>=64 && age <= 65)
    true ~ Dirac(age == ageVal)

    # true ~ Dirac(age>=60)
    # true ~ Dirac(cholesterol>=1)
    true ~ Dirac(cholesterol>=4.5 && cholesterol<=4.9) #  ??? I'm note sure about this
    #  true ~ Dirac(score<=0.5)

    oneMinusRisk ~ Dirac(1.0 - risk)
end

model = heart_disease_regression()

num_chains = 4

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
# chns = sample(model, PG(15), 10_000)
chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

# chns = sample(model, NUTS(), 10_000)
# chns = sample(model, Gibbs(PG(5,:sex),NUTS(1000,0.65,:cholesterol,:age,:score,:score_cholesterol,:score_sex,:score_age,:risk)), 10_000)

display(chns)
# display(plot(chns))


"""
    change_basis(basis,changeofbasis::AbstractMatrix)

# Examples

Compute the Lagrangian basis associated with a set of nodes

```jldoctests
using LinearAlgebra
using Gridap.Fields
using Gridap.Polynomials

D = 2
order = 1
f = MonomialBasis{D}(Float64,order)

nodes = Point{2,Int}[(0,0),(1,0),(0,1),(1,1)]
change = inv(evaluate(f,nodes))

g = change_basis(f,change)
println(evaluate(g,nodes))

# output
[1.0 0.0 0.0 0.0; 0.0 1.0 0.0 0.0; 0.0 0.0 1.0 0.0; 0.0 0.0 0.0 1.0]

```
"""
function change_basis(basis,changeofbasis::AbstractMatrix)
  BasisFromChangeOfBasis(basis,changeofbasis)
end

function return_type(::typeof(change_basis),prebasis,matrix_inv)
  typeof(change_basis(prebasis,matrix_inv))
end

struct BasisFromChangeOfBasis{B,M} <: AbstractVector{Field}
  basis::B
  change::M
  function BasisFromChangeOfBasis(basis,change::AbstractMatrix)
    B = typeof(basis)
    M = typeof(change)
    new{B,M}(basis,change)
  end
end

struct BasisTermFromChangeOfBasis end

Base.size(a::BasisFromChangeOfBasis) = (length(a.basis),)
Base.axes(a::BasisFromChangeOfBasis) = (axes(a.basis,1),)
# @santiagobadia : Not sure we want to create the real computation here
Base.getindex(a::BasisFromChangeOfBasis,i::Integer) = BasisTermFromChangeOfBasis()
Base.IndexStyle(::BasisFromChangeOfBasis) = IndexLinear()

function return_cache(b::BasisFromChangeOfBasis,x)
  cb = return_cache(b.basis,x)
  bx = evaluate!(cb,b.basis,x)
  c = CachedArray(bx*b.change)
  (c,cb)
end

function evaluate!(cache,b::BasisFromChangeOfBasis,x)
  c, cb = cache
  bx = evaluate!(cb,b.basis,x)
  setsize!(c,size(bx))
  mul!(c.array,bx,b.change)
  c.array
end

function return_gradient_cache(b::BasisFromChangeOfBasis,x)
  cb = return_gradient_cache(b.basis,x)
  bx = evaluate_gradient!(cb,b.basis,x)
  c = CachedArray(bx*b.change)
  (c,cb)
end

function evaluate_gradient!(cache,b::BasisFromChangeOfBasis,x)
  c, cb = cache
  bx = evaluate_gradient!(cb,b.basis,x)
  setsize!(c,size(bx))
  mul!(c.array,bx,b.change)
  c.array
end

function return_hessian_cache(b::BasisFromChangeOfBasis,x)
  cb = return_hessian_cache(b.basis,x)
  bx = evaluate_hessian!(cb,b.basis,x)
  c = CachedArray(bx*b.change)
  (c,cb)
end

function evaluate_hessian!(cache,b::BasisFromChangeOfBasis,x)
  c, cb = cache
  bx = evaluate_hessian!(cb,b.basis,x)
  setsize!(c,size(bx))
  mul!(c.array,bx,b.change)
  c.array
end


"""
    Geometric(p)

A *Geometric distribution* characterizes the number of failures before the first success in a sequence of independent Bernoulli trials with success rate `p`.

```math
P(X = k) = p (1 - p)^k, \\quad \\text{for } k = 0, 1, 2, \\ldots.
```

```julia
Geometric()    # Geometric distribution with success rate 0.5
Geometric(p)   # Geometric distribution with success rate p

params(d)      # Get the parameters, i.e. (p,)
succprob(d)    # Get the success rate, i.e. p
failprob(d)    # Get the failure rate, i.e. 1 - p
```

External links

*  [Geometric distribution on Wikipedia](http://en.wikipedia.org/wiki/Geometric_distribution)

"""
struct Geometric{T<:Real} <: DiscreteUnivariateDistribution
    p::T

    function Geometric{T}(p::T) where {T <: Real}
        new{T}(p)
    end
end

function Geometric(p::Real; check_args::Bool=true)
    @check_args Geometric (p, zero(p) < p < one(p))
    return Geometric{typeof(p)}(p)
end

Geometric() = Geometric{Float64}(0.5)

@distr_support Geometric 0 Inf

### Conversions
convert(::Type{Geometric{T}}, p::Real) where {T<:Real} = Geometric(T(p))
Base.convert(::Type{Geometric{T}}, d::Geometric) where {T<:Real} = Geometric{T}(T(d.p))
Base.convert(::Type{Geometric{T}}, d::Geometric{T}) where {T<:Real} = d

### Parameters

succprob(d::Geometric) = d.p
failprob(d::Geometric) = 1 - d.p
params(d::Geometric) = (d.p,)
partype(::Geometric{T}) where {T<:Real} = T

### Statistics

mean(d::Geometric) = failprob(d) / succprob(d)

median(d::Geometric) = -fld(logtwo, log1p(-d.p)) - 1

mode(d::Geometric{T}) where {T<:Real} = zero(T)

var(d::Geometric) = (1 - d.p) / abs2(d.p)

skewness(d::Geometric) = (2 - d.p) / sqrt(1 - d.p)

kurtosis(d::Geometric) = 6 + abs2(d.p) / (1 - d.p)

entropy(d::Geometric) = (-xlogx(succprob(d)) - xlogx(failprob(d))) / d.p

### Evaluations

function logpdf(d::Geometric, x::Real)
    insupport(d, x) ? log(d.p) + log1p(-d.p) * x : log(zero(d.p))
end

function cdf(d::Geometric, x::Int)
    p = succprob(d)
    n = max(x + 1, 0)
    p < 1/2 ? -expm1(log1p(-p)*n) : 1 - (1 - p)^n
end

ccdf(d::Geometric, x::Real) = ccdf_int(d, x)
function ccdf(d::Geometric, x::Int)
    p = succprob(d)
    n = max(x + 1, 0)
    p < 1/2 ? exp(log1p(-p)*n) : (1 - p)^n
end

logcdf(d::Geometric, x::Real) = logcdf_int(d, x)
logcdf(d::Geometric, x::Int) = log1mexp(log1p(-d.p) * max(x + 1, 0))

logccdf(d::Geometric, x::Real) = logccdf_int(d, x)
logccdf(d::Geometric, x::Int) =  log1p(-d.p) * max(x + 1, 0)

quantile(d::Geometric, p::Real) = invlogccdf(d, log1p(-p))

cquantile(d::Geometric, p::Real) = invlogccdf(d, log(p))

invlogcdf(d::Geometric, lp::Real) = invlogccdf(d, log1mexp(lp))

function invlogccdf(d::Geometric{T}, lp::Real) where T<:Real
    if (lp > zero(d.p)) || isnan(lp)
        return T(NaN)
    elseif isinf(lp)
        return T(Inf)
    elseif lp == zero(d.p)
        return zero(T)
    end
    max(ceil(lp/log1p(-d.p)) - 1, zero(T))
end

function mgf(d::Geometric, t::Real)
    p = succprob(d)
    p / (expm1(-t) + p)
end

function cf(d::Geometric, t::Real)
    p = succprob(d)
    # replace with expm1 when complex version available
    p / (exp(-t*im) - 1 + p)
end

### Sampling

rand(rng::AbstractRNG, d::Geometric) = floor(Int,-randexp(rng) / log1p(-d.p))

### Model Fitting

struct GeometricStats <: SufficientStats
    sx::Float64
    tw::Float64

    GeometricStats(sx::Real, tw::Real) = new(sx, tw)
end

suffstats(::Type{<:Geometric}, x::AbstractArray{T}) where {T<:Integer} = GeometricStats(sum(x), length(x))

function suffstats(::Type{<:Geometric}, x::AbstractArray{T}, w::AbstractArray{Float64}) where T<:Integer
    n = length(x)
    if length(w) != n
        throw(DimensionMismatch("Inconsistent argument dimensions."))
    end
    sx = 0.
    tw = 0.
    for i = 1:n
        wi = w[i]
        sx += wi * x[i]
        tw += wi
    end
    GeometricStats(sx, tw)
end

fit_mle(::Type{<:Geometric}, ss::GeometricStats) = Geometric(1 / (ss.sx / ss.tw + 1))


" Generate datasets "
# using JLD
using Random, Distributions

# N = 50
# d = 5
# n = 10
# k = 2

# zero error
# A = zeros(d, N)
# U = rand(d, k)
# V = rand(k, n)
# A[:, 1:n] = U*V
# A[:, n+1:N] = rand(d, N-n)
# save("datasets/N_$(N)_d_$(d).jld", "data", collect(eachcol(A)), "n_$(n)_k_$(k)", (0.0, collect(1:n)))

"""
generate N×d matrix A where the upper left p×n submatrix of A is of rank k, the rest Uniform[0,1] noise
"""
function matrix_with_subspace_noise(N, d, n, k, p)
    @assert p <= d
    @assert k < p
    A = rand(Uniform(-1.0, 1.0), d, N)
    U = rand(Uniform(-1.0, 1.0), p, k)
    V = rand(Uniform(-1.0, 1.0), k, n)
    A[1:p, 1:n] = U*V
    return A
end
# save("datasets/N_$(N)_d_$(d)_nonZeroError.jld", "data", collect(eachcol(A)), "n_$(n)_k_$(k)", (err, collect(1:n)))

"""
generate N×d matrix (A + M) where the the left d×n submatrix of A is of rank k,
the rest Uniform[0,1] noise, and M being a noise matrix with entries in Normal(0, std)
"""
function matrix_with_normal_noise(N, d, n, k, std)
    A = rand(Uniform(-1.0, 1.0), d, N)
    U = rand(Uniform(-1.0, 1.0), d, k)
    V = rand(Uniform(-1.0, 1.0), k, n)
    A[:, 1:n] = U*V
    noise = rand(Normal(0, std), d, N)
    return A + noise
end


## port methods of simplify module of SymPy
## from http://docs.sympy.org/0.7.2/modules/simplify/simplify.html

## simple methods (x, args) -> y (y coercion happens via PyCall)
simplify_sympy_meths = (:collect, :rcollect, :separate,
                        :separatevars,
                        :radsimp, :ratsimp, :trigsimp, :besselsimp,
                        :powsimp, :combsimp, :hypersimp,
                        :fraction,
                        :simplify, :nsimplify,
                        :posify, :powdenest, :sqrtdenest,
                        :logcombine, :hyperexpand)

expand_sympy_meths = (:expand_trig,
                      :expand_power_base, :expand_power_exp,
                      :expand_log,
                      :expand_func,
                      :hyperexpand
                      )

## special case cse due to output
## If this pattern is common, will need to collect and use metaprogramming.
@doc """
`$(cse)`: Perform common subexpression elimination on an expression.

[cf.](http://docs.sympy.org/latest/modules/simplify/simplify.html)

Example: (from man page)
```
@vars x w y z x0 x1
cse(((w + x + y + z)*(w + y + z))/(w + x)^3), ([(x0, y + z), (x1, w + x)], [(w + x0)*(x0 + x1)/x1^3]) # tuple of replacements and reduced expressions.

""" ->
function cse{T<:SymbolicObject}(ex::Union{T, AbstractArray{T}}, args...; kwargs...)
    a, b = sympy_meth(:cse, ex, args...; kwargs...)
    a, oftype(ex, b[1])
end
export(cse)

## didn't do traversal tools, EPath tools


import Base: rand,
             ∈,
             isempty

export Line,
       an_element,
       translate,
       translate!

"""
    Line{N, VN<:AbstractVector{N}} <: AbstractPolyhedron{N}

Type that represents a line in the form

```math
    \\{y ∈ \\mathbb{R}^n: y = p + λd, λ ∈ \\mathbb{R}\\}
```
where ``p`` is a point on the line and ``d`` is its direction vector (not necessarily
normalized).

### Fields

- `p` -- point on the line
- `d` -- direction

### Examples

The line passing through the point ``[-1, 2, 3]`` and parallel to the vector
``[3, 0, -1]``:

```jldoctest
julia> Line([-1, 2, 3.], [3, 0, -1.])
Line{Float64, Vector{Float64}}([-1.0, 2.0, 3.0], [3.0, 0.0, -1.0])
```
"""
struct Line{N, VN<:AbstractVector{N}} <: AbstractPolyhedron{N}
    p::VN
    d::VN

    # default constructor with length constraint
    function Line(p::VN, d::VN; check_direction::Bool=true) where {N, VN<:AbstractVector{N}}
        if check_direction
            @assert !iszero(d) "a line needs a non-zero direction vector"
        end
        return new{N, VN}(p, d)
    end
end

isoperationtype(::Type{<:Line}) = false
isconvextype(::Type{<:Line}) = true

"""
    Line(; from::AbstractVector, to::AbstractVector, normalize=true)

Constructor of a line given two points.

### Input

- `from`      -- point
- `to`        -- another point
- `normalize` -- (optional, default: `true`) if `true`, the direction of the line
                 has norm 1 (w.r.t the Euclidean norm)

### Output

The line which passes through `from` and `to`.

### Algorithm

Given two points ``p ∈ \\mathbb{R}^n`` and ``q ∈ \\mathbb{R}^n``, the line that
passes through these two points is
`L: `\\{y ∈ \\mathbb{R}^n: y = p + λ(q - p), λ ∈ \\mathbb{R}\\}``.
"""
function Line(; from::AbstractVector, to::AbstractVector, normalize=true)
    d = from - to
    if normalize && iszero(d)
        throw(ArgumentError("points `$from` and `$to` should be distinct"))
    end
    d_n = normalize ? d / dot(d, d) : d
    return Line(from, d_n)
end

"""
    Line(a::AbstractVector{N}, b::N; normalize=true) where {N}

Constructor of a two-dimensional line given `ax = b`.

### Input

- `a`         -- two-dimensional vector
- `b`         -- scalar
- `normalize` -- (optional, default: `true`) if `true`, the direction of the line
                 has norm 1 (w.r.t the Euclidean norm)

### Output

The `Line` that satisfies `a ⋅ x = b`.
"""
function Line(a::AbstractVector{N}, b::N; normalize=true) where {N}
    @assert length(a) == 2 "expected a normal vector of length two, but it is $(length(a))-dimensional"

    got_horizontal = iszero(a[1])
    got_vertical = iszero(a[2])

    if got_horizontal && got_vertical
        throw(ArgumentError("the vector $a must be non-zero"))
    end

    if got_horizontal
        α = b / a[2]
        p = [zero(N), α]
        q = [one(N), α]
    elseif got_vertical
        β = b / a[1]
        p = [β, zero(N)]
        q = [β, one(N)]
    else
        α = b / a[2]
        μ = a[1] / a[2]
        p = [zero(N), α]
        q = [one(N), α - μ]
    end
    return Line(from=p, to=q, normalize=normalize)
end

"""
    direction(L::Line)

Return the direction of the line.

### Input

- `L` -- line

### Output

The line's field corresponding to the direction to the line.

### Notes

The direction is not necessarily normalized.
See [`normalize(::Line, ::Real)`](@ref) / [`normalize!(::Line, ::Real)`](@ref)
for such operation.
"""
direction(L::Line) = L.d

"""
    normalize!(L::Line, p::Real=2.0)

Normalize the direction of a line storing the result in `L`.

### Input

- `L` -- line
- `p` -- (optional, default: `2.0`) vector `p`-norm used in the normalization

### Output

A line whose direction has unit norm w.r.t the given `p`-norm.
"""
function normalize!(L::Line, p::Real=2.0)
    normalize!(L.d, p)
    return L
end

"""
    normalize(L::Line, p::Real=2.0)

Normalize the direction of a line.

### Input

- `L` -- line
- `p` -- (optional, default: `2.0`) vector `p`-norm used in the normalization

### Output

A line whose direction has unit norm w.r.t the given `p`-norm.

### Notes

See also [`normalize!(::Line, ::Real)`](@ref) for the in-place version.
"""
function normalize(L::Line, p::Real=2.0)
    normalize!(copy(L), p)
end

# --- polyhedron interface functions ---

"""
    constraints_list(L::Line{N, VN}) where {N, VN}

Return the list of constraints of a line.

### Input

- `L` -- line

### Output

A list containing `2n-2` half-spaces whose intersection is `L`, where `n` is the
ambient dimension of `L`.
"""
function constraints_list(L::Line{N, VN}) where {N, VN}
    p = L.p
    n = length(p)
    d = reshape(L.d, 1, n)
    K = nullspace(d)
    m = size(K, 2)
    @assert m == n - 1 "expected $(n - 1) normal half-spaces, got $m"

    out = Vector{HalfSpace{N, VN}}(undef, 2m)
    idx = 1
    @inbounds for j in 1:m
        Kj = K[:, j]
        b = dot(Kj, p)
        out[idx] = HalfSpace(Kj, b)
        out[idx+1] = HalfSpace(-Kj, -b)
        idx += 2
    end
    return out
end

# --- LazySet interface functions ---

"""
    dim(L::Line)

Return the ambient dimension of a line.

### Input

- `L` -- line

### Output

The ambient dimension of the line.
"""
dim(L::Line) = length(L.p)

"""
    ρ(d::AbstractVector, L::Line)

Return the support function of a line in a given direction.

### Input

- `d` -- direction
- `L` -- line

### Output

The support function in the given direction.
"""
ρ(d::AbstractVector, L::Line) = _ρ(d, L)

function _ρ(d::AbstractVector, L::Line)
    if isapproxzero(dot(d, L.d))
        return dot(d, L.p)
    else
        return Inf
    end
end

"""
    σ(d::AbstractVector, L::Line)

Return the support vector of a line in a given direction.

### Input

- `d` -- direction
- `L` -- line

### Output

The support vector in the given direction.
"""
function σ(d::AbstractVector, L::Line)
    if isapproxzero(dot(d, L.d))
        return L.p
    else
        throw(ArgumentError("the support vector is undefined because the line is unbounded"))
    end
end

"""
    isbounded(L::Line)

Determine whether a line is bounded.

### Input

- `L` -- line

### Output

`false`.
"""
isbounded(::Line) = false

"""
    isuniversal(L::Line; [witness::Bool]=false)

Check whether a line is universal.

### Input

- `P`       -- line
- `witness` -- (optional, default: `false`) compute a witness if activated

### Output

* If `witness` is `false`: `true` if the ambient dimension is one, `false` otherwise

* If `witness` is `true`: `(true, [])` if the ambient dimension is one,
  `(false, v)` where ``v ∉ P`` otherwise
"""
isuniversal(L::Line; witness::Bool=false) = isuniversal(L, Val(witness))

# TODO implement case with witness
isuniversal(L::Line, ::Val{false}) = dim(L) == 1

"""
    an_element(L::Line)

Return some element of a line.

### Input

- `L` -- line

### Output

An element on the line.
"""
an_element(L::Line) = L.p

"""
    ∈(x::AbstractVector, L::Line)

Check whether a given point is contained in a line.

### Input

- `x` -- point/vector
- `L` -- line

### Output

`true` iff `x ∈ L`.

### Algorithm

The point ``x`` belongs to the line ``L : p + λd`` if and only if
``x - p`` is proportional to the direction ``d``.
"""
function ∈(x::AbstractVector, L::Line)
    @assert length(x) == dim(L) "expected the point and the line to have the same dimension, " *
                                "but they are $(length(x)) and $(dim(L)) respectively"
    _isapprox(x, L.p) && return true

    # TODO pass "minus" option to samedir
    return first(samedir(x - L.p, L.d)) || first(samedir(x - L.p, -L.d))
end

"""
    rand(::Type{Line}; [N]::Type{<:Real}=Float64, [dim]::Int=2,
         [rng]::AbstractRNG=GLOBAL_RNG, [seed]::Union{Int, Nothing}=nothing)

Create a random line.

### Input

- `Line` -- type for dispatch
- `N`    -- (optional, default: `Float64`) numeric type
- `dim`  -- (optional, default: 2) dimension
- `rng`  -- (optional, default: `GLOBAL_RNG`) random number generator
- `seed` -- (optional, default: `nothing`) seed for reseeding

### Output

A random line.

### Algorithm

All numbers are normally distributed with mean 0 and standard deviation 1.
"""
function rand(::Type{Line};
              N::Type{<:Real}=Float64,
              dim::Int=2,
              rng::AbstractRNG=GLOBAL_RNG,
              seed::Union{Int, Nothing}=nothing)

    rng = reseed(rng, seed)
    d = randn(rng, N, dim)
    while iszero(d)
        d = randn(rng, N, dim)
    end
    p = randn(rng, N, dim)
    return Line(p, d)
end

"""
    isempty(L::Line)

Return if a line is empty or not.

### Input

- `L` -- line

### Output

`false`.
"""
isempty(::Line) = false

"""
    translate(L::Line, v::AbstractVector)

Translate (i.e., shift) a line by a given vector.

### Input

- `L` -- line
- `v` -- translation vector

### Output

A translated line.

### Notes

See also `translate!` for the in-place version.
"""
function translate(L::Line, v::AbstractVector)
    return translate!(copy(L), v)
end

"""
    translate!(L::Line, v::AbstractVector)

Translate (i.e., shift) a line by a given vector storing the result in `L`.

### Input

- `L` -- line
- `v` -- translation vector

### Output

A translated line, modifying `L` in-place.
"""
function translate!(L::Line, v::AbstractVector)
    @assert length(v) == dim(L) "cannot translate a $(dim(L))-dimensional " *
                                "set by a $(length(v))-dimensional vector"

    L.p .+= v
    return L
end

"""
    distance(x::AbstractVector, L::Line; [p]::Real=2.0)

Compute the distance between point `x` and the line with respect to the given
`p`-norm.

### Input

- `x` -- vector
- `L` -- line
- `p` -- (optional, default: `2.0`) the `p`-norm used; `p = 2.0` corresponds to
         the usual Euclidean norm

### Output

A scalar representing the distance between `x` and the line `L`.
"""
function distance(x::AbstractVector, L::Line; p::Real=2.0)
    d = L.d  # direction of the line
    t = dot(x - L.p, d) / dot(d, d)
    return distance(x, L.p + t*d; p=p)
end

distance(L::Line, x::AbstractVector; p::Real=2.0) = distance(x, L; p=p)

"""
    linear_map(M::AbstractMatrix, L::Line)

Concrete linear map of a line.

### Input

- `M` -- matrix
- `L` -- line

### Output

The line obtained by applying the linear map to the point and direction of `L`.
"""
function linear_map(M::AbstractMatrix, L::Line)
    @assert dim(L) == size(M, 2) "a linear map of size $(size(M)) cannot be " *
                                 "applied to a set of dimension $(dim(L))"

    Mp = M * L.p
    Md = M * L.d
    return Line(Mp, Md)
end

function project(L::Line{N}, block::AbstractVector{Int}; kwargs...) where {N}
    d = L.d[block]
    if iszero(d)
        return Singleton(L.p[block])  # projected out all nontrivial dimensions
    elseif length(d) == 1
        return Universe{N}(1)  # special case: 1D line is a universe
    else
        return Line(L.p[block], d)
    end
end


using CSV, Statistics

rfile(name) = CSV.read(name, header=false)[:,1]
data = rfile.(["../data/machine1.csv",
		"../data/machine2.csv",
		"../data/machine3.csv"])
println("Sample means for each treatment: ",round.(mean.(data),digits=2))
println("Overall sample mean: ",round(mean(vcat(data...)),digits=2))