using Test
using Miletus
using Dates: today, Day

using CSV, DataFrames

s = SingleStock()
d1 = today()

bs = CSV.read("bs.csv")

# 1. Test with floats
calls = EuropeanCall.(d1 .+ Day.(bs.days), [s], float.(bs.strikeprice))
puts = EuropeanPut.(d1 .+ Day.(bs.days), [s], float.(bs.strikeprice))
m = GeomBMModel.([d1], float.(bs.startprice), bs.interestrate, bs.carryrate, bs.sigma)

atol=1e-6; rtol=1e-3

@test all(isapprox.(value.(m,calls), bs.c, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,puts), bs.p,  rtol=rtol,atol=atol))

# @test all(isapprox.(ivol.(m,calls,value.(m,calls)), bs.sigma))

crr = CSV.read("crr.csv")

eucalls = EuropeanCall.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice))
euputs  = EuropeanPut.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice ))
amcalls = AmericanCall.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice))
amputs  = AmericanPut.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice ))

m = CRRModel.(d1, d1 .+ Dates.Day.(crr.days), crr.nsteps, float.(crr.startprice), crr.interestrate, crr.carryrate, crr.sigma)

@test all(isapprox.(value.(m,eucalls) , crr.ce, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,euputs)  , crr.pe, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amcalls) , crr.ca, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amputs)  , crr.pa, rtol=rtol,atol=atol))

tian = CSV.read("tian.csv")

eucalls = EuropeanCall.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice))
euputs  = EuropeanPut.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice ))
amcalls = AmericanCall.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice))
amputs  = AmericanPut.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice ))

m = TianModel.([d1],d1 .+ Dates.Day.(tian.days), tian.nsteps, float.(tian.startprice), tian.interestrate, tian.carryrate, tian.sigma)

@test all(isapprox.(value.(m,eucalls) , tian.ce, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,euputs)  , tian.pe, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amcalls) , tian.ca, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amputs)  , tian.pa, rtol=rtol,atol=atol))

# 2. Test with currencies
calls = EuropeanCall.(d1 .+ Day.(bs.days), [s], float.(bs.strikeprice.*[USD]))
puts = EuropeanPut.(d1 .+ Day.(bs.days), [s], float.(bs.strikeprice.*[USD]))
m = GeomBMModel.([d1], float.(bs.startprice.*[USD]), bs.interestrate, bs.carryrate, bs.sigma)

atol=1e-6; rtol=1e-3

@test all(isapprox.(value.(m,calls) ./ [USD], bs.c, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,puts) ./ [USD], bs.p,  rtol=rtol,atol=atol))

crr = CSV.read("crr.csv")

eucalls = EuropeanCall.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice .* [USD]))
euputs  = EuropeanPut.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice .* [USD]))
amcalls = AmericanCall.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice .* [USD]))
amputs  = AmericanPut.(d1 .+ Dates.Day.(crr.days), [s], float.(crr.strikeprice .* [USD]))

m = CRRModel.([d1],d1 .+ Dates.Day.(crr.days), crr.nsteps, float.(crr.startprice.*[USD]), crr.interestrate, crr.carryrate, crr.sigma)

@test all(isapprox.(value.(m,eucalls) ./ [USD], crr.ce, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,euputs)  ./ [USD], crr.pe, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amcalls) ./ [USD], crr.ca, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amputs)  ./ [USD], crr.pa, rtol=rtol,atol=atol))

tian = CSV.read("tian.csv")

eucalls = EuropeanCall.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice .* [USD]))
euputs  = EuropeanPut.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice .* [USD]))
amcalls = AmericanCall.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice .* [USD]))
amputs  = AmericanPut.(d1 .+ Dates.Day.(tian.days), [s], float.(tian.strikeprice .* [USD]))

m = TianModel.([d1],d1 .+ Dates.Day.(tian.days), tian.nsteps, float.(tian.startprice.*[USD]), tian.interestrate, tian.carryrate, tian.sigma)

@test all(isapprox.(value.(m,eucalls) ./ [USD], tian.ce, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,euputs)  ./ [USD], tian.pe, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amcalls) ./ [USD], tian.ca, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amputs)  ./ [USD], tian.pa, rtol=rtol,atol=atol))

jr = CSV.read("jr.csv")

eucalls = EuropeanCall.(d1 .+ Dates.Day.(jr.days), [s], float.(jr.strikeprice .* [USD]))
euputs  = EuropeanPut.(d1 .+ Dates.Day.(jr.days), [s], float.(jr.strikeprice .* [USD]))
amcalls = AmericanCall.(d1 .+ Dates.Day.(jr.days), [s], float.(jr.strikeprice .* [USD]))
amputs  = AmericanPut.(d1 .+ Dates.Day.(jr.days), [s], float.(jr.strikeprice .* [USD]))

m = JRModel.([d1],d1 .+ Dates.Day.(jr.days), jr.nsteps, float.(jr.startprice.*[USD]), jr.interestrate, jr.carryrate, jr.sigma)

@test all(isapprox.(value.(m,eucalls) ./ [USD], jr.ce, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,euputs)  ./ [USD], jr.pe, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amcalls) ./ [USD], jr.ca, rtol=rtol,atol=atol))
@test all(isapprox.(value.(m,amputs)  ./ [USD], jr.pa, rtol=rtol,atol=atol))

# Test Black-76 model
d2 = d1 + Day(120)
cm = CoreForwardModel(d1, d2, 110.00USD, 0.02)
m = GeomBMModel(cm, 0.3)
o = EuropeanCall( d2, s, 100.00USD)
@test isapprox(value(m,o), 13.1833USD, rtol=1e-5)


"""
    nontrivial_pegiun(x, p, n) -> Function

Iterate a 2d discrete system with nonlinear, nontrivial coupling. This system
is from [Péguin-Feissolle & Teräsvirta (1999)](https://www.amse-aixmarseille.fr/sites/default/files/_dt/greqam/99a42.pdf), and
was also studied in [Chávez et al. (2003)](https://www.sciencedirect.com/science/article/pii/S0165027002003679).
The version implemented here allows for tweaking the parameters of the equations.

The difference equations are

```math
\\begin{aligned}
x(t+1) &= p_2 + p_3 x(t-2) + \\dfrac{p_4 - p_5 y(t-3)}{1 + e^{-p_6 y(t-3)}} + \\xi_1(t) \\
y(t+1) &= p_1 y(t) + \\xi_2(t).
\\end{aligned}
```

Here, ``\\xi_{1,2}(t)`` are two independent normally distributed noise processes
with zero mean and standard deviations ``\\sigma_1`` and ``\\sigma_2``. The
``\\xi_{1,2}(t)`` terms represent dynamical noise.

# References

Péguin-Feissolle, A., & Teräsvirta, T. (1999). A General Framework for
Testing the Granger Noncausaality Hypothesis. Universites d’Aix-Marseille II
et III. [https://www.amse-aixmarseille.fr/sites/default/files/_dt/greqam/99a42.pdf](https://www.amse-aixmarseille.fr/sites/default/files/_dt/greqam/99a42.pdf)

Chávez, M., Martinerie, J., & Le Van Quyen, M. (2003). Statistical
assessment of nonlinear causality: application to epileptic EEG signals.
Journal of Neuroscience Methods, 124(2), 113–128.
doi:10.1016/s0165-0270(02)00367-9
[https://www.sciencedirect.com/science/article/pii/S0165027002003679](https://www.sciencedirect.com/science/article/pii/S0165027002003679)
"""
function eom_nontrivial_pegiun(u, p, n)
    n = n + 10
    O = zeros(Float64, n + 3, 2)
    x, y = (u...,)
    p₁, p₂, p₃, p₄, p₅, p₆, σ₁, σ₂ = (p...,)

    # Propagate initial condition to the three first time steps.
    for i = 1:3
        O[i, 1] = x
        O[i, 2] = y
    end
    for i = 4:n
        y1 = O[i-1, 2]
        x2 = O[i-2, 1]
        y3 = O[i-3, 2]

        ξ₁ = rand(Normal(0, σ₁))
        ξ₂ = rand(Normal(0, σ₂))
        ynew = p₁*y1 + ξ₁
        xnew = p₂ + p₃*x2 + (p₄ - p₅*y3)/(1 + exp(-p₆*y3)) + ξ₂
        O[i, 1] = xnew
        O[i, 2] = ynew
    end
    O = O[10+3:end-10, :]
    O[:, 1] .= O[:, 1] .- mean(O[:, 1])
    O[:, 2] .= O[:, 2] .- mean(O[:, 2])
    O[:, 1] .= O[:, 1] ./ std(O[:, 1])
    O[:, 2] .= O[:, 2] ./ std(O[:, 2])
    return Dataset(O)
end

function nontrivial_pegiun(uᵢ, p₁, p₂, p₃, p₄, p₅, p₆, σ₁, σ₂, n::Int)
    p = [p₁, p₂, p₃, p₄, p₅, p₆, σ₁, σ₂]
    eom_nontrivial_pegiun(uᵢ, p, n)
end

"""
    nontrivial_pegiun(;uᵢ = rand(2), σ₁ = 0.1, σ₂ = 0.1,
        p₁ = 0.7, p₂ = 0.1, p₃ = 0.4, p₄ = 2.4, p₅ = 0.9, p₆ = 4, n = 100) -> Dataset

Create a 2d discrete systems with nonlinear, nontrivial coupling from [1] .
This version allows for tweaking the parameters of the equations. The variables
are each normalised to zero mean and unit variance.

The difference equations are

```math
\\begin{aligned}
x(t+1) &= p_2 + p_3 x(t-2) + \\dfrac{p_4 - p_5 y(t-3)}{1 + e^{-p_6 y(t-3)}} + \\xi_1(t) \\
y(t+1) &= p_1 y(t) + \\xi_2(t).
\\end{aligned}
```
Here, ``\\xi_{1,2}(t)`` are two independent normally distributed noise processes
with zero mean and standard deviations ``\\sigma_1`` and ``\\sigma_2``. The
``\\xi_{1,2}(t)`` terms represent dynamical noise.

# References
1. Chávez, M., Martinerie, J., & Le Van Quyen, M. (2003). Statistical
assessment of nonlinear causality: application to epileptic EEG signals.
Journal of Neuroscience Methods, 124(2), 113–128. doi:10.1016/s0165-0270(02)00367-9
"""
function nontrivial_pegiun(;uᵢ = rand(2), σ₁ = 0.1, σ₂ = 0.1,
        p₁ = 0.7, p₂ = 0.1, p₃ = 0.4, p₄ = 2.4, p₅ = 0.9, p₆ = 4, n = 100)
    eom_nontrivial_pegiun(uᵢ, [p₁, p₂, p₃, p₄, p₅, p₆, σ₁, σ₂], n)
end


isdefined(Base, :__precompile__) && __precompile__()

module PeaksAndValleys

using Compat

export PEAK, VALLEY
export identify_initial_pivot, peak_valley_pivots, max_drawdown
export compute_segment_returns, pivots_to_modes

@compat const PEAK, VALLEY = Int8(1), Int8(-1)

"""Quickly identify the xs[1] as a peak or valley."""
function identify_initial_pivot(xs, up_thresh, down_thresh)
    x_1 = xs[1]
    max_x = x_1
    max_t = 1
    min_x = x_1
    min_t = 1
    up_thresh += 1
    down_thresh += 1

    for (t, x_t) in enumerate(xs[2:end])
        t += 1  # I wish Julia had an enumerate(*,from=X)

        if x_t / min_x >= up_thresh
            return min_t == 1 ? VALLEY : PEAK
        end

        if x_t / max_x <= down_thresh
            return max_t == 1 ? PEAK : VALLEY
        end

        if x_t > max_x
            max_x = x_t
            max_t = t
        end

        if x_t < min_x
            min_x = x_t
            min_t = t
        end
    end
    x_1 < xs[length(xs)] ? VALLEY : PEAK
end

"""
Finds the peaks and valleys of a series.

Parameters
---
- **xs** : This is your series.
- **up_thresh** : The minimum relative change necessary to define a peak.
- **down_thesh** : The minimum relative change necessary to define a valley.

Returns
---
an array with 0 indicating no pivot and -1 and 1 indicating valley and peak
respectively.

The First and Last Elements
---
The first and last elements are guaranteed to be annotated as peak or
valley even if the segments formed do not have the necessary relative
changes. This is a tradeoff between technical correctness and the
propensity to make mistakes in data analysis. The possible mistake is
ignoring data outside the fully realized segments, which may bias analysis.
"""
function peak_valley_pivots(xs, up_thresh, down_thresh)
    assert(down_thresh < 0) # "It must be negative.

    initial_pivot = identify_initial_pivot(xs, up_thresh, down_thresh)

    t_n = length(xs)
    pivots = zeros(Int8, t_n)
    pivots[1] = initial_pivot

    # Adding one to the relative change thresholds saves operations. Instead
    # of computing relative change at each point as x_j / x_i - 1, it is
    # computed as x_j / x_1. Then, this value is compared to the threshold + 1.
    # This saves (t_n - 1) subtractions.
    up_thresh += 1
    down_thresh += 1

    trend = -initial_pivot
    last_pivot_t = 1
    last_pivot_x = xs[1]
    for (t, x) in enumerate(xs[2:end])
        t += 1
        r = x / last_pivot_x

        if trend == -1
            if r >= up_thresh
                pivots[last_pivot_t] = trend
                trend = 1
                last_pivot_x = x
                last_pivot_t = t
            elseif x < last_pivot_x
                last_pivot_x = x
                last_pivot_t = t
            end
        else
            if r <= down_thresh
                pivots[last_pivot_t] = trend
                trend = -1
                last_pivot_x = x
                last_pivot_t = t
            elseif x > last_pivot_x
                last_pivot_x = x
                last_pivot_t = t
            end
        end
    end

    pivots[t_n] == 0 && (pivots[t_n] = trend)

    return pivots
end

"""
Return the absolute value of the maximum drawdown of sequence X.

Note
---
If the sequence is increasing, 0 is returned."""
function max_drawdown(xs)
    mdd = 0
    peak = xs[1]
    for x in xs
        if x > peak
            peak = x
        end
        dd = (peak - x) / peak
        if dd > mdd
            mdd = dd
        end
    end
    return mdd
end

"""Return a numpy array of the pivot-to-pivot returns for each segment."""
function compute_segment_returns(xs, pivots)
    pivot_points = xs[pivots .!= 0]
    return pivot_points[2:end] ./ pivot_points[1:end-1] - 1.0
end

"""
Translate pivots into trend modes.

Parameters
---
pivots : the result of calling `peak_valley_pivots`

Returns
---
A vector of trend modes. That is, between (VALLEY, PEAK] it is 1 and
between (PEAK, VALLEY] it is -1.
"""
function pivots_to_modes(pivots)
    modes = zeros(Int8, length(pivots))
    modes[1] = pivots[1]
    mode = -modes[1]
    for (t, x) in enumerate(pivots[2:end])
        if x != 0
            modes[t+1] = mode
            mode = -x
        else
            modes[t+1] = mode
        end
    end
    modes
end

end


using DReal

a = Var(Float64,"aexpop2",0.,1.)
b2 = (a < 0.1) | (a > 0.9)
push_ctx!()
add!(b2)
is_satisfiable()
pop_ctx!()
push_ctx!()
b3 = !b2
#b3 = (a >= 0.1) & (a <= 0.9)
add!(b3)
is_satisfiable()


using MAT
using SparseArrays
using LinearAlgebra
include("../src/recursive_spectral.jl")
mat = matread("../data/emailEUcore.mat")
A = mat["A"]            # Largest connected component of email EU
ASim = mat["Asim"]      # Undirected version of same graph: ASim = spones(A+A')
m = sum(A)
label = vec(round.(Int64,mat["labels"]))

include("../src/Motif_Adj_Mat.jl")
Ata = ExtractTriMat(ASim)
Acc, p = largest_component(Ata)
core_inds = findall(x->x==true,p)
Ata = Ata[p,p]      # Just extracting triangles
ASim = ASim[p,p]    # Symmetric version of the graph
label = label[p]
n = size(ASim,1)

numpoints = 20
mins = 10 .^range(log10(5),log10(500),length=numpoints)
times = zeros(20,2)

SpecTri = zeros(n,numpoints)
SpecSim = zeros(n,numpoints)

for i = 1:length(mins)
    minsize = mins[i]

    # Run recursive spectral on the original graph
    start = time()
    @show minsize
    sets = recursive_spectral(ASim,minsize)
    c = set2clusvec(sets,n)
    times[i,1] = time()-start
    SpecSim[:,i] .= c

    # Run it on the motif adjacency matrix
    start = time()
    sets = recursive_spectral(Ata,minsize)
    c = set2clusvec(sets,n)
    times[i,2] = time()-start
    SpecTri[:,i] .= c

end

matwrite("Output/RecursiveSpectral_Email.mat", Dict("times"=>times,
"SpecTri"=>SpecTri, "SpecSim"=>SpecSim))


################################################################################
## Chi squared
################################################################################

#=---------------
Evaluate
---------------=#
function evaluate(dist::ChiSquared, a::T, b::T) where T <: AbstractFloat
    return (a-b)^2/(2*b)
end

function evaluate(dist::ChiSquared, a::AbstractVector{T}) where T <: AbstractFloat
    r = zero(T)
    @simd for i in eachindex(a)
        @inbounds r += (a[i] - 1.0)^2/2.0
    end
    return r
end

function evaluate(dist::ChiSquared, a::AbstractVector{T}, b::AbstractVector{T}) where T <: AbstractFloat
    if length(a) != length(b)
        throw(DimensionMismatch("first array has length $(length(a)) which does not match the length of the second, $(length(b))."))
    end
    r = zero(T)
    @simd for i in eachindex(a, b)
        @inbounds r += (a[i]-b[i])^2/(2*b[i])
    end
    return r
end

#=---------------
gradient
---------------=#
function gradient(dist::ChiSquared, a::T, b::T) where T <: AbstractFloat
    return (a/b) - 1.0
end

function gradient(dist::ChiSquared, a::T) where T <: AbstractFloat
    return a - 1.0
end

function gradient!(u::Vector{T}, dist::ChiSquared, a::AbstractVector{T}, b::AbstractVector{T}) where T <: AbstractFloat
    if length(a) != length(b)
        throw(DimensionMismatch("first array has length $(length(a)) which does not match the length of the second, $(length(b))."))
    end
    @simd for i = eachindex(a, b)
        ai = a[i]
        bi = b[i]
        @inbounds u[i] = (ai/bi)-1.0
    end
    u
end

function gradient!(u::Vector{T}, dist::ChiSquared, a::AbstractVector{T}) where T <: AbstractFloat
    @simd for i = eachindex(a)
        @inbounds u[i] = a[i]-1.0
    end
    u
end

#=---------------
hessian
---------------=#
function hessian(dist::ChiSquared, a::T, b::T) where T <: AbstractFloat
    return 1/b
end

function hessian(dist::ChiSquared, a::T) where T <: AbstractFloat
    return 1.0
end

function hessian!(u::Vector{T}, dist::ChiSquared, a::AbstractVector{T}, b::AbstractVector{T}) where T <: AbstractFloat
    if length(a) != length(b)
        throw(DimensionMismatch("first array has length $(length(a)) which does not match the length of the second, $(length(b))."))
    end
    @simd for i = eachindex(a)
         @inbounds u[i] = 1/b[i]
    end
    u
end

function hessian!(u::Vector{T}, dist::ChiSquared, a::AbstractVector{T}) where T <: AbstractFloat
    @simd for i = eachindex(a)
        @inbounds u[i] = 1.0
    end
    u
end


module TestMeans
using GaussianProcesses, Calculus
using Test, Statistics, Random

Random.seed!(1)

@testset "Means" begin
    n = 5 # number of observations
    d = 4 # dimension
    D = 3 # degree of polynomial mean function

    means = [MeanZero(),
             MeanConst(3.0),
             MeanLin(rand(d)),
             MeanPoly(rand(d, D)),
             MeanPeriodic(randn(d), randn(d), randn(d)),
             MeanConst(3.0) * MeanLin(rand(d)),
             MeanLin(rand(d)) + MeanPoly(rand(d, D))]

    X = rand(d, n)
    x = view(X, :, 1)

    @testset "Mean $(typeof(m))" for m in means
        println("\tTesting ", nameof(typeof(m)), "...")

        means = mean(m, X)
        params = GaussianProcesses.get_params(m)

        @testset "Run" begin
            mean(m, x)
            GaussianProcesses.grad_mean(m, x)
            GaussianProcesses.grad_stack(m, X)
            show(devnull, m)
            GaussianProcesses.get_param_names(m)
            set_params!(m, params)
        end

        @testset "Consistency" begin
            @test means ≈ invoke(mean, Tuple{GaussianProcesses.Mean, Matrix{Float64}}, m, X)
        end

        @testset "Gradients" begin
            @testset "Observation #$i" for i in 1:n
                Xi = view(X, :, i)
                theor_grad = GaussianProcesses.grad_mean(m, Xi)
                num_grad = Calculus.gradient(params) do params
                    set_params!(m, params)
                    mean(m, Xi)
                end
                @test theor_grad ≈ num_grad rtol=1e-5 atol=1e-5
            end
        end
    end
end
end


@testset "Demoralization user utils" begin
  @testset "vertexsetsize" begin
    partition_int = [[1, 4], [2, 3, 5], [6], [7, 8]]
    partition_vertex = VertexSet([Vertex(block) for block = partition_int])
    @test (QSWalk.vertexsetsize(partition_vertex)) == 8
    # error test
    @test_throws MethodError QSWalk.vertexsetsize(partition_int)

  end

  @testset "default_nm_loc_ham" begin
    @test QSWalk.default_nm_loc_ham(1) == spzeros(ComplexF64, 1, 1)
    @test QSWalk.default_nm_loc_ham(3) == sparse([0. im 0.; -im 0 im; 0 -im 0])
  end

  @testset "nm_loc_ham" begin
    #default option
    @test nm_loc_ham(VertexSet([[1], [2, 3]])) == sparse([0.0+0im 0 0;0 0 im;0 -im 0])
    @test nm_loc_ham(VertexSet([[1], [2], [3]])) == spzeros(ComplexF64, 3, 3)
    #by size version
    @test nm_loc_ham(VertexSet([[1, 2], [3, 4]]), Dict(2 =>[0 1; 1 0])) == sparse([0. 1 0 0; 1 0 0 0; 0 0 0 1; 0 0 1 0])
    @test nm_loc_ham(VertexSet([[1], [2], [3]]), Dict(1 =>ones(Float64, (1, 1)))) == SparseMatrixCSC{Float64}(I, 3, 3)
    #by index version
    M1 = [1. 2;3 5]
    M2 = ones(Float64, 1, 1)
    dict = Dict(Vertex([1, 3]) =>M1, Vertex([2]) =>M2)
    @test nm_loc_ham(VertexSet([[1, 3], [2]]), dict) ==
            sparse([ 1.0+0im 0 2; 0 1 0; 3 0 5 ])
  end

  @testset "Incidence and reverse incidence lists" begin
    A = [1 2 3; 0 3. 4.; 0 0 5.]
    @test QSWalk.incidence_list(A) == [[1], [1, 2], [1, 2, 3]]
    @test QSWalk.incidence_list(sparse(A)) == QSWalk.incidence_list(A)
    @test QSWalk.incidence_list(A; epsilon = 2.5) == [Int64[], [2], [1, 2, 3]]
    @test QSWalk.incidence_list(sparse(A)) == QSWalk.incidence_list(A)
    @test QSWalk.reversed_incidence_list(A) == [[1, 2, 3], [2, 3], [3]]
    @test QSWalk.reversed_incidence_list(sparse(A)) == QSWalk.reversed_incidence_list(A)
    @test QSWalk.reversed_incidence_list(A; epsilon = 2.5) == [[3], [2, 3], [3]]
    @test QSWalk.reversed_incidence_list(sparse(A); epsilon = 2.5) == QSWalk.reversed_incidence_list(A; epsilon = 2.5)
    #errors
    @test_throws ArgumentError QSWalk.reversed_incidence_list(A, epsilon = -1)
    @test_throws ArgumentError QSWalk.incidence_list(A, epsilon = -1)
    @test_throws ArgumentError QSWalk.reversed_incidence_list(sparse(A), epsilon = -1)
    @test_throws ArgumentError QSWalk.incidence_list(sparse(A), epsilon = -1)
  end

  @testset "vertexset creation" begin
    @test QSWalk.revinc_to_vertexset([[1, 3], [2, 3], Int[], [4, 6, 1]]) == VertexSet([[1, 2], [3, 4], [5], [6, 7, 8]])
    @test make_vertex_set([1 2 3; 0 3. 4.; 0 0 5.]) == VertexSet([[1, 2, 3], [4, 5], [6]])
  end

  @testset "Fourier matrix" begin
    @test QSWalk.fourier_matrix(4) ≈ [1 1 1 1; 1 1im -1 -1im;1 -1 1 -1; 1 -1im -1 1im]
    @test QSWalk.fourier_matrix(2) ≈ [1 1; 1 -1.]
    @test QSWalk.fourier_matrix(1) ≈ ones(Float64, (1, 1))
  end

  @testset "nm_lind" begin
    A = sparse([0.0+0.0im 1 0; 1 0 1; 0 1 0])
    B1, B2 = ones(1, 1), ones(2, 2)
    #default
    #needs to be roughly, since exp computing is inexact
    @test nm_lind(A)[1] ≈ [0 1 1 0;
                           1 0 0 1;
                           1 0 0 -1;
                           0 1 1 0]
    @test nm_lind(A)[2] == make_vertex_set(A)

    A = sparse([0.0+0.0im 0 0 0 1;
                0 0 1 0 1;
                0 0 0 0 0;
                0 1 1 0 0;
                0 0 0 0 0])
    L, vset = nm_lind(A)
    v1, v2, v3, v4, v5 = vlist(vset)
    @test nm_lind(A)[1] ≈ [0 0 0 0 0 0 1;
                                             0 0 0 1 0 0 1;
                                             0 0 0 1 0 0 -1;
                                             0 0 0 0 0 0 0;
                                             0 1 1 1 0 0 0;
                                             0 1 1 -1 0 0 0;
                                             0 0 0 0 0 0 0]
    @test nm_lind(A)[2] == make_vertex_set(A)
    @test nm_lind(A, Dict(v1 => B1, v2 => 2*B2, v3 => 3*B1, v4 => 4*B2, v5 => 5*B1))[1] ≈
             [0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  1.0+0.0im;
              0.0+0.0im  0.0+0.0im  0.0+0.0im  2.0+0.0im  0.0+0.0im  0.0+0.0im  2.0+0.0im;
              0.0+0.0im  0.0+0.0im  0.0+0.0im  2.0+0.0im  0.0+0.0im  0.0+0.0im  2.0+0.0im;
              0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              0.0+0.0im  4.0+0.0im  4.0+0.0im  4.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              0.0+0.0im  4.0+0.0im  4.0+0.0im  4.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im]

    A = [0 0 0; 0 0 0; 1 1 0]
    @test nm_lind(A)[1] ≈ [0 0 0 0;
                                             0 0 0 0;
                                             1 1 0 0;
                                             1 -1 0 0]
    @test nm_lind(A)[2] == make_vertex_set(A)

    @test nm_lind(A, Dict(1  => B1, 2 =>3*B2 ))[1] ≈
             [0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              3.0+0.0im  3.0+0.0im  0.0+0.0im  0.0+0.0im;
              3.0+0.0im  3.0+0.0im  0.0+0.0im  0.0+0.0im]
    @test nm_lind(A, Dict(1  => B1, 2 =>3*B2 ))[2] ==
      make_vertex_set(A)

    @test nm_lind(A, Dict(1  => B1, 2 =>3*B2 ))[1] ≈
             [0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              3.0+0.0im  3.0+0.0im  0.0+0.0im  0.0+0.0im;
              3.0+0.0im  3.0+0.0im  0.0+0.0im  0.0+0.0im]

    L, vset = nm_lind(A)
    v1, v2, v3 = vlist(vset)
    @test nm_lind(A, Dict(v1 => B1, v2 => 2*B1, v3 => 3*B2))[1] ≈
             [0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im;
              3.0+0.0im  3.0+0.0im  0.0+0.0im  0.0+0.0im;
              3.0+0.0im  3.0+0.0im  0.0+0.0im  0.0+0.0im]
    @test nm_lind(A, Dict(v1 => B1, v2 => 2*B1, v3 => 3*B2))[2] ==
      make_vertex_set(A)

      A = [2 0 0 3;
    im 0 3im 0;
    1 0 0 0;
    im -im 0 0]
    @test nm_lind(A,Dict(1=>ones(1,1), 2=> ones(2,2)))[1] ≈
              [2.0+0.0im 2.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im 3.0+0.0im 3.0+0.0im;
               2.0+0.0im 2.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im 3.0+0.0im 3.0+0.0im;
               0.0+1.0im 0.0+1.0im 0.0+0.0im 0.0+0.0im 0.0+3.0im 0.0+0.0im 0.0+0.0im;
               0.0+1.0im 0.0+1.0im 0.0+0.0im 0.0+0.0im 0.0+3.0im 0.0+0.0im 0.0+0.0im;
               1.0+0.0im 1.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im;
               0.0+1.0im 0.0+1.0im 0.0-1.0im 0.0-1.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im;
               0.0+1.0im 0.0+1.0im 0.0-1.0im 0.0-1.0im 0.0+0.0im 0.0+0.0im 0.0+0.0im]

  end

  @testset "nm_glob_ham" begin
    A = sparse([0 1 0;
               1 0 2;
               0 2 0])
    #default
    #needs to be roughly, since exp computing is inexact
    @test nm_glob_ham(A) ≈ [0 1 1 0;
                                   1 0 0 2;
                                   1 0 0 2;
                                   0 2 2 0]
    @test nm_glob_ham(A, Dict((1, 2) => (2+1im)*ones(1, 2), (2, 1) =>1im*ones(2, 1))) ≈
                                    [0 2+1im 2+1im 0;
                                     2-1im 0 0 2im;
                                     2-1im 0 0 2im;
                                     0 -2im -2im 0]

    v1, v2, v3 = vlist(make_vertex_set(A))
    @test nm_glob_ham(A, Dict{Tuple{Vertex,Vertex},AbstractMatrix{<:Number}}((v1, v2) =>2*ones(1, 2), (v2, v3) => -adjoint([1im 2im]))) ==
                                              [0 2 2 0;
                                               2 0 0 2im;
                                               2 0 0 4im;
                                               0 -2im -4im 0]
   A = [0  0  1;
        0  0  1;
        2  2  0]
  v1, v2, v3 = vlist(make_vertex_set(A))
  @test nm_glob_ham(A, Dict{Tuple{Vertex,Vertex},AbstractMatrix{<:Number}}((v1, v3) =>2*ones(1, 2), (v2, v3) =>[1im 2im;])) ≈
                          [0.0+0.0im  0.0+0.0im  3.0+0.0im  3.0+0.0im;
                           0.0+0.0im  0.0+0.0im  0.0+1.5im  0.0+3.0im;
                           3.0+0.0im  0.0-1.5im  0.0+0.0im  0.0+0.0im;
                           3.0+0.0im  0.0-3.0im  0.0+0.0im  0.0+0.0im]

  @test nm_glob_ham(A, Dict{Tuple{Vertex,Vertex},AbstractMatrix{<:Number}}((v1, v3) =>2*ones(1, 1), (v2, v3) =>ones(1,1)*1im),epsilon=3) ≈
                          [0.0+0.0im  0.0+0.0im  0.0+0.0im;
                           0.0+0.0im  0.0+0.0im  0.0+0.0im;
                           0.0+0.0im  0.0+0.0im  0.0+0.0im]
  a_directed = diagm(1=>ones(3))
  a_directed_transposed = diagm(-1=>ones(3))
  a_undirected = a_directed_transposed + a_directed
  @test nm_glob_ham(a_directed) ≈ a_undirected
  @test nm_glob_ham(a_directed_transposed) ≈ a_undirected
  @test_throws DimensionMismatch nm_glob_ham(a_undirected) ≈ a_undirected

  @test nm_glob_ham(Matrix(a_directed)) ≈ a_undirected
  @test nm_glob_ham(Matrix(a_directed_transposed)) ≈ a_undirected
  @test_throws DimensionMismatch nm_glob_ham(Matrix(a_undirected)) ≈ a_undirected
  end
end

@testset "Cannonical measurement" begin
  probability = [0.05, 0.1, 0.25, 0.3, 0.01, 0.20, 0.04, 0.05]
  vset = VertexSet([[1, 4], [2, 3, 5], [6], [7, 8]])
  result = [0.35, 0.36, 0.2, 0.09]
  @test nm_measurement(probability, vset) == result

  state = [1/6 1.0 1/6; 1. 2/3 1im; 1/6 -1im 1/6]
  vset = VertexSet([[1, 3], [2]])
  @test nm_measurement(state, vset) ≈ [1.0/3, 2.0/3]
  @test_throws AssertionError nm_measurement(Matrix{Float64}(I,2,2)/2., vset)
  @test_throws AssertionError nm_measurement([1.0/2, 1.0/3, 1.0/6, 0.], vset)

end

@testset "Initial states creation" begin
  vset = VertexSet([[1], [2, 3, 4], [5, 6, 7], [8, 9]])
  @test nm_init(vset[[1, 3, 4]], vset) ≈ spdiagm(0=>[1.0/3, 0, 0, 0, 1.0/9, 1.0/9, 1.0/9, 1.0/6, 1.0/6])
  A1 = ones(ComplexF64, 1, 1)/4
  A2 = [ 1/5+0im 0 1/5; 0 1/10 0 ; 1/5 0 1/5 ]
  A3 = [0.125 -0.125+0im; -0.125 0.125]
  dict = Dict(vset[1] =>A1, vset[3] =>A2, vset[4] =>A3 )
  @test nm_init(dict, vset) ==
        sparse([ 0.25+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im     0.0+0.0im     0.0+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im     0.0+0.0im     0.0+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im     0.0+0.0im     0.0+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im     0.0+0.0im     0.0+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.2+0.0im  0.0+0.0im  0.2+0.0im     0.0+0.0im     0.0+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.1+0.0im  0.0+0.0im     0.0+0.0im     0.0+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.2+0.0im  0.0+0.0im  0.2+0.0im     0.0+0.0im     0.0+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im   0.125+0.0im  -0.125+0.0im
                0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  0.0+0.0im  -0.125+0.0im   0.125+0.0im])

  dictwrong = Dict(vset[1] =>A2, vset[3] =>A2, vset[4] =>A3 )
  @test_throws AssertionError nm_init(vlist(VertexSet([[1, 2, 3], [4, 5]])), vset)
  @test_throws AssertionError nm_init(dictwrong, vset)
end


type SkewNormalSampler <: Sampleable{Univariate,Continuous}
    ξ::Float64
    ω::Float64
    aux::MvNormal
    function SkewNormalSampler(ξ::Real, ω::Real, α::Real)
        δ = α/(1 + α^2)
        aux = MvNormal([[1 δ]; [δ 1]])
        new(ξ, ω, aux)
    end
end

function rand(dist::SkewNormalSampler)
    x0, x = rand(dist.aux)
    dist.ω * ((x0 > 0) ? x : -x) + dist.ξ
end

type SkewNormal <: ContinuousUnivariateDistribution
    ξ::Float64  # Location
    ω::Float64  # Scale
    α::Float64  # Skewness
end

sampler(dist::SkewNormal) = SkewNormalSampler(dist.ξ, dist.ω, dist.α)

function pdf(dist::SkewNormal, x::Real)
    z = (x - dist.ξ)/dist.ω
    2.0 * (normpdf(z)/dist.ω) * normcdf(dist.α * z)
end

function cdf(dist::SkewNormal, x::Real)
    quadgk(t->pdf(dist,t), -Inf, x)[1]
end

function quantile(dist::SkewNormal, β::Float64)
    newton(x->cdf(dist,x) - β, dist.ξ)
end

minimum(dist::SkewNormal) = -Inf
maximum(dist::SkewNormal) = Inf

mean(dist::SkewNormal) = dist.ω*sqrt(2/π)*(dist.α/sqrt(1.0 + dist.α^2)) + dist.ξ
var(dist::SkewNormal) = (dist.ω^2) * (1.0 - (sqrt(2/π)*(dist.α/sqrt(1.0 + dist.α^2)))^2)

# Cumulant generating function

cgf(dist::SkewNormal, t::Real) = t*dist.ξ + 0.5*t^2*dist.ω^2 + log(2normcdf((dist.ω*t*dist.α)/sqrt(1 + dist.α^2)))
mgf(dist::SkewNormal, t::Real) = exp(cgf(dist, t))


# ---
# title: 1011. Capacity To Ship Packages Within D Days
# id: problem1011
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Array, Binary Search
# link: <https://leetcode.com/problems/capacity-to-ship-packages-within-d-days/description/>
# hidden: true
# ---
#
# A conveyor belt has packages that must be shipped from one port to another
# within `D` days.
#
# The `i`-th package on the conveyor belt has a weight of `weights[i]`.  Each
# day, we load the ship with packages on the conveyor belt (in the order given
# by `weights`). We may not load more weight than the maximum weight capacity of
# the ship.
#
# Return the least weight capacity of the ship that will result in all the
# packages on the conveyor belt being shipped within `D` days.
#
#
#
# **Example 1:**
#
#
#
#     Input: weights = [1,2,3,4,5,6,7,8,9,10], D = 5
#     Output: 15
#     Explanation:
#     A ship capacity of 15 is the minimum to ship all the packages in 5 days like this:
#     1st day: 1, 2, 3, 4, 5
#     2nd day: 6, 7
#     3rd day: 8
#     4th day: 9
#     5th day: 10
#
#     Note that the cargo must be shipped in the order given, so using a ship of capacity 14 and splitting the packages into parts like (2, 3, 4, 5), (1, 6, 7), (8), (9), (10) is not allowed.
#
#
# **Example 2:**
#
#
#
#     Input: weights = [3,2,2,4,1,4], D = 3
#     Output: 6
#     Explanation:
#     A ship capacity of 6 is the minimum to ship all the packages in 3 days like this:
#     1st day: 3, 2
#     2nd day: 2, 4
#     3rd day: 1, 4
#
#
# **Example 3:**
#
#
#
#     Input: weights = [1,2,3,1,1], D = 4
#     Output: 3
#     Explanation:
#     1st day: 1
#     2nd day: 2
#     3rd day: 3
#     4th day: 1, 1
#
#
#
#
# **Constraints:**
#
#   * `1 <= D <= weights.length <= 50000`
#   * `1 <= weights[i] <= 500`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


"""
    OceanGyre <: AbstractSimpleBoxProblem

Container structure for a simple box problem with wind-stress, coriolis force, and temperature forcing.
Lˣ = zonal (east-west) length
Lʸ = meridional (north-south) length
H  = height of the ocean
τₒ = maximum value of wind-stress (amplitude)
λʳ = temperature relaxation penetration constant (meters / second)
θᴱ = maximum surface temperature
"""
struct OceanGyre{T, BC} <: AbstractSimpleBoxProblem
    Lˣ::T
    Lʸ::T
    H::T
    τₒ::T
    λʳ::T
    θᴱ::T
    boundary_conditions::BC
    function OceanGyre{FT}(
        Lˣ,                  # m
        Lʸ,                  # m
        H;                   # m
        τₒ = FT(1e-1),       # N/m²
        λʳ = FT(4 // 86400), # m/s
        θᴱ = FT(10),         # K
        BC = (
            OceanBC(Impenetrable(NoSlip()), Insulating()),
            OceanBC(Impenetrable(NoSlip()), Insulating()),
            OceanBC(Penetrable(KinematicStress()), TemperatureFlux()),
        ),
    ) where {FT <: AbstractFloat}
        return new{FT, typeof(BC)}(Lˣ, Lʸ, H, τₒ, λʳ, θᴱ, BC)
    end
end

"""
    ocean_init_state!(::OceanGyre)

initialize u,v,η with 0 and θ linearly distributed between 9 at z=0 and 1 at z=H

# Arguments
- `p`: OceanGyre problem object, used to dispatch on and obtain ocean height H
- `Q`: state vector
- `A`: auxiliary state vector, not used
- `localgeo`: the local geometry information
- `t`: time to evaluate at, not used
"""
function ocean_init_state!(
    ::Union{HBModel, OceanModel},
    p::OceanGyre,
    Q,
    A,
    localgeo,
    t,
)
    coords = localgeo.coord
    @inbounds y = coords[2]
    @inbounds z = coords[3]
    @inbounds H = p.H

    Q.u = @SVector [-0, -0]
    Q.η = -0
    Q.θ = (5 + 4 * cos(y * π / p.Lʸ)) * (1 + z / H)

    return nothing
end

function ocean_init_state!(
    ::Union{SWModel, BarotropicModel},
    ::OceanGyre,
    Q,
    A,
    localgeo,
    t,
)
    Q.U = @SVector [-0, -0]
    Q.η = -0

    return nothing
end

"""
    kinematic_stress(::OceanGyre)

jet stream like windstress

# Arguments
- `p`: problem object to dispatch on and get additional parameters
- `y`: y-coordinate in the box
"""
@inline kinematic_stress(p::OceanGyre, y, ρ) =
    @SVector [(p.τₒ / ρ) * cos(y * π / p.Lʸ), -0]

@inline kinematic_stress(p::OceanGyre, y) =
    @SVector [-p.τₒ * cos(π * y / p.Lʸ), -0]

"""
    surface_flux(::OceanGyre)

cool-warm north-south linear temperature gradient

# Arguments
- `p`: problem object to dispatch on and get additional parameters
- `y`: y-coordinate in the box
- `θ`: temperature within element on boundary
"""
@inline function surface_flux(p::OceanGyre, y, θ)
    Lʸ = p.Lʸ
    θᴱ = p.θᴱ
    λʳ = p.λʳ

    θʳ = θᴱ * (1 - y / Lʸ)
    return λʳ * (θ - θʳ)
end


julia> print(j2(41,3,3))
Prisoner killing order: 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 32, 35, 38, 0, 4, 9, 13, 18, 22, 27, 31, 36, 40, 6, 12, 19, 25, 33, 39, 7, 16, 28, 37, 10, 24, 1, 21, 3.
Survivors: [15, 30, 34]


@testset "general reduction" begin
    ds = Dataset(g = [1, 1, 1, 2, 2],
                        x1_int = [0, 0, 1, missing, 2],
                        x2_int = [3, 2, 1, 3, -2],
                        x1_float = [1.2, missing, -1.0, 2.3, 10],
                        x2_float = [missing, missing, 3.0, missing, missing],
                        x3_float = [missing, missing, -1.4, 3.0, -100.0])
    @test byrow(ds, sum, r"int") == [3,2,2,3,0]
    @test byrow(repeat(ds,100), sum, r"int") == repeat([3,2,2,3,0], 100)

    @test isequal(byrow(ds, sum, r"float") , [1.2, missing, 0.6000000000000001,5.3,-90])
    @test isequal(byrow(repeat(ds, 100), sum, r"float") , repeat([1.2, missing, 0.6000000000000001,5.3,-90], 100))
    @test byrow(ds, mean, r"int") == [3/2,2/2,1.0,3.0,0]
    @test byrow(repeat(ds,100), mean, r"int") == repeat([3/2,2/2,1.0,3.0,0],100)
    @test isequal(byrow(ds, maximum, r"float") , [1.2, missing, 3,3.0,10.0])
    @test isequal(byrow(ds, maximum, r"float", by = abs) , [1.2, missing, 3,3.0,100.0])
    @test isequal(byrow(ds, minimum, r"float") , [1.2, missing, -1.4,2.3,-100.0])

    @test isequal(byrow(repeat(ds,100), maximum, r"float") , repeat([1.2, missing, 3,3.0,10.0], 100))
    @test isequal(byrow(repeat(ds,100), maximum, r"float", by = abs) , repeat([1.2, missing, 3,3.0,100.0], 100))
    @test isequal(byrow(repeat(ds,100), minimum, r"float") , repeat([1.2, missing, -1.4,2.3,-100.0], 100))

    @test byrow(ds, sum, r"int", threads = true) == [3,2,2,3,0]
    @test isequal(byrow(ds, sum, r"float", threads = true) , [1.2, missing, 0.6000000000000001,5.3,-90])
    @test byrow(ds, mean, r"int", threads = true) == [3/2,2/2,1.0,3.0,0]
    @test isequal(byrow(ds, maximum, r"float", threads = true) , [1.2, missing, 3,3.0,10.0])
    @test isequal(byrow(ds, maximum, r"float", by = abs, threads = true) , [1.2, missing, 3,3.0,100.0])
    @test isequal(byrow(ds, minimum, r"float", threads = true) , [1.2, missing, -1.4,2.3,-100.0])

    @test byrow(ds, argmax, :) == Symbol.(["x2_int", "x2_int", "x2_float", "x2_int", "x1_float"])
    @test isequal(byrow(ds, argmin, r"float"), [:x1_float, missing, :x3_float, :x1_float, :x3_float])
    @test isequal(byrow(ds, argmin, r"float", by = abs) , [:x1_float, missing, :x1_float, :x1_float, :x1_float])
    @test byrow(ds, coalesce, ["x2_float", "x1_float", "x1_int"]) == [1.2,0,3.0,2.3,10]
    @test isequal(byrow(ds, var, r"float"), [missing, missing, 5.92, 0.24499999999999922, 6050.0])
    @test isequal(byrow(ds, var, r"float", dof = false), [0.0, missing, 3.9466666666666663, 0.12249999999999961, 3025.0])
    @test byrow(ds, count, :, by=  ismissing) == [2,3,0,2,1]

    @test byrow(repeat(ds, 100), argmax, :) == repeat(Symbol.(["x2_int", "x2_int", "x2_float", "x2_int", "x1_float"]),100)
    @test isequal(byrow(repeat(ds, 100), argmin, r"float"), repeat([:x1_float, missing, :x3_float, :x1_float, :x3_float],100))
    @test isequal(byrow(repeat(ds, 100), argmin, r"float", by = abs) , repeat([:x1_float, missing, :x1_float, :x1_float, :x1_float],100))
    @test byrow(repeat(ds, 100), coalesce, ["x2_float", "x1_float", "x1_int"]) == repeat([1.2,0,3.0,2.3,10],100)
    @test isequal(byrow(repeat(ds, 100), var, r"float"), repeat([missing, missing, 5.92, 0.24499999999999922, 6050.0],100))
    @test isequal(byrow(repeat(ds, 100), var, r"float", dof = false), repeat([0.0, missing, 3.9466666666666663, 0.12249999999999961, 3025.0],100))
    @test byrow(repeat(ds, 100), count, :, by=  ismissing) == repeat([2,3,0,2,1],100)

    sds = view(ds, [5,5,5,3,3,3,2,2,2,4,4,4,4,4], [4,1,2,5])
    @test byrow(sds, sum, :) == [14.0,14.,14,4,4,4,1,1,1,4.3,4.3,4.3,4.3,4.3]
    @test byrow(sds, sum, [:g, :x2_float]) == [2,2.0,2,4,4,4,1,1,1,2,2,2,2,2]
    @test byrow(sds, argmax, [3,2,4,1]) == Symbol.(["x1_float","x1_float","x1_float","x2_float","x2_float","x2_float","g", "g", "g", "x1_float","x1_float","x1_float","x1_float","x1_float"])
    @test byrow(sds, argmax, [1,2,4,3], by = ismissing) == Symbol.(["x2_float","x2_float","x2_float", "x1_float","x1_float","x1_float", "x1_float","x1_float","x1_float", "x2_float", "x2_float", "x2_float", "x2_float", "x2_float"])
    @test isequal(byrow(sds, argmax, [3,4]), [:x1_int,:x1_int,:x1_int,:x2_float,:x2_float,:x2_float,:x1_int,:x1_int,:x1_int,missing, missing, missing, missing, missing])
    @test isequal(byrow(sds, argmin, [3,4], threads = true), [:x1_int,:x1_int,:x1_int,:x1_int,:x1_int,:x1_int,:x1_int,:x1_int,:x1_int,missing, missing, missing, missing, missing])

    @test byrow(sds, any, :, by = ismissing) == [true, true, true, false, false, false, true, true, true, true, true, true, true, true]

    ds = Dataset(x1 = [1,2,3,4,missing], x2 = [3,2,4,5, missing])
    @test byrow(ds, isequal, :) == [false, true, false, false, true]
    sds = view(ds, [1,2,2,1,3,4,5,5,5], [2,1])
    @test byrow(sds, isequal, :) == [0,1,1,0,0,0, 1,1,1]
    @test byrow(sds, isequal, [1]) == ones(9)

    ds = Dataset(x1 = [1,2,3,4,missing], x2 = [3,2,4,5, missing])
    @test byrow(ds, isequal, :, threads = true) == [false, true, false, false, true]
    sds = view(ds, [1,2,2,1,3,4,5,5,5], [2,1])
    @test byrow(sds, isequal, :, threads = true) == [0,1,1,0,0,0, 1,1,1]
    @test byrow(sds, isequal, [1], threads = true) == ones(9)

    ds = Dataset(x1 = [1,2,3,1,1], x2 =[2,2,3,missing, 1], x3 = [1,1,1,1,1])

    @test byrow(ds, isequal, 1:2, with = :x3) == [false, false, false, false, true]
    @test byrow(ds, isequal, 1:3, with = :x3) == [false, false, false, false, true]
    @test byrow(ds, isequal, [1,3], with = [1,1,1,1,1]) == [true, false, false, true, true]
    @test byrow(ds, isequal, 1:2, with = ds[!, :x3]) == [false, false, false, false, true]
    @test byrow(ds, isequal, 1:3, with = ds[:,3]) == [false, false, false, false, true]
    @test byrow(ds, isequal, 1, with = ds[:,3]) == [true, false, false, true, true]

    @test byrow(view(ds, :, :), isequal, 1:2, with = :x3) == [false, false, false, false, true]
    @test byrow(view(ds, :, :), isequal, 1:3, with = :x3) == [false, false, false, false, true]
    @test byrow(view(ds, :, :), isequal, [1,3], with = [1,1,1,1,1]) == [true, false, false, true, true]

    @test byrow(repeat(ds,100), isequal, 1:2, with = :x3) == repeat([false, false, false, false, true],100)
    @test byrow(repeat(ds,100), isequal, 1:3, with = :x3) == repeat([false, false, false, false, true],100)
    @test byrow(repeat(ds,100), isequal, [1,3], with = repeat([1,1,1,1,1],100)) == repeat([true, false, false, true, true],100)
    @test byrow(repeat(ds,100), isequal, 1:2, with = repeat(ds,100)[!, :x3]) == repeat([false, false, false, false, true],100)
    @test byrow(repeat(ds,100), isequal, 1:3, with = repeat(ds,100)[:,3]) == repeat([false, false, false, false, true],100)
    @test byrow(repeat(ds,100), isequal, 1, with = repeat(ds,100)[:,3]) == repeat([true, false, false, true, true],100)

    ds = Dataset(x1 = [1,2,3,4,missing], x2 = [3,2,4,5, missing])
    @test byrow(ds, issorted, :) == [true, true, true, true, true]
    @test byrow(ds, issorted, :, rev = true) == [false, true, false, false, true]

    ds = Dataset(randn(10000, 3), :auto)
    map!(ds, x->rand()<.1 ? missing : x, :)
    dsm = Matrix(ds)
    @test byrow(ds, issorted, :) == issorted.(eachrow(dsm))
    @test byrow(ds, issorted, :,  rev = true) == issorted.(eachrow(dsm), rev = true)
    insertcols!(ds, 1, :y=>rand(-1:1, nrow(ds)))
    dsm = Matrix(ds)
    @test byrow(ds, issorted, :) == byrow(ds, issorted, :, threads = false) == issorted.(eachrow(dsm))
    @test byrow(ds, issorted, :,  rev = true) == byrow(ds, issorted, :,  rev = true, threads = false) == issorted.(eachrow(dsm), rev = true)

    ds = Dataset(g = [1, 1, 1, 2, 2],
                        x1_int = [0, 0, 1, missing, 2],
                        x2_int = [3, 2, 1, 3, -2],
                        x1_float = [1.2, missing, -1.0, 2.3, 10],
                        x2_float = [missing, missing, 3.0, missing, missing],
                        x3_float = [missing, missing, -1.4, 3.0, -100.0])
    @test isequal(byrow(ds, findfirst, :, by = ismissing), [:x2_float, :x1_float, missing, :x1_int, :x2_float])
    @test isequal(byrow(ds, findlast, :, by = ismissing), [:x3_float, :x3_float, missing, :x2_float, :x2_float])
    @test isequal(byrow(ds, findfirst, :, by = x->isless(x,0)), [missing, missing, :x1_float, missing, :x2_int])
    @test isequal(byrow(ds, findlast, :, by = x->isless(x,0)), [missing, missing, :x3_float, missing, :x3_float])
    @test isequal(byrow(ds, findfirst, :, by = x->1), [:g,:g,:g, :g,:g])
    @test isequal(byrow(ds, findfirst, :), [:g,:g,:g, missing, missing])
    @test isequal(byrow(ds, findlast, :), [:g,:g,:x2_int, missing, missing])
    @test isequal(byrow(ds, findfirst, [3,2,1], by = isequal(2)) ,byrow(ds, findlast, 1:3, by = isequal(2)))
    @test isequal(byrow(ds, findfirst, 1:3, by = isequal(2)) ,byrow(ds, findlast, [3,2,1], by = isequal(2)))

    @test isequal(byrow(repeat(ds,100), findfirst, :, by = ismissing), repeat([:x2_float, :x1_float, missing, :x1_int, :x2_float],100))
    @test isequal(byrow(repeat(ds,100), findlast, :, by = ismissing), repeat([:x3_float, :x3_float, missing, :x2_float, :x2_float],100))
    @test isequal(byrow(repeat(ds,100), findfirst, :, by = x->isless(x,0)), repeat([missing, missing, :x1_float, missing, :x2_int],100))
    @test isequal(byrow(repeat(ds,100), findlast, :, by = x->isless(x,0)), repeat([missing, missing, :x3_float, missing, :x3_float],100))
    @test isequal(byrow(repeat(ds,100), findfirst, :, by = x->1), repeat([:g,:g,:g, :g,:g],100))
    @test isequal(byrow(repeat(ds,100), findfirst, :), repeat([:g,:g,:g, missing, missing],100))
    @test isequal(byrow(repeat(ds,100), findlast, :), repeat([:g,:g,:x2_int, missing, missing],100))
    @test isequal(byrow(repeat(ds,100), findfirst, [3,2,1], by = isequal(2)) ,byrow(repeat(ds,100), findlast, 1:3, by = isequal(2)))
    @test isequal(byrow(repeat(ds,100), findfirst, 1:3, by = isequal(2)) ,byrow(repeat(ds,100), findlast, [3,2,1], by = isequal(2)))

    sds = view(ds, rand(1:5, 100), [2,1,6,5,3,4])
    @test isequal(byrow(sds, findfirst,:, by = x->isless(x,0)), byrow(Dataset(sds), findfirst, :, by = x->isless(x,0)))
    @test isequal(byrow(sds, findlast,:, by = x->isless(x,0)), byrow(Dataset(sds), findlast, :, by = x->isless(x,0)))
    @test isequal(byrow(sds, findfirst,:, by = x->isless(x,0), threads = true), byrow(Dataset(sds), findfirst, :, by = x->isless(x,0)))
    @test isequal(byrow(sds, findlast,:, by = x->isless(x,0), threads = true), byrow(Dataset(sds), findlast, :, by = x->isless(x,0)))
    sds = view(ds, rand(1:5, 100), [2,1,6,5,3,4])
    @test isequal(byrow(sds, findfirst,:, by = x->isless(x,0)), byrow(Dataset(sds), findfirst, :, by = x->isless(x,0)))
    @test isequal(byrow(sds, findlast,:, by = x->isless(x,0)), byrow(Dataset(sds), findlast, :, by = x->isless(x,0)))
    @test isequal(byrow(sds, findfirst,:, by = x->isless(x,0), threads = true), byrow(Dataset(sds), findfirst, :, by = x->isless(x,0)))
    @test isequal(byrow(sds, findlast,:, by = x->isless(x,0), threads = true), byrow(Dataset(sds), findlast, :, by = x->isless(x,0)))

    ds = Dataset(g = [1, 1, 1, 2, 2],
                        x1_int = [0, 0, 1, missing, 2],
                        x2_int = [3, 2, 1, 3, -2],
                        x1_float = [1.2, missing, -1.0, 2.3, 10],
                        x2_float = [missing, missing, 3.0, missing, missing],
                        x3_float = [missing, missing, -1.4, 3.0, -100.0])
    @test isequal(byrow(ds, findfirst, r"int", item = [missing, missing, missing, missing, missing]), [missing, missing, missing, :x1_int, missing])
    @test isequal(byrow(ds, findfirst, r"int", item = [1,1,1,1,1]), [missing, missing, :x1_int, missing, missing])
    @test isequal(byrow(ds, findlast, r"int", item = [1,1,1,1,1]), [missing, missing, :x2_int, missing, missing])
    @test isequal(byrow(ds, findfirst, r"int", item = :x1_float, eq = isless), [:x2_int, missing, :x1_int, :x1_int, missing])
    @test isequal(byrow(ds, findlast, r"int", item = :x1_float, eq = isless), [:x2_int, missing, :x2_int, :x2_int, missing])

    repeat!(ds, 2)
    sds1 = view(ds, [6,7,8,9,10], [6,5,4,2,3,1])
    sds2 = view(ds, 6:10, 1:6)

    @test isequal(byrow(sds1, findfirst, r"int", item = [missing, missing, missing, missing, missing]), [missing, missing, missing, :x1_int, missing])
    @test isequal(byrow(sds1, findfirst, r"int", item = [1,1,1,1,1]), [missing, missing, :x1_int, missing, missing])
    @test isequal(byrow(sds1, findlast, r"int", item = [1,1,1,1,1]), [missing, missing, :x2_int, missing, missing])
    @test isequal(byrow(sds1, findfirst, r"int", item = :x1_float, eq = isless), [:x2_int, missing, :x1_int, :x1_int, missing])
    @test isequal(byrow(sds1, findlast, r"int", item = :x1_float, eq = isless), [:x2_int, missing, :x2_int, :x2_int, missing])

    @test isequal(byrow(sds2, findfirst, r"int", item = [missing, missing, missing, missing, missing]), [missing, missing, missing, :x1_int, missing])
    @test isequal(byrow(sds2, findfirst, r"int", item = [1,1,1,1,1]), [missing, missing, :x1_int, missing, missing])
    @test isequal(byrow(sds2, findlast, r"int", item = [1,1,1,1,1]), [missing, missing, :x2_int, missing, missing])
    @test isequal(byrow(sds2, findfirst, r"int", item = :x1_float, eq = isless), [:x2_int, missing, :x1_int, :x1_int, missing])
    @test isequal(byrow(sds2, findlast, r"int", item = :x1_float, eq = isless), [:x2_int, missing, :x2_int, :x2_int, missing])

    sds = view(ds, rand(1:5, 100), [2,1,3,4])
    @test isequal(byrow(sds, findfirst,[1,4,3,2], by = x->isless(x,0)), byrow(Dataset(sds), findfirst, [1,4,3,2], by = x->isless(x,0)))
    @test isequal(byrow(sds, findlast,[1,4,3,2], by = x->isless(x,0)), byrow(Dataset(sds), findlast, [1,4,3,2], by = x->isless(x,0)))
    @test isequal(byrow(sds, findfirst,[1,4,3,2], by = x->isless(x,0), threads = true), byrow(Dataset(sds), findfirst, [1,4,3,2], by = x->isless(x,0)))
    @test isequal(byrow(sds, findlast,[1,4,3,2], by = x->isless(x,0), threads = true), byrow(Dataset(sds), findlast, [1,4,3,2], by = x->isless(x,0)))

    ds = Dataset(x1 = [1,2,3,4], x2 = [1.5,6.5,3.4,2.4], x3 = [true, false, true, false], y1 = ["x2", "x1", missing, "x2"], y2 = [:x2, :x1, missing, :x2], y3 = [3,1,1,2])
    @test isequal(byrow(ds, select, 1:2, with = :y1), [1.5, 2,missing,2.4])
    @test isequal(byrow(ds, select, 1:2, with = :y2), [1.5, 2,missing,2.4])
    @test isequal(byrow(ds, select, 1:2, with = ds[!, :y1]), [1.5, 2,missing,2.4])
    @test isequal(byrow(ds, select, 1:2, with = ds[:, :y1]), [1.5, 2,missing,2.4])
    @test isequal(byrow(ds, select, [2,1], with = ds[:, :y1]), [1.5, 2,missing,2.4])
    @test isequal(byrow(ds, select, [2,1,3], with = ds[:, :y2]), [1.5, 2,missing,2.4])
    @test isequal(byrow(ds, select, [2,1,3], with = :y3), [1, 6.5,3.4,4])
    @test isequal(byrow(ds, select, [2,1,3], with = [3,1,1,2]), [1, 6.5,3.4,4])
    @test isequal(byrow(ds, select, [2,1,3], with = ds[!,  :y3]), [1, 6.5,3.4,4])

    @test isequal(byrow(repeat(ds,100), select, 1:2, with = :y1), repeat([1.5, 2,missing,2.4],100))
    @test isequal(byrow(repeat(ds,100), select, 1:2, with = :y2), repeat([1.5, 2,missing,2.4],100))
    @test isequal(byrow(repeat(ds,100), select, 1:2, with = repeat(ds,100)[!, :y1]), repeat([1.5, 2,missing,2.4],100))
    @test isequal(byrow(repeat(ds,100), select, 1:2, with = repeat(ds,100)[:, :y1]), repeat([1.5, 2,missing,2.4],100))
    @test isequal(byrow(repeat(ds,100), select, [2,1], with = repeat(ds,100)[:, :y1]), repeat([1.5, 2,missing,2.4],100))
    @test isequal(byrow(repeat(ds,100), select, [2,1,3], with = repeat(ds,100)[:, :y2]), repeat([1.5, 2,missing,2.4],100))
    @test isequal(byrow(repeat(ds,100), select, [2,1,3], with = :y3), repeat([1, 6.5,3.4,4],100))
    @test isequal(byrow(repeat(ds,100), select, [2,1,3], with = repeat([3,1,1,2],100)), repeat([1, 6.5,3.4,4],100))
    @test isequal(byrow(repeat(ds,100), select, [2,1,3], with = repeat(ds,100)[!,  :y3]), repeat([1, 6.5,3.4,4],100))

    repeat!(ds, 2)
    sds = view(ds, [5,6,7,8], [2,1,3,4])
    @test isequal(byrow(sds, select, 1:2, with = :y1), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, 1:2, with = sds[!, :y1]), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, 1:2, with = sds[:, :y1]), [1.5, 2,missing,2.4])

    sds = view(ds, 1:4, 1:6)
    @test isequal(byrow(sds, select, 1:2, with = :y1), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, 1:2, with = :y2), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, 1:2, with = sds[!, :y1]), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, 1:2, with = sds[:, :y1]), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, [2,1], with = sds[:, :y1]), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, [2,1,3], with = sds[:, :y2]), [1.5, 2,missing,2.4])
    @test isequal(byrow(sds, select, [2,1,3], with = :y3), [1, 6.5,3.4,4])
    @test isequal(byrow(sds, select, [2,1,3], with = [3,1,1,2]), [1, 6.5,3.4,4])
    @test isequal(byrow(sds, select, [2,1,3], with = sds[!,  :y3]), [1, 6.5,3.4,4])

    ds = Dataset(x1 = [1.0,missing,3,4], x2 = [1.5,6.5,missing,2.4], y3 = [3,1,1,2])
    @test byrow(ds, fill, r"x", with = :y3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(ds, fill, [1, 2], with = :y3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(ds, fill, 1:2, with = 3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(ds, fill, r"x", with = :y3, rolling = true) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,3,2.4], y3 = [3,1,1,2])
    @test byrow(ds, fill, r"x", with = :y3, by = x->isless(2,x)) == Dataset(x1 = [1.0,1,1,2], x2 = [1.5,1,1,2], y3 = [3,1,1,2])
    @test byrow(ds, fill, r"x", with = :y3, by = x->isless(2,x), rolling = true) == Dataset(x1 = [1.0,1,1,2.0], x2 = [1.5,1,1,2], y3 = [3,1,1,2])
    @test byrow(ds, fill, r"x", with = [missing, missing, missing, missing], by = x->isless(2,x), rolling = true) == Dataset(x1 = [1.0,missing, missing, missing], x2 = [1.5,missing, missing, missing], y3 = [3,1,1,2])

    @test byrow(repeat(ds,100), fill, r"x", with = :y3) == repeat(Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2]),100)
    @test byrow(repeat(ds,100), fill, [1, 2], with = :y3) == repeat(Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2]),100)
    @test byrow(repeat(ds,100), fill, 1:2, with = 3) == repeat(Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2]),100)
    @test byrow(repeat(ds,100), fill, r"x", with = :y3, rolling = true) == repeat(Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,3,2.4], y3 = [3,1,1,2]),100)
    @test byrow(repeat(ds,100), fill, r"x", with = :y3, by = x->isless(2,x)) == repeat(Dataset(x1 = [1.0,1,1,2], x2 = [1.5,1,1,2], y3 = [3,1,1,2]),100)
    @test byrow(repeat(ds,100), fill, r"x", with = :y3, by = x->isless(2,x), rolling = true) == repeat(Dataset(x1 = [1.0,1,1,2.0], x2 = [1.5,1,1,2], y3 = [3,1,1,2]),100)
    @test byrow(repeat(ds,100), fill, r"x", with = repeat([missing, missing, missing, missing],100), by = x->isless(2,x), rolling = true) == repeat(Dataset(x1 = [1.0,missing, missing, missing], x2 = [1.5,missing, missing, missing], y3 = [3,1,1,2]),100)

    repeat!(ds, 2)
    sds = view(ds, [5,6,7,8], [1,2,3])
    @test byrow(sds, fill, r"x", with = :y3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, [1, 2], with = :y3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, 1:2, with = 3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = :y3, rolling = true) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,3,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = :y3, by = x->isless(2,x)) == Dataset(x1 = [1.0,1,1,2], x2 = [1.5,1,1,2], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = :y3, by = x->isless(2,x), rolling = true) == Dataset(x1 = [1.0,1,1,2.0], x2 = [1.5,1,1,2], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = [missing, missing, missing, missing], by = x->isless(2,x), rolling = true) == Dataset(x1 = [1.0,missing, missing, missing], x2 = [1.5,missing, missing, missing], y3 = [3,1,1,2])

    sds = view(ds, 1:4, 1:3)
    @test byrow(sds, fill, r"x", with = :y3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, [1, 2], with = :y3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, 1:2, with = 3) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,1,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = :y3, rolling = true) == Dataset(x1 = [1.0,1,3,4], x2 = [1.5,6.5,3,2.4], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = :y3, by = x->isless(2,x)) == Dataset(x1 = [1.0,1,1,2], x2 = [1.5,1,1,2], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = :y3, by = x->isless(2,x), rolling = true) == Dataset(x1 = [1.0,1,1,2.0], x2 = [1.5,1,1,2], y3 = [3,1,1,2])
    @test byrow(sds, fill, r"x", with = [missing, missing, missing, missing], by = x->isless(2,x), rolling = true) == Dataset(x1 = [1.0,missing, missing, missing], x2 = [1.5,missing, missing, missing], y3 = [3,1,1,2])

    ds = Dataset(x1 = [1.0,missing,3,4], x2 = [1.5,6.5,missing,2.4], y3 = [3,1,1,2])
    setformat!(ds,:x1=>sqrt)
    byrow(ds, fill!, r"x", with = :y3)
    @test getformat(ds,:x1) == sqrt
    ds = Dataset(x1 = [1.0,missing,3,4], x2 = [1.5,6.5,missing,2.4], y3 = [3,1,1,2])
    setformat!(ds,:x1=>sqrt)
    repeat!(ds, 2)
    sds = view(ds, [5,6,7,8], [1,2,3])
    byrow(sds, fill!, r"x", with = :y3)
    @test getformat(ds,:x1) == sqrt

    ds = Dataset(x1 = [1.0,missing,3,4], x2 = [1.5,6.5,missing,2.4], y3 = [3,1,1,2])
    sort!(ds, 2)
    byrow(ds, fill!, r"x", with = :y3)
    @test IMD.index(ds).sortedcols == []
    ds = Dataset(x1 = [1.0,missing,3,4], x2 = [1.5,6.5,missing,2.4], y3 = [3,1,1,2])
    repeat!(ds, 2)
    sort!(ds, 2)
    sds = view(ds, [5,6,7,8], [3,1,2])
    byrow(sds, fill!, r"x", with = [0,0,0,0])
    @test IMD.index(ds).sortedcols == []

    ds = Dataset(x1 = [1.0,missing,3,4], x2 = [1.5,6.5,missing,2.4], y3 = [3,1,1,2])
    sort!(ds, 3)
    byrow(ds, fill!, 1:2, with = 3)
    @test IMD.index(ds).sortedcols == [3]

    ds = Dataset(rand(1:10, 10000, 4), :auto)
    @test byrow(ds, fill, :, with = missings(Int, nrow(ds)), by = isequal(1), threads = false) == byrow(ds, fill, :, with = missings(Int, nrow(ds)), by = isequal(1), threads = true)
    byrow(ds, fill!, :, with = missings(Int, nrow(ds)), by = isequal(1))
    @test IMD.minimum.(eachcol(ds)) == [2,2,2,2]

    ds = Dataset(g = [1, 1, 1, 2, 2],
                        x1_int = [0, 0, 1, missing, 2],
                        x2_int = [3, 2, 1, 3, -2],
                        x1_float = [1.2, missing, -1.0, 2.3, 10],
                        x2_float = [missing, missing, 3.0, missing, missing],
                        x3_float = [missing, missing, -1.4, 3.0, -100.0])
    @test byrow(ds, isless, r"int", with = :x1_float) == [0,1,0,0,1]
    @test byrow(ds, isless, r"int", with = :x1_float, rev = true) == [0,0,1,1,0]
    @test byrow(ds, isless, r"int", with = ds[!,:x1_float]) == [0,1,0,0,1]
    @test byrow(ds, isless, r"int", with = ds[!,:x1_float], rev = true) == [0,0,1,1,0]
    @test byrow(ds, isless, r"int", with = ds[:,:x1_float]) == [0,1,0,0,1]
    @test byrow(ds, isless, r"int", with = ds[:,:x1_float], rev = true) == [0,0,1,1,0]
    @test byrow(view(ds, :, :), isless, r"int", with = :x1_float) == [0,1,0,0,1]
    @test byrow(view(ds, :, :), isless, r"int", with = :x1_float, rev = true) == [0,0,1,1,0]
    @test byrow(view(ds, :, :), isless, r"int", with = view(ds, :, :)[!,:x1_float]) == [0,1,0,0,1]
    @test byrow(view(ds, :, :), isless, r"int", with = ds[!,:x1_float], rev = true) == [0,0,1,1,0]

    ds = Dataset(rand(100, 10), :auto)
    _tval = byrow(view(ds, :, 1:9) .> ds[!, 10], all)
    @test byrow(ds, isless, 1:9, with=:x10, rev = true) == _tval

    ds = Dataset(x1 = [1,2,3,1,1], x2 = [4,3,1,2,5], x3 = 1:5)
    @test byrow(ds, in, 1:2, item = :x3) == [1,1,1,0,1]
    @test byrow(ds, in, 1:2, item = ds[!, 3])== [1,1,1,0,1]
    @test byrow(ds, in, 1:2, item = ds[:, 3])== [1,1,1,0,1]

    repeat!(ds, 2)
    sds1 = view(ds, [6,7,8,9,10], [2,1,3])
    sds2 = view(ds, 6:10, 1:3)
    @test byrow(sds1, in, 1:2, item = :x3) == [1,1,1,0,1]
    @test byrow(sds1, in, 1:2, item = sds1[!, 3])== [1,1,1,0,1]
    @test byrow(sds1, in, 1:2, item = sds1[:, 3])== [1,1,1,0,1]
    @test byrow(sds2, in, 1:2, item = :x3) == [1,1,1,0,1]
    @test byrow(sds2, in, 1:2, item = sds2[!, 3])== [1,1,1,0,1]
    @test byrow(sds2, in, 1:2, item = sds2[:, 3])== [1,1,1,0,1]

    ds = Dataset(x1 = ["A", "B,"], x2 =["TEA", "TOOOOL"])
    @test byrow(ds, join, r"x") == ["ATEA", "B,TOOOOL"]
    @test byrow(ds, join, r"x", delim = ",") == ["A,TEA", "B,,TOOOOL"]
    @test byrow(ds, join, r"x", last = ".") == ["A.TEA", "B,.TOOOOL"]
    @test byrow(ds, join, r"x", last = "end", delim = "/-/") == ["AendTEA", "B,endTOOOOL"]

    @test byrow(view(ds, [1,2], :), join, r"x") == ["ATEA", "B,TOOOOL"]
    @test byrow(view(ds, [1,2], :), join, r"x", delim = ",") == ["A,TEA", "B,,TOOOOL"]
    @test byrow(view(ds, [1,2], :), join, r"x", last = ".") == ["A.TEA", "B,.TOOOOL"]
    @test byrow(view(ds, [1,2], :), join, r"x", last = "end", delim = "/-/") == ["AendTEA", "B,endTOOOOL"]

    repeat!(ds, 1000)
    @test byrow(ds, join, r"x", threads = true) == repeat(["ATEA", "B,TOOOOL"], 1000)
    @test byrow(ds, join, r"x", threads = true, delim = ",") == repeat(["A,TEA", "B,,TOOOOL"], 1000)
    @test byrow(ds, join, r"x", threads = true, last = ".") == repeat(["A.TEA", "B,.TOOOOL"], 1000)
    @test byrow(ds, join, r"x", threads = true, last = "end", delim = "/-/") == repeat(["AendTEA", "B,endTOOOOL"], 1000)

    ds = Dataset(x = [1,10], x2 = ["A", "BC"], x3 = [2.0,4.54])
    @test byrow(ds, join, :, delim = "--", last = "-") == ["1--A-2.0", "10--BC-4.54"]

    ds = Dataset(x = ["α1", "β∘1"], y = [1,2], z = [1.4,5.6])
    @test byrow(ds, join, :) == ["α111.4", "β∘125.6"]
    @test byrow(ds, join, :, delim = ",") == ["α1,1,1.4", "β∘1,2,5.6"]
    @test byrow(ds, join, :, delim = ":×:") == ["α1:×:1:×:1.4", "β∘1:×:2:×:5.6"]
end

@testset "cum*/! - sort/!" begin
    ds = Dataset(x1 = [1,missing,3,missing], x2 = [1.0,2.0,missing,4.0], x3 = [1,missing,3,4])
    @test byrow(ds, cumsum, :) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [2.0, 2.0, 3.0, 4.0], x3 = [3.0, 2.0, 6.0, 8.0])
    @test byrow(ds, cumsum, :, missings = :skip) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [2.0, 2.0, missing, 4.0], x3=[3.0, missing, 6.0, 8.0])
    @test byrow(ds, cumprod, :) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, 3.0, 4.0], x3=[1.0, 2.0, 9.0, 16.0])
    @test byrow(ds, cumprod, :, missings = :skip) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, missing, 4.0], x3=[1.0, missing, 9.0, 16.0])

    @test byrow(repeat(ds,100), cumsum, :) == repeat(Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [2.0, 2.0, 3.0, 4.0], x3 = [3.0, 2.0, 6.0, 8.0]),100)
    @test byrow(repeat(ds,100), cumsum, :, missings = :skip) == repeat(Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [2.0, 2.0, missing, 4.0], x3=[3.0, missing, 6.0, 8.0]),100)
    @test byrow(repeat(ds,100), cumprod, :) == repeat(Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, 3.0, 4.0], x3=[1.0, 2.0, 9.0, 16.0]),100)
    @test byrow(repeat(ds,100), cumprod, :, missings = :skip) == repeat(Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, missing, 4.0], x3=[1.0, missing, 9.0, 16.0]),100)

    byrow(ds, cumsum!, :, missings = :skip)
    @test ds == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [2.0, 2.0, missing, 4.0], x3=[3.0, missing, 6.0, 8.0])
    byrow(ds, cumsum!, :, missings = :ignore)
    @test ds == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [3.0, 2.0, 3.0, 4.0], x3=[6.0, 2.0, 9.0, 12.0])
    ds = Dataset(x1 = [1,missing,3,missing], x2 = [1.0,2.0,missing,4.0], x3 = [1,missing,3,4])
    byrow(ds, cumprod!, :, missings = :skip)
    @test ds == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, missing, 4.0], x3=[1.0, missing, 9.0, 16.0])
    byrow(ds, cumprod!, :, missings = :ignore)
    @test ds == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, 3.0, 4.0], x3=[1.0, 2.0, 27.0, 64.0])

    ds = Dataset(x1 = [1,missing,3,missing], x2 = [1.0,2.0,missing,4.0], x3 = [1,missing,3,4])
    sds = view(ds, [1,2,1,3,1,4], [3,1,2])
    @test byrow(sds, cumsum, :) == Dataset(x3 = [1.0, missing, 1,3,1,4], x1 = [2, missing, 2.0, 6, 2,4], x2 = [3.0, 2.0, 3, 6, 3, 8])
    @test byrow(sds, cumsum, :, missings = :skip) == Dataset(x3 = [1.0, missing, 1,3,1,4], x1 = [2, missing, 2.0, 6, 2,missing], x2 = [3.0, 2.0, 3, missing, 3, 8])
    @test byrow(sds, cumprod, :) == Dataset(x3 = [1.0, missing, 1,3,1,4], x1 = [1, missing, 1.0, 9, 1,4], x2 = [1.0, 2.0, 1, 9, 1, 16])
    @test byrow(sds, cumprod, :, missings = :skip) == Dataset(x3 = [1.0, missing, 1,3,1,4], x1 = [1, missing, 1.0, 9, 1,missing], x2 = [1.0, 2.0, 1, missing, 1, 16])

    ds = Dataset(x1 = [1,missing,3,missing], x2 = [1.0,2.0,missing,4.0], x3 = [1,missing,3,4])
    @test byrow(ds, cummax, :) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, 3.0, 4.0], x3 = [1.0, 2.0, 3.0, 4.0])
    @test byrow(ds, cummax, :, missings = :skip) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, missing, 4.0], x3 = [1.0, missing, 3.0, 4.0])
    @test byrow(ds, cummin, :) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, 3.0, 4.0], x3 = [1.0, 2.0, 3.0, 4.0])
    @test byrow(ds, cummin, :, missings = :skip) == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, missing, 4.0], x3 = [1.0, missing, 3.0, 4.0])

    byrow(ds, cummax!, :)
    @test ds == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, 3.0, 4.0], x3 = [1.0, 2.0, 3.0, 4.0])
    ds = Dataset(x1 = [1,missing,3,missing], x2 = [1.0,2.0,missing,4.0], x3 = [1,missing,3,4])
    byrow(ds, cummin!, :, missings = :skip)
    @test ds == Dataset(x1 = [1.0, missing, 3.0, missing], x2 = [1.0, 2.0, missing, 4.0], x3 = [1.0, missing, 3.0, 4.0])

    ds = Dataset(x1 = [1,missing,3,missing], x2 = [1.0,2.0,missing,4.0], x3 = [1,missing,3,6])
    sds = view(ds, [1,2,1,3,1,4], [3,1,2])
    @test byrow(sds, cummax, :) == Dataset(x3 = [1.0, missing, 1,3,1,6], x1 = [1, missing, 1,3,1,6.0], x2 = [1.0,2.0, 1.0,3.0,1.0,6.0])
    @test byrow(sds, cummax, :, missings = :skip) == Dataset(x3 = [1.0, missing, 1,3,1,6], x1 = [1, missing, 1,3,1.0, missing], x2 = [1.0,2.0, 1.0,missing,1.0,6.0])

    ds = Dataset(rand(1:100, 1000, 10), :auto)
    ds2 = byrow(ds, sort, :)
    @test all(byrow(ds2, issorted, :))
    @test Matrix(ds2) == sort(Matrix(ds), dims = 2)

    ds = Dataset(rand(1:100, 1000, 10), :auto)
    ds2 = byrow(view(ds, :, :), sort, :)
    @test all(byrow(ds2, issorted, :))
    @test Matrix(ds2) == sort(Matrix(ds), dims = 2)
end


using IncrementalInference
using Test

##

@testset "test Sphere1D" begin

##

fg = initfg()
getSolverParams(fg).useMsgLikelihoods = true

addVariable!.(fg, [Symbol("x$i") for i=0:4], Sphere1)
addFactor!(fg, [:x0], PriorSphere1(Normal(0.0,0.1)))
map(i->addFactor!(fg, [Symbol("x$i"),Symbol("x$(i+1)")], Sphere1Sphere1(Normal(1.0, 0.1))), 0:3)

solveTree!(fg);

##

sppes = map(var->getPPE(var).suggested[1], sortDFG(getVariables(fg),by=getLabel))

gt = rem2pi.(collect(0:4), RoundNearest)

@show sppes
@test all(isapprox.(sppes, gt, atol=0.35))

# test packing converters also
d = "/tmp/caesar/random/testfg"
saveDFG(d,fg)
lfg = loadDFG(d)
Base.rm(d*".tar.gz")
# check loaded fg for all variable and factors
@test issetequal(ls(fg), ls(lfg))
@test issetequal(lsf(fg), lsf(lfg))

##

end


include("../src/quasirandom_module.jl")

module qtest
using ..quasirandom
using PyPlot
PyPlot.pygui(true)

# h = low.halton(1; base = 2)
# h = low.halton(10; base = 2)

n_points = 21
n_dim = 2
hseq = quasirandom.halton_filling_seq(n_points, n_dim)

mmc = quasirandom.morris_mitchell_criterion(hseq)

PyPlot.figure()
PyPlot.grid()
PyPlot.plot(hseq[:,1], hseq[:,2], linestyle = :none, marker = :., markersize = 10)

end


using Test, NLSolvers, LinearAlgebra
@testset "truncated SPR solver" begin
m = NLSolvers.TCG()

H = [0.3 0.0; 0.0 0.9]
g = [0.2,0.4]

m(g, H, 0.7, rand(2), 1)

# Since -H is negative definite the solution is guaranteed
# to be at the boundary unless g = 0
for Δ in range(0, 100; step=0.1)
    @test norm(m(g, -H, Δ, rand(2), 1).p, 2) ≈ Δ
end

# Small gradient entries
g .= [1e-12, 1e-9]
for Δ in range(0, 100; step=0.1)
    @test norm(m(g, -H, Δ, rand(2), 1).p, 2) ≈ Δ
end

# Mixed gradient entries
g .= [1e12, 1e-9]
for Δ in range(0, 100; step=0.1)
    @test norm(m(g, -H, Δ, rand(2), 1).p, 2) ≈ Δ
end

# Zero case
g = [0.0, 0.0]
for Δ in range(0, 100; step=0.1)
    @test norm(m(g, -H, Δ, rand(2), 1).p, 2) == 0
end
end


P = 23800
b = P * 0.05 + 300
i = 0.1
N = 1

F = P * (1 + i)^N
ic = (F / (P - b))^(1 / N) - 1


@testset "farmer" begin
    NS = 3  # number of scenarios
    probability = ones(3) / 3
    CROPS = 1:3 # set of crops (wheat, corn and sugar beets, resp.)
    PURCH = 1:2 # set of crops to purchase (wheat and corn, resp.)
    SELL = 1:4  # set of crops to sell (wheat, corn, sugar beets under 6K and those over 6K)
    Cost = [150 230 260]    # cost of planting crops
    Budget = 500            # budget capacity
    Purchase = [238 210]    # purchase price
    Sell = [170 150 36 10]  # selling price
    Yield = [3.0 3.6 24.0; 2.5 3.0 20.0; 2.0 2.4 16.0]
    Minreq = [200 240 0]    # minimum crop requirement

    @testset "MILP" begin
        # This creates a Lagrange dual problem for each scenario s.
        function create_scenario_model(s::Int64)
            m = Model(GLPK.Optimizer)
            @variable(m, 0 <= x[i=CROPS] <= 500, Int)
            @variable(m, y[j=PURCH] >= 0)
            @variable(m, w[k=SELL] >= 0)

            @objective(m, Min,
                probability[s] * sum(Cost[i] * x[i] for i=CROPS)
                + probability[s] * sum(Purchase[j] * y[j] for j=PURCH)
                - probability[s] * sum(Sell[k] * w[k] for k=SELL))

            @constraint(m, sum(x[i] for i=CROPS) <= Budget)
            @constraint(m, [j=PURCH], Yield[s,j] * x[j] + y[j] - w[j] >= Minreq[j])
            @constraint(m, Yield[s,3] * x[3] - w[3] - w[4] >= Minreq[3])
            @constraint(m, w[3] <= 6000)
            return m
        end

        models = Dict{Int,JuMP.Model}(s => create_scenario_model(s) for s in 1:NS)
        coupling_variables = Vector{DD.CouplingVariableRef}()
        for s in 1:NS
            model = models[s]
            xref = model[:x]
            for i in CROPS
                push!(coupling_variables, DD.CouplingVariableRef(s, i, xref[i]))
            end
        end

        @testset "Subgradient Method" begin
            # Create DualDecomposition instance.
            algo = DD.LagrangeDual()

            # Add Lagrange dual problem for each scenario s.
            for s in 1:NS
                DD.add_block_model!(algo, s, models[s])
            end

            # Set nonanticipativity variables as an array of symbols.
            DD.set_coupling_variables!(algo, coupling_variables)

            # Lagrange master method
            LM = DD.SubgradientMaster()
            LM.maxiter = 1000
            LM.step_size = (method) -> (abs(-108390 - method.f) / norm(method.∇f)^2) # Polyak's

            @testset "objective limit" begin
                DD.set_obj_limit!(LM, -110000.)
                DD.run!(algo, LM)
                @show DD.dual_objective_value(algo)
                @test DD.dual_objective_value(algo) > -110000.
            end
            DD.set_obj_limit!(LM, -100000.)

            # Solve the problem with the solver; this solver is for the underlying bundle method.
            DD.run!(algo, LM)

            @show DD.dual_objective_value(algo)
            @show DD.dual_solution(algo)
            @test isapprox(DD.dual_objective_value(algo), -108390, rtol=1e-3)
        end

        @testset "Proximal Method" begin
            # Create DualDecomposition instance.
            algo = DD.LagrangeDual()

            # Add Lagrange dual problem for each scenario s.
            for s in 1:NS
                DD.add_block_model!(algo, s, models[s])
            end

            # Set nonanticipativity variables as an array of symbols.
            DD.set_coupling_variables!(algo, coupling_variables)

            # Lagrange master method
            LM = DD.BundleMaster(BM.ProximalMethod, optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))

            @testset "objective limit" begin
                DD.set_obj_limit!(LM, -111000.)
                DD.run!(algo, LM)
                @show DD.dual_objective_value(algo)
                @test DD.dual_objective_value(algo) > -111000.
            end
            DD.set_obj_limit!(LM, -100000.)

            # Solve the problem with the solver; this solver is for the underlying bundle method.
            DD.run!(algo, LM)

            @show DD.dual_objective_value(algo)
            @show DD.dual_solution(algo)
            @test isapprox(DD.dual_objective_value(algo), -108390, rtol=1e-3)
        end

        @testset "Trust Region Method" begin
            # Create DualDecomposition instance.
            algo = DD.LagrangeDual()

            # Add Lagrange dual problem for each scenario s.
            for s in 1:NS
                DD.add_block_model!(algo, s, models[s])
            end

            # Set nonanticipativity variables as an array of symbols.
            DD.set_coupling_variables!(algo, coupling_variables)

            # Lagrange master method
            LM = DD.BundleMaster(BM.TrustRegionMethod, GLPK.Optimizer)

            @testset "objective limit" begin
                DD.set_obj_limit!(LM, -111000.)
                DD.run!(algo, LM)
                @show DD.dual_objective_value(algo)
                @test DD.dual_objective_value(algo) > -111000.
            end
            DD.set_obj_limit!(LM, -100000.)

            # Solve the problem with the solver; this solver is for the underlying bundle method.
            DD.run!(algo, LM)

            @show DD.dual_objective_value(algo)
            @show DD.dual_solution(algo)
            @test isapprox(DD.dual_objective_value(algo), -108390, rtol=1e-3)
        end
    end

    @testset "QP" begin
        function deterministic_model_bound()
            m = Model(optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))
            @variable(m, 0 <= x[i=CROPS] <= 500)
            @variable(m, y[s=1:NS,j=PURCH] >= 0)
            @variable(m, w[s=1:NS,k=SELL] >= 0)

            @objective(m, Min,
                sum(Cost[i] * x[i]^2 for i=CROPS)
                + sum(probability[s] * Purchase[j] * y[s,j] for s=1:NS, j=PURCH)
                - sum(probability[s] * Sell[k] * w[s,k] for s=1:NS, k=SELL))

            @constraint(m, sum(x[i] for i=CROPS) <= Budget)
            @constraint(m, [s=1:NS,j=PURCH], Yield[s,j] * x[j] + y[s,j] - w[s,j] >= Minreq[j])
            @constraint(m, [s=1:NS], Yield[s,3] * x[3] - w[s,3] - w[s,4] >= Minreq[3])
            @constraint(m, [s=1:NS], w[s,3] <= 6000)
            optimize!(m)
            objval = Inf
            if termination_status(m) in [MOI.OPTIMAL, MOI.LOCALLY_SOLVED]
                objval = objective_value(m)
            end
            return objval
        end

        # This creates a Lagrange dual problem for each scenario s.
        function create_scenario_model(s::Int64)
            m = Model(optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))
            @variable(m, 0 <= x[i=CROPS] <= 500)
            @variable(m, y[j=PURCH] >= 0)
            @variable(m, w[k=SELL] >= 0)

            @objective(m, Min,
                probability[s] * sum(Cost[i] * x[i]^2 for i=CROPS)
                + probability[s] * sum(Purchase[j] * y[j] for j=PURCH)
                - probability[s] * sum(Sell[k] * w[k] for k=SELL))

            @constraint(m, sum(x[i] for i=CROPS) <= Budget)
            @constraint(m, [j=PURCH], Yield[s,j] * x[j] + y[j] - w[j] >= Minreq[j])
            @constraint(m, Yield[s,3] * x[3] - w[3] - w[4] >= Minreq[3])
            @constraint(m, w[3] <= 6000)
            return m
        end

        models = Dict{Int,JuMP.Model}(s => create_scenario_model(s) for s in 1:NS)
        coupling_variables = Vector{DD.CouplingVariableRef}()
        for s in 1:NS
            model = models[s]
            xref = model[:x]
            for i in CROPS
                push!(coupling_variables, DD.CouplingVariableRef(s, i, xref[i]))
            end
        end

        objval = deterministic_model_bound()
        @show objval
        @test objval < Inf

        @testset "Proximal Method" begin
            # Create DualDecomposition instance.
            algo = DD.LagrangeDual()

            # Add Lagrange dual problem for each scenario s.
            for s in 1:NS
                DD.add_block_model!(algo, s, models[s])
            end

            # Set nonanticipativity variables as an array of symbols.
            DD.set_coupling_variables!(algo, coupling_variables)

            # Change parameters
            params = BM.Parameters()
            BM.set_parameter(params, "ϵ_s", 1.e-4)
            @test BM.get_parameter(params, "ϵ_s") == 1.e-4

            # Lagrange master method
            LM = DD.BundleMaster(BM.ProximalMethod, optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0), params)

            # Solve the problem with the solver; this solver is for the underlying bundle method.
            DD.run!(algo, LM)

            @show DD.dual_objective_value(algo)
            @show DD.dual_solution(algo)
            @test isapprox(DD.dual_objective_value(algo), objval, rtol=1e-3)
        end

        @testset "Trust Region Method" begin
            # Create DualDecomposition instance.
            algo = DD.LagrangeDual()

            # Add Lagrange dual problem for each scenario s.
            for s in 1:NS
                DD.add_block_model!(algo, s, models[s])
            end

            # Set nonanticipativity variables as an array of symbols.
            DD.set_coupling_variables!(algo, coupling_variables)

            # Lagrange master method
            LM = DD.BundleMaster(BM.TrustRegionMethod, GLPK.Optimizer)

            # Solve the problem with the solver; this solver is for the underlying bundle method.
            DD.run!(algo, LM)

            @show DD.dual_objective_value(algo)
            @show DD.dual_solution(algo)
            @test isapprox(DD.dual_objective_value(algo), objval, rtol=1e-3)
        end
    end
end


using StructuresKit

#362S162-54, Mcrd, no springs, no moment gradient, Lm>>Lcrd

CorZ=0
t=0.0566

#out to out dimensions
ho=3.625
b=1.625
d=0.5
θc=90.0

E=29500.0
μ=0.30
G=E/(2*(1+μ))
f1=1
f2=-1
M1=1.0
M2=1.0
Lm=100.0
kϕ=0.0

Sf=0.9085/(3.625/2)

CurvatureSign = -1.0

Mcrd = AISIS10016.app23331(CorZ, t, ho, b, d, θc, E, μ, G, f1, f2, M1, M2, CurvatureSign, Lm, kϕ, Sf)

#compare to CFSEI TechNote G100-08 (or G100-09?), Table 3
#https://cfsei.memberclicks.net/technical-notes
@test Mcrd ≈ 89.2*Sf atol=0.01*Mcrd


using FFTW
using MAT

@testset "calculateSNR" begin
    # MATLAB output
    test = matread("resources/calculateSNR.mat")

    # low pass
    u = 0.5 * sin.(2*pi*85/8192 * (0:8191))
    ntf = synthesizeNTF(5, 32, 1)
    v, = simulateDSM(u, ntf)
    spec = fft(v .* ds_hann(8192)/2048)
    snr = calculateSNR(spec[3:129], 83)
    @test isapprox(test["snr_low"], snr)

    # band pass
    ntf = synthesizeNTF(8, 64, 1, 1.5, 0.125)
    u = 0.5 * sin.(2*pi*1045/8192 * (0:8191))
    v, = simulateDSM(u, ntf)
    spec = fft(v .* ds_hann(8192)/2048)
    snr = calculateSNR(spec[992:1056], 54)
    @test isapprox(test["snr_band"], snr)
end


function compute_char(board_array, row, col)
    board_array[row, col] && return '*'
    rows, cols = size(board_array)
    row_range = max(1, row-1):min(rows, row+1)
    col_range = max(1, col-1):min(cols, col+1)
    count = sum(board_array[row_range, col_range])
    count > 0 ? '0' + count : ' '
end

function annotate(board)
    isempty(board) && return []
    isempty(board[1]) && return [""]
    board_array = map(c -> c == '*', mapreduce(collect, hcat, board))'
    board_array = map(i -> compute_char(board_array, i[1], i[2]), CartesianIndices(board_array))
    map(row -> join(board_array[row,:]), 1:length(board))
end


### A Pluto.jl notebook ###
# v0.12.18

using Markdown
using InteractiveUtils

# ╔═╡ 621fa420-5392-11eb-1c1e-d35ed6a9ddc0
begin
	using OffsetArrays
	using BenchmarkTools
	using Memoization
end

# ╔═╡ d35a7228-5394-11eb-29b1-43cd660605f0
md"
This notebook shows a translation of [this notebook](https://www.kaggle.com/lewisfogden/the-life-modelling-problem) into Julia and an example of memoization of actuarial calculations.

To *not* memoize, remove the `@memoize` placed before various function definitions.

When memoized, the calculations benchmark goes from `~650µs` to `~15µs`
"

# ╔═╡ 135c48c4-5391-11eb-1941-07f9ffd5e22d
q = OffsetArray([0.001,
     0.002,
     0.003,
     0.003,
     0.004,
     0.004,
     0.005,
     0.007,
     0.009,
     0.011],-1)

# ╔═╡ 3e501fda-5391-11eb-2c84-dfc93c9b8871
w = OffsetArray([0.05,
     0.07,
     0.08,
     0.10,
     0.14,
     0.20,
     0.20,
     0.20,
     0.10,
     0.04],-1)

# ╔═╡ d6f1f9da-5395-11eb-2608-a10b027a9721
md" Note that all of the loose parameters in the source notebook are combined into one parameters tuple:"

# ╔═╡ 55440cf6-5391-11eb-06f7-ed0c64129bde
params = (
	prem = 100,
	face = 25_000,
	term = 10,
	w = w,
	q = q,
	int_rate = 0.02
	)

# ╔═╡ 6d50caa0-5391-11eb-0572-9de6d4af4b85
begin
	@memoize function num_in_force(t,params)
		if t == 0
			return 1.0
		elseif t >= params.term
			return 0.0
		else
			return num_in_force(t-1,params) - num_deaths(t-1,params) - num_lapses(t-1,params)
		end
	end

	@memoize function num_deaths(t,params)
		if t < params.term
			return num_in_force(t,params) * params.q[t]
		else
			return 0
		end
	end

	@memoize function num_lapses(t,params)
		if t < params.term
			return num_in_force(t,params) * params.w[t]
		end
	end

end

# ╔═╡ e275cd2a-5392-11eb-2e41-a5eddbdaa5e9
begin
	claims(t,params) = num_deaths(t,params) * params.face
	premiums(t, params) = num_in_force(t,params) * params.prem
	net_cashflows(t,params) = return premiums(t,params) - claims(t,params)
end

# ╔═╡ 9478fbba-5392-11eb-017b-91c4a8ddde90
map(0:10) do t
	(
		time = t,
		ℓ = num_in_force(t,params),
		claims = claims(t,params),
		premiums = premiums(t,params),
		net_cfs = net_cashflows(t,params),
	)
end

# ╔═╡ 4109cc9c-5393-11eb-3e82-59e4b89f6e05
function npv(cashflow, params)
	val = 0
	for t in 0:params.term
		val = (val + cashflow(t, params)) / (1 + params.int_rate)
	end
	return val
end

# ╔═╡ 83ce88a6-5393-11eb-0c7c-551d48ada79d
@benchmark npv(net_cashflows,params)

# ╔═╡ 24013174-5395-11eb-0ed6-2f9b874a2d20

# ╔═╡ Cell order:
# ╟─d35a7228-5394-11eb-29b1-43cd660605f0
# ╠═621fa420-5392-11eb-1c1e-d35ed6a9ddc0
# ╠═135c48c4-5391-11eb-1941-07f9ffd5e22d
# ╠═3e501fda-5391-11eb-2c84-dfc93c9b8871
# ╟─d6f1f9da-5395-11eb-2608-a10b027a9721
# ╠═55440cf6-5391-11eb-06f7-ed0c64129bde
# ╠═6d50caa0-5391-11eb-0572-9de6d4af4b85
# ╠═e275cd2a-5392-11eb-2e41-a5eddbdaa5e9
# ╠═9478fbba-5392-11eb-017b-91c4a8ddde90
# ╠═4109cc9c-5393-11eb-3e82-59e4b89f6e05
# ╠═83ce88a6-5393-11eb-0c7c-551d48ada79d
# ╠═24013174-5395-11eb-0ed6-2f9b874a2d20


# ---
# title: 914. X of a Kind in a Deck of Cards
# id: problem914
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Array, Math
# link: <https://leetcode.com/problems/x-of-a-kind-in-a-deck-of-cards/description/>
# hidden: true
# ---
#
# In a deck of cards, each card has an integer written on it.
#
# Return `true` if and only if you can choose `X >= 2` such that it is possible
# to split the entire deck into 1 or more groups of cards, where:
#
#   * Each group has exactly `X` cards.
#   * All the cards in each group have the same integer.
#
#
#
# **Example 1:**
#
#
#
#     Input: deck = [1,2,3,4,4,3,2,1]
#     Output: true
#     **Explanation** : Possible partition [1,1],[2,2],[3,3],[4,4].
#
#
# **Example 2:**
#
#
#
#     Input: deck = [1,1,1,2,2,2,3,3]
#     Output: false´
#     **Explanation** : No possible partition.
#
#
# **Example 3:**
#
#
#
#     Input: deck = [1]
#     Output: false
#     **Explanation** : No possible partition.
#
#
# **Example 4:**
#
#
#
#     Input: deck = [1,1]
#     Output: true
#     **Explanation** : Possible partition [1,1].
#
#
# **Example 5:**
#
#
#
#     Input: deck = [1,1,2,2,2,2]
#     Output: true
#     **Explanation** : Possible partition [1,1],[2,2],[2,2].
#
#
#
#
# **Constraints:**
#
#   * `1 <= deck.length <= 10^4`
#   * `0 <= deck[i] < 10^4`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


# This file was generated, do not modify it.

using RDatasets, DataFrames

boston = dataset("MASS", "Boston");

typeof(boston)

names(boston)

first(boston, 4)

boston.Crim[1:5]

boston[3, 5]

boston[1:5, [:Crim, :Zn]]

boston[1:5, 1:2]

b1 = select(boston, [:Crim, :Zn, :Indus])
first(b1, 2)

b2 = select(boston, Not(:NOx))
first(b2, 2)

select!(b1, Not(:Crim))
first(b1, 2)

using StatsBase
describe(boston, :min, :max, :mean, :median, :std)

foo(x) = sum(abs.(x)) / length(x)
d = describe(boston, :mean, :median, :foo => foo)
first(d, 3)

using Statistics

mat = convert(Matrix, boston)
mat[1:3, 1:3]

boston.Crim_x_Zn = boston.Crim .* boston.Zn;

mao = dataset("gap", "mao")
describe(mao, :nmissing)

std(mao.Age)

std(skipmissing(mao.Age))

iris = dataset("datasets", "iris")
first(iris, 3)

unique(iris.Species)

gdf = groupby(iris, :Species);

subdf_setosa = gdf[1]
describe(subdf_setosa, :min, :mean, :max)

by(iris, :Species, :PetalLength => mean)

by(iris, :Species, MPL = :PetalLength => mean, SPL = :PetalLength => std)

DataFrames.aggregate(iris, :Species, std)


function duftau(t,
		fft_u,  fft_v,
		fft_du, fft_dv,
		A1, A2,
                matr, conjmatr,
		sigma, llambda, Ntaumm)

    sigma = 1

    u = ifft(exp.(1im * t .* A1) .* fft_u .* matr)
    v = ifft(exp.(1im * t .* A1) .* fft_v .* matr)

    du = ifft(exp(1im * t .* A1) .* fft_du .* matr)
    dv = ifft(exp(1im * t .* A1) .* fft_dv .* matr)

    z  = (u  .+ conj(v) ) / 2
    dz = (du .+ conj(dv)) / 2

    fz1 = 2 * abs(z) .^ 2 * dz .+ z .^ 2 .* conj(dz)

    champu = -1im * llambda * A2 .* exp.(-1im * t .* A1) .* fft(conjmatr .* fz1)
    champv = -1im * llambda * A2 .* exp.(-1im * t .* A1) .* fft(conjmatr .* conj(fz1))

    return champu, champv

end


using LinearAlgebra
using DataFrames
using CSV
using Plots

A = CSV.File("../../DATA/hald_ingredients.csv", header = false) |> Tables.matrix
b = CSV.File("../../DATA/hald_heat.csv", header = false) |> Tables.matrix

U, S, V = svd(A)
x = (V*inv(diagm(S))*U'*b) # Solve Ax=b using the SVD

p1 = plot(b, label = "Heat data") # Plot data
plot!(A*x, label = "Regression") # Plot regression


###############################################
#FEniCS tutorial demo program: Incompressible Navier-Stokes equations
#for flow around a cylinder using the Incremental Pressure Correction
#Scheme (IPCS).
#  u' + u . nabla(u)) - div(σ(u, p)) = f
#                                 div(u) = 0
###############################################

module ft08

using FenicsPy

# 参数：ρ,μ,Δt
T = 5
num_steps = 5000
Δt = T / num_steps
μ = 0.001
ρ = 1

# Create mesh
channel = Rectangle(Point([0.0, 0.0]), Point([2.2, 0.41]))
cylinder = Circle(Point([0.2, 0.2]), 0.05)
Ω = channel - cylinder
mesh = generate_mesh(Ω, 64)

# Define function spaces
V = VectorFunctionSpace(mesh, "P", 2)
Q = FunctionSpace(mesh, "P", 1)

# Define boundaries
inflow   = "near(x[0], 0)"
outflow  = "near(x[0], 2.2)"
walls    = "near(x[1], 0) || near(x[1], 0.41)"
cylinder = "on_boundary && x[0]>0.1 && x[0]<0.3 && x[1]>0.1 && x[1]<0.3"

# Define inflow profile
inflow_profile = ("4.0*1.5*x[1]*(0.41 - x[1]) / pow(0.41, 2)", "0")

# Define boundary conditions
bcu_inflow = DirichletBC(V, Expression(inflow_profile, degree=2), inflow)
bcu_walls = DirichletBC(V, Constant((0, 0)), walls)
bcu_cylinder = DirichletBC(V, Constant((0, 0)), cylinder)
bcp_outflow = DirichletBC(Q, Constant(0), outflow)
bcu = [bcu_inflow,bcu_walls,bcu_cylinder]
bcp = [bcp_outflow]

u = TrialFunction(V)
v = TestFunction(V)
p = TrialFunction(Q)
q = TestFunction(Q)

# Define functions for solutions at previous and current time steps
u_n = FeFunction(V)
u_ = FeFunction(V)
p_n = FeFunction(Q)
p_ = FeFunction(Q)

U  = 0.5*(u_n + u)
n  = FacetNormal(mesh)
f  = Constant((0, 0))

# Define symmetric gradient
function ϵ(u)
    return sym(nabla_grad(u))
end

# Define stress tensor
function σ(u, p)
    return 2*μ*ϵ(u) - p*Identity(len(u))
end

# Define variational problem for step 1
F1 = ρ*dot((u - u_n) / Δt, v)*dx  + ρ*dot(dot(u_n, nabla_grad(u_n)), v)*dx +
            inner(σ(U, p_n), ϵ(v))*dx  + dot(p_n*n, v)*ds -
            dot(2*μ*nabla_grad(U)*n, v)*ds   - ρ*dot(f, v)*dx
a1 = lhs(F1)
L1 = rhs(F1)

# Define variational problem for step 2
a2 = dot(nabla_grad(p), nabla_grad(q))*dx
L2 = dot(nabla_grad(p_n), nabla_grad(q))*dx - (ρ/Δt)*div(u_)*q*dx

# Define variational problem for step 3
a3 = dot(u, v)*dx
L3 = dot(u_, v)*dx - (Δt/ρ)*dot(nabla_grad(p_ - p_n), v)*dx

# Assemble matrices
A1 = assemble(a1)
A2 = assemble(a2)
A3 = assemble(a3)

# Apply boundary conditions to matrices
[bc.apply(A1) for bc in bcu]
[bc.apply(A2) for bc in bcp]

# Create XDMF files for visualization output
xdmffile_u = XDMFFile("navier_stokes_cylinder/velocity.xdmf")
xdmffile_p = XDMFFile("navier_stokes_cylinder/pressure.xdmf")

# Create time series (for use in reaction_system.py)
timeseries_u = TimeSeries("navier_stokes_cylinder/velocity_series")
timeseries_p = TimeSeries("navier_stokes_cylinder/pressure_series")

# Save mesh to file (for use in reaction_system.py)
File("navier_stokes_cylinder/cylinder.xml.gz") << mesh

t = 0
for i in 1:num_steps

    # Update current time
    global t += Δt

    # Step 1: Tentative velocity step
    b1 = assemble(L1)
    [bc.apply(b1) for bc in bcu]
    solve(A1, u_.vector(), b1, "bicgstab", "hypre_amg")

    b2 = assemble(L2)
    [bc.apply(b2) for bc in bcp]
    solve(A2, p_.vector(), b2, "bicgstab", "hypre_amg")

    b3 = assemble(L3)
    solve(A3, u_.vector(), b3, "cg", "sor")

    #update values
    u_n.assign(u_)
    p_n.assign(p_)

    if (i-1)%25 == 0 || i == num_steps

        xdmffile_u.write(u_, t)
        xdmffile_p.write(p_, t)

        # Save nodal values to file
        timeseries_u.store(u_.vector(), t)
        timeseries_p.store(p_.vector(), t)

        println(i, "/", num_steps, " \t max:", max(array(u_)...))

    end

    # Plot solution
    #plot(u_)
    #plot(p_)
end

xdmffile_u.close()
xdmffile_p.close()

end # module ft08


# Example with 3 independent variables (constant, dummy, and continuous variable)
using StatsPlots

N = 1000;
X = [ones(N) rand(0:1, N) randn(N)];
Time, Indicator, λ, β, α = simulate([0.5, 1.0, -1.0], .5, X);
b, a = estimate(Time, Indicator, X, MvNormal([0, 0, 0], 100I), Exponential(1), 2_500);

# Plotting
gr(size = (800, 600))
plot(layout = (3, 1), label = false)

plot!(b, color = [:blue :red :green], subplot = 1, label = false)
hline!(β', color = [:blue :red :green], label = false, subplot = 1)

plot!(a, color = :grey, subplot = 2, label = false)
hline!([α], color = :black, label = false, linewidth = 2, subplot = 2)

Fit = lambda(mean(b[501:2500, :]; dims = 1)[:], mean(a[501:2500]), X)
#scatter!(Time, Fit, subplot = 3, label = false)
#plot!(0:10, 0:10, subplot = 3, label = false)
plot!([Time Fit], subplot = 3, label = false)


using Roots

export libration_dist, gcrf_to_libration_norm, libration_norm_to_gcrf

l1dist(x, μ) = x^3 - μ*(1 - x)^2 / (3 - 2*μ - x*(3 - μ - x))
l2dist(x, μ) = x^3 - μ*(1 + x)^2 / (3 - 2*μ + x*(3 - μ + x))
l3dist(x, μ) = x^3 - (1 - μ)*(1 + x)^2 / (1 + 2*μ + x*(2 + μ + x))

function libration_dist(μ_prime, μ_sec, point)
    μ_lib = 1 / (1 + μ_prime/μ_sec)
    if point == :L1
        initial = (μ_lib / 3 / (1 - μ_lib))^(1/3)
        return fzero((x) -> l1dist(x, μ_lib), initial)
    elseif point == :L2
        initial = (μ_lib / 3 / (1 - μ_lib))^(1/3)
        return fzero((x) -> l2dist(x, μ_lib), initial)
    elseif point == :L3
        initial = 1 - (7*μ_lib/12)
        return fzero((x) -> l3dist(x, μ_lib), initial)
    end
end

function rotation_params(rv)
    M = zeros(3,3)
    rm = norm(rv[1:3])
    M[:,1] = rv[1:3] / rm
    ω = cross(rv[1:3], rv[4:6])
    M[:,3] = ω / norm(ω)
    M[:,2] = cross(M[:,3], M[:,1])

    Ω = norm(ω) / rm^2
    return M, rm, Ω
end

function gcrf_to_libration_norm(rv, epoch, primary, secondary, point)
    γ = libration_dist(μ(primary), μ(secondary), point)
    rv_sec = state(DATA.ephemeris, naif_id(primary)÷100, naif_id(secondary), epoch.jd, epoch.jd1)
    rv_lib = rv - rv_sec*(1 + γ)

    M, rm, Ω = rotation_params(rv_sec)
    rv_lib[1:3] = M'*rv_lib[1:3]
    rv_lib[4:6] = M'*rv_lib[4:6] - [-Ω*rv_lib[2], Ω*rv_lib[1], 0.0]
    rv_lib[1:3] /= rm
    rv_lib[4:6] /= rm*Ω
    rv_lib
end

function libration_norm_to_gcrf(rv_lib, epoch, primary, secondary, point)
    γ = libration_dist(μ(primary), μ(secondary), point)
    rv_sec = state(DATA.ephemeris, naif_id(primary)÷100, naif_id(secondary), epoch.jd, epoch.jd1)

    M, rm, Ω = rotation_params(rv_sec)
    rv_eci = rv_lib
    rv_eci[4:6] += [-rv_lib[2], rv_lib[1], 0.0]
    rv_eci[1:3] = M*rv_eci[1:3]
    rv_eci[4:6] = M*rv_eci[4:6]
    rv_eci[1:3] *= rm
    rv_eci[4:6] *= Ω*rm
    rv_eci .+ rv_sec .* (1 + γ)
end


using Plots
using HDF5
using Pkg
using Statistics

data=h5open("test/mnist.hdf5","r")
xtrain=transpose(read(data,"x_train"))
ytrain=read(data,"t_train")
xvalid=transpose(read(data,"x_valid"))
yvalid=read(data,"t_valid")

n = size(xtrain,1)
xtrain = [ones(n,1) xtrain]
d = size(xtrain,2)

# Choose network structure and randomly initialize weights
include("NeuralNet.jl")
nHidden = [3,3,4,3]
# nHidden = [5,5,5,5]
nParams = NeuralNet_nParams(d,nHidden)
w = randn(nParams,1)

# Train with stochastic gradient
include("sgdfunc.jl")
maxIter = 10000

#----Uncomment below for hyperparameter search:----#

# aTest = [0.1, 0.01, 0.001, 0.0001, 0.00001]
# BTest = [1, 5, 10, 20, 100] #200 500 1000
#
# bestvalid = Inf
# bestw = []
# bestparams = [0,0]
# for B in BTest
#     for a in aTest
#         global bestvalid, bestw, bestparams
#         wb, validerr = VanillaSGD(NeuralNet_backprop, NeuralNet_predict, maxIter, nHidden,
#                                     nParams, xtrain, ytrain, xvalid, yvalid, a, B)
#         if (validerr[end]<bestvalid)
#             bestvalid=validerr[end]
#             bestw=wb
#             bestparams = [a,B]
#         end
#     end
# end

#best hyperparams found to be [0.0001, 20]

wbSGDB, validSGDB = SGDBabysitter(NeuralNet_backprop, NeuralNet_predict,
maxIter, nHidden, nParams, xtrain, ytrain, xvalid, yvalid)
savefig("SGDB_MNIST.pdf")

wbVan, validVan = VanillaSGD(NeuralNet_backprop, NeuralNet_predict, maxIter, nHidden, nParams, xtrain, ytrain, xvalid, yvalid, bestparams[1], convert(Int64, bestparams[2]))
savefig("Vanilla_MNIST.pdf")


using MathLink
sphericalbesselj_M(n, x) = weval(W"SphericalBesselJ"(W"n", W"x"); n, x)

@testset "_bessels.jl" begin
    @testset "domain" begin
        @test_throws DomainError SparseIR.sphericalbesselj(-2, 0.3)
        @test !isnan(SparseIR.sphericalbesselj(4, 1e20))
    end

    @testset "accuracy of sphericalbesselj(2^$pn, 2^$px)" for pn in 0:7, px in -1:25
        n = 2^pn
        x = 2.0^px
        @test SparseIR.sphericalbesselj(n, x) ≈ sphericalbesselj_M(n, x)
        @test SparseIR.sphericalbesselj(n + 1, x) ≈ sphericalbesselj_M(n + 1, x)
    end
end


using TropicalYao.Reversible.LogLikeNumbers

@testset "Tropical algebra" begin
	Random.seed!(4)
	@test check_inv(i_muladd, (Tropical(2.0), Tropical(3.0), Tropical(4.0), false))
	@test check_inv(i_muladd, (Tropical(2.0), Tropical(3.0), Tropical(-5.0), false))
	x = Tropical.(randn(100, 100))
	y = Tropical.(randn(100, 100))
	v = Tropical.(randn(100))
	out1 = x * y
	out2 = one.(out1)
	i_gemm!(out2, x, y)
	out4 = Tropical.(zeros(100))
	out3 = x * v
	bk = zeros(Bool, 100)
	i_gemv!(out4, x, v, bk)
	@test out1 ≈ out2
	@test out3 ≈ out4
end

@testset "i_sum" begin
	i_sum(TropicalF32(0), TropicalF32.([3.0,10.0,2.0]))[1].n == 10
end


immutable FDist <: ContinuousUnivariateDistribution
    ndf::Float64
    ddf::Float64
    function FDist(d1::Real, d2::Real)
    	if d1 > 0 && d2 > 0
    		new(float64(d1), float64(d2))
    	else
    		error("Both numerator and denominator degrees of freedom must be positive")
    	end
    end
end

@_jl_dist_2p FDist f

insupport(d::FDist, x::Number) = isreal(x) && isfinite(x) && 0.0 <= x

mean(d::FDist) = 2.0 < d.ddf ? d.ddf / (d.ddf - 2.0) : NaN

function var(d::FDist)
	if 4.0 < d.ddf
		return 2.0 * d.ddf^2 *
		       (d.ndf + d.ddf - 2.0) /
		       (d.ndf * (d.ddf - 2.0)^2 * (d.ddf - 4.0))
	else
		return NaN
	end
end


using Distributions
using LinearAlgebra
using Random

# Define the DGP
function simulate_sample(N0, N1)
    X0 = [ones(N0) rand([0, 1], N0, 4) rand(N0)]
    X1 = kron([ones(2, 4) [1, 0] [0, 0]], ones(N1))
    α = 4
    δ = α / √(1 + α^2)
    ω = 1
    ξ = - ω * δ * √(2 / π)
    ϵ = √(1 - δ^2) .* randn(N0) + δ .* abs.(randn(N0)) .+ ξ
    y0 = 1.0 .+ 0.5 * X0[:, end - 1] .+ 0.5 * X0[:, end] .+ ϵ
    y1 = LinRange(-3, 3, N1) |> collect |> x -> repeat(x, 2)
    return y0, X0, y1, X1, ϵ
end

# # Not run
# Random.seed!(1);
# N0, N1 = 500, 0;
# y0, X0, y1, X1, ϵ = simulate_sample(N0, N1);
# mean(ϵ)


abstract type LSHIndex end

function _get_partition(threshold, max_n_hashes)
    min_error = Inf
    opt = (0, 0)
    bestfp, bestfn = (0,0)
    for b in 1 : max_n_hashes
        max_r = Int(floor(max_n_hashes / b))
        for r in 1 : max_r
            fp = _integrate(s -> 1 - (1 - s^r)^b, 0.0, threshold)
            fn = _integrate(s -> (1 - s^r)^b, threshold, 1.0)
            error = fp + fn
            if error < min_error
                min_error = error
                opt = (b, r)
                bestfp, bestfn = fp, fn
            end
        end
    end
    @show bestfp, bestfn
    return opt
end

function _integrate(f, a, b)
    p = 0.001
    area = 0.0
    x = a
    while x < b
        area = area + f(x+0.5*p)*p
        x = x + p
    end
    return area
end

"""
Find all pairs of similar strings in index.
"""
function similar_pairs(mh::LSHIndex)::Vector{Tuple{Int, Int}}
    sims = Set{Tuple{Int, Int}}()
    for table in mh.tables
        for similars in values(table)
            for i in 1:length(similars), j in i + 1 : length(similars)
                push!(sims, (similars[i],similars[j]))
            end
        end
    end
    collect(sims)
end