using LightGraphs
using LightGraphs.SimpleGraphs
using Combinatorics
using LinearAlgebra

"""
    removesorted(collection, item) -> contains(collection, item)

Remove item from sorted collection.
"""
function removesorted!(n, v)
    i = searchsorted(n, v)
    isempty(i) && return false   # not found
    deleteat!(n, first(i))
    true
end

"""
    skeleton(n, I) -> g, S

Perform the undirected PC skeleton algorithm for a set of 1:n variables using the test I.
Returns skeleton graph and separating set
"""
function skeleton(n::V, I, par...; kwargs...) where {V}
    g = CompleteGraph(n)
    S = Dict{edgetype(g),Vector{V}}()
    d = 0 # depth

    while true
        isdone = true
        for e0 in collect(edges(g)) # cannot remove edges while iterating
            for e in (e0, reverse(e0))
                nb0 = neighbors(g, src(e))
                if length(nb0) > d  # i.e. |nb\{dst(e)}| >= d
                    i = searchsorted(nb0, dst(e))
                    isempty(i) && continue # the edge does not exist anymore
                    nb = copy(nb0)
                    deleteat!(nb, first(i))
                    isdone = false
                    for s in combinations(nb, d)
                        if I(src(e), dst(e), s, par...; kwargs...)
                            @debug "Removing edge $(e0) given $(s)"
                            rem_edge!(g, e0)
                            if !(e0 in keys(S))
                                S[e0] = s
                            end
                            break
                        end
                    end
                end
            end
        end
        d = d + 1
        if isdone
            return g, S
        end
    end
end

"""
    dseporacle(i, j, s, g)

Oracle for the `skeleton` and `pcalg` functions using [`dsep`](@ref) on the true graph `g`
"""
function dseporacle(i, j, s, g; sel=[])
    dsep(g, i, j, vcat(s,sel))
end

"""
    partialcor(i, j, s, C)

Compute the partial correlation of nodes `i` and `j` given list of nodes `s`
using the correlation matrix `C`.
"""
function partialcor(i, j, s, C)
    n = length(s)
    if n == 0
        C[i,j]
    elseif n == 1
        k = s[1]
        (C[i, j] - C[i, k]*C[j, k])/sqrt((1 - C[j, k]^2)*(1 - C[i, k]^2))
    else
        is = zeros(Int, n+2)
        is[1] = i
        is[2] = j
        for k in 1:n
            is[k+2] = s[k]
        end
        C0 = C[is, is]
        P = pinv(C0, 1.5e-8)
        -P[1, 2]/sqrt(P[1, 1]*P[2, 2])
    end
end

"""
    gausscitest(i, j, s, (C,n), c)

Test for conditional independence of variable no i and j given variables in s with
Gaussian test at the critical value c. C is covariance of n observations.

"""
@inline function gausscitest(i, j, s, stat, c)
    C, n = stat
    r = partialcor(i, j, s, C)
    r = clamp(r, -1, 1)
    n - length(s) - 3 <= 0 && return true # remove edges which cannot be tested for
    t = sqrt(n - length(s) - 3)*atanh(r)
    #@debug "testing $(i)-$(j) given $(s): $(abs(t)) -- $(c)"
    abs(t) < c
end

"""
    cmitest(i,j,s,data,crit; kwargs...)

Test for conditional independence of variables i and j given variables in s with
permutation test using nearest neighbor conditional mutual information estimates
at p-value crit.

keyword arguments:
kwargs...: keyword arguments passed to independence tests
"""
@inline function cmitest(i, j, s, data, crit; kwargs...)
    x=collect(transpose(convert(Array, data[i])))
    y=collect(transpose(convert(Array, data[j])))

    if length(s)==0
        res = kl_perm_mi_test(x, y; kwargs...)
    else
        z = reduce(vcat, map(c->collect(transpose(convert(Array, data[c]))), s))
        res = kl_perm_cond_mi_test(x, y, z; kwargs...)
    end

    #@debug "CMI test for $(i)-$(j) given $(s): $(res) compared to $(crit)"
    return res>crit
end

truetest(i, j, s) = true
falsetest(i, j, s) = false


function isotonic_regression!(y::Vector{Float64}, weights::Vector{Float64})

    n = length(y)
    if n <= 1
        return y
    end
    if n != length(weights)
        throw(DimensionMismatch("Lengths of values and weights mismatch"))
    end

    @inbounds begin
        n -= 1
        while true
            i = 1
            pooled = 0
            while i <= n
                k = i
                while k <= n && y[k] >= y[k+1]
                    k += 1
                end

                # Find a decreasing subsequence, and update
                # all points in the sequence to the weighted average.
                if y[i] != y[k]
                    numerator = 0.0
                    denominator = 0.0
                    for j in i : k
                        numerator += y[j] * weights[j]
                        denominator += weights[j]
                    end

                    for j in i : k
                        y[j] = numerator / denominator
                    end
                    pooled = 1
                end
                i = k + 1
            end
            if pooled == 0
                break
            end
        end
    end
    return y
end

isotonic_regression!(y::Vector{Float64}) = isotonic_regression!(y, ones(size(y, 1)))

# non-mutating versions
isotonic_regression(y::Vector{Float64}, weights::Vector{Float64}) = isotonic_regression!(copy(y), weights)
isotonic_regression(y::Vector{Float64}) = isotonic_regression(y, ones(size(y,1)))


# definition of lyapunov
using DrWatson
@quickactivate "NonlinearDynamicsTextbook"
include(srcdir("style.jl"))
using DynamicalSystems, PyPlot, Random

using LinearAlgebra
lo = Systems.lorenz([20,20,20.0])
X₁ = trajectory(lo, 45)
u₂ = get_state(lo) .+ 1e-6
X₂ = trajectory(lo, 45, u₂)
δ  = norm.(X₂.data .- X₁.data)
λ = lyapunov(lo, 100000)

fig = figure(figsize=(figx/2,figy/2))
ax = gca()
ax.plot(0:0.01:45, log.(δ), c = COLORS[1], label ="\$\\ln(\\delta(t)))\$")
ax.set_yticks(-12:4:4)
ax.set_xlabel("\$t\$", labelpad=-20)
# Lyapunov
ax.plot([0, 15] .+ 4, λ .* [0, 15] .- 13, color = COLORS[2])
ax.text(12, -9, "\$\\lambda_1\$=$(round(λ;digits=2))", color = COLORS[2])
ax.legend(;handlelength = 1.0)
xticks(0:15:45)
tight_layout(pad = 0.25)
subplots_adjust(bottom = 0.24, left = 0.12)
wsave(plotsdir("3", "lyapunov"), fig)


using Probably
using Probably: Bits, apply!, H, ⊗

# (N = 2^n)th root of unity
ω(n) = MathConstants.e^(-2π*im/(2^n))

# The Fourier transform is represented by the matrix W.
# We could use W directly, but it's more interesting
# to build it from basic (one or two qubit) gates.
W(n) = ω(n) .^ [i*j for i = 0:2^n-1, j = 0:2^n-1] ./ √(2^n)

# We will use the same recursive decomposition that powers the FFT.

# W(k+1) = 1|I D(k)|  * |W(k) 0  | * R(k+1)
#         √2|I D(k)|    | 0  W(k)|
# Where
# D(k) = diagm(ω(k+1) .^ (0:2^k-1))
# R(k)[i,j] = 2i == j || 2(i-2^k)+1 == j

# R moves the least significant bit to the front
function R(buf)
  for i = length(buf):-1:2
    swap(buf[i],buf[i-1])
  end
end

# Phase shift gates that make up D
Ds(k) = [[1 0; 0 ω(i)] for i = 1:k]

# Apply |I D(k)| by decomposition
#       |I D(k)|
function D(buf)
  ds = Ds(length(buf))
  for i = 2:length(buf)
    apply!(control(ds[i]), buf[1], buf[i])
  end
  hadamard(buf[1])
end

function QFT(buf::QBuffer)
  if length(buf) == 1
    hadamard(buf[1])
  else
    R(buf)
    QFT(buf[2:end])
    D(buf)
  end
  return buf
end

# Test it

buf = QBuffer(4)
# Set up an interesting state
hadamard(buf[1])
cnot(buf[1], buf[2])
# Cheat and check what the result will be
cheat_fft = W(4)*buf.bits.state.ψ
# Apply the QFT
QFT(buf)
buf.bits.state.ψ ≈ cheat_fft


"""
    Computes the volume of a polytope.

# Arguments

`polytope_vertices::AbstractArray{Float64, 2}`    Contains all the intersecting points between all the boundaries of the simplices. Each row represents some intersecting point. Matrix of size D-by-N, where D = total number of intersecting points. Each row is an intersecting point.
`convexp::AbstractArray{Float64, 2}`  Convex expansions of the vertices in `polytope_vertices` in terms of the original simplices. The size of this array is D-by-(2N + 2), where D = total number of intersecting points. (2N + 2) corresponds to the number of vertices in each simplex * 2. The first n+1 columns correspond to the convex expansion coefficients of the intersecting points in terms of the vertices generating simplex 1. The remaining n+1 to (2N+2) columns correspond to the convex expansion coefficients of the intersecting points in terms of the vertices generating simplex 2. The faces of each simplex are numbered according to the column labels.
"""
function VolumeComputation(polytope_vertices::AbstractArray{Float64, 2},
                           convexp::AbstractArray{Float64, 2})

    num_generators = size(polytope_vertices, 1) # The number of polytope generators (vertices)
    dim = size(polytope_vertices, 2)
    intvol = 0.0

    if num_generators == dim + 1 # Intersection is a simplex
        return abs(det(vcat(ones(1, dim + 1), copy(transpose(polytope_vertices)))))
    elseif num_generators > dim + 1 # Intersection is a polytope.
        # The polytope faces and how many of them are nonsingular.
        triangulation_polytope_faces, n_nonsingular_faces = TriangulationPolytopeFaces(convexp, num_generators, dim)

        if n_nonsingular_faces >= dim + 1
            polytope_centroid = ones(1, num_generators) * polytope_vertices / num_generators

            for a = 1:size(triangulation_polytope_faces, 1)
                Ind = round.(Int64, triangulation_polytope_faces[a, :])
                vertices = [polytope_vertices[vec(Ind), :]; polytope_centroid]
                intvol += abs(det([ones(dim + 1, 1) vertices]))
            end
        end
    end
    return intvol
end


# Estimate cost of first-stage resource allocation decision given scenarios of the demands

function estimateCostOfSoln(z_soln::Array{Float64},demand_scen::Array{Float64,2})

	mod = Model(solver=GurobiSolver(gurobi_env,Presolve=0,OutputFlag=0))

	@variable(mod, v[1:numResources,1:numCustomers] >= 0)
	@variable(mod, w[1:numCustomers] >= 0)

	@objective(mod, Min, sum(recourseCostCoeff[j]*w[j] for j=1:numCustomers))

	@constraint(mod, [i=1:numResources], sum(v[i,j] for j = 1:numCustomers) <= rho[i]*z_soln[i])
	@constraint(mod, con1[j=1:numCustomers], sum(mu[i,j]*v[i,j] for i = 1:numResources) + w[j] >= 0)

	numScenarios::Int64 = size(demand_scen,1)
	solnCost::Float64 = 0.0

	for s = 1:numScenarios
		for j = 1:numCustomers
			JuMP.setRHS(con1[j],demand_scen[s,j])
		end

		status = solve(mod)
		solnCost += getobjectivevalue(mod)
	end

	solnCost /= numScenarios
	solnCost += sum(costVector[i]*z_soln[i] for i=1:numResources)

	return solnCost
end


""" `numerical_deriv( x, y)`
Calculate simple estimate of numerical derivative
"""
function numerical_deriv( x::AA1, y::AA2 )   where { T1<:Real, AA1<:AbstractArray{T1,1}, T2<:Real, AA2<:AbstractArray{T2,1}  }
	@assert length(x) == length(y)
	dfluxdlnλ = zeros(size(y))
	dfluxdlnλ[1] = (y[2]-y[1])/(x[2]-x[1])
	dfluxdlnλ[2:end-1] .= (y[3:end].-y[1:end-2])./(x[3:end].-x[1:end-2])
	dfluxdlnλ[end] = (y[end]-y[end-1])/(x[end]-x[end-1])
	return dfluxdlnλ
end

""" Estimate numerical derivative of fluxes given wavelengths. """
function calc_dfluxdlnlambda(flux::AbstractArray{T1,1}, λ::AbstractArray{T2,1}) where { T1<:Real, T2<:Real }
    @assert size(flux) == size(λ)
    @assert length(flux) >= 3
    dfdlogλ = Array{T1,1}(undef,length(flux))
    calc_dfluxdlnlambda!(dfdlogλ,flux,λ)
    #dfdlogλ[1] = 0.5 * (flux[2]-flux[1])/(λ[2]-λ[1])*(λ[2]+λ[1])
    #dfdlogλ[2:end-1] .= 0.5 .* (flux[3:end].-flux[1:end-2])./(λ[3:end].-λ[1:end-2]).*(λ[3:end].+λ[1:end-2])
    #dfdlogλ[end] = 0.5 * (flux[end]-flux[end-1])/(λ[end]-λ[end-1])*(λ[end]+λ[end-1])
    return dfdlogλ
end

""" Estimate numerical derivative of fluxes given wavelengths. """
function calc_dfluxdlnlambda!(dfdlogλ::AbstractArray{T1,1}, flux::AbstractArray{T2,1}, λ::AbstractArray{T3,1}) where { T1<:Real, T2<:Real, T3<:Real }
    @assert size(flux) == size(λ)
    @assert size(dfdlogλ) == size(flux)
    @assert length(flux) >= 3
    #dfdlogλ = Array{T1,1}(undef,length(flux))
    dfdlogλ[1] = 0.5 * (flux[2]-flux[1])/(λ[2]-λ[1])*(λ[2]+λ[1])
    dfdlogλ[2:end-1] .= 0.5 .* (flux[3:end].-flux[1:end-2])./(λ[3:end].-λ[1:end-2]).*(λ[3:end].+λ[1:end-2])
    dfdlogλ[end] = 0.5 * (flux[end]-flux[end-1])/(λ[end]-λ[end-1])*(λ[end]+λ[end-1])
    return dfdlogλ
end

""" Estimate numerical second derivative of fluxes given wavelengths. """
function calc_d2fluxdlnlambda2(flux::AbstractArray{T1,1}, λ::AbstractArray{T2,1}) where { T1<:Real, T2<:Real }
    @assert size(flux) == size(λ)
    @assert length(flux) >= 3
    logλ = log.(λ)
    d2fdlogλ2 = Array{T1,1}(undef,length(flux))
    #d2fdlogλ2[2:end-1] .= 0.25*(flux[3:end].+flux[1:end-2].-2.0.*flux[2:end-1])./(λ[3:end].+λ[1:end-2].-2.0*λ[2:end-1]).*(λ[3:end].+λ[end-2]).^2
    #d2fdlogλ2[2:end-1] .= 0.5 * (flux[3:end].+flux[1:end-2].-2.0.*flux[2:end-1]).* ((λ[3:end].+λ[1:end-2])./(logλ[3:end].-logλ[1:end-2])).^2
    #d2fdlogλ2[end] = d2fdlogλ2[end-1]
    #d2fdlogλ2[1] = d2fdlogλ2[2]
    calc_d2fluxdlnlambda2!(d2fdlogλ2,flux,λ)
    return d2fdlogλ2
end

""" Estimate numerical second derivative of fluxes given wavelengths. """
function calc_d2fluxdlnlambda2!(d2fdlogλ2::AbstractArray{T1,1}, flux::AbstractArray{T1,1}, λ::AbstractArray{T2,1}) where { T1<:Real, T2<:Real, T3<:Real }
    @assert size(flux) == size(λ)
    @assert size(d2fdlogλ2) == size(flux)
    @assert length(flux) >= 3
    #logλ = log.(λ)
    #d2fdlogλ2 = Array{T1,1}(undef,length(flux))
    #d2fdlogλ2[2:end-1] .= 0.25*(flux[3:end].+flux[1:end-2].-2.0.*flux[2:end-1])./(λ[3:end].+λ[1:end-2].-2.0*λ[2:end-1]).*(λ[3:end].+λ[end-2]).^2
    d2fdlogλ2[2:end-1] .= 0.5 * (flux[3:end].+flux[1:end-2].-2.0.*flux[2:end-1]).* ((λ[3:end].+λ[1:end-2]).^2 ./ (2.0.*log.(λ[3:end]./λ[1:end-2])))
    d2fdlogλ2[end] = d2fdlogλ2[end-1]
    d2fdlogλ2[1] = d2fdlogλ2[2]
    return d2fdlogλ2
end

function calc_weights_for_running_mean(n::Integer; decay_factor::Float64 = 2.0)
	@assert 1<=n<=100
	@assert 1 <= decay_factor < Inf
	w = zeros(n)
	if isodd(n)
		center = convert(Int,1 + (n-1)//2)
		w[center] = 1
		for i in 1:(center-1)
			w[center+i] = w[center-i] = decay_factor^(-i)
		end
	else
		center = convert(Int,n//2)
		for i in 1:center
			w[center+i] = w[center+1-i] = decay_factor^(-(i-0.5))
		end
	end
	w ./= sum(w)
	return w
end

""" Smooth array via weighted running mean of n consecutive points """
function calc_running_mean_weighted(x::AbstractVector{T1}; n::Integer = 5, decay_factor::Float64 = 2.0,
				weights::AbstractVector{T2} = calc_weights_for_running_mean(n, decay_factor=decay_factor) ) where { T1<:Real, T2<:Real }
	@assert 2 <= n < length(x)/2
	@assert isodd(n)   # Could generalize, but haven't
	smoothed = Array{T1,1}(undef,length(x))
	n = length(weights)
	m = floor(Int,n//2)
	for i in 1:m
		start_idx = 2+m-i
		smoothed[i] = sum(view(x,1:i+m).*view(weights,start_idx:n)) /sum(view(weights,start_idx:n))
	end
	for i in (m+1):(length(x)-m-1)
		smoothed[i] = sum(view(x,i-m:i+m) .* weights)
	end
	for i in (length(x)-m):length(x)
		stop_idx = length(x)-(i-m)+1
		smoothed[i] = sum(view(x,i-m:length(x)).*view(weights,1:stop_idx)) /sum(view(weights,1:stop_idx))
	end
	return smoothed
end


## Residual Constrained Alternating Minimization
# Function Definitions

function AltMinChol{ET<:Number}(b::AbstractArray{ET}, r::Int, K::Int)
# Performs RCAM on a single matricised monochromatic slice
# Calls: minFrobQR

    # Generate L0
    m,n = size(b)
    L = collect(ET, randn(m,r))

    indi = (1:max(m,n))::UnitRange{Int64}
    local R

    # Begin Alterations
    for k = 1:K

        # Solve for R
     @time R = minFrobChol(L,b,indi)

        # Solve for L
     @time L = minFrobChol(R,b',indi)

    end

    return L, R
end
export AltMinChol

function minFrobChol{ET<:Number}(L::AbstractArray{ET}, b::AbstractArray{ET}, indi::UnitRange{Int64})
# Solves for each row of input
# Calls: minFrobRsub

    m = size(b,2)::Int
    r = size(L,2)::Int

    # Initialize R
    R = zeros(ET,m,r)

    for k =1:length(indi)

        if indi[k]<=m

             R[indi[k],:] = minFrobRsubChol(L, b, indi[k])

        end

    end

    R = conj(R)

    return R
end

function minFrobRsubChol{ET<:Number}(L::AbstractArray{ET},b::AbstractArray{ET},indi::Int)
# Solves a single row

    # Solve for the kth row of R
    bsub = b[:,indi]

    # Find the non zero rows of the kth column of b
    ind = find(x -> x != 0, b[:,indi])

    # Keep only the non zero
    L2 = L[ind,:]
    bsub = transpose(L2)*bsub[ind]

    # Chol fact
    L2 = L2'*L2
    Q = chol(L2)

    bsub2 = fwdsub(Q',bsub)

    v = backsub(Q,bsub2)

    return v

end

function backsub{ET<:Number}(A::AbstractArray{ET}, y::AbstractArray{ET})

    m = size(A,2)
    x = zeros(ET,m,1)

    x[m] = y[m]/A[m,m]

    for i = m-1:-1:1
        x[i] = (y[i]-dot(A[i,i+1:m], x[i+1:m]))/A[i,i]
    end

    return x
end

function fwdsub{ET<:Number}(A::AbstractArray{ET},y::AbstractArray{ET})

    m = size(A,2)
    x = zeros(ET,m,1)
    x[1] = y[1]/A[1,1]

    for i = 2:1:m
        x[i] = (y[i]-dot(A[i,i-1:-1:1], x[i-1:-1:1]))/A[i,i]
    end

    return x
end


module ZeroMeanFESpacesTests

using Test
using Gridap.Geometry
using Gridap.Fields
using Gridap.FESpaces
using Gridap.CellData
using Gridap.ReferenceFEs

domain = (0,1,0,1)
partition = (4,4)
model = CartesianDiscreteModel(domain,partition)

order = 2

trian = get_triangulation(model)
degree = order
dΩ = Measure(trian,degree)

_V = FESpace(model,ReferenceFE(lagrangian,Float64,order);conformity=:L2)

V = ZeroMeanFESpace(_V,dΩ)

matvecdata = ([],[],[])
matdata = ([],[],[])
vecdata = ([],[])
test_single_field_fe_space(V,matvecdata,matdata,vecdata)

U = TrialFESpace(V)
test_single_field_fe_space(U,matvecdata,matdata,vecdata)
@test isa(U,ZeroMeanFESpace)

fun(x) = sin(4*pi*(x[1]+x[2]^2)) + 3
uh = interpolate(fun, U)

mean1 = sum(∫(uh)*dΩ)

tol = 1.0e-10
@test abs(mean1) < tol

V = FESpace(model,ReferenceFE(lagrangian,Float64,order);conformity=:L2,constraint=:zeromean)
@test isa(V,ZeroMeanFESpace)

end # module


module TestExpressionTrees

using Test

using Catlab.Programs
using Catlab.WiringDiagrams

# Conversion
############

# Convert Julia expressions to formulas.
@test to_formula(:(sin(x))) == Formula(:sin, :x)
@test to_formula(:(sin(2x))) == Formula((:sin, (:*, 2, :x)))
@test to_formula(:(2*sin(2x))) == Formula((:*, 2, (:sin, (:*, 2, :x))))
@test to_formula(:(x == y)) == Formula(:(==), :x, :y)
@test to_formula(:(x -> 2x)) == Formula((:->, :x, (:*, 2, :x)))
@test to_formula(:(A && B)) == Formula(:&, :A, :B)
@test to_formula(:(A || B)) == Formula(:|, :A, :B)

# Convert algebraic networks to formulas.
R = Ob(AlgebraicNet, :R)
f = Hom(:sin, R, R)
@test to_formula(f, [:x]) == Formula(:sin, :x)

f = compose(linear(2,R,R), Hom(:sin,R,R))
@test to_formula(f,[:x]) == Formula((:sin, (:*, 2, :x)))
f = compose(linear(2,R,R), Hom(:sin,R,R), linear(2,R,R))
@test to_formula(f,[:x]) == Formula((:*, 2, (:sin, (:*, 2, :x))))

f = compose(Hom(:cos,R,R), Hom(:sin,R,R), Hom(:tan,R,R))
@test to_formula(f,[:x]) == Formula((:tan, (:sin, (:cos, :x))))

f = compose(otimes(id(R),constant(1,R)), mmerge(R))
@test to_formula(f,[:x]) == Formula(:+, :x, 1)
@test to_formula(mmerge(R,1),[:x]) == :x
@test to_formula(mmerge(R,3),[:x,:y,:z]) == Formula(:+, :x, :y, :z)

f = compose(otimes(Hom(:cos,R,R), Hom(:sin,R,R)), mmerge(R))
@test to_formula(f,[:x,:y]) == Formula((:+, (:cos, :x), (:sin, :y)))
f = compose(mcopy(R), otimes(Hom(:cos,R,R), Hom(:sin,R,R)), mmerge(R))
@test to_formula(f,[:x]) == Formula((:+, (:cos, :x), (:sin, :x)))

w = [ 1 => 1, 2 => 2, 3 => 1, 4 => 2 ]
f = compose(wiring(w, otimes(R,R,R,R), otimes(R,R)), Hom(:*, otimes(R,R), R))
@test to_formula(f,[:w,:x,:y,:z]) == Formula((:*, (:+, :w, :y), (:+, :x, :z)))

f = wiring([ 1 => 1, 1 => 1, 2 => 1 ], otimes(R,R), R)
@test to_formula(f,[:x,:y]) == Formula((:+, (:*, 2, :x), :y))

# Convert formulas to wiring diagrams.
make_box = (value, arity) -> Box(value, repeat([nothing], arity), [nothing])

d = to_wiring_diagram(Formula(:*, :x, :y), [:x, :y])
@test boxes(d) == [ make_box(:*, 2) ]
v_times, = box_ids(d)
@test wires(d) == map(Wire, [
  (input_id(d), 1) => (v_times, 1),
  (input_id(d), 2) => (v_times, 2),
  (v_times, 1) => (output_id(d), 1),
])

d = to_wiring_diagram(Formula((:+, (:cos, :x), (:sin, :x))), [:x])
@test boxes(d) == [ make_box(:cos, 1), make_box(:sin, 1), make_box(:+, 2) ]
v_cos, v_sin, v_plus = box_ids(d)
@test wires(d) == map(Wire, [
  (input_id(d), 1) => (v_cos, 1),
  (input_id(d), 1) => (v_sin, 1),
  (v_cos, 1) => (v_plus, 1),
  (v_sin, 1) => (v_plus, 2),
  (v_plus, 1) => (output_id(d), 1),
])

d = to_wiring_diagram(Formula((:*, (:cos, :x), (:cos, :x))), [:x])
@test boxes(d) == [ make_box(:cos, 1), make_box(:*, 2) ]
v_cos, v_times = box_ids(d)
@test wires(d) == map(Wire, [
  (input_id(d), 1) => (v_cos, 1),
  (v_cos, 1) => (v_times, 1),
  (v_cos, 1) => (v_times, 2),
  (v_times, 1) => (output_id(d), 1),
])

# Evaluation
############

# Standard evaluation.
@test evaluate(Formula(:sin, :x), Dict(:x => pi/2)) == sin(pi/2)
@test evaluate(Formula(:(==), :x, :y), Dict(:x => 1, :y => 1)) == true

# Vectorized evaluation.
x = collect(range(0, stop=2pi, length=10))
@test evaluate(Formula(:sin, :x), Dict(:x => x)) == sin.(x)
@test evaluate(Formula((:*, (:sin, :x), (:cos, :x))), Dict(:x => x)) ==
  @. sin(x) * cos(x)

# Pretty-print
##############

latex(form::Formula) = sprint(show_latex, form)
sexpr(form::Formula) = sprint(show_sexpr, form)

# LaTeX pretty-print.
@test latex(Formula(:f, :x , :y)) == "f\\left(x,y\\right)"
@test latex(Formula(:cos, :x)) == "\\cos\\left(x\\right)"
@test latex(Formula(:+, :x, :y)) == "x + y"
@test latex(Formula(:+, :x, :y, :z)) == "x + y + z"
@test latex(Formula(:+, :alpha, :beta)) == "\\alpha + \\beta"
@test latex(Formula(:-, :x, :y)) == "x - y"
@test latex(Formula(:-, :x)) == "- x"
@test latex(Formula(:*, :x, :y)) == "x y"
@test latex(Formula(:.*, :x, :y)) == "x y"
@test latex(Formula(:*, 2, :x)) == "2 x"
@test latex(Formula(:*, 2, 2)) == "2 \\cdot 2"
@test latex(Formula(:/, :x, :y)) == "\\frac{x}{y}"
@test latex(Formula(:./, :x, :y)) == "\\frac{x}{y}"
@test latex(Formula((:*, :x, (:+, :y, :z)))) == "x \\left(y + z\\right)"
@test latex(Formula(:^, :x, :y)) == "x^{y}"
@test latex(Formula(:.^, :x, :y)) == "x^{y}"
@test latex(Formula(:+, Inf, 1)) == "\\infty + 1"
@test latex(Formula((:^, (:+, :x, :y), 2))) == "\\left(x + y\\right)^{2}"
@test latex(Formula((:^, 2, (:+, :x, :y)))) == "2^{x + y}"
@test latex(Formula((:^, (:^, :x, :a), :b))) == "\\left(x^{a}\\right)^{b}"
@test latex(Formula(:factorial, :x)) == "x !"
@test latex(Formula(:(==), :x, :y)) == "x = y"
@test latex(Formula(:<, :x, :y)) == "x < y"
@test latex(Formula(:<=, :x, :y)) == "x \\leq y"
@test latex(Formula(:&, :A, :B)) == "A \\wedge B"
@test latex(Formula(:|, true, false)) == "\\top \\vee \\bot"
@test latex(Formula(:!, :A)) == "\\neg A"

# Operator precedence in LaTeX pretty-print.
@test latex(Formula((:+, (:*, :a, :b), (:*, :c, :d)))) == "a b + c d"
@test latex(Formula((:*, (:+, :a, :b), (:+, :c, :d)))) ==
  "\\left(a + b\\right) \\left(c + d\\right)"
@test latex(Formula((:(==), (:+, :x, :y), (:+, :y, :x)))) == "x + y = y + x"
@test latex(Formula((:&, (:(==), :x, :y), (:(==), :z, :w)))) ==
  "x = y \\wedge z = w"
@test latex(Formula((:-, (:+, :x, :y)))) == "- \\left(x + y\\right)"
@test_skip latex(Formula((:+, :x, (:-, :y)))) == "x + \\left(- y\\right)"
@test latex(Formula((:factorial, (:*, :m, :n)))) == "\\left(m n\\right) !"
@test latex(Formula(:^, -1, :n)) == "\\left(- 1\\right)^{n}"

# S-expression pretty-print.
@test sexpr(Formula(:f)) == "(:f)"
@test sexpr(Formula(:+, :x, :y)) == "(:+ :x :y)"
@test sexpr(Formula((:*, :x, (:+, :y, 1)))) == "(:* :x (:+ :y 1))"

end


######################################################################
# dynamics.jl
# Each trajectory manager will have a type of dynamics
######################################################################
export Dynamics, LinearDynamics, DubinsDynamics, linearize
export IntegrationScheme, ForwardEuler, SymplecticEuler
export forward_euler
export GroupDynamics
export vvf2vvvf

type LinearDynamics <: Dynamics
    n::Int
    m::Int
    A::Matrix{Float64}
    B::Matrix{Float64}

    function LinearDynamics(A, B)
        n,m = size(B)
        return new(n,m,deepcopy(A),deepcopy(B))
    end
end

type GroupDynamics <: Dynamics
	n::Int
	m::Int
	num_agents::Int
	array

	function GroupDynamics(array)
		num_agents = length(array)
		n = m = 0
		for j = 1:num_agents
			n += array[j].n
			m += array[j].m
		end
		return new(n, m, num_agents, array)
	end
end

type DubinsDynamics <: Dynamics
	n::Int
	m::Int

	v0::Float64		# constant speed
	r::Float64		# minimum turn radius

	# Constructors
	DubinsDynamics() = DubinsDynamics(1.0, 1.0)
	DubinsDynamics(v0::Real, r::Real) = new(3, 1, v0, r)
end

# Not really types,
export SingleIntegrator, DoubleIntegrator
function SingleIntegrator(n::Int, h::Float64)
	A = eye(n)
	B = h*eye(n)
	return LinearDynamics(A, B)
end
function DoubleIntegrator(n::Int, h::Float64)
	A = eye(2*n)
	for i = 1:n
		A[i, n+i] = h
	end

	B = zeros(2*n, n)
	for i = 1:n
		B[n+i, i] = h
	end

	return LinearDynamics(A, B)
end

"""
`dynamics!(tm::TrajectoryManager, d::Dynamics)`

Not only sets `tm.dynamics = d`, but also sets reward matrices `tm.Qn`, `tm.R`, and `tm.Rn` to default matrices of the correct sizes. These defaults are:

`tm.Qn = eye(d.n)`

`tm.R = 0.01 * eye(d.m)`

`tm.Rn = eye(d.m)`

"""
function dynamics!(tm::TrajectoryManager, d::Dynamics)
	tm.dynamics = d
	tm.Qn = eye(d.n)
	tm.R = 0.01 * eye(d.m)
	tm.Rn = eye(d.m)
end

######################################################################
# linearization
######################################################################

# linearizes about a trajectory
function linearize(d::Dynamics, x::VVF, u::VVF, h::Float64)
	N = length(u)
	A = VMF()
	B = VMF()
	for n = 1:N
		An, Bn = linearize(d, x[n], u[n], h)
		push!(A, An)
		push!(B, Bn)
	end
	return A,B
end

#function linearize(vtm::Vector{TrajectoryManager}, xds::VVVF, uds::VVVF)
#	num_agents = length(vtm)
#	N = length(xds[1]) - 1
#
#	# Make A and B a matrix with N rows and num_agents cols
#	A, B = linearize(vmt[1].dynamics, xds[1], uds[1], vmt[1].h)
#	for j = 2:num_agents
#		tm = vtm[j]
#		At, Bt = linearize(vtm[j].dynamics, xds[j], uds[j], vtm[j].h)
#		A = hcat(A, At)
#		B = hcat(B, Bt)
#	end
#
#	# now concatenate along each of the rows
#	dims = ones(Int, num_agents)
#	dims[1] = 1
#	for n = 1:N
#		push!(bigA, cat(dims, A[n]...))
#		push!(bigB, cat(dims, B[n]...))
#	end
#
#	return bigA, bigB
#end

function linearize(ld::LinearDynamics, x::VF, u::VF, h::Float64)
	return ld.A, ld.B
end

function linearize(ld::DubinsDynamics, x::VF, u::VF, h::Float64)
	A = eye(3)
	A[1,3] = -h * sin(x[3]) * ld.v0
	A[2,3] = h * cos(x[3]) * ld.v0

	B = zeros(3,1)
	B[3] = h/ld.r

	return A, B
end

function gsplit(gd::GroupDynamics, x::VF)
	xarr = VVF()
	xind = 1
	for j = 1:gd.num_agents
		d = gd.array[j]
		push!(xarr, x[xind + d.n - 1])
		xind += d.n
	end
	return xarr
end

function gsplit(gd::GroupDynamics, x::VF, u::VF)
	xarr = VVF()
	uarr = VVF()
	xind = uind = 1
	for j = 1:gd.num_agents
		d = gd.array[j]
		push!(xarr, x[xind:(xind + d.n - 1)])
		push!(uarr, u[uind:(uind + d.m - 1)])
		xind += d.n
		uind += d.m
	end
	return xarr, uarr
end

function linearize(gd::GroupDynamics, x::VF, u::VF, h::Float64)

	xarr, uarr = gsplit(gd, x, u)

	Aarr = VMF()
	Barr = VMF()
	for j = 1:gd.num_agents
		A, B = linearize(gd.array[j], xarr[j], uarr[j], h)
		push!(Aarr, A)
		push!(Barr, B)
	end

	dims = 2*ones(Int, gd.num_agents)
	dims[1] = 1

	return cat(dims, Aarr...), cat(dims, Barr...)
end

######################################################################
# integration
######################################################################
type ForwardEuler <: IntegrationScheme end
type SymplecticEuler <: IntegrationScheme end

function integrate(tm::TrajectoryManager, x::VF, u::VF)
	integrate(tm.int_scheme, tm.dynamics, x, u, tm.h)
end

function integrate(::ForwardEuler, d::Dynamics, x::VF, u::VF, h::Float64)
	forward_euler(d, x, u, h)
end

function integrate(::SymplecticEuler, d::Dynamics, x::VF, u::VF, h::Float64)
	symplectic_euler(d, x, u, h)
end

function integrate(tm::TrajectoryManager, ud::VVF)
	xd = Array(Vector{Float64}, tm.N+1)

	xd[1] = deepcopy(tm.x0)
	for i = 1:tm.N
		xd[i+1] = integrate(tm.int_scheme, tm.dynamics, x, ud[i], tm.h)
	end

	return xd
end

######################################################################
# forward_euler
######################################################################
function forward_euler(tm::TrajectoryManager, x::VF, u::VF)
	forward_euler(tm.dynamics, x, u, tm.h)
end

function forward_euler(ld::LinearDynamics, x::VF, u::VF, h::Float64)
	return ld.A*x + ld.B*u
end
function forward_euler(gd::GroupDynamics, x::VF, u::VF, h::Float64)
	n_ind = m_ind = 1
	xp = VVF()
	xarr, uarr = gsplit(gd, x, u)
	for j = 1:gd.num_agents
		push!(xp, forward_euler(gd.array[j], xarr[j], uarr[j], h))
	end
	return vcat(xp...)
end

function forward_euler(dd::DubinsDynamics, x::VF, u::VF, h::Float64)
	xp = deepcopy(x)
	xp[1] += cos(x[3]) * dd.v0 * h
	xp[2] += sin(x[3]) * dd.v0 * h
	u_val = u[1]
	if u_val > 1.
		u_val = 1.
	end
	if u_val < -1.
		u_val = -1.
	end
	xp[3] += u_val / dd.r * h
	return xp
end

######################################################################
# symplectic_euler
######################################################################
function symplectic_euler(d::Dynamics, x::VF, u::VF, h::Float64)
	#A,B = linearize(d, x, u, h)
	A = zeros(4,4)
	A[1,3] = 1
	A[2,4] = 1
	A[3,1] = -1
	A[4,2] = -1
	B = zeros(4,2)
	B[3,1] = 1
	B[4,2] = 1

	#n2 = round(Int, d.n/2)
	n2 = 2

	A11 = A[1:n2, 1:n2]
	A12 = A[1:n2, n2+1:d.n]
	A21 = A[n2+1:d.n, 1:n2]
	A22 = A[n2+1:d.n, n2+1:d.n]

	B1 = B[1:n2,:]
	B2 = B[n2+1:d.n,:]

	# block matrices
	Ad22 = inv(eye(n2)-h*A22)
	Ad11 = eye(n2) + h*A11 + h*h*A12*Ad22*A21
	Ad12 = h*A12*Ad22
	Ad21 = h*Ad22*A21

	Bd1 = h*h*A12*Ad22*B2 + h*B1
	Bd2 = h*Ad22*B2

	Ase = [Ad11 Ad12; Ad21 Ad22]
	Bse = [Bd1; Bd2]

	xp = Ase*x + Bse*u
	return xp
end

# split one vector per state up
function vvf2vvvf(xd::VVF, ud::VVF, vtm::Vector{TrajectoryManager})
	N = length(xd) - 1
	num_agents = length(vtm)

	x_start = u_start = 1
	xds = VVVF(num_agents)
	uds = VVVF(num_agents)
	for j = 1:num_agents
		x_end = x_start + vtm[j].dynamics.n - 1
		u_end = u_start + vtm[j].dynamics.m - 1
		xds[j] = VVF()
		uds[j] = VVF()
		for n = 1:N
			push!(xds[j], xd[n][x_start:x_end])
			push!(uds[j], ud[n][u_start:u_end])
		end
		push!(xds[j], xd[N+1][x_start:x_end])
		x_start = x_end + 1
		u_start = u_end + 1
	end
	return xds, uds
end

function vvf2vvvf(xd::VVF, vtm::Vector{TrajectoryManager})
	N = length(xd) - 1
	num_agents = length(vtm)

	x_start = 1
	xds = VVVF(num_agents)
	for j = 1:num_agents
		x_end = x_start + vtm[j].dynamics.n - 1
		xds[j] = VVF()
		for n = 1:N+1
			push!(xds[j], xd[n][x_start:x_end])
		end
		x_start = x_end + 1
	end
	return xds
end

function vvf2vvvf(xd::VVF, gd::GroupDynamics)
	N = length(xd) - 1

	x_start = 1
	xds = VVVF(gd.num_agents)
	for j = 1:gd.num_agents
		x_end = x_start + gd.array[j].n - 1
		xds[j] = VVF()
		for n = 1:(N+1)
			push!(xds[j], xd[n][x_start:x_end])
		end
		x_start = x_end + 1
	end
	return xds
end


using DynamicEnergyBudgets, Unitful, Test

using DynamicEnergyBudgets: tempcorr

reftemp = 310.0u"K"
arrtemp = 2000.0u"K"
lowerbound = 280.0u"K"
arrlower = 20000.0u"K"
upperbound = 315.0u"K"
arrupper = 70000.0u"K"

tc = TempCorrLowerUpper(
    reftemp=reftemp,
    arrtemp=arrtemp,
    tbelow=lowerbound,
    arrlower=arrlower,
    tabove=upperbound,
    arrupper=arrupper,
)

@test_broken tempcorr(tc, 100.0u"°C" |> u"K") ≈ 0.0 atol=1e-10
@test tempcorr(tc, -100.0u"°C" |> u"K") ≈ 0.0 atol=1e-10
@test tempcorr(tc, reftemp) == 1.0

tc = TempCorrLower(reftemp, arrtemp, lowerbound, arrlower)
@test tempcorr(tc, 100.0u"°C") > 1.0
@test tempcorr(tc, -100.0u"°C") ≈ 0.0 atol=1e-10
@test tempcorr(tc, reftemp) == 1.0

tc = TempCorr(reftemp, arrtemp)
@test tempcorr(tc, 100.0u"°C") > 1.0
@test tempcorr(tc, -100.0u"°C") > 0.0
@test tempcorr(tc, -100.0u"°C") < 1.0
@test tempcorr(tc, reftemp) == 1.0


ack2(m::Integer, n::Integer) = m == 0 ? n + 1 : n == 0 ? ack2(m - 1, 1) : ack2(m - 1, ack2(m, n - 1))


struct Spectrum_29
    cyclefrequency
    spectralamplitude
end
function Spectrum_29(spect::Spectrogram; unitfreq = 1, unitdens = 1)
    binwidth = spect.freq.multiplier
    amplitudes = sqrt.( 2 * spect.power * binwidth) * unitdens
    # amplitudes can be a matrix if the sample size was larger than the window length
    avgamplitudes = mean(amplitudes, dims = 2)
    Spectrum_29(spect.freq * unitfreq, avgamplitudes)
end

function sampledspectrum_29(v, sampledistance; minimumcyclefrequency = NaN, maximumwavelength = NaN)
    welldefined = xor(isnan(minimumcyclefrequency), isnan(maximumwavelength))
    @assert welldefined "either minimumcyclefrequency or maximumwavelength must be given"
    minimumcyclefrequency = isnan(minimumcyclefrequency) ? 1 / maximumwavelength : minimumcyclefrequency
    @assert dimension(sampledistance * minimumcyclefrequency) == NoDims
    unitλ =  oneunit(sampledistance)
    unitv = oneunit(eltype(v))
    maxcyclesamples =  round(Int, upreferred( 1 / (minimumcyclefrequency * sampledistance)))
    windowlength =  min(nextfastfft(maxcyclesamples * 8), length(v))
    # Calculate parameter for a Tukey window with 10 % of window
    # length outside of flat region
    x_flat = 0.5 * 0.9
    α = 1 - 2x_flat
    wind = tukey(windowlength, α)
    onsid = !(eltype(v) <: ComplexQuantity)
    spectul = spectrogram(v / unitv, windowlength,
        onesided = onsid,
        fs = 1 / (sampledistance / unitλ),
        window = wind
        )
    Spectrum_29(spectul, unitfreq = 1 / unitλ, unitdens = unitv)
end

function spectrumbars_29(spect::Spectrum_29)
    iter = zip(spect.cyclefrequency[2:end], spect.spectralamplitude[2:end])
    pts = Vector{Point}()
    maxa = maximum(spect.spectralamplitude[2:end])
    for (f, a) in iter
        λ = 1 / f
        y = a * 2EM * scale_pt_to_unit(m) / maxa
        push!(pts,  Point(λ, 0y))
        push!(pts,  Point(λ, y))
        push!(pts,  Point(λ, 0y))
    end
    pts
end

function draw_spectrumbars_29(origo, spect::Spectrum_29)
    spectpoints = spectrumbars_29(spect)
    prettypoly(origo .+ spectpoints, :stroke, vertexlabels = (n,l ) -> begin
        mod(n, 3) == 2 && circle(O, 0.006m, :stroke)
    end)
end

function draw_spectrum_29(origo, spect::Spectrum_29)
    draw_spectrumbars_29(origo, spect)
    λ_max = 1 / spect.cyclefrequency[2]
    @layer begin
        sethue(PALETTE[3])
        arrow(origo, origo + (0.0,  -2EM))
        settext("Amplitude", origo + (0.0,  -2EM ), markup=true)
        arrow(origo, origo + (min(1.05λ_max, 0.95 * scale_pt_to_unit(m) * WI), 0.0m))
        settext("Wave length, λ [m]", origo + ( 0.8 * scale_pt_to_unit(m) * WI, 0.0m), markup=true)
    end
end

islocalmax(v, i) = i > 1 && i < length(v) && v[i - 1] < v[i] > v[i + 1]

"""
   two_largest_maxima_amplitude_wavelength(spect::Spectrum_29)

   Returns two tuples, (λ_1, a_1), (λ_2, a_2)

where
   λ_1 is the wavelength of the peak amplitude in the spectrum
   a_1 is the peak amplitude in the spectrum
   (λ_2, a_2) is the second largest peak
"""
function two_largest_maxima_amplitude_wavelength(spect::Spectrum_29)
    amps = spect.spectralamplitude
    ua = eltype(amps)
    freqs = spect.cyclefrequency
    uλ = typeof(1 / freqs[1])

    local_maxima_indices = [i for i= 1:length(amps) if islocalmax(amps, i)]
    local_maxima_values = [amps[i] for i in local_maxima_indices]
    sorted_maxima_indices = sortperm(local_maxima_values, rev = true )
    ind1, ind2 = local_maxima_indices[sorted_maxima_indices[1]], local_maxima_indices[sorted_maxima_indices[2]]

    round3(typ, x) = typ <: Quantity ? round(typ, x, digits = 3) : round(x, digits = 3)

    λ_1 = round3(uλ, 1 / freqs[ind1])
    a_1 = round3(ua, amps[ind1])
    λ_2 = round3(uλ, 1 / freqs[ind2])
    a_2 = round3(ua, amps[ind2])

    return (λ_1, a_1), (λ_2, a_2)
end


if VERSION >= v"0.7"
  using Test
else
  using Base.Test
  include("random.jl")
end
include("../src/define_constants.jl")
include("loglinspace.jl")
include("test_cel_vec.jl")
include("test_s2_gradient.jl")
include("test_transit_poly.jl")

include("test_IJv_derivative.jl")
#include("test_sn_jacobian2.jl")

skip_plots = false
include("test_transit_poly_gradient2.jl")


export expandPolygon, insidePolygon

function expandPolygon( x::Vector{Float64}, y::Vector{Float64},  # polygon points
	                     LL )  # distance
# Expand the polygon.  Each line segment goes out a distance of LL
# in the normal direction.

npts = length(x)

if length(y) != npts
   error("length(y) != npts")
end

xs = Array{Float64}(npts*2)
ys = Array{Float64}(npts*2)

idx = 1

for i = 1:npts-1

   theta = atan2(y[i+1] - y[i], x[i+1] - x[i])
   theta += pi/2  # rotate by 90 for normal direction.

   lct = LL * cos(theta)
   lst = LL * sin(theta)

   xs[idx] = x[i] + lct
   ys[idx] = y[i] + lst

   xs[idx+1] = x[i+1] + lct
   ys[idx+1] = y[i+1] + lst

   idx += 2
end # i

# end point
theta = atan2(y[1] - y[npts], x[1] - x[npts])
theta += pi/2

lct = LL * cos(theta)
lst = LL * sin(theta)

xs[idx] = x[npts] + lct
ys[idx] = y[npts] + lst

xs[idx+1] = x[1] + lct
ys[idx+1] = y[1] + lst

return xs, ys
end # function expandPolygon

#---------------------------------------------------------------------

function insidePolygon( x::Vector{Float64}, y::Vector{Float64},  # polygon points
  	                     xp,yp)
#  Is point (xp,yp) inside polygon defined by x,y ?

n = length(x)

if length(y) != n
   error("length(y) != n")
end

inside = false

for i = 1:n
   i1 = i + 1
   if i1 > n
      i1 = 1
   end
   dy = y[i1] - y[i]

   if dy != 0.0
      y1 = (yp <= y[i])
      y2 = (yp >  y[i1])

      if y1 == y2

         rslope = (x[i1] - x[i]) / dy
         d = (xp-x[i]) - (yp-y[i]) * rslope

         if abs(d) < 1.e-4
         	 d = 0.0
         end

         if d <= 0.0
            inside = !inside
         end

      end  # y1 == y2

   end # (dy != 0.0)

end # i

return inside
end # function insidePolygon


export KnightTourDrawing

"""
`KnightTourDrawing(r,c)` illustrates a Knight's tour of an r-by-c
chessboard. Throws an error if no such tour exists.
"""
function KnightTourDrawing(r::Int = 6, c::Int = 6)
    G = Knight(r, c)
    @info "Searching for a Hamiltonian cycle in a $r-by-$c Knight's move graph"
    @time h = hamiltonian_cycle(G)
    @info "Finished"

    if length(h) == 0
        throw(error("There is no such Knight's move tour"))
    end

    T = eltype(G)
    H = SimpleGraph{T}()
    for v in G.V
        add!(H, v)
    end

    for k = 1:NV(H)-1
        add!(H, h[k], h[k+1])
    end

    add!(H, h[1], h[end])

    xy = getxy(H)

    for v in H.V
        xy[v] = collect(v)
    end

    embed(H, xy)

    draw(H)

    for a = 0:c
        plot!([0.5, r + 0.5], [a + 0.5, a + 0.5], color = "black", linestyle = :dash)
    end

    for b = 0:r
        plot!([b + 0.5, b + 0.5], [0.5, c + 0.5], color = "black", linestyle = :dash)
    end

    finish()
end


"""
    whittaker1(y, w, lambda) -> z

Perform a first-order differences Whittaker-Henderson smoothing and interpolation.
# Citation
'Smoothing and interpolation with finite differences' [Eilers P. H. C, 1994]
(URL: http://dl.acm.org/citation.cfm?id=180916)
"""
function whittaker1{T <: Number}(y::Vector{Float64}, w::Vector{T}, lambda::Float64)
  z = similar(y)
  return smooth1!(y, w, lambda, z)
end

"""
    whittaker1!(y, w, lambda) -> y

Perform an in-place first-order differences Whittaker-Henderson smoothing and interpolation on `y`.
# Citation
'Smoothing and interpolation with finite differences' [Eilers P. H. C, 1994]
(URL: http://dl.acm.org/citation.cfm?id=180916)
"""
function whittaker1!{T <: Number}(y::Vector{Float64}, w::Vector{T}, lambda::Float64)
  return smooth1!(y, w, lambda, y)
end

"""
    smooth1!(y, w, lambda, z) -> z

Smooth and interpolate with first-order differences.
"""
function smooth1!{T <: Number}(y::Vector{Float64}, w::Vector{T}, lambda::Float64, z::Vector{Float64})
  # init
  m = length(y)
  c = Array{Float32}(m)
  d = Array{Float32}(m)
  @inbounds @fastmath begin
    w0 = w[1]
    d1 = d[1] = w0 + lambda
    c1 = c[1] = -lambda/d1
    z1 = z[1] = w0*y[1]
  end

  # compute intermediate values
  @inbounds @fastmath for i = 2:(m-1)
    w0 = w[i]
    d1 = d[i] = w0 + 2*lambda - c1^2*d1
    z1 = z[i] = w0*y[i] - c1*z1
    c1 = c[i] = -lambda/d1
  end

  # compute last values
  @inbounds @fastmath begin
    w0 = w[m]
    d1 = d[m] = w0 + lambda - c1^2*d1
    z1 = z[m] = (w0*y[m] - c1*z1)/d1
  end

  # compute result
  @inbounds @fastmath for i = (m-1):-1:1
    z1 = z[i] = z[i]/d[i] - c[i]*z1
  end

  return z
end


function saliency_map(network, input; coeffs = [-0.74, -0.44])
    """ Computes the gradient of coeffs' * y with respect to the input x
        Args:
            - network: flux network
            - input: input to get saliency map for
            ------------------
            - coeffs: coefficients to get output (linear combination of network outputs)
    """
    grad_function(x) = coeffs' * network(x)
    _, b = Flux.pullback(() -> grad_function(input), Flux.params(input))
    return b(1)[input]
end

function integrated_gradients(network, input, baseline; coeffs = [-0.74, -0.44], res = 20)
    """
    Args:
        - network: flux network
        - input: input to get integrated gradients for
        - baseline: IG baseline (should be same size as input)
        ----------------
        - coeffs: coefficients to get output (linear combination of network outputs)
        - number of samples used to approximate integral
    """
    grad_function(x) = coeffs' * network(x)
    αs = collect(range(0.0, stop = 1.0, length = res))
    attrs = zeros(length(input))
    for i = 1:res
        interp_inp = αs[i] .* input .+ (1 .- αs[i]) .* baseline
        attrs += saliency_map(network, interp_inp)
    end
    return attrs ./ res
end

function get_mask(attrs, x; threshold = 0.9, masked_opacity = 0.2)
    """ Returns a masked image that highlights the pixels with the highest
        absolute attributions
        Args:
            - attrs: vector of attributions (same size as x)
            - x: input associated with the attrs (vector)
            -------------
            - threshold: highlight pixels above this percentile
            - masked_opacity: opacity for non-highlighted pixels
    """
    perc = quantile(attrs, threshold)
    mask = ones(length(attrs))
    mask[attrs .< perc] .= masked_opacity
    masked_input = mask .* x
    return masked_input
end


import SpecialFunctions: digamma, loggamma

@node Dirichlet Stochastic [ out, a ]

@average_energy Dirichlet (q_out::Dirichlet, q_a::PointMass) = -loggamma(sum(mean(q_a))) + sum(loggamma.(mean(q_a))) - sum((mean(q_a) .- 1.0) .* mean(log, q_out))


@testset "detectors" begin
  Random.seed!(42)
  x = vcat(rand(8,2), 5*rand(1,2), 30*rand(1,2))
  x1 = vcat(rand(20,3), 5*rand(1,3), 30*rand(1,3))
  @test rxdetect(x, 0.9) == [false, false, false, false, false, false, false, false, false, true]
  @test hosvdc4detect(x, 4., 2; b=2) == [false, false, false, false, false, false, false, false, true, true]
  @test hosvdc4detect(x1, 3.9, 1; b=1) == vcat(fill(false, 20), [true, true])
  @test rxdetect(x1, 0.9) == vcat(fill(false, 20), [true, true])
  ls = fill(true, 10)
  ls1 = [true, true, true, true, true, true, true, true, false, false]
  c = cumulants(x, 4)
  @test hosvdstep(x, ls, 4., 2, c[4])[1] == ls1
  @test hosvdstep(x, ls, 3., 1, c[4])[2] ≈ 1.147775879385989
  @test hosvdstep(x, ls, 3., 2, c[4])[2] ≈ 1.2715241637233354
  c = cumulants(x[ls1,:], 4)
  @test hosvdstep(x, ls1, 4., 2, c[4])[1] == ls1
  @test hosvdstep(x, ls1, 4., 2, c[4])[2] ≈ 1.6105157709082383
  m = momentarray(x,4,2)
  m1, t1 = updatemoments(m, size(x,1), x, ls1, ls)
  @test t1 == 8
  @test Array(m1[4]) ≈ Array(moment(x[1:8,:], 4))
end


module BFZ

using CompEcon: qnwnorm
using Interpolations

type BFZModel
    A::Float64
    B::Float64
    vbar::Float64
    lgbar::Float64
    ρ::Float64
    α::Float64
    β::Float64
    η::Float64
    ν::Float64
    δ::Float64
    ϵ::Vector{Float64}
    Π::Vector{Float64}
    x::AbstractVector
    k::AbstractVector
    xp::Matrix{Float64}
    gp::Matrix{Float64}
end

function BFZModel(;ρ=-1.0, α=-9.0, β=0.99, η=1/3, ν=0.1, δ=0.025, lgbar=0.004,
                   A=0.0, B=1.0, vbar=0.015^2, nk=35, nx=15, nϵ1=5)

    # quadrature weights and nodes
    ϵ, Π = qnwnorm(nϵ1, 0.0, 1.0)

    # grids for x and k
    x = linspace(-0.05, 0.05, nx)
    k = linspace(15.0, 30.0, nk)

    # grids for x_{t+1} and g_{t+1} -- apply law of motion
    xp = Array(Float64, nϵ1, nx)
    gp = Array(Float64, nϵ1, nx)

    for i=1:nx
        _x = x[i]
        for j=1:nϵ1
            xp[j, i] = A*_x + B*sqrt(vbar)*ϵ[j]
            gp[j, i] = exp(lgbar + xp[j, i])
        end

    end

    BFZModel(A, B, vbar, lgbar, ρ, α, β, η, ν, δ, ϵ, Π, x, k, xp, gp)
end

## Helpful functions
_Y(m::BFZModel, k) = (m.η .* k.^m.ν + (1 - m.η)).^(1/m.ν)

"""
Initializes the guess of the value function on the grid to be a small fraction
of consuming production each period. Use the same value for all x
"""
function initialize(m::BFZModel)
    JT = (((1-m.β) * (_Y(m, m.k) + (1-m.δ)*m.k).^(m.ρ)).^(1/m.ρ)) ./ 500
    Jvals = repeat(JT, inner=[1, length(m.x)])
    itp = scale(interpolate(Jvals, BSpline(Quadratic(Line())), OnGrid()),
                m.k, m.x)
    Jvals, itp
end

function envelope_method(m::BFZModel;maxiter=500, tol=1e-4)
    # simplify notation
    β, δ, ρ, η, ν, α = m.β, m.δ, m.ρ, m.η, m.ν, m.α
    nshocks = length(m.ϵ)
    nk, nx = length(m.k), length(m.x)

    # initialize interpolant for value function
    Jvals, itp = initialize(m)
    χpolicy = Array(Float64, nk, nx)
    Jupd = similar(χpolicy)

    # intermediate helper variables
    err = 100.0
    iter = 0
    start_time = time()

    # Solve this bad boy
    while err>tol && iter<maxiter

        iter += 1
        # Given the current vf, find the χ that solves the env condition
        for i_x=1:nx
            # Pull out current x
            xt = m.x[i_x]

            for i_k=1:nk
                # Pull out current k
                kt = m.k[i_k]

                yt = _Y(m, kt)  # production

                dJ_k = gradient(itp, kt, xt)[1]  # evaluate dJ/dk
                dJ_k = dJ_k < 1e-12 ? 0. : dJ_k

                Jt = itp[kt, xt] # evaluate J

                # use envelope and budget constraint to get c and χ, then save χ
                # in policy matrix
                ct = (dJ_k/((1-β)*Jt^(1-ρ)*(yt^(1-ν)*η*kt^(ν-1)+1-δ)))^(1./(ρ-1))
                χt = yt + (1 - δ)*kt - ct
                χpolicy[i_k, i_x] = χt

                # calculate the certainty equivalent
                expterm = 0.
                for i_eps=1:nshocks
                    xp = m.xp[i_eps, i_x]
                    gp = m.gp[i_eps, i_x]
                    jp = itp[χt/gp, xp]
                    expterm += m.Π[i_eps] * (jp * gp)^α
                end
                μ = expterm^(1/α)

                # evaluate value function
                Jupd[i_k, i_x] = ((1 - β)*ct^ρ + β*μ^ρ)^(1/ρ)

            end
        end

        err = maxabs(Jvals - Jupd)
        copy!(Jvals, Jupd)
        itp = scale(interpolate(Jupd, BSpline(Quadratic(Line())), OnGrid()),
                    m.k, m.x)
        @show iter, err, time() - start_time

    end

    return itp
end

end  # module


# Realization and dynamic range scaling
using RSDeltaSigmaPort
using RSDeltaSigmaPort.EasyPlot #set, cons
import Printf: @sprintf
j=im

function stringify(vA, digits=6)
	#ctx=IOContext(stdout, :compact=>true)
	#IOContext does not support # of digits
	#_s(v) = @sprintf("%.6f", v) #Cannot parameterize format specifier
	return [round(v, digits=6) for v in vA]
end

#==
===============================================================================#
println("\n*** Modulator realization and scaling")
OSR=42
order=5; opt=1

NTF = synthesizeNTF(order, OSR, opt=opt)
plot = plotNTF(NTF, OSR)
plot.title = string(order, "th-Order Modulator")
#saveimage(:png, "dsdemo3_ntf.png", plot, AR=2/1, width=900)
displaygui(plot)

@info("Realizing NTF..."); flush(stdout); flush(stderr)
#-------------------------------------------------------------------------------
a,g,b,c = realizeNTF(NTF)
b = [b[1] zeros(1,length(b)-1)] #Use a single feed-in for the input

println("\nUnscaled modulator")
println("   DAC feedback coefficients = ", join(stringify(a), ", "))
println("   resonator feedback coefficients = ", join(stringify(g), ", "))

println()
@info("Calculating the state maxima...")
#-------------------------------------------------------------------------------
ABCD = stuffABCD(a,g,b,c)
u = range(0, stop=0.6, length=30)
N = 10^4
plot = plotStateMaxima(u, ABCD, N=N)
displaygui(plot)

println()
@info("Calculating the scaled coefficients...")

#-------------------------------------------------------------------------------
ABCDs, umax = scaleABCD(ABCD, N=N)
as, gs, bs, cs = mapABCD(ABCDs)
println("\nScaled modulator, umax=", @sprintf("%.2f", umax))
println("   DAC feedback coefficients = ", join(stringify(as), ", "))
println("   resonator feedback coefficients = ", join(stringify(gs), ", "))
println("   interstage coefficients = ", join(stringify(cs), ", "))
println("   feed-in coefficients = ", join(stringify(bs), ", "))

println()
@info("Calculating the state maxima...")
#-------------------------------------------------------------------------------
u = range(0, stop=umax, length=30)
N = 10^4

plot = plotStateMaxima(u, ABCDs, N=N)
displaygui(plot)

:END_OF_DEMO


@testset "KPLS Pediction Tests (in sample)" begin

    @testset "Test KPLS Single Non Linear Target" begin

        Random.seed!(1)

        z(x)     = 4.26 * (exp.(-x) - 4 * exp.(-2.0*x) + 3 * exp.(-3.0*x))
        x_values = Array(range(0.0,step=3.5,length=100))
        z_pure   = z(x_values)
        noise    = Random.randn(100)
        z_noisy  = z_pure + noise
        X        = MLJBase.table(collect(x_values)[:,:])
        Y        = z_noisy #[:,:] #z_pure

        pls_pipe       = MLJBase.@pipeline prediction_type=:deterministic Stand.Standardizer() PartialLeastSquaresRegressor.KPLSRegressor(n_factors=1,kernel="rbf",width=0.01)  target = Stand.Standardizer()

		pls_machine    = MLJBase.machine(pls_pipe, X, Y)

        train = range(1,stop=length(X))
        MLJBase.fit!(pls_machine, rows=train,force=true)
        yhat = MLJBase.predict(pls_machine, rows=train);
        @test MLJBase.mae(yhat, Y[train]) |> mean < 1e-2

    end

    @testset "Test KPLS Single Target (Linear Target)" begin

        X        = MLJBase.table([1 2; 2 4; 4.0 6])
        Y        = [-2; -4; -6.0] #[:,:]

        pls_pipe       = MLJBase.@pipeline prediction_type=:deterministic Stand.Standardizer() PartialLeastSquaresRegressor.KPLSRegressor(n_factors=1,kernel="rbf",width=0.01)  target = Stand.Standardizer()

		pls_machine    = MLJBase.machine(pls_pipe, X, Y)

        train = range(1,stop=length(X))
        MLJBase.fit!(pls_machine, rows=train,force=true)
        yhat = MLJBase.predict(pls_machine, rows=train);
        @test MLJBase.mae(yhat, Y[train]) |> mean < 1e-2

        X        = MLJBase.table([1 2; 2 4; 4.0 6])
        Y        = [2; 4; 6.0]#[:,:]

        pls_pipe       = MLJBase.@pipeline prediction_type=:deterministic Stand.Standardizer() PartialLeastSquaresRegressor.KPLSRegressor(n_factors=1,kernel="rbf",width=0.01)  target = Stand.Standardizer()

		pls_machine    = MLJBase.machine(pls_pipe, X, Y)

        train = range(1,stop=length(X))
        MLJBase.fit!(pls_machine, rows=train,force=true)
        yhat = MLJBase.predict(pls_machine, rows=train);
        @test MLJBase.mae(yhat, Y[train]) |> mean < 1e-2

    end

    @testset "Test KPLS Multiple Target (Linear Target)" begin

        X        = MLJBase.table([1; 2; 3.0][:,:])
        Y        = MLJBase.table([1 1; 2 2; 3 3.0]) #[:,:]

        pls_pipe       = MLJBase.@pipeline prediction_type=:deterministic Stand.Standardizer() PartialLeastSquaresRegressor.KPLSRegressor(n_factors=1,kernel="rbf",width=0.01)  target = Stand.Standardizer()

		pls_machine    = MLJBase.machine(pls_pipe, X, Y)

        train = range(1,stop=length(X))
        MLJBase.fit!(pls_machine, rows=train,force=true)
        yhat = MLJBase.predict(pls_machine, rows=train);
        @test abs.(MLJBase.matrix(yhat) .- MLJBase.matrix(Y)[train,:]) |> mean < 1e-6

        X        = MLJBase.table([1; 2; 3.0][:,:])
        Y        = MLJBase.table([1 -1; 2 -2; 3 -3.0])#[:,:]

        pls_pipe       = MLJBase.@pipeline prediction_type=:deterministic Stand.Standardizer() PartialLeastSquaresRegressor.KPLSRegressor(n_factors=1,kernel="rbf",width=0.01)  target = Stand.Standardizer()

		pls_machine    = MLJBase.machine(pls_pipe, X, Y)

        train = range(1,stop=length(X))
        MLJBase.fit!(pls_machine, rows=train,force=true)
        yhat = MLJBase.predict(pls_machine, rows=train);
        @test abs.( MLJBase.matrix(yhat) .- MLJBase.matrix(Y)[train,:]) |> mean < 1e-6

        @testset "Linear Prediction Tests " begin

        X        = MLJBase.table([1 2; 2 4; 4 6.0])
        Y        = MLJBase.table([4 2;6 4;8 6.0])#[:,:]

        pls_pipe       = MLJBase.@pipeline prediction_type=:deterministic Stand.Standardizer() PartialLeastSquaresRegressor.KPLSRegressor(n_factors=1,kernel="rbf",width=0.01)  target = Stand.Standardizer()

		pls_machine    = MLJBase.machine(pls_pipe, X, Y)

        train = range(1,stop=length(X))
        MLJBase.fit!(pls_machine, rows=train,force=true)
        yhat = MLJBase.predict(pls_machine, rows=train);
        @test abs.( MLJBase.matrix(yhat) .- MLJBase.matrix(Y)[train,:]) |> mean < 1e-6

        X           = MLJBase.table([1 -2; 2 -4; 4 -6.0])
        Y           = MLJBase.table([-4 -2;-6 -4;-8 -6.0])#[:,:]

        pls_pipe       = MLJBase.@pipeline prediction_type=:deterministic Stand.Standardizer() PartialLeastSquaresRegressor.KPLSRegressor(n_factors=1,kernel="rbf",width=0.01)  target = Stand.Standardizer()

		pls_machine    = MLJBase.machine(pls_pipe, X, Y)

        train = range(1,stop=length(X))
        MLJBase.fit!(pls_machine, rows=train,force=true)
        yhat = MLJBase.predict(pls_machine, rows=train);
        @test abs.(MLJBase.matrix(yhat) .- MLJBase.matrix(Y)[train,:]) |> mean < 1e-6

        end

    end

end;


using Test,RSCG
using SparseArrays
using LinearMaps

function test1(n,M)
    σ = zeros(ComplexF64,M)
    σmin = -10.0*im
    σmax = 10.0*im
    for i=1:M
        σ[i] = (i-1)*(σmax-σmin)/(M-1) + σmin
    end
    A = make_mat(n)
    i = 1
    j = 1
    Gij = greensfunctions(i,j,σ,A)

    Ii = spzeros(ComplexF64,n,n)
    for ii=1:n
        Ii[ii,ii] = σ[1]
    end
    G = inv(Matrix(Ii-A))

    @test abs(G[1,1]-Gij[1]) < 1e-7
end

function test1(n,M)
    σ = zeros(ComplexF64,M)
    σmin = -10.0*im
    σmax = 10.0*im
    for i=1:M
        σ[i] = (i-1)*(σmax-σmin)/(M-1) + σmin
    end
    A = make_mat(n)
    i = 1
    j = 1
    Gij = greensfunctions(i,j,σ,A)

    Ii = spzeros(ComplexF64,n,n)
    for ii=1:n
        Ii[ii,ii] = σ[1]
    end
    G = inv(Matrix(Ii-A))
    println("Residual ",abs(G[1,1]-Gij[1]))
    @test abs(G[1,1]-Gij[1]) < 1e-7
end

function test2(n,M)
    σ = zeros(ComplexF64,M)
    σmin = -10.0*im
    σmax = 10.0*im
    for i=1:M
        σ[i] = (i-1)*(σmax-σmin)/(M-1) + σmin
    end
    A2 = make_matc(n)
    i = 1
    j = 1
    Gij = greensfunctions(i,j,σ,A2)

    Ii = spzeros(ComplexF64,2n,2n)
    for ii=1:2n
        Ii[ii,ii] = σ[1]
    end
    G = inv(Matrix(Ii-A2))
    println("Residual ",abs(G[1,1]-Gij[1]))
    @test abs(G[1,1]-Gij[1]) < 1e-7
end

function test3(n,M)
    σ = zeros(ComplexF64,M)
    σmin = -4.0*im+0.2
    σmax = 4.0*im -0.3
    for i=1:M
        σ[i] = (i-1)*(σmax-σmin)/(M-1) + σmin
    end
    A2 = make_matc(n)
    vec_i = [1,4,div(n,2),n]
    j = div(n,2)
    Gij = greensfunctions(vec_i,j,σ,A2)

    Ii = spzeros(ComplexF64,2n,2n)
    for ii=1:2n
        Ii[ii,ii] = σ[div(M,2)]
    end
    G = inv(Matrix(Ii-A2))

    for ii = 1:length(vec_i)
        i = vec_i[ii]
        println("Residual ",abs(G[i,j]-Gij[div(M,2),ii]) )
        @test abs(G[i,j]-Gij[div(M,2),ii]) < 1e-7
    end
end

function set_diff(v)
    function calc_diff!(y::AbstractVector, x::AbstractVector)
        n = length(x)
        length(y) == n || throw(DimensionMismatch())
        μ = -1.5
        for i=1:n
            dx = 1
            jp = i+dx
            jp += ifelse(jp > n,-n,0) #+1方向
            dx = -1
            jm = i+dx
            jm += ifelse(jm < 1,n,0) #-1方向
            y[i] = v*(x[jp]+x[jm])-μ*x[i]
        end

        return y
    end
    (y,x) -> calc_diff!(y,x)
end

function test4(n,M)
    σ = zeros(ComplexF64,M)
    σmin = -4.0*im-1.2
    σmax = 4.0*im +4.3
    for i=1:M
        σ[i] = (i-1)*(σmax-σmin)/(M-1) + σmin
    end
    A = set_diff(-1.0)
    D = LinearMap(A, n; ismutating=true,issymmetric=true)
    A2 = make_mat(n)

    vec_i = [2,4,div(n,2),n]
    j = n
    Gij = greensfunctions(vec_i,j,σ,D)

    Ii = spzeros(ComplexF64,n,n)
    for ii=1:n
        Ii[ii,ii] = σ[div(M,2)]
    end
    G = inv(Matrix(Ii-A2))

    for ii = 1:length(vec_i)
        i = vec_i[ii]
        println("Residual ",abs(G[i,j]-Gij[div(M,2),ii]) )
        @test abs(G[i,j]-Gij[div(M,2),ii]) < 1e-7
    end
end

function make_mat(n)
    A = spzeros(Float64,n,n)
    t = -1.0
    μ = -1.5
    for i=1:n
        dx = 1
        jp = i+dx
        jp += ifelse(jp > n,-n,0) #+1方向
        dx = -1
        jm = i+dx
        jm += ifelse(jm < 1,n,0) #-1方向
        A[i,jp] = t
        A[i,i] = -μ
        A[i,jm] = t
    end
    return A
end

function make_matc(n)
    A = make_mat(n)
    A2 = spzeros(ComplexF64,2n,2n)

    Delta = spzeros(ComplexF64,n,n)
    phi = rand(n)
    for i=1:n
        Delta[i,i] = exp(im*phi[i])
    end

    A2[1:n,1:n] = A[1:n,1:n]
    A2[1+n:2n,1+n:2n] = -conj(A[1:n,1:n])
    A2[1:n,1+n:2n] = Delta[1:n,1:n]
    A2[1+n:2n,1:n] = Delta'[1:n,1:n]

    return A2
end

@testset "SparseArrays" begin
    @testset "real" begin
        M = 100
        σ = zeros(M)
        n = 10
        println("n = $(n)")
        @time test1(n,M)
        n = 100
        println("n = $(n)")
        @time test1(n,M)
        n=  1000
        println("n = $(n)")
        @time test1(n,M)
    end

    @testset "complex" begin
        M = 100
        σ = zeros(M)
        n = 5
        println("n = $(2n)")
        @time  test2(n,M)
        n = 50
        println("n = $(2n)")
        @time  test2(n,M)
        n=  500
        println("n = $(2n)")
        @time  test2(n,M)
    end

    @testset "complex: multi values" begin
        M = 100
        σ = zeros(M)
        n = 5
        println("n = $(2n)")
        @time  test3(n,M)
        n = 50
        println("n = $(2n)")
        @time  test3(n,M)
        n=  500
        println("n = $(2n)")
        @time  test3(n,M)
    end
end

@testset "Linearmaps" begin
    @testset "complex: multi values" begin
        M = 100
        σ = zeros(M)
        n = 10
        println("n = $(n)")
        @time  test4(n,M)
        n = 100
        println("n = $(n)")
        @time  test4(n,M)
        n=  1000
        println("n = $(n)")
        @time  test4(n,M)
    end
end


immutable Exponential <: ContinuousUnivariateDistribution
    β::Float64 		# note: scale not rate

    function Exponential(β::Real)
        β > zero(β) || error("scale must be positive")
        new(float64(β))
    end

    Exponential() = new(1.0)
end

@distr_support Exponential 0.0 Inf

#### Parameters

scale(d::Exponential) = d.β
rate(d::Exponential) = 1.0 / d.β

params(d::Exponential) = (d.β,)

#### Statistics

mean(d::Exponential) = scale(d)

median(d::Exponential) = logtwo * scale(d)

mode(d::Exponential) = 0.0

var(d::Exponential) = scale(d)^2

skewness(d::Exponential) = 2.0

kurtosis(d::Exponential) = 6.0

entropy(d::Exponential) = 1.0 + log(scale(d))

#### Evaluation

zval(d::Exponential, x::Float64) = x / d.β
xval(d::Exponential, z::Float64) = z * d.β

pdf(d::Exponential, x::Float64) = (λ = rate(d); x < 0.0 ? 0.0 : λ * exp(-λ * x))
logpdf(d::Exponential, x::Float64) =  (λ = rate(d); x < 0.0 ? -Inf : log(λ) - λ * x)

cdf(d::Exponential, x::Float64) = x > 0.0 ? -expm1(-zval(d, x)) : 0.0
ccdf(d::Exponential, x::Float64) = x > 0.0 ? exp(-zval(d, x)) : 0.0
logcdf(d::Exponential, x::Float64) = x > 0.0 ? log1mexp(-zval(d, x)) : -Inf
logccdf(d::Exponential, x::Float64) = x > 0.0 ? -zval(d, x) : 0.0

quantile(d::Exponential, p::Float64) = -xval(d, log1p(-p))
cquantile(d::Exponential, p::Float64) = -xval(d, log(p))
invlogcdf(d::Exponential, lp::Float64) = -xval(d, log1mexp(lp))
invlogccdf(d::Exponential, lp::Float64) = -xval(d, lp)

gradlogpdf(d::Exponential, x::Float64) = x > 0.0 ? -rate(d) : 0.0

mgf(d::Exponential, t::Real) = 1.0/(1.0 - t * scale(d))
cf(d::Exponential, t::Real) = 1.0/(1.0 - t * im * scale(d))

#### Sampling

rand(d::Exponential) = xval(d, randexp())

#### Fit model

immutable ExponentialStats <: SufficientStats
    sx::Float64   # (weighted) sum of x
    sw::Float64   # sum of sample weights

    ExponentialStats(sx::Real, sw::Real) = new(float64(sx), float64(sw))
end

suffstats(::Type{Exponential}, x::Array) = ExponentialStats(sum(x), length(x))
suffstats(::Type{Exponential}, x::Array, w::Array) = ExponentialStats(dot(x, w), sum(w))

fit_mle(::Type{Exponential}, ss::ExponentialStats) = Exponential(ss.sx / ss.sw)


# coding: utf-8

# In[1]:

#this is a more advanced knapsack problem
#we have a new variable called groups
#we are only allowed to select at most one item from each group
#please note julia version is a lot more complex than python version
#simply becuz julia starts index at 1 and doesnt allow negative number indexing
#please check the following link for the basic knapsack problem
# https://github.com/je-suis-tm/recursion-and-dynamic-programming/blob/master/knapsack.jl

# In[2]:

#solve multiple choice knapsack via dynamic programming
function knapsack_multichoice(total_weight,values,weights,groups)

    #julia starts index at 1 which is why we use length(values)
    #create a nested list with size of number of items*weights
    array=[[0 for _ in 1:total_weight] for _ in 1:length(values)]
    path=[[[] for _ in 1:total_weight] for _ in 1:length(values)]

    #now we begin our traversal on all elements in matrix
    for i in 1:length(values)
        for j in 1:total_weight

            #-1 doesnt work as index in julia
            #have to find a way to get around
            if i==1
                if weights[1]<=j
                    array[1][j]=values[1]
                    path[1][j]=[1]
                end
                continue
            end

            #this is the part to check if adding item i would exceed the current capacity j
            #if it does,we go to the previous status
            #if not,we shall find out whether adding item i would be the new optimal
            if weights[i]<j

                #initialize
                subset_max=0
                target=1

                #we only select one item from each group
                #we will find the item that maximizes the value in each group
                prev_group=groups[i]-1

                #get column of the matrix
                subset=[row[j-weights[i]] for row in array]

                #find the item that maximizes the value in the previous group
                for k in 1:length(values)
                    if groups[k]==prev_group && subset[k]>subset_max
                        subset_max=subset[k]
                        target=k
                    end
                end

                #dynamic programming
                if subset_max+values[i]>array[i-1][j]
                    array[i][j]=subset_max+values[i]
                    path[i][j]=[path[target][j-weights[i]];[i]]
                elseif subset_max+values[i]==array[i-1][j] && weights[i]<weights[path[i-1][j][end]]
                    array[i][j]=subset_max+values[i]
                    path[i][j]=[path[target][j-weights[i]];[i]]
                else
                    array[i][j]=array[i-1][j]
                    path[i][j]=path[i-1][j]
                end

            elseif weights[i]==j

                #dynamic programming
                if values[i]>array[i-1][j]
                    array[i][j]=values[i]
                    path[i][j]=[i]
                elseif values[i]==array[i-1][j] && weights[i]<weights[path[i-1][j][end]]
                    array[i][j]=values[i]
                    path[i][j]=[i]
                else
                    array[i][j]=array[i-1][j]
                    path[i][j]=path[i-1][j]
                end

            else
                array[i][j]=array[i-1][j]
                path[i][j]=path[i-1][j]
            end
        end
    end
    return array,path
end

# In[3]:

#a simple test
values=[60,80,100,110,120,150]
weights=[10,15,20,25,30,35]
groups=[0,0,1,1,2,2]
total_weight=50

# In[4]:

array,path=knapsack_multichoice(total_weight,values,
                                weights,groups)

println(array[length(weights)][total_weight])
println(path[length(weights)][total_weight])


function mrp_quaternion_map(mrp)
	n2 = mrp[1] * mrp[1] + mrp[2] * mrp[2] + mrp[3] * mrp[3]
    M = 2/(1+n2)
    return SVector{4}([(1-n2)/(1+n2), M*mrp[1], M*mrp[2], M*mrp[3]])
end

mrp_rotation_matrix(mrp) = quaternion_rotation_matrix(mrp_quaternion_map(mrp))

mrp = rand(3)

norm(MRP(mrp...) - mrp_rotation_matrix(mrp))

@variables mrp_sym[1:3]

MRP(mrp_sym...)

mrp_rotation_matrix(mrp_sym)


if !(VERSION.major == 0 && VERSION.minor <= 6)
    using Test
else
    using Base.Test
end
using FinancialToolbox

print_colored("Starting Complex Number Test\n", :green)

#Test Parameters
testToll = 1e-14;
spot = 10;
K = 10;
r = 0.02;
T = 2.0;
sigma = 0.2;
d = 0.01;
assert_(value, toll) = @test abs(value) < toll

#EuropeanCall Option
PriceCall = blsprice(spot, K, r, T, sigma, d);
DeltaCall = blsdelta(spot, K, r, T, sigma, d);
ThetaCall = blstheta(spot, K, r, T, sigma, d);
LambdaCall = blslambda(spot, K, r, T, sigma, d);
RhoCall = blsrho(spot, K, r, T, sigma, d);

#EuropeanPut Option
PricePut = blsprice(spot, K, r, T, sigma, d, false);
DeltaPut = blsdelta(spot, K, r, T, sigma, d, false);
ThetaPut = blstheta(spot, K, r, T, sigma, d, false);
RhoPut = blsrho(spot, K, r, T, sigma, d, false);
LambdaPut = blslambda(spot, K, r, T, sigma, d, false);

#Equals for both Options
Gamma = blsgamma(spot, K, r, T, sigma, d);
Vega = blsvega(spot, K, r, T, sigma, d);

## Complex Test with Complex Step Approximation for European Call
#Test parameters
DerToll = 1e-13;
di = 1e-15;
df(f, x) = f(x + 1im * di) / di;
#Function definition
Fcall1(spot) = blsprice(spot, K, r, T, sigma, d);
Gcall1(r) = blsprice(spot, K, r, T, sigma, d);
Hcall1(T) = blsprice(spot, K, r, T, sigma, d);
Lcall1(sigma) = blsprice(spot, K, r, T, sigma, d);
Pcall1(spot) = blsprice(spot, K, r, T, sigma, d) * spot.re / blsprice(spot.re, K, r, T, sigma, d);
#TEST
print_colored("--- European Call Sensitivities: Complex Step Approximation\n", :yellow)
print_colored("-----Testing Delta\n", :blue);
assert_(df(Fcall1, spot).im - DeltaCall, DerToll)
print_colored("-----Testing Rho\n", :blue);
assert_(df(Gcall1, r).im - RhoCall, DerToll)
print_colored("-----Testing Theta\n", :blue);
assert_(-df(Hcall1, T).im - ThetaCall, DerToll)
print_colored("-----Testing Lambda\n", :blue);
assert_(df(Pcall1, spot).im - LambdaCall, DerToll)

## Complex Test with Complex Step Approximation for European Put
#Function definition
Fput1(spot) = blsprice(spot, K, r, T, sigma, d, false);
Gput1(r) = blsprice(spot, K, r, T, sigma, d, false);
Hput1(T) = blsprice(spot, K, r, T, sigma, d, false);
Pput1(spot) = blsprice(spot, K, r, T, sigma, d, false) * spot.re / blsprice(spot.re, K, r, T, sigma, d, false);
#TEST
print_colored("--- European Put Sensitivities: Complex Step Approximation\n", :yellow)
print_colored("-----Testing Delta\n", :blue);
assert_(df(Fput1, spot).im - DeltaPut, DerToll)
print_colored("-----Testing Rho\n", :blue);
assert_(df(Gput1, r).im - RhoPut, DerToll)
print_colored("-----Testing Theta\n", :blue);
assert_(-df(Hput1, T).im - ThetaPut, DerToll)
print_colored("-----Testing Lambda\n", :blue);
assert_(df(Pput1, spot).im - LambdaPut, DerToll)

print_colored("-----Testing Vega\n", :blue);
assert_(df(Lcall1, sigma).im - Vega, DerToll)
print_colored("Complex Number Test Passed\n", :green)
println("")

#TEST OF INPUT VALIDATION
print_colored("Starting Input Validation Test Complex\n", :magenta)
print_colored("----Testing Negative  Spot Price \n", :cyan)
@test_throws(ErrorException, blsprice(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blkprice(-spot + 1im, K, r, T, sigma))
@test_throws(ErrorException, blsdelta(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blsgamma(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blstheta(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blsrho(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blspsi(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blslambda(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blsvanna(-spot + 1im, K, r, T, sigma, d))
@test_throws(ErrorException, blsvega(-spot + 1im, K, r, T, sigma, d))

print_colored("----Testing Negative  Strike Price \n", :cyan)
@test_throws(ErrorException, blsprice(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blkprice(spot, -K + 1im, r, T, sigma))
@test_throws(ErrorException, blsdelta(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blsgamma(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blstheta(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blsrho(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blspsi(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blslambda(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blsvanna(spot, -K + 1im, r, T, sigma, d))
@test_throws(ErrorException, blsvega(spot, -K + 1im, r, T, sigma, d))

print_colored("----Testing Negative  Time to Maturity \n", :cyan)
@test_throws(ErrorException, blsprice(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blkprice(spot, K, r, -T + 1im, sigma))
@test_throws(ErrorException, blsdelta(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blsgamma(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blstheta(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blsrho(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blspsi(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blslambda(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blsvanna(spot, K, r, -T + 1im, sigma, d))
@test_throws(ErrorException, blsvega(spot, K, r, -T + 1im, sigma, d))

print_colored("----Testing Negative  Volatility \n", :cyan)
@test_throws(ErrorException, blsprice(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blkprice(spot, K, r, T, -sigma + 1im))
@test_throws(ErrorException, blsdelta(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blsgamma(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blstheta(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blsrho(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blspsi(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blslambda(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blsvanna(spot, K, r, T, -sigma + 1im, d))
@test_throws(ErrorException, blsvega(spot, K, r, T, -sigma + 1im, d))

print_colored("Complex Input Validation Test Passed\n", :magenta)


p = parse.(Int, readlines("Dec1_data.txt"))

# Question 1
fun(x) = div(x, 3) - 2

# test cases
fun(1969)    # should be 654
fun(100756)  # should be 33583

# result 1
sum(fun, p)

# Question 2
function fun2(rem)
    ret = fun(rem)
    ret <= 0 ? 0 : ret += fun2(ret)
end

# test cases
fun2(1969)    # should be 966
fun2(100756)  # should be 100756

# result 2
sum(fun2, p)


# Tests for interaction information function

# Test interaction information formula
@test get_interaction_information(arr1, arr2, arr3) ≈ get_conditional_mutual_information(arr1, arr2, arr3) - get_mutual_information(arr1, arr2)
println("Interaction information formula passed.")


"""
A package for dealing with the estimation of (co)variance matrices.

    1. Define CovarianceEstimator(s) which are robust to heteroskedasticity, time-dependence, amd spatial dependence (cluster)
    2. Optionally, interface with GLM.jl to provide estimates of coefficients' standard errors which are robust

These use cases utilize different parts of this package, make sure you read the
documentation.
"""

module CovarianceMatrices

using CategoricalArrays
using LinearAlgebra
using Requires: @require
using StatsBase: CovarianceEstimator
using StatsModels
using Statistics
include("types.jl")
include("HAC.jl")
include("HC.jl")
include("CRHC.jl")
include("VARHAC.jl")
include("lrvar.jl")
include("CovarianceMatrix.jl")
# using GLM
# include("glm.jl")

function __init__()
    @require GLM="38e38edf-8417-5370-95a0-9cbb8c7f171a" include("glm.jl")
end

export Andrews,
       BartlettKernel,
       CRHC0,
       CRHC1,
       CRHC2,
       CRHC3,
       CovarianceMatrix,
       HC0,
       HC1,
       HC2,
       HC3,
       HC4,
       HC4m,
       HC5,
       NeweyWest,
       ParzenKernel,
       QuadraticSpectralKernel,
       TruncatedKernel,
       TukeyHanningKernel,
       lrvar,
       lrvarmatrix,
       optimalbandwidth,
       #stderror,
       #vcov,
       vcovmatrix

end # module


function modalparami(A,B,C,dt,dofu)
	#
	# Compute natural frequencies, damping ratios, mode shapes
	# and modal masses from A, C, D and dt
	#
	# dofu: degrees of freedon where inputs are applied
	# 	example : consider we measure at points (a,b,c,d,e) and
	# 	the inputs are located at points b,e => dofu = [2,5]
	#
	# javier.cara@upm.es, 2016-02
	#

	# size
	no,ns = size(C)

	# invariants
	# ---------------------------------
	# eigenvalues and eigenvectors
	D,V = eigen(A)
	# L
	Lambda = log.(D)/dt

	# invariante V^{-1}*B
	VinvB = V\B

	# invariante C*V
	CV = C*V

	# modal parameters
	# ---------------------------------
	# allocating
	wm1 = zeros(ns)
	zm1 = zeros(ns)
	vm1 = zeros(no,ns)
	mm1 = zeros(ns)

	s = 1
	ss = 1
	while s < ns
		if abs(imag(D[s])) > 1e-10 # complex number
			wm1[ss] = abs(Lambda[s])
			zm1[ss] = -real(Lambda[s])/wm1[ss]

			vm0 = CV[:,s]/Lambda[s]^2
			# maximum element = 1
			maxval = vm0[1]
			for j in 2:no
				if abs(vm0[j]) > abs(maxval)
					maxval = vm0[j]
				end
			end
			vm1[:,ss] = real(vm0/maxval) # in theory, it should be a real vector

			c = (D[s] - 1.0)/( Lambda[s]*(Lambda[s] - conj(Lambda[s])) ) * (vm1[dofu,ss:ss]')/(VinvB[s:s,:]*maxval) # c is a vector with 1 component
			mm1[ss] = abs(real( c[1] ) )

			# next value
			s = s+2
			ss = ss+1

		 else
		    # real eigenvalue
		    s = s+1
		 end
	end

	# deleting zero values
	wm = wm1[1:ss-1]
	zm = zm1[1:ss-1]
	vm = vm1[:,1:ss-1]
	mm = mm1[1:ss-1]

	# sorting frequencies
	pos = sortperm(wm)
	wm = wm[pos]
	zm = zm[pos]
	vm = vm[:,pos]
	mm = mm[pos]

	return wm,zm,vm,mm

end


# Plot ROC curve
function plot_ROC_curve(y_cond::Vector{Int}, y_prob::Vector{Float64})

    # Fixed counts
    population = length(y_cond)
    cond_pos = sum(y_cond .== 1)
    cond_neg = sum(y_cond .== 0)
    accuracy = round(sum(y_cond .== round(Int, y_prob)) / population, 2)

    # Number of plot points
    points = 1001
    max = points - 1

    # Calculate TPR and FPR for varied discrimination
    TPR, FPR = zeros(points), zeros(points)
    for d = 0:1:max
        TP, FP = 0, 0
        for i=1:population
            if y_prob[i] >= (max - d) / max
                y_cond[i] == 1 ? TP += 1 : FP += 1
            end
        end
        TPR[d + 1] = TP / cond_pos
        FPR[d + 1] = FP / cond_neg
    end

    # Accuracy
    TP, FP = 0, 0
    for i = 1:population
        if y_prob[i] >= 0.5
            y_cond[i] == 1 ? TP += 1 : FP += 1
        end
    end
    TPR1 = TP / cond_pos
    FPR1 = FP / cond_neg
    df1 = DataFrame(x = FPR1, y = TPR1, label = "Accuracy = $accuracy")

    # AUC
    AUC = 0
    for i=2:length(TPR)
        AUC += (TPR[i] + TPR[i - 1]) * 0.5 * (FPR[i] - FPR[i - 1])
    end
    AUC = round(AUC, 2)
    df2 = DataFrame(x = [0.65], y = [0.3], label = "AUC = $AUC")

    Sigma = Char(931)

    # Plot ROC curve
    plot(
         layer(df1, x="x", y="y", label="label", Geom.point, Geom.label,
               Stat.xticks(ticks=[FPR1]), Stat.yticks(ticks=[TPR1]),
               Theme(default_color=colorant"red", default_point_size=3pt)),
         layer(x=FPR, y=TPR, Geom.step),
         layer(x=[0.0; 1.0], y=[0.0; 1.0], Geom.line,
               Stat.xticks(ticks=[0.0, 0.5, 1.0]), Stat.yticks(ticks=[0.0, 0.5, 1.0]),
               Theme(default_color=colorant"grey", line_style=Gadfly.get_stroke_vector(:dash))),
         layer(df2, x="x", y="y", label="label", Geom.label),
         Scale.x_continuous(minvalue=0., maxvalue=1., labels=x -> @sprintf("%0.2f", x)),
         Scale.y_continuous(minvalue=0., maxvalue=1., labels=y -> @sprintf("%0.2f", y)),
         Guide.ylabel("True Positive Rate \n
($Sigma True Positives / $Sigma Condition Positive)"),
         Guide.xlabel("False Positive Rate \n
($Sigma False Positives / $Sigma Condition Negative)")
         )
end


"""
    PerspectiveMap()

Construct a perspective transformation. The perspective transformation takes,
e.g., a point in 3D space and "projects" it onto a 2D virtual screen of an ideal
pinhole camera (at distance `1` away from the camera). The camera is oriented
towards the positive-Z axis (or in general, along the final dimension) and the
sign of the `x` and `y` components is preserved for objects in front of the
camera (objects behind the camera are also projected and therefore inverted - it
is up to the user to cull these as necessary).

This transformation is designed to be used in composition with other coordinate
transformations, defining e.g. the position and orientation of the camera. For
example:

    cam_transform = PerspectiveMap() ∘ inv(AffineMap(cam_rotation, cam_position))
    screen_points = map(cam_transform, points)

(see also `cameramap`)
"""
struct PerspectiveMap <: Transformation
end

function (::PerspectiveMap)(v::AbstractVector)
    scale =  1/v[end]
    return [v[i] * scale for i in 1:length(v)-1]
end

@inline function (::PerspectiveMap)(v::StaticVector)
    return pop(v) * inv(v[end])
end

Base.isapprox(::PerspectiveMap, ::PerspectiveMap; kwargs...) = true

"""
    cameramap()
    cameramap(scale)
    cameramap(scale, offset)

Create a transformation that takes points in real space (e.g. 3D) and projects
them through a perspective transformation onto the focal plane of an ideal
(pinhole) camera with the given properties.

The `scale` sets the scale of the screen. For a standard digital camera, this
would be `scale = focal_length / pixel_size`. Non-square pixels are supported
by providing a pair of scales in a tuple, `scale = (scale_x, scale_y)`. Positive
scales represent a camera looking in the +z axis with a virtual screen in front
of the camera (the x,y coordinates are not inverted compared to 3D space). Note
that points behind the camera (with negative z component) will be projected
(and inverted) onto the image coordinates and it is up to the user to cull
such points as necessary.

The `offset = (offset_x, offset_y)` is used to define the origin in the imaging
plane. For instance, you may wish to have the point (0,0) represent the top-left
corner of your imaging sensor. This measurement is in the units after applying
`scale` (e.g. pixels).

(see also `PerspectiveMap`)
"""
cameramap() = PerspectiveMap()
cameramap(scale::Number) =
    LinearMap(UniformScaling(scale)) ∘ PerspectiveMap()
cameramap(scale::Tuple{Number, Number}) =
    LinearMap(@SMatrix([scale[1] 0; 0 scale[2]])) ∘ PerspectiveMap()
cameramap(scale::Number, offset::Tuple{Number,Number}) =
    AffineMap(UniformScaling(scale), SVector(-offset[1], -offset[2])) ∘ PerspectiveMap()
cameramap(scale::Tuple{Number, Number}, offset::Tuple{Number,Number}) =
    AffineMap(@SMatrix([scale[1] 0; 0 scale[2]]), SVector(-offset[1], -offset[2])) ∘ PerspectiveMap()


doc"""
    Rayleigh(σ)

The *Rayleigh distribution* with scale `σ` has probability density function

$f(x; \sigma) = \frac{x}{\sigma^2} e^{-\frac{x^2}{2 \sigma^2}}, \quad x > 0$

It is related to the [`Normal`](:func:`Normal`) distribution via the property that if $X, Y \sim \operatorname{Normal}(0,\sigma)$, independently, then $\sqrt{X^2 + Y^2} \sim \operatorname{Rayleigh}(\sigma)$.

```julia
Rayleigh()       # Rayleigh distribution with unit scale, i.e. Rayleigh(1.0)
Rayleigh(s)      # Rayleigh distribution with scale s

params(d)        # Get the parameters, i.e. (s,)
scale(d)         # Get the scale parameter, i.e. s
```

External links

* [Rayleigh distribution on Wikipedia](http://en.wikipedia.org/wiki/Rayleigh_distribution)

"""
immutable Rayleigh <: ContinuousUnivariateDistribution
    σ::Float64

    Rayleigh(σ::Real) = (@check_args(Rayleigh, σ > zero(σ)); new(σ))
    Rayleigh() = new(1.0)
end

@distr_support Rayleigh 0.0 Inf

#### Parameters

scale(d::Rayleigh) = d.σ
params(d::Rayleigh) = (d.σ,)

#### Statistics

mean(d::Rayleigh) = sqrthalfπ * d.σ
median(d::Rayleigh) = 1.177410022515474691 * d.σ   # sqrt(log(4.0)) = 1.177410022515474691
mode(d::Rayleigh) = d.σ

var(d::Rayleigh) = 0.429203673205103381 * d.σ^2   # (2.0 - π / 2) = 0.429203673205103381
std(d::Rayleigh) = 0.655136377562033553 * d.σ

skewness(d::Rayleigh) = 0.631110657818937138
kurtosis(d::Rayleigh) = 0.245089300687638063

entropy(d::Rayleigh) = 0.942034242170793776 + log(d.σ)

#### Evaluation

function pdf(d::Rayleigh, x::Float64)
	σ2 = d.σ^2
	x > 0.0 ? (x / σ2) * exp(- (x^2) / (2.0 * σ2)) : 0.0
end

function logpdf(d::Rayleigh, x::Float64)
	σ2 = d.σ^2
	x > 0.0 ? log(x / σ2) - (x^2) / (2.0 * σ2) : -Inf
end

logccdf(d::Rayleigh, x::Float64) = - (x^2) / (2.0 * d.σ^2)
ccdf(d::Rayleigh, x::Float64) = exp(logccdf(d, x))

cdf(d::Rayleigh, x::Float64) = 1.0 - ccdf(d, x)
logcdf(d::Rayleigh, x::Float64) = log1mexp(logccdf(d, x))

quantile(d::Rayleigh, p::Float64) = sqrt(-2.0 * d.σ^2 * log1p(-p))

#### Sampling

rand(d::Rayleigh) = d.σ * sqrt(2.0 * randexp())


using Pkg
Pkg.activate(@__DIR__)

using ProgressiveHedging
import JuMP
import Ipopt

function two_stage_model(scenario_id::ScenarioID)

    model = JuMP.Model(()->Ipopt.Optimizer())
    JuMP.set_optimizer_attribute(model, "print_level", 0)
    JuMP.set_optimizer_attribute(model, "tol", 1e-12)
    JuMP.set_optimizer_attribute(model, "acceptable_tol", 1e-12)

    scen = value(scenario_id)

    ref = JuMP.@variable(model, x >= 0.0)
    stage1 = [ref]

    ref = JuMP.@variable(model, y >= 0.0)
    stage2 = [ref]

    b_s = scen == 0 ? 11.0 : 4.0
	c_s = scen == 0 ? 0.5 : 10.0

    JuMP.@constraint(model, x + y == b_s)

    JuMP.@objective(model, Min, 1.0*x + c_s*y)

    return JuMPSubproblem(model,
	                  scenario_id,
                          Dict(stid(1) => stage1,
			       stid(2) => stage2)
                          )
end

scen_tree = two_stage_tree(2)
(niter, abs_res, rel_res, obj, soln_df, phd) = solve(scen_tree,
                                                     two_stage_model,
                                                     ScalarPenaltyParameter(1.0)
                                                     )
@show niter
@show abs_res
@show rel_res
@show obj
@show soln_df

dual_df = retrieve_w(phd)
@show dual_df

raw_values_df = retrieve_no_hats(phd)
@show raw_values_df


# Least Squares Tests

# Test absolute state constructors
linear_sys = LinearSystem(1.0, 1.0)
linear_obs = LinearObserver(1.0, 1.0)
initial_est = ContinuousState(1.0)
lse = LeastSquaresEstimator(linear_sys, linear_obs, initial_est)
@test typeof(lse.estimate) <: UncertainContinuousState
initial_est = DiscreteState(1.0)
lse = LeastSquaresEstimator(linear_sys, linear_obs, initial_est)
@test typeof(lse.estimate) <: UncertainDiscreteState

# Test continuous-time linear least squares
srand(1)
linear_sys = LinearSystem(-eye(2), 0.0001*eye(2))
linear_obs = LinearObserver(eye(2), 0.01*eye(2))
initial_est = UncertainContinuousState([1.0, 2.0], 0.1*eye(2))
lse = LeastSquaresEstimator(linear_sys, linear_obs, initial_est)
simulator = make_simulator(lse)
correct_solution = deepcopy(simulator.true_state)
for idx = 1:10
    true_state, measurement = simulate(simulator, (idx-1)*0.1)
    add!(lse, measurement)
end
@test mahalanobis(correct_solution, solve(lse)) < 3

# Test continuous-time linear least squares with archiving
srand(1)
linear_sys = LinearSystem(-eye(2), 0.0001*eye(2))
linear_obs = LinearObserver(eye(2), 0.01*eye(2))
initial_est = UncertainContinuousState([1.0, 2.0], 0.1*eye(2))
lse = LeastSquaresEstimator(linear_sys, linear_obs, initial_est)
simulator = make_simulator(lse)
correct_solution = deepcopy(simulator.true_state)
for idx = 1:10
    true_state, measurement = simulate(simulator, (idx-1)*0.1)
    add!(lse, measurement)
end
archive = EstimatorHistory(lse)
@test mahalanobis(correct_solution, solve(lse, archive)) < 3
@test (length(archive.states) == 10) & (length(archive.residuals) == 10)

# Test discrete-time linear least squares
srand(1)
linear_sys = LinearSystem(eye(3), 0.0001*eye(3))
linear_obs = LinearObserver(eye(3), 0.01*eye(3))
initial_est = UncertainDiscreteState([1.0, 2.0, 3.0], 0.1*eye(3))
lse = LeastSquaresEstimator(linear_sys, linear_obs, initial_est)
simulator = make_simulator(lse)
correct_solution = deepcopy(simulator.true_state)
measurements = []
for idx = 1:10
    true_state, measurement = simulate(simulator, idx-1)
    push!(measurements, measurement)
end
add!(lse, measurements)
solve!(lse)
@test mahalanobis(correct_solution, lse.estimate) < 3

# Test discrete-time linear least squares with archiving
srand(1)
linear_sys = LinearSystem(eye(3), 0.0001*eye(3))
linear_obs = LinearObserver(eye(3), 0.01*eye(3))
initial_est = UncertainDiscreteState([1.0, 2.0, 3.0], 0.1*eye(3))
lse = LeastSquaresEstimator(linear_sys, linear_obs, initial_est)
simulator = make_simulator(lse)
correct_solution = deepcopy(simulator.true_state)
measurements = []
for idx = 1:10
    true_state, measurement = simulate(simulator, idx-1)
    push!(measurements, measurement)
end
add!(lse, measurements)
archive = EstimatorHistory(lse)
solve!(lse, archive)
@test mahalanobis(correct_solution, lse.estimate) < 3
@test (length(archive.states) == 10) & (length(archive.residuals) == 10)

# Test continuous-time nonlinear least squares
srand(1)
nl_sys_fcn(t, x::Vector) = -x;
nl_sys_jac(t, x::Vector) = -eye(length(x))
nonlin_sys = NonlinearSystem(nl_sys_fcn, nl_sys_jac, 0.0001*eye(2))
nl_obs_fcn(t, x::Vector) = x;
nl_obs_jac(t, x::Vector) = eye(length(x))
nonlin_obs = NonlinearObserver(nl_obs_fcn, nl_obs_jac, 0.01*eye(2))
initial_est = UncertainContinuousState([1.0, 3.0], eye(2))
nlse = NonlinearLeastSquaresEstimator(nonlin_sys, nonlin_obs, initial_est, 1e-4)
simulator = make_simulator(nlse)
correct_solution = deepcopy(simulator.true_state)
for idx = 1:10
    true_state, measurement = simulate(simulator, idx*0.1)
    add!(nlse, measurement)
end
solve!(nlse)
@test mahalanobis(correct_solution, nlse.estimate) < 3


include("../../iron_line_profile.jl")
include("../../arbitrary_f/iron_line_profile_approx.jl")

verticle_line_x = fill(6.4, 100)
verticle_line_y = LinRange(0, 1.05, 100)

x_vals_exact, bins_exact = iron_line_profile(
                                            spin=0.8,
                                            obs_angle=30,
                                            fov=12,
                                            tolerance=1e-12,
                                            size_multiplier=5,
                                            dtmax=0.5,
                                            output="data",
                                            nbins=200,
                                            normalised=true
                                            )

x_vals_approx, bins_approx = iron_line_profile_approx(
                                                    spin=0.8,
                                                    obs_angle=30,
                                                    fov=12,
                                                    tolerance=1e-12,
                                                    size_multiplier=5,
                                                    dtmax=0.5,
                                                    output="data",
                                                    nbins=200,
                                                    normalised=true
                                                    )

plt = plot(
            x_vals_exact,
            bins_exact,
            xlabel="Energy (keV)",
            ylabel="Flux (Arbitrary Units)",
            label="Analytic f",
            grid=false,
            framestyle=:box,
            ylims=(0,1.05)
            )
plot!(
        x_vals_approx,
        bins_approx,
        label="Numerical f",
        legend=:topleft,
        linestyle =:dash
        )
# adding vertical line at 6.4 keV
plot!(
    verticle_line_x,
    verticle_line_y,
    linestyle=:dash,
    linecolor=:black,
    label=false,
    ylims=(0,1.05),
    lw=2
    )
png(plt, "iron_line_approx_ana_comp.png")

#changing ϵ_3
vars = [-2,0,2]

labels = []
x_vals_vals = []
bins_vals = []

for var in vars
    x_vals_approx, bins_approx = iron_line_profile_approx(
                                                        ϵ_3=var,
                                                        spin=0.8,
                                                        obs_angle=30,
                                                        fov=12,
                                                        tolerance=1e-12,
                                                        size_multiplier=5,
                                                        dtmax=0.5,
                                                        output="data",
                                                        nbins=200,
                                                        normalised=false
                                                        )
    push!(bins_vals, bins_approx)
    push!(x_vals_vals, x_vals_approx)
    push!(labels, "ϵ₃ = $var")
end
fixed_labels=permutedims(labels)
plt = plot(
            x_vals_vals,
            bins_vals,
            xlabel="Energy (keV)",
            ylabel="Flux (Arbitrary Units)",
            label=fixed_labels,
            grid=false,
            framestyle=:box
            )
png(plt, "iron_line_eps_comp.png")

#changing α_22
vars = [-5,0,5]

labels = []
x_vals_vals = []
bins_vals = []

for var in vars
    x_vals_approx, bins_approx = iron_line_profile_approx(
                                                        α_22=var,
                                                        spin=0.8,
                                                        obs_angle=30,
                                                        fov=12,
                                                        tolerance=1e-12,
                                                        size_multiplier=5,
                                                        dtmax=0.5,
                                                        output="data",
                                                        nbins=200,
                                                        normalised=false
                                                        )
    push!(bins_vals, bins_approx)
    push!(x_vals_vals, x_vals_approx)
    push!(labels, "α₂₂ = $var")
end
fixed_labels=permutedims(labels)
plt = plot(
            x_vals_vals,
            bins_vals,
            xlabel="Energy (keV)",
            ylabel="Flux (Arbitrary Units)",
            label=fixed_labels,
            grid=false,
            framestyle=:box
            )
png(plt, "iron_line_a22_comp.png")

#changing α_13
vars = [-2,0,2]

labels = []
x_vals_vals = []
bins_vals = []

for var in vars
    x_vals_approx, bins_approx = iron_line_profile_approx(
                                                        α_13=var,
                                                        spin=0.8,
                                                        obs_angle=30,
                                                        fov=12,
                                                        tolerance=1e-12,
                                                        size_multiplier=5,
                                                        dtmax=0.5,
                                                        output="data",
                                                        nbins=200,
                                                        normalised=false
                                                        )
    push!(bins_vals, bins_approx)
    push!(x_vals_vals, x_vals_approx)
    push!(labels, "α₁₃ = $var")
end
fixed_labels=permutedims(labels)
plt = plot(
            x_vals_vals,
            bins_vals,
            xlabel="Energy (keV)",
            ylabel="Flux (Arbitrary Units)",
            label=fixed_labels,
            grid=false,
            framestyle=:box
            )
png(plt, "iron_line_a13_comp.png")

#changing α_52
vars = [-5,0,5]

labels = []
x_vals_vals = []
bins_vals = []

for var in vars
    x_vals_approx, bins_approx = iron_line_profile_approx(
                                                        α_52=var,
                                                        spin=0.8,
                                                        obs_angle=30,
                                                        fov=12,
                                                        tolerance=1e-12,
                                                        size_multiplier=5,
                                                        dtmax=0.5,
                                                        output="data",
                                                        nbins=200,
                                                        normalised=false
                                                        )
    push!(bins_vals, bins_approx)
    push!(x_vals_vals, x_vals_approx)
    push!(labels, "α₅₂ = $var")
end
fixed_labels=permutedims(labels)
plt = plot(
            x_vals_vals,
            bins_vals,
            xlabel="Energy (keV)",
            ylabel="Flux (Arbitrary Units)",
            label=fixed_labels,
            grid=false,
            framestyle=:box
            )
png(plt, "iron_line_a52_comp.png")


function loglikelihood(tree::Tree,
                       mod::T,
                       node_data::N;
                       output_calculations::Bool=false) where {T <: NASM, N <: Union{NodeDNA, NodeRNA}}
  # Error checking
  if !all(map(x -> x in keys(node_data), findleaves(tree)))
    error("Some leaves are missing sequence data")
  elseif length(findroots(tree)) > 1
    error("More than one root detected")
  end

  # Create a Dict to store likelihood calculations
  calculations = Dict{Int64, Array{Float64, 2}}()

  # Find node visit order for postorder traversal
  visit_order = postorder(tree)
  for i in visit_order
    if isleaf(tree, i)
      calculations[i] = node_data[i].data
    else
      branches = tree.nodes[i].out
      for j in branches
        branch_length = tree.branches[j].length
        child_node = tree.branches[j].target
        p = P(mod, branch_length)
        if !haskey(calculations, i)
          calculations[i] = p * calculations[child_node]
        else
          calculations[i] .*= p * calculations[child_node]
        end
      end
    end
  end
  if output_calculations
    return sum(log.(_π(mod)' * calculations[visit_order[end]])), calculations, visit_order
  else
    return sum(log.(_π(mod)' * calculations[visit_order[end]]))
  end
end


using MultivariateStats
using GLMakie
using Statistics
using Random
using Distributions
using LinearAlgebra

include("utils.jl")
include("decompositions.jl")

n_latent = 1
n_out = 1
n_samples = 10000
s_noise = 0.00001
sigma = 0.1

# Atest = rand(n_out,n_latent)
Atest = Matrix(I,n_out,n_latent)
data = Atest*rand(Laplace(0,sigma),n_latent,n_samples) + randn(n_out,n_samples)*s_noise

ica = fit(ICA, data, n_latent; maxiter=100000, tol=0.001)


#!/usr/bin/env julia
using Random
function getMass(nBodies)
    mList=zeros(nBodies)
    N=(1.5^(-1.3)-150^(-1.3))/1.3 #crude approximation of IMF integral assuming alpha = 2.3, stellar mass range of 0.5:150 solar masses
    rescale=1e6
    max=floor(Int,N*rescale)
    for i=1:nBodies
        intTarget=rand(0:max,1)[1]/rescale
        m=(1.5^(-1.3)-intTarget*1.3)^(-1/1.3) #just algebra from above
        mList[i]=round(m,digits=2)
    end
    return mList
end