export computeTensorizedSP

function computeTensorizedSP(m::Int64,
                             α::Vector{Vector{Float64}},β::Vector{Vector{Float64}},
                             nodes::Vector{Vector{Float64}},weights::Vector{Vector{Float64}},
                             ind::Matrix{Int64};
                             issymmetric::BitArray=falses(length(α)))
    @assert m>=1 "`dimension` has to be positive"
    l,p = size(ind)
    l-=1
    T = spzeros(reduce(+,[l^i for i in 0:m]))
    # @show length(collect(with_replacement_combinations(0:l,m)))
    for tensor_ind in collect(with_replacement_combinations(0:l,m))
        index = 1 + reduce(+,[idx*l^(m-i) for (i,idx) in enumerate(tensor_ind)])
        T[index] = computeSP(tensor_ind,α,β,nodes,weights,ind;issymmetric=issymmetric)
    end
    return T
end
function computeTensorizedSP(m::Int64,op::Vector{OrthoPoly},q::Vector{Quad},ind::Matrix{Int64})
    @assert length(op)==length(q) "Inconsistent lengh of polynomials and quadrature rules"
    @assert issymmetric.(op)==issymmetric.(q)
    α,β = coeffs(op)
    n,w = nw(q)
    computeTensorizedSP(m,α,β,n,w,ind;issymmetric=issymmetric.(op))
end
function computeTensorizedSP(m::Int64,mOP::MultiOrthoPoly)
    @assert typeof(mOP.uni)==Vector{OrthoPolyQ} "no quadrature rules provided"
    ops = [ mOP.uni[i].op for i=1:length(mOP.uni) ]
    quads = [ mOP.uni[i].quad for i=1:length(mOP.uni) ]
    computeTensorizedSP(m,ops,quads,mOP.ind)
end

function computeTensorizedSP(m::Int64,opq::OrthoPolyQ)
    computeTensorizedSP(m,[opq.op],[opq.quad],calculateMultiIndices(1,deg(opq)) )
end


import PySCF: pyimport

using TensorOperations: @tensor

#########################################################################
# Additional Concepts
# also see: "Hartree-Fock methods of Quantum Chemistry" by Janos Angyan
#########################################################################
# One-electron properties
function dipole_moment(D, mol)
   # eq. (365) Janos
   # It does the same as PySCF does.
   #ao_dip = @pywith mol.with_common_orig([0,0,0]) begin
   #   mol.intor_symmetric("int1e_r", comp=3)
   #end
   common_orig_bak = mol._env[1:3]
   mol = mol.set_common_orig([0,0,0])
   ao_dip = mol.intor_symmetric("int1e_r", comp=3)
   mol = mol.set_common_orig(common_orig_bak)

   @tensor el_dip[x] := ao_dip[x,i,j] * D[i,j]
   el_dip = real.(el_dip)
   charges  = mol.atom_charges()
   coords   = mol.atom_coords()
   nucl_dip = vec(sum(charges .* coords, dims=1))
   mol_dip  = nucl_dip - el_dip
   _au2debye = 2.5417464157449032
   mol_dip *= _au2debye
end

# Population Analysis/Atomic Charges
@doc raw"""
   `population_analysis(D,S,α,mol)`

Generalized Mulliken, Lowdin, ... population analysis
with `D` the AO density matrix, `S` the overlap matrix,
`α` defines the analysis type, and mol is a PySCF molecule.

Qa = Za - ∑μ∈a [S^α P S^(1-α)]μμ  with 0 < α < 1.

In particular:
* α = 1 => Mulliken doi:10.1063/1.1740588
* α = ½ => Löwdin
"""
function population_analysis(D,S,α,mol;verbose=true)
   s_α, s_mα = S^α, S^(1-α) # Matrix powers (via squaring)
   @assert isapprox(mol.nelectron, tr(s_α*D*s_mα), atol=1e-8) # eq. (363) Janos
   PopAO = diag(s_α * D * s_mα)
   Q = zeros(mol.natm)
   for (μ,record) in enumerate(mol.ao_labels(fmt=nothing))
      a = record[1] + 1 |> n -> convert(Int,n)
      Q[a] -= PopAO[μ] # eqs. (362,364) Janos
   end
   Q += mol.atom_charges()
   if verbose
      for a in eachindex(Q)
         println(" Atom $a has charge $(Q[a])")
      end
   end
   @assert isapprox(sum(Q), mol.charge, atol=1e-8)
   Q
end

function _covalent_bond_order(P,S,mol)
   natm = mol.natm
   nelec = mol.nelectron
   @show nelec
   @show Npairs = sum( (2P * S) * (2P * S) ) # RHF
   ao_atom = first.(mol.ao_labels(fmt=nothing))
   B_ab = zeros(natm,natm)
   for a in 1:natm, b in a:natm
      inds = findall( x -> isequal(x+1,a) || isequal(x+1,b), ao_atom)
      #@show inds
      #Pab = reshape(P[inds,inds],length(inds),length(inds))
      #Sab = reshape(S[inds,inds],length(inds),length(inds))
      Pab = P[inds,inds]
      Sab = S[inds,inds]
      # eq. (375) Janos
      #Bab = sum( (Pab * Sab)*(Pab * Sab)' + (Pab * Sab)*(Pab * Sab)' ) # RHF
      Bab = 2dot( Pab * Sab, Pab * Sab) # RHF
      println("$a  $b  $Bab")
   end
end

# Hartree-Fock Reduced Density Matrices
@doc raw"""
   fock_dirac_denmat(C,S,D)

1RDM in MO basis. For Hartree-Fock it is idempotent, diagonal with 1 or 0 elements.
Therefore it is idempotent. And it is called Fock-Dirac density matrix.

`C` is the matrix of cannonical coefficients, `S` is the overlap matrix
and `D` is the density matrix in the basis of AO.
 """
function fock_dirac_denmat(C,S,D)
   C' * S * D * S * C
end

"""
Nj^2. McWeeny convention (wo. 1/2)
# Arguments
- D in AO basis
- S is ovlp in AO basis
"""
function number_pairs_j(D,S)
   #@tensor scalar(D[i,j] * D[k,l] * S[i,j] * S[k,l])
   #@tensor scalar( (D[i,j] * S[i,j]) * (D[k,l] * S[k,l]))
   @tensor scalar(D[i,j] * S[i,j] * D[k,l] * S[k,l])
end

"""
Nxc. McWeeny convention (wo. 1/2)

# Arguments
- D in AO basis
- S is ovlp in AO basis
"""
function number_pairs_xc(D,S)
   #@tensor 0.5scalar(D[i,j] * D[k,l] * S[i,l] * S[k,j])
   #@tensor 0.5scalar(D[i,j] * D[k,l] * S[l,i] * S[k,j]) # D is symmetric
   #@tensor 0.5scalar(D[i,j] * S[l,i] * D[k,l] * S[k,j]) # D is symmetric
   @tensor 0.5scalar(D[i,j] * S[i,l] * D[k,l] * S[k,j]) # j ↔ l
end

function number_pairs(D,S)
   number_pairs_j(D,S) - number_pairs_xc(D,S)
end

"""
2RD
"""
function _second_reduced_density(rdm1)
   #rdm2D = zeros(nao,nao)
   #for i in 1:nao, j in 1:nao
   rdm2D = similar(rdm1)
   for (i,j) in Tuple.(CartesianIndices(rdm1))
      rdm2D[i,j] = rdm1[i,i]*rdm1[j,j] - rdm1[i,j]*rdm1[j,i]
   end
   rdm2D
end

"""
2RDM
"""
function _second_denmat(D)
   # Choose convention: McWeeny or Löwdin (0.5 factor)
   # Clearly explained in sections 11.15, 13.3 of Janos Angyan notes
   #rdm2 = zeros(nsmo,nsmo,nsmo,nsmo)
   #for i in 1:nao, j in 1:nao, k in 1:nao, l in 1:nao
   #for i in 1:nao,j in 1:nao,k in 1:nao,l in 1:nao
      # 1st RHS is the Coulomb component ∫ = N^2
      # 2nd RHS is the exchange component ∫ = N
      #rdm2[l,k,i,j] =     D[l,i] * D[k,j] - D[l,j] * D[k,i] # eq. (338) Janos
      #lk, ij = index(l,k), index(i,j)
      #rdm2[l,k,i,j] = D[l,i]*D[j,k] - D[l,j]*D[k,i]
      #rdm2[l,j,i,k] = D[l,i]*D[k,j] - D[l,k]*D[j,i]
      #rdm2[i,j,k,l] = 0.5(D[i,j] * D[k,l] - D[i,l] * D[k,j]) # eq. (6) J. of Math. Chem. 28, Nos. 1–3, 2000
   #end
   #rdm2 = zeros(size(D)...,size(D)...)
   #for (i,j,k,l) in Tuple.(CartesianIndices(rdm2))
   #   rdm2[i,j,k,l] = D[i,j] * D[k,l] - D[i,l] * D[k,j]
   #end
   #rdm2
   @tensor rdm2[i,j,k,l] := D[i,j] * D[k,l] - D[i,l] * D[k,j]
end

"""
2RDM Coulomb part (J to distinguish from c for correlation)
"""
function _second_denmat_j(D)
   @tensor rdm2_j[i,j,k,l] := D[i,j] * D[k,l]
end


function plotintersect(S::Session, c; its=20000, h=.05, rkonly=false)
	local data
	rk4((t,x)->S.core.f(t,[x;c]), .0, 10rand(3), h, predCount(its), init=()->(data=Array{Float64}(0,3)), callback=(t,y,f)->(data = [data;y']))

	f = figure()
	axes(projection="3d")
	hold(true)
	plot(data[end÷4:end,1], data[end÷4:end,2], data[end÷4:end,3], color="k", alpha=rkonly?1.0:.3)
	f[:show]()
	PyPlot.draw()
	rkonly && return
	for B in S.P
		col = get(S.viz.color, B, "k")
		for (a,b) in zip(B,B[2:end])
			if sign(a[end] - c) ≠ sign(b[end] - c)
				# k = abs((a[end]-c)/(a[end]-b[end]))
				# x = k.*a.data + (1-k).*b.data
				x = abs(a[end]-c) < abs(b[end]-c) ? a.data : b.data
				Htmp(V) = S.core.H([V; c])
				Jtmp(V) = S.core.J([V; c])[:, 1:end-1]
				x = newton(Htmp, Jtmp, x[1:end-1], predCount(25) ∧ predEps(1e-8))
				x = [x;c]
				X,ω,ℵ = unwrap(x)
				m = size(X,1)-1
				f = interp(X)
				T = linspace(0,2pi,8m)
				F = f(T)
				plot(F[:,1], F[:,2], F[:,3], color=col, alpha=.7)
			end
		end
	end
	PyPlot.draw()
end


using FFTW

export QFTCircuit, QFTBlock, invorder_firstdim

export openbox

"""
    openbox(block::AbstractBlock) -> AbstractBlock

For a black box, like QFTBlock, you can get its white box (loyal simulation) using this function.
"""
function openbox end

CRk(i::Int, j::Int, k::Int) = control([i, ], j=>shift(2π/(1<<k)))
CRot(n::Int, i::Int) = chain(n, i==j ? kron(i=>H) : CRk(j, i, j-i+1) for j = i:n)
QFTCircuit(n::Int) = chain(n, CRot(n, i) for i = 1:n)

struct QFTBlock{N} <: PrimitiveBlock{N} end
YaoBlocks.mat(::Type{T}, q::QFTBlock{N}) where {T, N} = T.(applymatrix(q))

YaoBlocks.apply!(reg::DefaultRegister{B}, ::QFTBlock) where B = (reg.state = ifft!(invorder_firstdim(reg |> state), 1)*sqrt(1<<nactive(reg)); reg)
YaoBlocks.apply!(reg::DefaultRegister{B}, ::Daggered{N, <:QFTBlock}) where {B,N} = (reg.state = invorder_firstdim(fft!(reg|>state, 1)/sqrt(1<<nactive(reg))); reg)

# traits
YaoBlocks.ishermitian(q::QFTBlock{N}) where N = N==1
YaoBlocks.isreflexive(q::QFTBlock{N}) where N = N==1
YaoBlocks.isunitary(q::QFTBlock{N}) where N = true

openbox(q::QFTBlock{N}) where N = QFTCircuit(N)
openbox(q::Daggered{<:QFTBlock, N}) where {N} = QFTCircuit(N)'

function print_block(io::IO, pb::QFTBlock{N}) where N
    printstyled(io, "QFT(1-$N)"; bold=true, color=:blue)
end

function print_block(io::IO, pb::Daggered{N,<:QFTBlock}) where {N, T}
    printstyled(io, "IQFT(1-$N)"; bold=true, color=:blue)
end

function invorder_firstdim(v::Matrix)
    w = similar(v)
    n = size(v, 1) |> log2i
    n_2 = n ÷ 2
    mask = [bmask(i, n-i+1) for i in 1:n_2]
    @simd for b in basis(n)
        @inbounds w[breflect(b, mask; nbits=n)+1,:] = v[b+1,:]
    end
    w
end

function invorder_firstdim(v::Vector)
    n = length(v) |> log2i
    n_2 = n ÷ 2
    w = similar(v)
    #mask = SVector{n_2, Int}([bmask(i, n-i+1)::Int for i in 1:n_2])
    mask = [bmask(i, n-i+1)::Int for i in 1:n_2]
    @simd for b in basis(n)
        @inbounds w[breflect(b, mask; nbits=n)+1] = v[b+1]
    end
    w
end


module InsidePallet

include("./constants.jl")
using JuMP
using .Constants

export cons_inside_pallet

#Constraints ensuring that the only allowed rotations are those of Currence Robotics
function cons_inside_pallet(model)

    #Check not exceeding in X-axis
    @constraint(model,
    xnel_x_not_exceed_length[i=1:length(boxes)],
    (model[:flbx_front_left_bot_x][i] +
    boxes[i][1]*model[:lpx_length_of_box_parallel_to_x][i] +
    boxes[i][2]*model[:wpx_width_of_box_parallel_to_x][i] +
    boxes[i][3]*model[:hpx_height_of_box_parallel_to_x][i] <=
    L)
    )

    #Check not exceeding in Y-axis
    @constraint(model,
    ynew_y_not_exceed_width[i=1:length(boxes)],
    (model[:flby_front_left_bot_y][i] +
    boxes[i][1]*model[:lpy_length_of_box_parallel_to_y][i] +
    boxes[i][2]*model[:wpy_width_of_box_parallel_to_y][i] +
    boxes[i][3]*model[:hpy_height_of_box_parallel_to_y][i] <=
    W)
    )

    #Check not exceeding in Z-axis
    @constraint(model,
    zneh_z_not_exceed_height[i=1:length(boxes)],
    (model[:flbz_front_left_bot_z][i] +
    boxes[i][1]*model[:lpz_length_of_box_parallel_to_z][i] +
    boxes[i][2]*model[:wpz_width_of_box_parallel_to_z][i] +
    boxes[i][3]*model[:hpz_height_of_box_parallel_to_z][i] <=
    H)
    )

    return model
end

end


module Norms

# A norm

using Base.Test

import Base.+, Base.norm

export Norm
export norm, +
export avg, norm2

immutable Norm
    sum::Float64
    sum2::Float64
    count::Float64
    min::Float64
    max::Float64
    maxabs::Float64
end
Norm() = Norm(0.0, 0.0, 0.0, Inf, -Inf, 0.0)

norm(val::Float64) = Norm(val, val*val, 1.0, val, val, abs(val))
+(x::Norm, y::Norm) = Norm(x.sum + y.sum, x.sum2 + y.sum2, x.count + y.count,
                           min(x.min, y.min), max(x.max, y.max),
                           max(x.maxabs, y.maxabs))

@test norm(+2.0) + norm(+3.0) == Norm( 5.0, 13.0, 2.0,  2.0,  3.0, 3.0)
@test norm(+2.0) + norm(-3.0) == Norm(-1.0, 13.0, 2.0, -3.0,  2.0, 3.0)
@test norm(-2.0) + norm(+3.0) == Norm( 1.0, 13.0, 2.0, -2.0,  3.0, 3.0)
@test norm(-2.0) + norm(-3.0) == Norm(-5.0, 13.0, 2.0, -3.0, -2.0, 3.0)

avg(x::Norm) = x.sum / x.count
norm2(x::Norm) = sqrt(x.sum2 / x.count)

@test avg(norm(2.0)) == 2.0
@test norm2(norm(2.0)) == 2.0

end


"""
Type for correlated squared Euclidean
"""
mutable struct P_misfit_xcorr{T<:Real}
	pxcorr::Conv.P_xcorr{T}
	cy::Vector{Matrix{T}}
	dcg::Vector{Matrix{T}}
end

function P_misfit_xcorr(nt::Int, nr::Int,
			pxcorr=P_xcorr(nt, nr, norm_flag=false); y=nothing, cy=nothing)

	if(cy===nothing)
		cy=deepcopy(pxcorr.cg) # allocate cy
		(y===nothing) && error("need y or cy")
		(size(y)≠size(pxcorr.g)) && error("size y")

		Conv.mod!(pxcorr, cg=cy, g=y)
	end
	dcg=deepcopy(pxcorr.cg)
	fill!.(dcg, 0.0)
	return P_misfit_xcorr(pxcorr, cy, dcg)
end

function func_grad!(dfdx,  x,  pa::P_misfit_xcorr)
	nt=size(x,1)
	nr=size(x,2)
	cg=pa.pxcorr.cg
	cy=pa.cy
	dfdcg=pa.dcg

	# copy x
	copyto!(pa.pxcorr.g,x)

	# mod xcorr
	mod!(pa.pxcorr)

	J=0.0
	for ir in 1:length(cg)
		cgx=cg[ir]
		cyy=cy[ir]
		dfdcgx=dfdcg[ir]

		if(dfdx===nothing)
			JJ=Misfits.error_squared_euclidean!(nothing,  cgx,   cyy,   nothing, norm_flag=false)
		else
			JJ=Misfits.error_squared_euclidean!(dfdcgx,  cgx,   cyy,   nothing, norm_flag=false)
		end
		J+=JJ
	end
	if(!(dfdx===nothing))
		mod_grad!(dfdx, pa.pxcorr, dcg=dfdcg)
	end

	return J
end

mutable struct P_misfit_weighted_acorr{T<:Real}
	p_conv::Conv.P_conv{T,2,2,2}
	ds::Matrix{Float64}
end

function P_misfit_weighted_acorr(nt::Int, nr::Int)
	p_conv=P_conv(gsize=[nt,nr], dsize=[nt,nr], ssize=[2*nt-1,nr], slags=[nt-1, nt-1])
	ds=zero(p_conv.s)
	return P_misfit_weighted_acorr(p_conv, ds)
end

"""
Each column of the matrix x is correlated with itself,
the energy at non-zero las will be penalized and returned as J.
"""
function func_grad!(dfdx, x, pa::P_misfit_weighted_acorr)
	nt=size(x,1)
	nr=size(x,2)

	copyto!(pa.p_conv.g,x)
	copyto!(pa.p_conv.d,x)

	Conv.mod!(pa.p_conv, S())
	s=pa.p_conv.s
	J=0.0
	for ir in 1:nr
		for it in 1:size(s,1)
			J += abs2(s[it,ir]) * abs2((nt-it)/(nt-1))
		end
	end

	if(!(dfdx===nothing))
		fill!(pa.ds, 0.0)
		for ir in 1:nr
			for it in 1:size(s,1)
				pa.ds[it,ir] = 2.0 * (s[it,ir]) * abs2((nt-it)/(nt-1))
			end
		end

		Conv.mod!(pa.p_conv, G(), g=dfdx, s=pa.ds)

		rmul!(dfdx, 2.)
	end

	return J

end


# fisher information
export fisher_det

# TODO: mega wrong, don't pass in the uav's sensor, pass in the filter's
fisher_det(uav::Vehicle, df::DF) = fisher_det(uav.sensor, df)

fisher_det(df) = fisher_det(df.sensor, df)

function fisher_det(bo::BearingOnly, df::DF)
	fd = zeros(df.n, df.n)
	ns2 = bo.noise_sigma * bo.noise_sigma

    # Loop over possible vehicle states
	for xi = 1:df.n, yi = 1:df.n
        x = (xi - 0.5) * df.cell_size
        y = (yi - 0.5) * df.cell_size

        # Loop over possible target states
        a = bc = d = 0.0
        for txi = 1:df.n, tyi = 1:df.n
            tx = (txi - 0.5) * df.cell_size
            ty = (tyi - 0.5) * df.cell_size
            p = df.b[txi,tyi]

            dx = tx - x
            dy = ty - y

            # ensure dx, dy are non-zero (to avoid singularities)
            dx == 0.0 && (dx = 1e-6)
            dy == 0.0 && (dy = 1e-6)

            r2 = dx*dx + dy*dy
            r4 = r2*r2

            a += p*dy*dy / r4
            bc -= p*dx*dy / r4
            d += p*dx*dx / r4
        end
        a /= ns2
        bc /= ns2
        d /= ns2

        fd[xi, yi] = a*d - bc*bc
	end

	return fd
end

function fisher_det(bo::RangeOnly, df::DF)
	fd = zeros(df.n, df.n)
	ns2 = bo.noise_sigma * bo.noise_sigma
	for xind = 1:df.n
		x = (xind - 0.5) * df.cell_size
		for yind = 1:df.n
			y = (yind - 0.5) * df.cell_size

			# now we must loop over belief
			a = bc = d = 0.0
			for txi = 1:df.n
				tx = (txi - 0.5) * df.cell_size
				for tyi = 1:df.n
					ty = (tyi - 0.5) * df.cell_size
					p = df.b[txi,tyi]
					dx = tx - x
					dy = ty - y
					if dx == 0.0
						dx = 1e-6
					end
					if dy == 0.0
						dy = 1e-6
					end
					r2 = dx*dx + dy*dy

					a += p*dx*dx / r2
					bc += p*dx*dy / r2
					d += p*dy*dy / r2
				end
			end
			a /= ns2
			bc /= ns2
			d /= ns2

			fd[xind, yind] = a*d - bc*bc

		end
	end
	return fd
end


"""
Kibaek Kim - ANL MCS 2016
  Updated in 2020
Farmer example from Birge and Louveaux book.

To use MPI, add
- MPI.Init()
- DSPopt.parallelize(comm)
- MPI.Finalize()
"""

using MPI
using StructJuMP
using DSPopt

MPI.Init()

# Initialize DSPopt.jl with the communicator.
DSPopt.parallelize(MPI.COMM_WORLD)

NS = 3;                        # number of scenarios
probability = [1/3, 1/3, 1/3]; # probability

CROPS  = 1:3 # set of crops (wheat, corn and sugar beets, resp.)
PURCH  = 1:2 # set of crops to purchase (wheat and corn, resp.)
SELL   = 1:4 # set of crops to sell (wheat, corn, sugar beets under 6K and those over 6K)

Cost     = [150 230 260]   # cost of planting crops
Budget   = 500             # budget capacity
Purchase = [238 210];      # purchase price
Sell     = [170 150 36 10] # selling price
Yield    = [3.0 3.6 24.0;
            2.5 3.0 20.0;
            2.0 2.4 16.0]
Minreq   = [200 240 0]     # minimum crop requirement

m = StructuredModel(num_scenarios = NS)

@variable(m, 0 <= x[i=CROPS] <= 500, Int)
@objective(m, Min, sum(Cost[i] * x[i] for i=CROPS))
@constraint(m, const_budget, sum(x[i] for i=CROPS) <= Budget)

# TODO: Distributing blocks is not supported yet.
# This is because of the DSPopt implementation.
for s in 1:NS
    blk = StructuredModel(parent = m, id = s, prob = probability[s])

    @variable(blk, y[j=PURCH] >= 0)
    @variable(blk, w[k=SELL] >= 0)

    @objective(blk, Min, sum(Purchase[j] * y[j] for j=PURCH) - sum(Sell[k] * w[k] for k=SELL))

    @constraint(blk, const_minreq[j=PURCH], Yield[s,j] * x[j] + y[j] - w[j] >= Minreq[j])
    @constraint(blk, const_minreq_beets, Yield[s,3] * x[3] - w[3] - w[4] >= Minreq[3])
    @constraint(blk, const_aux, w[3] <= 6000)
end

status = optimize!(m,
    is_stochastic = true, # Needs to indicate that the model is of the stochastic program.
    solve_type = DSPopt.DW, # see instances(DSPopt.Methods) for other methods
    param = "examples/params.txt" # This path assumes running from the one-level upper directory (i.e., ../).
    )

if DSPopt.myrank() == 0 && status == MOI.OPTIMAL
    @show objective_value(m)
    @show dual_objective_value(m)
    @show value.(x)
    @show dual() # This is available only for solve_type = DSPopt.Dual.
end

MPI.Finalize()


#!/usr/bin/env julia --startup-file=no

"""
Linear-Stability-Calculators: Quasi-Hydrostatic Model
=====================================================

Linear Stability Calculation of an inertially unstable jet using spectral methods

The basic state is that of a barotropic Bickley jet (to be generalized).
The perturbation is assumed to be periodic in both the zonal (x) and vertical (z) directions.

Calculations are done in serial.  See inertial_instability_mpi.py for parallel execution.
"""

include("src/quasihydrostatic_setup.jl")
include("src/parameters.jl")
include("src/Compute_Eigenvalues.jl")
include("src/Plotting_Functions.jl")

files   = Files(         nc = "data_spectrum_serial.nc",
                       json = "data_parameters_serial.json",
                 plotgrowth = "growth_serial.png",
                 plotslicem = "growth_slice_m_serial.png",
                plotmodes1D = "plot_modes1D_serial",
                plotmodes2D = "plot_modes2D_serial"
                )
grid    = Grid(Ly = 1000e3, Lz = 3e3, Ny = 100, θ₀ = π/32, method=Cheb())
physics = Physics(N = 1e-2, ν = 0.26, θ₀ = grid.θ₀, NT = 1)
jet     = Jet(grid, physics)

Output_Parameters(grid, physics, jet, files.json)

Neigs  = 10
dk, Nk = 1e-6, 41
dm, Nm = 1e-4, 150

ks = collect(0 :dk:(Nk-1)*dk)
ms = collect(dm:dm:Nm*dm)

ωs    = zeros(Complex, (Neigs, Nm, Nk))
modes = zeros(Complex, (3*grid.Ny+1, Neigs, Nm, Nk))

for (ik, k) in enumerate(ks)
    for (im, m) in enumerate(ms)
        A = build_matrix(k, m, grid, physics, jet)

        ωs[:, im, ik], modes[:, :, im, ik] = compute_spectrum(A, Neigs)

        #print("k = ", ks[ik], " m = ", ms[im], " growth = ", imag(ωs[1, im, ik]), "\n")
    end
end

save_spectrum(ωs, modes, ks, ms, Neigs, grid.y, grid.Ny, files.nc)

plot_growth_slice(files.nc, files.json, files.plotslicem)
plot_growth(      files.nc, files.json, files.plotgrowth)

plot_modes_1D(files.nc, files.json, files.plotmodes1D, Neigs)
plot_modes_2D(files.nc, files.json, files.plotmodes2D, Neigs)


""" 2-category of categories, functors, and natural transformations.

Categories in mathematics appear in the large, often as categories of sets with
extra structure, and in the small, as algebraic structures that generalize
groups, monoids, preorders, and graphs. This division manifests in Catlab as
well. Large categories (in spirit, if not in the [technical
sense](https://ncatlab.org/nlab/show/large+category)) occur throughout Catlab as
`@instance`s of the theory of categories. For computational reasons, small
categories are usually presented by generators and relations.

This module provides a minimal interface to accomodate both situations. Category
instances are supported through the wrapper type [`TypeCat`](@ref). Finitely
presented categories are provided by another module, [`FinCats`](@ref).
"""
module Categories
export Cat, TypeCat, Functor, Transformation, dom, codom, compose, id,
  ob, hom, is_hom_equal, ob_map, hom_map, dom_ob, codom_ob, component

using AutoHashEquals

using ...GAT
import ...Theories: Category2, ob, hom, dom, codom, compose, ⋅, ∘, id,
  composeH, *

# Categories
############

""" Abstract base type for a category.

The objects and morphisms in the category have Julia types `Ob` and `Hom`,
respectively. Note that these types do *not* necessarily form an `@instance` of
the theory of categories, as they may not meaningfully form a category outside
the context of this object. For example, a finite category regarded as a
reflexive graph with a composition operation might have type `Cat{Int,Int}`,
where the objects and morphisms are numerical identifiers for vertices and edges
in the graph.

The basic operations available in any category are: [`dom`](@ref),
[`codom`](@ref), [`id`](@ref), [`compose`](@ref).
"""
abstract type Cat{Ob,Hom} end

""" Coerce or look up object in category.
"""
ob(::Cat, x) = x

""" Coerce or look up morphism in category.
"""
hom(::Cat, f) = f

""" Domain of morphism in category.
"""
dom(C::Cat, f) = dom(f)

""" Codomain of morphism in category.
"""
codom(C::Cat, f) = codom(f)

""" Identity morphism on object in category.
"""
id(C::Cat, x) = id(x)

""" Compose morphisms in a category.
"""
compose(C::Cat, fs...) = compose(fs...)

""" Are two morphisms in a category equal?

By default, just checks for equality of Julia objects using ``==``. In some
categories, checking equality of morphisms may involve nontrivial reasoning.
"""
is_hom_equal(C::Cat, f, g) = is_hom_equal(f, g)
is_hom_equal(f, g) = f == g

# Instances
#----------

""" Pair of Julia types regarded as a category.

The Julia types should form an `@instance` of the theory of categories
(`Theories.Category`).
"""
struct TypeCat{Ob,Hom} <: Cat{Ob,Hom} end

TypeCat(Ob::Type, Hom::Type) = TypeCat{Ob,Hom}()

# FIXME: This isn't practical because types are often too tight.
#ob(::TypeCat{Ob,Hom}, x) where {Ob,Hom} = convert(Ob, x)
#hom(::TypeCat{Ob,Hom}, f) where {Ob,Hom} = convert(Hom, f)

Base.show(io::IO, ::TypeCat{Ob,Hom}) where {Ob,Hom} =
  print(io, "TypeCat($Ob, $Hom)")

# Functors
##########

""" Abstract base type for a functor between categories.

A functor has a domain and a codomain ([`dom`](@ref) and [`codom`](@ref)), which
are categories, and object and morphism maps, which can be evaluated using
[`ob_map`](@ref) and [`hom_map`](@ref). The functor object can also be called
directly when the objects and morphisms have distinct Julia types. This is
sometimes but not always the case (see [`Cat`](@ref)), so when writing generic
code one should prefer the `ob_map` and `hom_map` functions.
"""
abstract type Functor{Dom<:Cat,Codom<:Cat} end

""" Evaluate functor on object.
"""
@inline ob_map(F::Functor, x) = do_ob_map(F, x)

""" Evaluate functor on morphism.
"""
@inline hom_map(F::Functor, f) = do_hom_map(F, f)

""" Identity functor on a category.
"""
@auto_hash_equals struct IdentityFunctor{Dom<:Cat} <: Functor{Dom,Dom}
  dom::Dom
end

codom(F::IdentityFunctor) = F.dom

do_ob_map(F::IdentityFunctor, x) = ob(F.dom, x)
do_hom_map(F::IdentityFunctor, f) = hom(F.dom, f)

function Base.show(io::IO, F::IdentityFunctor)
  print(io, "id(")
  show_domains(io, F, codomain=false)
  print(io, ")")
end

""" Composite of functors.
"""
@auto_hash_equals struct CompositeFunctor{Dom,Codom,
    F<:Functor{Dom,<:Cat},G<:Functor{<:Cat,Codom}} <: Functor{Dom,Codom}
  fst::F
  snd::G
end
Base.first(F::CompositeFunctor) = F.fst
Base.last(F::CompositeFunctor) = F.snd

dom(F::CompositeFunctor) = dom(first(F))
codom(F::CompositeFunctor) = codom(last(F))

do_ob_map(F::CompositeFunctor, x) = ob_map(F.snd, ob_map(F.fst, x))
do_hom_map(F::CompositeFunctor, f) = hom_map(F.snd, hom_map(F.fst, f))

function Base.show(io::IO, F::CompositeFunctor)
  print(io, "compose(")
  show(io, first(F))
  print(io, ", ")
  show(io, last(F))
  print(io, ")")
end

show_type_constructor(io::IO, ::Type{<:Functor}) = print(io, "Functor")

function show_domains(io::IO, f; domain::Bool=true, codomain::Bool=true,
                      recurse::Bool=true)
  if get(io, :hide_domains, false)
    print(io, "…")
  else
    if domain
      show(IOContext(io, :compact=>true, :hide_domains=>!recurse), dom(f))
    end
    if codomain
      domain && print(io, ", ")
      show(IOContext(io, :compact=>true, :hide_domains=>!recurse), codom(f))
    end
  end
end

# Instances
#----------

(F::Functor{TypeCat{Ob,Hom}})(x::Ob) where {Ob,Hom} = ob_map(F, x)
(F::Functor{TypeCat{Ob,Hom}})(f::Hom) where {Ob,Hom} = hom_map(F, f)

# Natural transformations
#########################

""" Abstract base type for a natural transformation between functors.

A natural transformation ``α: F ⇒ G`` has a domain ``F`` and codomain ``G``
([`dom`](@ref) and [`codom`](@ref)), which are functors ``F,G: C → D`` having
the same domain ``C`` and codomain ``D``. The transformation consists of a
component ``αₓ: Fx → Gx`` in ``D`` for each object ``x ∈ C``, accessible using
[`component`](@ref) or indexing notation (`Base.getindex`).
"""
abstract type Transformation{C<:Cat,D<:Cat,Dom<:Functor,Codom<:Functor} end

""" Component of natural transformation.
"""
function component end

@inline Base.getindex(α::Transformation, c) = component(α, c)

""" Domain object of natural transformation.

Given ``α: F ⇒ G: C → D``, this function returns ``C``.
"""
dom_ob(α::Transformation) = dom(dom(α)) # == dom(codom(α))

""" Codomain object of natural transformation.

Given ``α: F ⇒ G: C → D``, this function returns ``D``.
"""
codom_ob(α::Transformation) = codom(dom(α)) # == codom(codom(α))

@auto_hash_equals struct IdentityTransformation{C<:Cat,D<:Cat,Dom<:Functor{C,D}} <:
     Transformation{C,D,Dom,Dom}
  dom::Dom
end

codom(α::IdentityTransformation) = α.dom

function component(α::IdentityTransformation, x)
  F = dom(α)
  id(codom(F), ob_map(F, x))
end

const IdIdTransformation{C<:Cat} = IdentityTransformation{C,C,IdentityFunctor{C}}

# 2-category of categories
##########################

@instance Category2{Cat,Functor,Transformation} begin
  dom(F::Functor) = F.dom
  codom(F::Functor) = F.codom
  id(C::Cat) = IdentityFunctor(C)

  function compose(F::Functor, G::Functor; strict::Bool=true)
    !strict || codom(F) == dom(G) ||
      error("Domain mismatch in composition $F ⋅ $G")
    compose_id(F, G)
  end

  dom(α::Transformation) = α.dom
  codom(α::Transformation) = α.codom
  id(F::Functor) = IdentityTransformation(F)

  function compose(α::Transformation, β::Transformation; strict::Bool=true)
    !strict || codom(α) == dom(β) ||
      error("Domain mismatch in vertical composition $α ⋅ $β")
    compose_id(α, β)
  end
  function composeH(α::Transformation, β::Transformation; strict::Bool=true)
    !strict || codom_ob(α) == dom_ob(β) ||
      error("Domain mismatch in horizontal composition $α * $β")
    composeH_id(α, β)
  end
  function composeH(α::Transformation, H::Functor; strict::Bool=true)
    !strict || codom_ob(α) == dom(H) ||
      error("Domain mismatch in whiskering $α * $H")
    composeH_id(α, H)
  end
  function composeH(F::Functor, β::Transformation; strict::Bool=true)
    !strict || codom(F) == dom_ob(β) ||
      error("Domain mismatch in whiskering $F * $β")
    composeH_id(F, β)
  end
end

# XXX: Is this normalization of identities using multiple dispatch a good idea?
# Unlike in `Sets`, it doesn't feel great since it requires so much boilerplate.

@inline compose_id(F::Functor, G::Functor) = do_compose(F, G)
@inline compose_id(F::Functor, ::IdentityFunctor) = F
@inline compose_id(::IdentityFunctor, G::Functor) = G
@inline compose_id(F::IdentityFunctor, ::IdentityFunctor) = F

@inline compose_id(α::Transformation, β::Transformation) = do_compose(α, β)
@inline compose_id(α::Transformation, ::IdentityTransformation) = α
@inline compose_id(::IdentityTransformation, β::Transformation) = β
@inline compose_id(α::IdentityTransformation, ::IdentityTransformation) = α

@inline composeH_id(α::Transformation, β::Transformation) = do_composeH(α, β)
@inline composeH_id(α::Transformation, ::IdIdTransformation) = α
@inline composeH_id(::IdIdTransformation, β::Transformation) = β
@inline composeH_id(α::IdIdTransformation, ::IdIdTransformation) = α

@inline composeH_id(α::Transformation, H::Functor) = do_composeH(α, H)
@inline composeH_id(α::Transformation, ::IdentityFunctor) = α
@inline composeH_id(α::IdentityTransformation, H::Functor) =
  id(compose_id(dom(α), H))
@inline composeH_id(α::IdentityTransformation, ::IdentityFunctor) = α

@inline composeH_id(F::Functor, β::Transformation) = do_composeH(F, β)
@inline composeH_id(::IdentityFunctor, β::Transformation) = β
@inline composeH_id(F::Functor, β::IdentityTransformation) =
  id(compose_id(F, dom(β)))
@inline composeH_id(::IdentityFunctor, β::IdentityTransformation) = β

do_compose(F::Functor, G::Functor) = CompositeFunctor(F, G)

@inline function do_composeH(α::Transformation, β::Transformation)
  do_composeH(α, β, Val{:covariant})
end
function do_composeH(α::Transformation, β::Transformation,
                     ::Type{Val{:covariant}})
  G, H = codom(α), dom(β)
  compose_id(composeH_id(α, H), composeH_id(G, β))
end
function do_composeH(α::Transformation, β::Transformation,
                     ::Type{Val{:contravariant}})
  F, K = dom(α), codom(β)
  compose_id(composeH_id(F, β), composeH_id(α, K))
end

end


if !@isdefined ExoplanetsSysSim
    using ExoplanetsSysSim
end
using Distributions
using Roots #if we want to use the 'find_zero()' function
using Optim

##### This code is a translation of 'MRpredict.R' at 'https://github.com/Bo-Ning/Predicting-exoplanet-mass-and-radius-relationship/blob/master/MR-predict/MRpredict.R'

struct MR_param_Ning2018
    Mass_min::Float64 #log10 scale
    Mass_max::Float64 #log10 scale
    Radius_min::Float64 #log10 scale
    Radius_max::Float64 #log10 scale
    deg::Int64
    weights_mle::Vector{Float64}
    indices_keep::Vector{Int64}
    indices_div::Vector{Int64}
    indices_rem::Vector{Int64}
end

function pdfnorm_beta(x::Float64, x_obs::Float64, x_sd::Float64, x_max::Float64, x_min::Float64, shape1::Int64, shape2::Int64 ; log::Bool=true)
    if log == true
        norm_dist = Normal(10^x, x_sd)
    else
        norm_dist = Normal(x, x_sd)
    end
    #beta_dist = Beta(shape1, shape2)
    return pdf.(norm_dist, x_obs) .* pdf.(Beta(shape1, shape2), (x-x_min)/(x_max-x_min)) / (x_max-x_min)
end

function fn_for_integrate(x_obs::Float64, x_sd::Float64, deg::Int, degree::Int, x_max::Float64, x_min::Float64 ; log::Bool=false, abs_tol::Float64=1e-10)
    shape1, shape2 = degree, deg-degree+1
    return quadgk(x -> pdfnorm_beta(x, x_obs, x_sd, x_max, x_min, shape1, shape2; log=log), x_min, x_max, abstol=abs_tol)[1]
end

function conditional_density(y::Float64, y_max::Float64, y_min::Float64, x_max::Float64, x_min::Float64, deg::Int64, w_hat::Vector{Float64}, indices_keep::Vector{Int64}, indices_div::Vector{Int64}, indices_rem::Vector{Int64}; y_sd::Union{Nothing,Float64}=nothing, qtl::Vector{Float64}=[0.16, 0.84])
    deg_vec = 1:deg

    #To compute conditional mean, variance, quantile, distribution:
    y_beta_indv = zeros(length(indices_rem))
    if y_sd == nothing
        map!(degree -> pdf.(Beta(degree, deg-degree+1), (y-y_min)/(y_max-y_min)), y_beta_indv, deg_vec[indices_rem])
        y_beta_indv /= (y_max-y_min)
    else
        map!(degree -> fn_for_integrate(y, y_sd, deg, degree, y_max, y_min), y_beta_indv, deg_vec[indices_rem])
    end
    denominator = sum(w_hat .* y_beta_indv)

    #Mean:
    mean = sum(w_hat::Vector{Float64} .* (((deg_vec::UnitRange{Int64})/(deg+1)*(x_max-x_min) .+ x_min)[indices_div] .* y_beta_indv::Vector{Float64})::Vector{Float64}) / denominator::Float64

    #Variance:
    var = sum(w_hat::Vector{Float64} .* (((deg_vec::UnitRange{Int64}) .* (deg .- deg_vec .+ 1) / ((deg+1)^2*(deg+2))*(x_max-x_min)^2)[indices_div] .* y_beta_indv::Vector{Float64})::Vector{Float64}) / denominator::Float64

    #Quantile:
    function pbeta_conditional_density(x::Float64)::Float64
        function mix_density(j::Float64)::Float64
            @fastmath @inbounds @views return sum(w_hat::Vector{Float64} .* (map(degree -> cdf.(Beta(degree, deg::Int64-degree+1), (j-x_min)/(x_max-x_min)), (deg_vec::UnitRange{Int64})[indices_div]) .* y_beta_indv::Vector{Float64})) / denominator::Float64
        end

        return mix_density(x)
    end

    function conditional_quantile(q::Float64)::Float64
        function g(x::Float64)::Float64
            return pbeta_conditional_density(x) - q
        end
        function g2(x::Float64)::Float64
            return (pbeta_conditional_density(x) - q)^2
        end

        #return find_zero(g, (x_min, x_max), FalsePosition())
        #return find_zero(g, (x_min, x_max))::Float64 ###'find_zero' is from 'using Roots' (is this a good function to use?); Eric suggested brent's method in the Optim package

        result = optimize(g2, x_min, x_max, Brent(), rel_tol=1e-3, abs_tol=1e-3) #Brent(); GoldenSection()
        if Optim.converged(result)
            return Optim.minimizer(result)::Float64
        else
            return find_zero(g, (x_min, x_max))::Float64
        end
    end

    quantile = map(conditional_quantile, qtl)

    return (mean, var, quantile, denominator, y_beta_indv)
end

function conditional_density(y::Float64, y_max::Float64, y_min::Float64, x_max::Float64, x_min::Float64, deg::Int64, w_hat::Vector{Float64}, indices_keep::Vector{Int64}; y_sd::Union{Nothing,Float64}=nothing, qtl::Vector{Float64}=[0.16, 0.84])
    deg_vec = 1:deg

    #To compute conditional mean, variance, quantile, distribution:
    y_beta_indv = zeros(deg)
    if y_sd == nothing
        map!(degree -> pdf.(Beta(degree, deg-degree+1), (y-y_min)/(y_max-y_min)), y_beta_indv, deg_vec)
        y_beta_indv /= (y_max-y_min)
    else
        map!(degree -> fn_for_integrate(y, y_sd, deg, degree, y_max, y_min), y_beta_indv, deg_vec)
    end
    y_beta_pdf = kron(ones(deg), y_beta_indv)[indices_keep]
    denominator = sum(w_hat .* y_beta_pdf)

    #Mean:
    mean = sum(w_hat::Vector{Float64} .* kron(deg_vec::UnitRange{Int64}/(deg+1)*(x_max-x_min)+x_min, y_beta_indv::Vector{Float64})[indices_keep]) / denominator::Float64

    #Variance:
    var = sum(w_hat::Vector{Float64} .* kron(deg_vec::UnitRange{Int64} .* (deg-deg_vec+1) / ((deg+1)^2*(deg+2))*(x_max-x_min)^2, y_beta_indv::Vector{Float64})[indices_keep]) / denominator::Float64

    #Quantile:
    function pbeta_conditional_density(x::Float64)::Float64
        function mix_density(j::Float64)::Float64
            @fastmath @inbounds @views return sum(w_hat::Vector{Float64} .* kron(map(degree -> cdf.(Beta(degree, deg-degree+1), (j-x_min)/(x_max-x_min)), deg_vec::UnitRange{Int64}), y_beta_indv::Vector{Float64})[indices_keep]) / denominator::Float64
            #@fastmath @inbounds @views return sum(w_hat::Vector{Float64} .* (map(degree -> cdf.(Beta(degree, deg::Int64-degree+1), (j-x_min)/(x_max-x_min)), (deg_vec::UnitRange{Int64})[indices_div]) .* y_beta_indv::Vector{Float64})) / denominator::Float64
        end

        return mix_density(x)
    end

    function conditional_quantile(q::Float64)::Float64
        function g(x::Float64)::Float64
            return pbeta_conditional_density(x) - q
        end
        function g2(x::Float64)::Float64
            return (pbeta_conditional_density(x) - q)^2
        end

        #return find_zero(g, (x_min, x_max), FalsePosition())
        #return find_zero(g, (x_min, x_max))::Float64 ###'find_zero' is from 'using Roots' (is this a good function to use?); Eric suggested brent's method in the Optim package

        result = optimize(g2, x_min, x_max, Brent(), rel_tol=1e-3, abs_tol=1e-3) #Brent(); GoldenSection()
        if Optim.converged(result)
            return Optim.minimizer(result)::Float64
        else
            return find_zero(g, (x_min, x_max))::Float64
        end
    end

    quantile = map(conditional_quantile, qtl)

    return (mean, var, quantile, denominator, y_beta_indv)
end

function predict_mass_given_radius(Radius::Float64, param::MR_param_Ning2018 ; R_sigma::Union{Nothing,Float64}=nothing, posterior_sample::Bool=false, qtl::Vector{Float64}=[0.16, 0.84])
    #Convert data to log scale:
    l_radius = log10.(Radius)

    #The 'posterior_sample == false' condition can deal with two cases:
    #Case I: if input data do not have measurement errors
    #Case II: if input data have measurement error
    if posterior_sample == false
        predicted_value = conditional_density(l_radius, param.Radius_max, param.Radius_min, param.Mass_max, param.Mass_min, param.deg, param.weights_mle, param.indices_keep, param.indices_div, param.indices_rem; y_sd=R_sigma, qtl=qtl)
        ###predicted_value = conditional_density(l_radius, param.Radius_max, param.Radius_min, param.Mass_max, param.Mass_min, param.deg, param.weights_mle, param.indices_keep; y_sd=R_sigma, qtl=qtl)
        predicted_mean = predicted_value[1]
        predicted_quantiles = predicted_value[3]
    elseif posterior_sample == true
        #Case III: if the input are posterior samples
        radius_sample = log10(Radius)
        k = length(radius_sample)
        denominator_sample = zeros(k)
        mean_sample = zeros(k)
        y_beta_indv_sample = zeros(k, param.deg)

        #Calculate mean:
        for i in 1:k
            results = cond_density_estimation(y=radius_sample[i], y_max=param.Radius_max, y_min=param.Radius_min, x_max=param.Mass_max, x_min=param.Mass_min, deg=param.deg, w_hat=param.weights_mle, qtl=quantile, only_output_mean=true) ###what function is this??? Is it part of some CRAN package...
            mean_sample[i] = (results[1]) ###why are there parentheses here? It makes no difference...
            denominator_sample[i] = results[2]
            y_beta_indv_sample[i,:] = results[3:57]
        end
        predicted_mean = mean(mean_sample)

        #Calculate 16% and 84% quantiles:
        #Mixture of the CDF of k conditional densities
        function pbeta_conditional_density(x)
            function mix_density(j)
                deg_vec = 1:param.deg
                x_indv_cdf = map(degree -> cdf.(Beta(degree, param.deg-degree+1), (j-x_min)/(x_max-x_min)), deg_vec)
                quantile_numerator = zeros(k)
                p_beta_sample = zeros(k)
                for ii in 1:k
                    quantile_numerator[ii] = sum(param.weights_mle .* kron(x_indv_cdf, y_beta_indv_sample[ii,:]))
                    p_beta_sample[ii] = quantile_numerator[ii]/denominator_sample[ii]
                end
                return p_beta = mean(p_beta_sample)
            end

            return map(mix_density, x)
        end

        function mixture_conditional_quantile(q, x_min, x_max)
            function g(x)
                return pbeta_conditional_density(x) - q
            end
            return find_zero(g, (x_min, x_max)) ###see note above
            #return Optim.minimum(optimize(g, x_min, x_max))
        end

        predicted_quantiles = map(q -> mixture_conditional_quantile(q, param.Mass_min, param.Mass_max), qtl)
    end

    #Return the output:
    return (predicted_mean, predicted_quantiles)::Tuple{Float64, Vector{Float64}}
end

function draw_planet_mass_from_radius_Ning2018(Radius::Float64, param::MR_param_Ning2018)
    #This function takes in a Radius (in solar radii) and draws a mass (returning in solar masses) probabilistically
    Radius_in_earths = Radius/ExoplanetsSysSim.earth_radius
    log_Radius_in_earths = log10.(Radius_in_earths)
    @assert param.Radius_min < log_Radius_in_earths < param.Radius_max
    q::Float64 = rand()
    l_mass = predict_mass_given_radius(Radius_in_earths, param; qtl=[q])[2][1]
    return (10^l_mass)*ExoplanetsSysSim.earth_mass
end

function generate_planet_mass_from_radius_Ning2018(Radius::Float64, sim_param::SimParam)
    global MR_param
    return draw_planet_mass_from_radius_Ning2018(Radius, MR_param)
end

function interpolate_planet_mass_from_radius_quantile_Ning2018_table(log_Radius::Float64, quantile::Float64)
    #If using "GridInterpolations" package (very slow for repeated use):
    #=
    global grid, gridData
    x = [log_Radius, quantile]
    return interpolate(grid, gridData, x)
    =#

    #If using "Interpolations" package (fastest):
    global scaled_itp
    return scaled_itp(log_Radius, quantile)
end

function generate_planet_mass_from_radius_Ning2018_table(Radius::Float64, sim_param::SimParam)
    Radius_in_earths = Radius/ExoplanetsSysSim.earth_radius
    log_Radius_in_earths = log10.(Radius_in_earths)
    q::Float64 = rand()
    log_Mass = interpolate_planet_mass_from_radius_quantile_Ning2018_table(log_Radius_in_earths, q)
    return (10^log_Mass)*ExoplanetsSysSim.earth_mass
end

##### To initialize the M-R model parameters:

weights_mle = CSV.read("weights.mle.csv")[:x]
degrees = 55

N_keep = 25
indices_keep = sortperm(weights_mle, rev=true)[1:N_keep]
indices_div = 1 .+ div.(indices_keep .- 1, degrees)
indices_rem = 1 .+ rem.(indices_keep .- 1, degrees)
weights = weights_mle[indices_keep]
MR_param = MR_param_Ning2018(-1., 3.809597, -0.302, 1.357509, degrees, weights, indices_keep, indices_div, indices_rem) #(-1., 3.809597, -0.3, 1.357509, 55, weights_mle)

##### Examples:

#=
#Observation without measurement errors:

Radius = 5. #original scale, not log scale
predict_result = predict_mass_given_radius(Radius, MR_param)
println(predict_result)

#Observation with a measurement error:

Radius = 5. #original scale, not log scale
R_sigma = 0.1
predict_result = predict_mass_given_radius(Radius, MR_param; R_sigma=0.1)
println(predict_result)

#If want to change 16% and 84% quantiles to 5% and 95% quantiles:

Radius = 5. #original scale, not log scale
predict_result = predict_mass_given_radius(Radius, MR_param; qtl=[0.05, 0.95])
println(predict_result)
=#

#Input are posterior samples: ###currently broken because the function 'cond_density_estimation' is undefined
#=
Radius_dist = Normal(5, 0.5) #original scale, not log scale
Radius_samples = rand(Radius_dist, 100)
predict_result = predict_mass_given_radius(Radius_samples, MR_param; R_sigma=0.1, posterior_sample=true)
println(predict_result)
=#

##### For plotting and timing the model:
#=
#Pkg.add("PyPlot")
using PyPlot

N_keep_array = [3025, 19, 18]
colors = ["blue", "green", "red"]
Radii = 10 .^(range(MR_param.Radius_min+0.01, stop=MR_param.Radius_max-0.01, length=1000))

fig, ax = subplots()
for (i,N_keep) in enumerate(N_keep_array)
    indices_keep = sortperm(weights_mle, rev=true)[1:N_keep]
    indices_div = 1 .+ div.(indices_keep .- 1, degrees)
    indices_rem = 1 .+ rem.(indices_keep .- 1, degrees)
    weights = weights_mle[indices_keep]
    MR_param = MR_param_Ning2018(-1., 3.809597, -0.302, 1.357509, degrees, weights, indices_keep, indices_div, indices_rem)

    @time Masses_predict = map(r -> predict_mass_given_radius(r, MR_param; qtl=[0.16, 0.84]), Radii)

    Masses_mean = 10 .^[x[1] for x in Masses_predict]
    Masses_lower = 10 .^[x[2][1] for x in Masses_predict]
    Masses_upper = 10 .^[x[2][2] for x in Masses_predict]

    loglog(Radii, Masses_mean, color=colors[i], linewidth=2.0, linestyle="-", label="M = "*string(N_keep))
    loglog(Radii, Masses_lower, color=colors[i], linewidth=1.0, linestyle="--")
    loglog(Radii, Masses_upper, color=colors[i], linewidth=1.0, linestyle="--")
end
xlabel(L"$R_p (R_\oplus)$")
ylabel(L"$M_p (M_\oplus)$")
legend(loc="upper left")
=#

##### If we want to generate and save a pre-computed table of masses on a radius vs. quantile grid:
#=
N_keep = 3025
indices_keep = sortperm(weights_mle, rev=true)[1:N_keep]
indices_div = 1 .+ div.(indices_keep .- 1, degrees)
indices_rem = 1 .+ rem.(indices_keep .- 1, degrees)
weights = weights_mle[indices_keep]
MR_param = MR_param_Ning2018(-1., 3.809597, -0.302, 1.357509, degrees, weights, indices_keep, indices_div, indices_rem)

N_radii = 1001
N_quantiles = 1001
log_Radii = collect(range(MR_param.Radius_min, stop=MR_param.Radius_max, length=N_radii))
Radii = 10 .^log_Radii
quantiles = collect(range(0., stop=1.0, length=N_quantiles))

file_name = "MRpredict_table_weights"*string(N_keep)*"_R"*string(N_radii)*"_Q"*string(N_quantiles)*".txt"
f = open(file_name, "w")
println(f, "# All masses are in log10; weights used = ", N_keep)
println(f, "# First uncommented line is the header with the column labels (first column contains the radii in log10 while the remaining columns correspond to the quantiles)")
writedlm(f, reshape(append!(["log_R"], "q".*string.(quantiles)), (1,:)), ", ")

log_Mass_table = zeros(N_radii, N_quantiles + 1)
for (i,r) in enumerate(Radii)
    log_Mass_quantiles = predict_mass_given_radius(r, MR_param; qtl=quantiles)[2]
    log_Mass_table[i,:] = append!([log_Radii[i]], log_Mass_quantiles)
    println(i, ", Radius = ", r)
end
writedlm(f, log_Mass_table, ", ")
close(f)
=#

##### To load a pre-computed table for interpolating the mass radius relation:

log_Mass_table = CSV.read("MRpredict_table_weights3025_R1001_Q1001.txt", header=3)

#One way to do the interpolation is to use "GridInterpolations" but this is very slow:
#=
#Pkg.add("GridInterpolations")
using GridInterpolations

N_quantiles = 1001
quantiles = collect(range(0., stop=1.0, length=N_quantiles))
grid = RectangleGrid(log_Mass_table[:,1], quantiles)
gridData = convert(Array, log_Mass_table[:,2:end])

#x = [0., 0.5] #an example data point to interpolate at
#interpolate(grid, gridData, x) #interpolate at the example data point
=#

#A faster way to do the interpolation is to first construct an interpolation object:

#Pkg.add("Interpolations") #NOTE: the README is somewhat out-dated so some of the examples/syntax do not actually work as the way they are shown
using Interpolations

N_radii = 1001
N_quantiles = 1001
log_Radii = range(MR_param.Radius_min, stop=MR_param.Radius_max, length=N_radii)
quantiles = range(0., stop=1.0, length=N_quantiles)

table_data = convert(Array, log_Mass_table[:,2:end])
itp = interpolate(table_data, BSpline(Cubic(Line(OnGrid())))) #interpolation object where the x and y axes are indices
scaled_itp = Interpolations.scale(itp, log_Radii, quantiles) #scaled interpolation object where the x and y axes are scaled to their physical units

#scaled_itp(0., 0.5) #calls the interpolation object to perform an interpolation at a given point

##### For timing the functions:

sim_param = setup_sim_param_model()
#Radii = ones(10000)*ExoplanetsSysSim.earth_radius
Radii = (10 .^(range(MR_param.Radius_min+0.01, stop=MR_param.Radius_max-0.01, length=10000)))*ExoplanetsSysSim.earth_radius
@time Masses = map(r -> generate_planet_mass_from_radius_Ning2018(r, sim_param), Radii)

@time Masses = map(r -> generate_planet_mass_from_radius_Ning2018_table(r, sim_param), Radii)


module CovSel

using ProximalBase
using DataStructures: BinaryMaxHeap
using CoordinateDescent, Distributions
using Printf, SparseArrays, LinearAlgebra

export
  ADMMOptions,
  covsel!, covselpath,
  #covsel_refit!, covselpath_refit

  # difference estimation
  #
  fusedGraphicalLasso, fusedGraphicalLasso!,   # Danaher et al.
  fusedNeighborhoodSelection,

  # Direct Difference Estimation and Inference
  differencePrecisionActiveShooting, differencePrecisionActiveShooting!, differencePrecisionRefit,
  CDDirectDifferenceLoss,

  differencePrecisionIHT,
  differenceLatentPrecisionIHT

include("utils.jl")

include("admmCovSel.jl")
include("tvCovSel.jl")
include("LatentCovSel_IHT.jl")

include("directDifference.jl")
include("fusedDifference.jl")

include("generate.jl")
include("evaluation.jl")

# alternative implementations
include("altImplementation.jl")

end


"""
```julia
h = EllipsoidAltitude(Body::T, PosRsph::Vector{Float64}) where {T <: abstractCelestialBody}
```
Computes the altitude above the reference ellipsoid with equatorial radius `Re`,
and a flattening `flat`.
"""

function EllipsoidAltitude(Body::T, PosRsph::Vector{Float64}) where {T <: abstractCelestialBody}
  return PosRsph[1] - equatorial_radius(Body)*(1-flattening(Body)*sin(PosRsph[3])^2)
end


#===================================================================================================
  Kernel Affinity
===================================================================================================#

doc"KernelAffinity(κ;a,c) = a⋅κ + c"
immutable KernelAffinity{T<:AbstractFloat} <: KernelOperation{T}
    a::HyperParameter{T}
    c::HyperParameter{T}
    kappa::Kernel{T}
    KernelAffinity(a::Variable{T}, c::Variable{T}, κ::Kernel{T}) = new(
        HyperParameter(a, leftbounded(zero(T), :open)),
        HyperParameter(c, leftbounded(zero(T), :closed)),
        κ
    )
end
function KernelAffinity{T<:AbstractFloat}(a::Argument{T}, c::Argument{T}, κ::Kernel{T})
    KernelAffinity{T}(Variable(a), Variable(c), κ)
end

function ==(ψ1::KernelAffinity, ψ2::KernelAffinity)
    (ψ1.a == ψ2.a) && (ψ1.c == ψ2.c) && (ψ1.kappa == ψ2.kappa)
end

ismercer(ψ::KernelAffinity) = ismercer(ψ.kappa)
isnegdef(ψ::KernelAffinity) = isnegdef(ψ.kappa)

attainszero(ψ::KernelAffinity)     = attainszero(ψ.kappa)
attainspositive(ψ::KernelAffinity) = attainspositive(ψ.kappa)
attainsnegative(ψ::KernelAffinity) = attainsnegative(ψ.kappa)

function description_string(κ::KernelAffinity, showtype::Bool = true)
    obj_str = string("KernelAffinity", showtype ? string("{", eltype(κ), "}") : "")
    kernel_str = description_string(κ.kappa)
    string(obj_str, "(a=$(κ.a.value),c=$(κ.c.value),", kernel_str, ")")
end

function convert{T<:AbstractFloat}(::Type{KernelAffinity{T}}, ψ::KernelAffinity)
    KernelAffinity(convert(T, ψ.a.value), convert(T, ψ.c.value), convert(Kernel{T}, ψ.kappa))
end

@inline phi{T<:AbstractFloat}(ψ::KernelAffinity{T}, z::T) = ψ.a*z + ψ.c

# Operations

+{T<:AbstractFloat}(κ::Kernel{T}, c::Real) = KernelAffinity(one(T), convert(T, c), κ)
+(c::Real, κ::Kernel) = +(κ, c)

*{T<:AbstractFloat}(κ::Kernel{T}, a::Real) = KernelAffinity(convert(T, a), zero(T), κ)
*(a::Real, κ::Kernel) = *(κ, a)

function +{T<:AbstractFloat}(κ::KernelAffinity{T}, c::Real)
    KernelAffinity(κ.a.value, κ.c + convert(T,c), κ.kappa)
end
+(c::Real, κ::KernelAffinity) = +(κ, c)

function *{T<:AbstractFloat}(κ::KernelAffinity{T}, a::Real)
    a = convert(T, a)
    KernelAffinity(a * κ.a, a * κ.c, κ.kappa)
end
*(a::Real, κ::KernelAffinity) = *(κ, a)

function ^{T<:AbstractFloat}(ψ::KernelAffinity{T}, d::Integer)
    KernelComposition(PolynomialClass(ψ.a.value, ψ.c.value, convert(T,d)), ψ.kappa)
end

function ^{T<:AbstractFloat}(ψ::KernelAffinity{T}, γ::AbstractFloat)
    KernelComposition(PowerClass(ψ.a.value, ψ.c.value, convert(T,γ)), ψ.kappa)
end

function exp{T<:AbstractFloat}(ψ::KernelAffinity{T})
    KernelComposition(ExponentiatedClass(ψ.a.value, ψ.c.value), ψ.kappa)
end

function tanh{T<:AbstractFloat}(ψ::KernelAffinity{T})
    KernelComposition(SigmoidClass(ψ.a.value, ψ.c.value), ψ.kappa)
end

#===================================================================================================
  Kernel Product and Sum
===================================================================================================#

# Kernel Product

immutable KernelProduct{T<:AbstractFloat} <: KernelOperation{T}
    a::HyperParameter{T}
    kappa1::Kernel{T}
    kappa2::Kernel{T}
    function KernelProduct(a::Variable{T}, κ1::Kernel{T}, κ2::Kernel)
        if !(ismercer(κ1) && ismercer(κ2))
            error("Kernels must be Mercer for closure under multiplication.")
        end
        new(HyperParameter(a, leftbounded(zero(T), :open)), κ1, κ2)
    end
end
function KernelProduct{T<:AbstractFloat}(a::Argument{T}, κ1::Kernel{T}, κ2::Kernel{T})
    KernelProduct{T}(Variable(a), κ1, κ2)
end

# Kernel Sum

immutable KernelSum{T<:AbstractFloat} <: KernelOperation{T}
    c::HyperParameter{T}
    kappa1::Kernel{T}
    kappa2::Kernel{T}
    function KernelSum(c::Variable{T}, κ1::Kernel{T}, κ2::Kernel{T})
        if !(ismercer(κ1) && ismercer(κ2)) && !(isnegdef(κ1) && isnegdef(κ2))
            error("All kernels must be Mercer or negative definite for closure under addition")
        end
        new(HyperParameter(c, leftbounded(zero(T), :closed)), κ1, κ2)
    end
end
function KernelSum{T<:AbstractFloat}(c::Argument{T}, κ1::Kernel{T}, κ2::Kernel{T})
    KernelSum{T}(Variable(c), κ1, κ2)
end

# Common Functions

for (kernel_obj, kernel_op, identity, scalar) in (
        (:KernelProduct, :*, :1, :a),
        (:KernelSum,     :+, :0, :c)
    )
    other_identity = identity == :1 ? :0 : :1
    scalar_str = string(scalar)
    @eval begin
        function description_string(ψ::$kernel_obj, showtype::Bool = true)
            constant_str = string($scalar_str,"=", ψ.$scalar.value)
            kernel1_str = "kappa1=" * description_string(ψ.kappa1, false)
            kernel2_str = "kappa2=" * description_string(ψ.kappa2, false)
            obj_str = string($kernel_obj.name.name, showtype ? string("{", eltype(ψ), "}") : "")
            string(obj_str, "(", constant_str, ",", kernel1_str, ",", kernel2_str, ")")
        end

        function ==(ψ1::$kernel_obj, ψ2::$kernel_obj)
            (ψ1.$scalar == ψ2.$scalar) && (ψ1.kappa1 == ψ2.kappa1) && (ψ1.kappa2 == ψ2.kappa2)
        end

        function convert{T<:AbstractFloat}(::Type{($kernel_obj){T}}, ψ::$kernel_obj)
            $kernel_obj(Variable(convert(T, ψ.$scalar.value), ψ.$scalar.isfixed),
                           convert(Kernel{T}, ψ.kappa1), convert(Kernel{T}, ψ.kappa2))
        end

        ismercer(ψ::$kernel_obj) = ismercer(ψ.kappa1) && ismercer(ψ.kappa2)
        isnegdef(ψ::$kernel_obj) = isnegdef(ψ.kappa1) && isnegdef(ψ.kappa2)
    end
end

for (kernel_obj, kernel_op, identity, scalar, op2_identity, op2_scalar) in (
        (:KernelProduct, :*, :1, :a, :0, :c),
        (:KernelSum,     :+, :0, :c, :1, :a)
    )
    @eval begin

        function $kernel_op{T}($scalar::Real, ψ::$kernel_obj{T})
            $kernel_obj($kernel_op(convert(T, $scalar), ψ.$scalar.value), ψ.kappa1, ψ.kappa2)
        end
        $kernel_op(ψ::$kernel_obj, $scalar::Real) = $kernel_op($scalar, ψ)

        $kernel_op{T}(κ1::Kernel{T}, κ2::Kernel{T}) = $kernel_obj(convert(T, $identity), κ1, κ2)

        function ($kernel_op){T}(κ1::KernelAffinity{T}, κ2::KernelAffinity{T})
            if κ1.$op2_scalar == $op2_identity && κ2.$op2_scalar == $op2_identity
                $kernel_obj($kernel_op(κ1.$scalar.value, κ2.$scalar.value), κ1.kappa, κ2.kappa)
            else
                $kernel_obj(convert(T, $identity), κ1, κ2)
            end
        end

        function ($kernel_op){T}(κ1::KernelAffinity{T}, κ2::StandardKernel{T})
            if κ1.$op2_scalar == $op2_identity
                $kernel_obj(κ1.$scalar.value, κ1.kappa, κ2)
            else
                $kernel_obj(convert(T, $identity), κ1, κ2)
            end
        end

        function ($kernel_op){T}(κ1::StandardKernel{T}, κ2::KernelAffinity{T})
            if κ2.$op2_scalar == $op2_identity
                $kernel_obj(κ2.$scalar.value, κ1, κ2.kappa)
            else
                $kernel_obj(convert(T, $identity), κ1, κ2)
            end
        end
    end
end


mutable struct linear_solver_JULIA <: abstract_linear_system_solver
	_factor # TO DO, give type
	_factor_defined::Bool

	# options
	sym::Symbol # this determines if we use LU, cholesky or LDL.
	safe_mode::Bool
	recycle::Bool

  function linear_solver_JULIA(sym::Symbol, safe_mode::Bool, recycle::Bool)
			this = new();
			this.sym = sym
			this.safe_mode = safe_mode
			this.recycle = recycle
			this._factor_defined = false
			return this
  end
end

function ls_factor!(solver::linear_solver_JULIA, SparseMatrix::SparseMatrixCSC{Float64,Int64}, n::Int64, m::Int64, timer::class_advanced_timer)
			inertia_status_val = 1;

			start_advanced_timer(timer, "JULIA/factorize")
			if solver.sym == :unsymmetric
				# incomplete ---- how does one compute inertia????
				 solver._factor = lufact(SparseMatrix);
			elseif solver.sym == :definite
				# do a cholesky factorization
				@assert(m == 0)
				try
					if !solver.recycle || !solver._factor_defined
						#solver._factor = cholfact(Symmetric(SparseMatrix,:L));
						solver._factor = cholesky(Symmetric(SparseMatrix,:L));
						solver._factor_defined = true
					else
						cholfact!(solver._factor,Symmetric(SparseMatrix,:L));
					end
				catch(e)
					if typeof(e) == LinearAlgebra.PosDefException
						inertia_status_val = 0
					else
						println("ERROR in linear_solver_JULIA.ls_factor!")
						throw(e)
					end
				end
			elseif solver.sym == :symmetric
				# LDL factorization
				try
					if !solver.recycle || !solver._factor_defined
						#solver._factor = ldltfact(Symmetric(SparseMatrix,:L));
						solver._factor = ldlt(Symmetric(SparseMatrix,:L));
						solver._factor_defined = true
					else
						ldltfact!(solver._factor,Symmetric(SparseMatrix,:L));
					end
				catch(e)
					#PosDefException was added when updating Julia from 0.6.4 to 0.7.0 by Fadi Hamad
					#Because ArgumentError is not thrown anymore and it is being replaced by PosDefException
					#This is related to (PosDefException: matrix is not positive definite; Cholesky factorization failed.)
					if typeof(e) == ArgumentError || typeof(e) == LinearAlgebra.PosDefException || typeof(e) == ZeroPivotException
					#if typeof(e) == LinearAlgebra.PosDefException
						inertia_status_val = 0
					else
						println("ERROR in saddle_solver_JULIA.ls_factor!")
						throw(e)
					end
				end

				if inertia_status_val == 1
					# do something !!!!!!
					di = diag(solver._factor)
					tol = 1e-20
					pos_eigs =  sum(di .> tol)
					zero_eigs = sum((-tol .<= di) .& (di .<= tol))
					neg_eigs = sum(di .< -tol)
					nan_eigs = sum(isnan.(di))
					inf_eigs = sum(isinf.(di))
					if nan_eigs + inf_eigs <= 0
						inertia_status_val = inertia_status(pos_eigs, neg_eigs, zero_eigs, n, m)
					else
						if inf_eigs > 0
							warn("Inf appears in diagonal")
						end
						if nan_eigs > 0
							warn("NaN appears in diagonal")
						end
						inertia_status_val = 0
					end
				end
			else
				error("this.options.sym = " * string(solver.sym) * " not supported")
			end
			pause_advanced_timer(timer, "JULIA/factorize")

			return inertia_status_val; # inertia
end

function ls_solve!(solver::linear_solver_JULIA, my_rhs::Array{Float64,1}, my_sol::Array{Float64,1}, timer::class_advanced_timer)
	start_advanced_timer(timer, "JULIA/ls_solve")
  my_sol[1:length(my_sol)] = solver._factor \ my_rhs; #::UmfpackLU{Float64,Int64}
	pause_advanced_timer(timer, "JULIA/ls_solve")
end

function ls_solve(solver::linear_solver_JULIA, my_rhs::AbstractArray, timer::class_advanced_timer)
    start_advanced_timer(timer, "JULIA/ls_solve")
    #Vector was added when updating from Julia 0.6.4 to Julia 0.7.0 by Fadi Hamad
    #This is becasue my_rhs in somecases is being a SparseVector instead of being array.
    #Due to the code in shur.jl jac_res = eval_jac_T_prod and hess_res = hess_product are returning SparseMatrix now instead of Array
    sol = solver._factor \ Vector(my_rhs);
    pause_advanced_timer(timer, "JULIA/ls_solve")
    return sol
end


const σ_thomson = (e^4 / (6π * ε_0^2 * m_e^2 * c_0^4)) |> u"m^2"

"""
    thomson(electron_density::NumberDensity)

Compute the Thomson extinction as a function of electron density.
"""
thomson(electron_density::NumberDensity)::PerLength = σ_thomson * electron_density


using TropicalTensors
using TropicalTensors: getbonds, getvertices, rand_Js, rand_hs
using Test
using Random

@testset "spinglass" begin
    lt = CubicLattice(4,4,4)
    sg = Spinglass(lt, rand_Js(Float32, Ferro(), lt), rand_hs(Float32, Zero(), lt))
    res = solve(sg; usecuda=false)
    @test res.n == 144
end

@testset "counting tropical" begin
    lt = CubicLattice(4,4,4)
    sg = Spinglass(lt, rand_Js(Float32, Ferro(), lt), rand_hs(Float32, Zero(), lt))
    res = solve(CountingTropicalF32, sg; usecuda=false)
    @test res.n == 144
    @test res.c == 2
end

using BitBasis
function count_degeneracy_exact(sg::Spinglass{LT,T}) where {LT, T}
    elist = T[]
    bonds = getbonds(sg.lattice)
    lt = sg.lattice
    LI = LinearIndices((lt.Nx, lt.Ny, lt.Nz))
    for b in basis(BitStr64{length(sg.lattice)})
        eng = zero(T)
        for ((i, j), J) in sg.Js
            eng += (b[LI[i...]] == b[LI[j...]]) ? J : -J
        end
        push!(elist, eng)
    end
    findall(==(minimum(elist)), elist) |> length
end

@testset "counting tropical 2" begin
    Random.seed!(3)
    lt = CubicLattice(2, 3, 3)
    sg = Spinglass(lt, rand_Js(Float32, Randpm(), lt), rand_hs(Float32, Zero(), lt))
    res = solve(CountingTropical{Int,Int}, sg; usecuda=false)
    @test res.c == count_degeneracy_exact(sg)
end


## circulant_embedding.jl : Gaussian random field generator using fft; only for uniformly spaced GRFs

"""
CirculantEmbedding <: GaussianRandomFieldGenerator

A [`GaussiandRandomFieldGenerator`](@ref) that uses FFT to compute samples of the Gaussian random field. Circulant embedding can only be applied if the points are specified on a structured grid.

# Examples
```jldoctest
julia> m = Matern(0.1,1.0)
Matérn (λ=0.1, ν=1.0, σ=1.0, p=2.0)

julia> c = CovarianceFunction(2,m)
2d Matérn covariance function (λ=0.1, ν=1.0, σ=1.0, p=2.0)

julia> pts1 = 0:0.02:1; pts2 = 0:0.02:1
0.0:0.02:1.0

julia> grf = GaussianRandomField(c,CirculantEmbedding(),pts1,pts2)
Gaussian random field with 2d Matérn covariance function (λ=0.1, ν=1.0, σ=1.0, p=2.0) on a 51x51 structured grid, using a circulant embedding

julia> contourf(grf)
[...]

julia> plot_eigenvalues(grf)
[...]

```
With appropriate ordering, the covariance matrix of a Gaussian random field is a (nested block) Toeplitz matrix. This matrix can be embedded into a larger (nested block) circulant matrix, whose eigenvalues can be rapidly computed using FFT. A difficulty here is that although the covariance matrix is positive semi-definite, its circulant extension in general is not. As a remedy, one can add so-called *ghost points* outside the domain of interest using the optional flag `padding`.
```jldoctest
julia> m = Matern(1,1)
Matérn (λ=1.0, ν=1.0, σ=1.0, p=2.0)

julia> c = CovarianceFunction(1,m)
1d Matérn covariance function (λ=1.0, ν=1.0, σ=1.0, p=2.0)

julia> pts = range(0,stop = 1,length = 256)
0.0:0.00392156862745098:1.0

julia> grf = GaussianRandomField(c,CirculantEmbedding(),pts)
WARNING: negative eigenvalue -0.001465931113950698 detected, Gaussian random field will be approximated (ignoring all negative eigenvalues)
WARNING: increase padding if possible
Gaussian random field with 1d Matérn covariance function (λ=1.0, ν=1.0, σ=1.0, p=2.0) on a 256-point structured grid, using a circulant embedding

julia> grf = GaussianRandomField(c,CirculantEmbedding(),pts,padding=5)
Gaussian random field with 1d Matérn covariance function (λ=1.0, ν=1.0, σ=1.0, p=2.0) on a 256-point structured grid, using a circulant embedding

```
For faster sampling (but slower initialization), use the optional argument `measure` (default=true).

See also: [`Cholesky`](@ref), [`Spectral`](@ref), [`KarhunenLoeve`](@ref)
"""
struct CirculantEmbedding <: GaussianRandomFieldGenerator end

const CirculantGRF = GaussianRandomField{C,CirculantEmbedding} where {C}

function _GaussianRandomField(mean,cov::CovarianceFunction{d},method::CirculantEmbedding,pts...;padding=1,measure=true) where {d}

    # add ghost points by padding
    padded_pts = pad.(pts,padding)
    n = length.(padded_pts)
    n2 = n[2:d]
    padded_pts = normalize.(padded_pts)
    padded_pts = (padded_pts[1],mirror.(padded_pts[2:d])...) # do not mirror dimension 1

    # compute eigenvalues of circulant matrix
    c = zeros(n[1],(2 .*n2.-1)...)
    for (i,idx) in enumerate(Base.product(padded_pts...))
        c[i] = apply(cov.cov,collect(idx))
    end
    c̃ = zeros(n[1],(2 .* n2)...)
    c̃[:,broadcast(:,2,2 .*n2)...] = c
    c̃ = fftshift(c̃,2:d)
    Λ = irfft(c̃,2*size(c̃,1)-1)
    Λ⁺ = zeros(size(Λ))
    Λ⁻ = zeros(size(Λ))
    for i in 1:length(Λ)
        if Λ[i] < 0
            Λ⁻[i] = -Λ[i]
        else
            Λ⁺[i] = Λ[i]
        end
    end
    Λ_max = maximum(Λ⁻)
    Λ_max > 0. && ( warn("negative eigenvalue $(-Λ_max) detected, Gaussian random field will be approximated (ignoring all negative eigenvalues)"); warn("increase padding if possible") )

    # optimize
    Σ = sqrt.(Λ⁺)
    P = measure ? plan_fft(Σ) : plan_fft(Σ,flags=FFTW.MEASURE)

    GaussianRandomField{typeof(cov),CirculantEmbedding,typeof(pts)}(mean,cov,pts,(Σ,P))
end

normalize(pts::R where {R<:AbstractRange}) = pts.-pts[1]
mirror(pts::R where {R<:AbstractRange}) = -pts[end]:(pts[2]-pts[1]):pts[end]

function pad(x,n)
    x0 = x[1]
    xn = x[end]
    dx = x[2]-x[1]
    x0:dx:n*xn
end

# returns the required dimension of the random points
randdim(grf::CirculantGRF) = length(grf.data[1])

# sample function
function _sample(grf::CirculantGRF, xi)
    v = grf.data[1]
    y = v.*reshape(xi,size(v))
    P = grf.data[2]
    w = P*y # fft
    w = real(w) + imag(w)
    n = length.(grf.pts)
    z = w[broadcast(:,1,n)...] # select appropriate elements

    grf.mean + std(grf.cov)*z
end

show(io::IO,::CirculantEmbedding) = print(io,"circulant embedding")


struct Circuit{T}
    netlist::Netlist
    conduction_matrix::AbstractMatrix{T}
    currents_matrix#::Union{AbstractVector{T},SparseVector{T,Int},SparseVector{Number,Int}}
    parameters
 end

 function Circuit(net::Netlist,s::Num)
    N = nnodes(net)
    M = spzeros(N,N)
    I = spzeros(N)
    Ps = [s]
    for component in net.components
        m,i,p = SymStamp(component,N,s)
        M += m
        I += i
        push!(Ps,p...)
    end

    Gm = M[2:end,2:end]
    Im = I[2:end]
    #println(Ps)
    Circuit(net,Gm,Im,Ps)
 end

 function Circuit(net::Netlist)
    N = nnodes(net)
    M = spzeros(N,N)
    I = spzeros(N)
    Ps = []
    for component in net.components
        m,i,p = SymStamp(component,N)
        M += m
        I += i
        push!(Ps,p...)
    end

    Gm = M[2:end,2:end]
    Im = I[2:end]
    #println(Ps)
    Circuit(net,Gm,Im,Ps)
end

function SymbolicMatrixes(ckt::Circuit)
    N = nnodes(ckt.netlist)-1
    Vvector = @variables V[1:N]
    Vvector  = Symbolics.scalarize(Vvector)
    M = copy(ckt.conduction_matrix)
    I = copy(ckt.currents_matrix)
    return Matrix(M),Vector(I),Vector(Vvector[1])
end

function SymbolicAuxEqs(ckt::Circuit,V)
    eq = []
    for comp in ckt.netlist.components
        dd = symSubs(comp,V)
        for d in dd
            push!(eq,first(d)~last(d))
        end
    end
    return eq
end

function CircuitFunction(ckt::Circuit)
    N = nnodes(ckt.netlist)-1 #exclude ground
    Vnames = map(x->x.name,ckt.netlist.nodes)[2:end]
    test = Symbolics.Symbol.(Vnames)
    Vvector = Vector(Symbolics.Variable.(test))
    M = copy(ckt.conduction_matrix)
    I = copy(ckt.currents_matrix)
    S = size(M)
    for comp in ckt.netlist.components
        for l in 1:S[1]
            I[l] = substitute(I[l],symSubs(comp,Vvector))
            for c in 1:S[2]
                M[l,c] = substitute(M[l,c],symSubs(comp,Vvector))
            end
        end
    end
    return Matrix(M),Vector(I),Vector(Vvector)
end

"""
 dc_op(ckt::Circuit; maxiter=1000,abs_tol=1e-6,rel_tol=1e-2)
 Calculates the DC Operating Point of a given circuit using NewtonRaphson method
"""
function dc_op(ckt::Circuit; maxiter=1000,abs_tol=1e-6,rel_tol=1e-2)
    Mm,Im,Vm = CircuitFunction(ckt)

    M_f = eval(build_function(Mm,Vm)[2])
    I_f = eval(build_function(Im,Vm)[2])

    v₀ = zeros(length(Vm))
    Mm = zeros(size(Mm))
    Im = zeros(size(Im))
    Base.invokelatest(M_f,Mm,v₀)
    Base.invokelatest(I_f,Im,v₀)
    v₁ = Mm\Im
    iter = 0
    abserror = 100
    relerror = 100
    while iter < maxiter && abserror > abs_tol && relerror > rel_tol
        v₀ = v₁
        Base.invokelatest(M_f,Mm,v₀)
        Base.invokelatest(I_f,Im,v₀)
        v₁ = Mm\Im
        iter += 1
        abserror = abs(maximum(v₁-v₀))
        relerror = abs(maximum((v₁-v₀)/v₁))
        if iter == maxiter
            @warn "Maximum iteration achieved"
        end
    end
    return v₁
end

struct CircuitProperties{T}
    ckt::Circuit
    Vi::Int
    Vo::Int
    Z::AbstractMatrix{T}
    Y::AbstractMatrix{T}
    T::AbstractMatrix{T}
    H::AbstractMatrix{T}
    VoltageGain::T
    TransconductanceGain::T
    Rin::T
    Rout::T
end

"""
CircuitProperties(ckt::Circuit,Vi::Int,Vo::Int)
Calculate The Circuit Parameters such as Voltage Gain, TransconductanceGain, R_in, R_out for a circuit with input node Vi and output node Vo
# Parameters
ckt::Circuit - Circuit to be anaylized
Vi::Int - Input Node number
Vo::Int - Output Node number
"""

function CircuitProperties(ckt::Circuit,Vi::Int,Vo::Int)
    Z = inv(Matrix(ckt.conduction_matrix))
    Z = Z[[Vi,Vo],[Vi,Vo]]
    Y = Z
    try
        Y = inv(Z)
    catch
        Y = Matrix(ckt.conduction_matrix)[[Vi,Vo],[Vi,Vo]]
    end
    # T Matrix
    y11 = Y[1,1]
    y12 = Y[1,2]
    y21 = Y[2,1]
    y22 = Y[2,2]
    A = -y22/y21
    B = (-1/y21)
    C = ((y12*y21-y11*y22)/y21)
    D = (-y11/y21)
    T = [A B;C D]
    #H Matrix

    A = 1/y11
    B = (-y12/y11)
    C = y21/y11
    D = ((y11*y22-y12*y21)/y11)
    H = [A B;C D]

    VoltageGain = simplify(1/T[1,1])
    TransconductanceGain = simplify(1/T[1,2])
    Rin = simplify(1/Y[1,1])
    Rout = simplify(Z[2,2])

    CircuitProperties(ckt,Vi,Vo,Z,Y,T,H,VoltageGain,TransconductanceGain,Rin,Rout)
end


import GLPK

using SparseArrays: spzeros

@testset "Minimum-cost flow" begin

    # bipartite oriented + source & sink
    # source 5, sink 6, v1 {1, 2} v2 {3, 4}
    g = Graphs.DiGraph(6)
    Graphs.add_edge!(g, 5, 1)
    Graphs.add_edge!(g, 5, 2)
    Graphs.add_edge!(g, 3, 6)
    Graphs.add_edge!(g, 4, 6)
    Graphs.add_edge!(g, 1, 3)
    Graphs.add_edge!(g, 1, 4)
    Graphs.add_edge!(g, 2, 3)
    Graphs.add_edge!(g, 2, 4)
    cost = zeros(6,6)
    cost[1,3] = 10.
    cost[1,4] = 5.
    cost[2,3] = 2.
    cost[2,4] = 2.
    # v2 -> sink have demand of one
    demand = spzeros(6,6)
    demand[3,6] = 1
    demand[4,6] = 1
    capacity = ones(6,6)

    o = GLPK.Optimizer
    node_demand = spzeros(Graphs.nv(g))
    flow = mincost_flow(g, node_demand, capacity, cost, o, edge_demand=demand, source_nodes=[5], sink_nodes=[6])
    @test flow[5,1] ≈ 1
    @test flow[5,2] ≈ 1
    @test flow[3,6] ≈ 1
    @test flow[4,6] ≈ 1
    @test flow[1,4] ≈ 1
    @test flow[2,4] ≈ 0
    @test flow[2,3] ≈ 1
    @test flow[1,3] ≈ 0

    # flow conservation property
    for n1 = 1:4
        @test sum(flow[n1, nout] for nout in outneighbors(g, n1)) ≈ sum(flow[nin, n1] for nin in inneighbors(g, n1))
    end

    #--- Same test without edge_demands & positive cost should result in null flow
    flow = mincost_flow(g, node_demand, capacity, cost, o, source_nodes=[5], sink_nodes=[6])

    @test flow[5,1] ≈ 0
    @test flow[5,2] ≈ 0
    @test flow[3,6] ≈ 0
    @test flow[4,6] ≈ 0
    @test flow[1,4] ≈ 0
    @test flow[2,4] ≈ 0
    @test flow[2,3] ≈ 0
    @test flow[1,3] ≈ 0

    # flow conservation property
    for n1 = 1:4
        @test sum(flow[n1,:]) ≈ sum(flow[:,n1])
    end

    #run same test again with nodal demands
    node_demand=spzeros(6)
    node_demand[5]=-2
    node_demand[6]=2
    #--- Same test without edge_demands
    flow = mincost_flow(g, node_demand, capacity, cost, o, source_nodes=[5],sink_nodes=[6])
    @test flow[5,1] ≈ 1
    @test flow[5,2] ≈ 1
    @test flow[3,6] ≈ 1
    @test flow[4,6] ≈ 1
    @test flow[1,4] ≈ 1
    @test flow[2,4] ≈ 0
    @test flow[2,3] ≈ 1
    @test flow[1,3] ≈ 0
    @test sum(diag(flow)) ≈ 0

    # flow conservation property
    for n1 = 1:4
        @test sum(flow[n1,:]) ≈ sum(flow[:,n1])
    end

    # no demand => null flow
    d2 = spzeros(6,6)
    flow = mincost_flow(g, spzeros(Graphs.nv(g)), capacity, cost, o, edge_demand=d2, source_nodes=[5], sink_nodes=[6])
    for i in 1:6
        for j in 1:6
            @test flow[i,j] ≈ 0.0
        end
    end

    # graph without sink or source
    g = Graphs.DiGraph(6)
    # create circuit
    for s in 1:5
        Graphs.add_edge!(g, s, s+1)
    end
    Graphs.add_edge!(g, 6, 1)
    Graphs.add_edge!(g, 2, 5) # shortcut

    capacity = ones(6,6)
    demand = spzeros(6,6)
    demand[1,2] = 1
    costs = ones(6,6)
    flow = mincost_flow(g,spzeros(Graphs.nv(g)), capacity, costs, o, edge_demand=demand)
    active_flows = [(1,2), (2,5), (5,6),(6,1)]
    for s in 1:6
        for t in 1:6
            if (s,t) in active_flows
                @test flow[s,t] ≈ 1.
            else
                @test flow[s,t] ≈ 0.
            end
        end
    end
    # flow conservation property
    for n1 = 1:6
        @test sum(flow[n1,:]) ≈ sum(flow[:,n1])
    end
    # higher short-circuit cost
    costs[2,5] = 10.
    flow = mincost_flow(g,spzeros(Graphs.nv(g)), capacity, costs, o, edge_demand=demand)
    active_flows = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,1)]
    for s in 1:6
        for t in 1:6
            if (s,t) in active_flows
                @test flow[s,t] ≈ 1.
            else
                @test flow[s,t] ≈ 0.
            end
        end
    end
    # flow conservation property
    for n1 = 1:6
        @test sum(flow[n1,:]) ≈ sum(flow[:,n1])
    end
end


using LinearAlgebraicRepresentation
Lar = LinearAlgebraicRepresentation
using Plasm

V,EV = Lar.randomcuboids(250, .2)
V = Plasm.normalize(V,flag=true)
Plasm.view(Plasm.numbering(.05)((V,[[[k] for k=1:size(V,2)], EV])))

W = convert(Lar.Points, V')
cop_EV = Lar.coboundary_0(EV::Lar.Cells)
cop_EW = convert(Lar.ChainOp, cop_EV)
V, copEV, copFE = Lar.Arrangement.planar_arrangement(W::Lar.Points, cop_EW::Lar.ChainOp)

triangulated_faces = Lar.triangulate2D(V, [copEV, copFE])
FVs = convert(Array{Lar.Cells}, triangulated_faces)
V = convert(Lar.Points, V')
Plasm.viewcolor(V::Lar.Points, FVs::Array{Lar.Cells})

W, copEV, copFE = Lar.Arrangement.planar_arrangement(W::Lar.Points, cop_EW::Lar.ChainOp)
EVs = Lar.FV2EVs(copEV, copFE) # polygonal face fragments
V = convert(Lar.Points, W')
Plasm.viewcolor(V::Lar.Points, EVs::Array{Lar.Cells})

model = V,EVs
Plasm.view(Plasm.lar_exploded(model)(1.2,1.2,1.2))


# Hierarchical Gaussian Filter

module HGF1Benchmark

using Rocket
using ReactiveMP
using GraphPPL
using Distributions

@model function hgf()

    real_k     = constvar(1.0)
    real_w     = constvar(-5.0)
    y_variance = constvar(0.01)
    z_variance = constvar(1.0)

    xt_min_mean = datavar(Float64)
    xt_min_var  = datavar(Float64)

    zt_min_mean = datavar(Float64)
    zt_min_var  = datavar(Float64)

    xt_min ~ NormalMeanVariance(xt_min_mean, xt_min_var)
    zt_min ~ NormalMeanVariance(zt_min_mean, zt_min_var)

    zt ~ NormalMeanVariance(zt_min, z_variance) where { q = q(zt_min)q(z_variance)q(zt) }

    gcv_node, xt ~ GCV(xt_min, zt, real_k, real_w) where { q = q(xt, xt_min)q(zt)q(κ)q(ω) }

    y = datavar(Float64)

    y ~ NormalMeanVariance(xt, y_variance)

    return zt, xt, y, gcv_node, xt_min_mean, xt_min_var, zt_min_mean, zt_min_var
end

function generate_input(rng, n)
    real_k = 1.0
    real_w = -5.0

    z_prev = 0.0
    x_prev = 0.0

    z = Vector{Float64}(undef, n)
    v = Vector{Float64}(undef, n)
    x = Vector{Float64}(undef, n)
    y = Vector{Float64}(undef, n)

    y_variance = 0.01
    z_variance = 1.0

    for i in 1:n
        z[i] = rand(rng, Normal(z_prev, sqrt(z_variance)))
        v[i] = exp(real_k * z[i] + real_w)
        x[i] = rand(rng, Normal(x_prev, sqrt(v[i])))
        y[i] = rand(rng, Normal(x[i], sqrt(y_variance)))

        z_prev = z[i]
        x_prev = x[i]
    end

    return y
end

function benchmark(input)
    iters      = 10
    real_k     = 1.0
    real_w     = -5.0
    y_variance = 0.01
    z_variance = 1.0

    n = length(input)

    ms_scheduler = PendingScheduler()
    fe_scheduler = PendingScheduler()

    mz = keep(Marginal)
    mx = keep(Marginal)
    fe = keep(Float64)

    model, (zt, xt, y, gcv_node, xt_min_mean, xt_min_var, zt_min_mean, zt_min_var) = hgf()

    s_mz = subscribe!(getmarginal(zt) |> schedule_on(ms_scheduler), mz)
    s_mx = subscribe!(getmarginal(xt) |> schedule_on(ms_scheduler), mx)
    s_fe = subscribe!(score(Float64, BetheFreeEnergy(), model, fe_scheduler), fe)

    # Initial prior messages
    current_zt = NormalMeanVariance(0.0, 10.0)
    current_xt = NormalMeanVariance(0.0, 10.0)

    # Prior marginals
    setmarginal!(gcv_node, :y_x, MvNormalMeanCovariance([ 0.0, 0.0 ], [ 5.0, 5.0 ]))
    setmarginal!(gcv_node, :z, NormalMeanVariance(0.0, 5.0))

    for i in 1:n

        for _ in 1:iters
            update!(y, input[i])
            update!(zt_min_mean, mean(current_zt))
            update!(zt_min_var, var(current_zt))
            update!(xt_min_mean, mean(current_xt))
            update!(xt_min_var, var(current_xt))

            release!(fe_scheduler)
        end

        release!(ms_scheduler)

        current_zt = getvalues(mz)[end]
        current_xt = getvalues(mx)[end]
    end

    unsubscribe!(s_mz)
    unsubscribe!(s_mx)
    unsubscribe!(s_fe)

    return map(getvalues, (mz, mx, fe))
end

end


doc"""
    GeneralizedPareto(ξ, σ, μ)

The *Generalized Pareto distribution* with shape parameter `ξ`, scale `σ` and location `μ` has probability density function

$f(x; \xi, \sigma, \mu) = \begin{cases}
        \frac{1}{\sigma}(1 + \xi \frac{x - \mu}{\sigma} )^{-\frac{1}{\xi} - 1} & \text{for } \xi \neq 0 \\
        \frac{1}{\sigma} e^{-\frac{\left( x - \mu \right) }{\sigma}} & \text{for } \xi = 0
    \end{cases}~,
    \quad x \in \begin{cases}
        \left[ \mu, \infty \right] & \text{for } \xi \geq 0 \\
        \left[ \mu, \mu - \sigma / \xi \right] & \text{for } \xi < 0
    \end{cases}$

```julia
GeneralizedPareto()             # Generalized Pareto distribution with unit shape and unit scale, i.e. GeneralizedPareto(1.0, 1.0, 0.0)
GeneralizedPareto(k, s)         # Generalized Pareto distribution with shape k and scale s, i.e. GeneralizedPareto(k, s, 0.0)
GeneralizedPareto(k, s, m)      # Generalized Pareto distribution with shape k, scale s and location m.

params(d)       # Get the parameters, i.e. (k, s, m)
shape(d)        # Get the shape parameter, i.e. k
scale(d)        # Get the scale parameter, i.e. s
location(d)     # Get the location parameter, i.e. m
```

External links

* [Generalized Pareto distribution on Wikipedia](https://en.wikipedia.org/wiki/Generalized_Pareto_distribution)

"""

immutable GeneralizedPareto <: ContinuousUnivariateDistribution
    ξ::Float64
    σ::Float64
    μ::Float64

    function GeneralizedPareto(ξ::Real, σ::Real, μ::Real)
        @check_args(GeneralizedPareto, σ > zero(σ))
        new(ξ, σ, μ)
    end
    GeneralizedPareto(ξ::Real, σ::Real) = GeneralizedPareto(ξ::Real, σ::Real, 0.0)
    GeneralizedPareto() = new(1.0, 1.0, 0.0)
end

minimum(d::GeneralizedPareto) = d.μ
maximum(d::GeneralizedPareto) = d.ξ < 0.0 ? d.μ - d.σ / d.ξ : Inf

#### Parameters

shape(d::GeneralizedPareto) = d.ξ
scale(d::GeneralizedPareto) = d.σ
location(d::GeneralizedPareto) = d.μ
params(d::GeneralizedPareto) = (d.ξ, d.σ, d.μ)

#### Statistics

median(d::GeneralizedPareto) = d.μ + d.σ * expm1(d.ξ * log(2.0)) / d.ξ

function mean(d::GeneralizedPareto)
    if d.ξ < 1.0
        return d.μ + d.σ / (1.0 - d.ξ)
    else
        return Inf
    end
end

function var(d::GeneralizedPareto)
    if d.ξ < 0.5
        return d.σ^2 / ((1.0 - d.ξ)^2 * (1.0 - 2.0 * d.ξ))
    else
        return Inf
    end
end

function skewness(d::GeneralizedPareto)
    (ξ, σ, μ) = params(d)

    if ξ < (1.0 / 3.0)
        return 2.0 * (1.0 + ξ) * sqrt(1.0 - 2.0 * ξ) / (1.0 - 3.0 * ξ)
    else
        return Inf
    end
end

function kurtosis(d::GeneralizedPareto)
    (ξ, σ, μ) = params(d)

    if ξ < 0.25
        k1 = (1.0 - 2.0 * ξ) * (2.0 * ξ^2 + ξ + 3.0)
        k2 = (1.0 - 3.0 * ξ) * (1.0 - 4.0 * ξ)
        return 3.0 * k1 / k2 - 3.0
    else
        return Inf
    end
end

#### Evaluation

function logpdf(d::GeneralizedPareto, x::Float64)
    (ξ, σ, μ) = params(d)

    # The logpdf is log(0) outside the support range.
    p = -Inf

    if x >= μ
        z = (x - μ) / σ
        if abs(ξ) < eps()
            p = -z - log(σ)
        elseif ξ > 0.0 || (ξ < 0.0 && x < maximum(d))
            p = (-1.0 - 1.0 / ξ) * log1p(z * ξ) - log(σ)
        end
    end

    return p
end

pdf(d::GeneralizedPareto, x::Float64) = exp(logpdf(d, x))

function logccdf(d::GeneralizedPareto, x::Float64)
    (ξ, σ, μ) = params(d)

    # The logccdf is log(0) outside the support range.
    p = -Inf

    if x >= μ
        z = (x - μ) / σ
        if abs(ξ) < eps()
            p = -z
        elseif ξ > 0.0 || (ξ < 0.0 && x < maximum(d))
            p = (-1.0 / ξ) * log1p(z * ξ)
        end
    end

    return p
end

ccdf(d::GeneralizedPareto, x::Float64) = exp(logccdf(d, x))
cdf(d::GeneralizedPareto, x::Float64) = -expm1(logccdf(d, x))

function quantile(d::GeneralizedPareto, p::Float64)
    (ξ, σ, μ) = params(d)

    if p == 0.0
        z = 0.0
    elseif p == 1.0
        z = ξ < 0.0 ? -1.0 / ξ : Inf
    elseif 0.0 < p < 1.0
        if abs(ξ) < eps()
            z = -log1p(-p)
        else
            z = expm1(-ξ * log1p(-p)) / ξ
        end
    else
      z = NaN
    end

    return μ + σ * z
end

#### Sampling

function rand(d::GeneralizedPareto)
    # Generate a Float64 random number uniformly in (0,1].
    u = 1.0 - rand()

    if abs(d.ξ) < eps()
        rd = -log(u)
    else
        rd = expm1(-d.ξ * log(u)) / d.ξ
    end

    return d.μ + d.σ * rd
end


using Test
using ParameterizedFunctions
using DiffEqBiological

# align
f = @ode_def TestAlignFeedback2 begin
    dx = y*c_1 - x
    dy = x^c_2 - y
end c_1 c_2

@test latexify(f; env=:align, cdot=false) ==
raw"\begin{align}
\frac{dx}{dt} =& y c_{1} - x \\
\frac{dy}{dt} =& x^{c_{2}} - y
\end{align}
"

@test latexify(f; env=:align, cdot=true) ==
raw"\begin{align}
\frac{dx}{dt} =& y \cdot c_{1} - x \\
\frac{dy}{dt} =& x^{c_{2}} - y
\end{align}
"

rn = @reaction_network begin
    hillr(D₂,α,K,n), ∅ --> m₁
    hillr(D₁,α,K,n), ∅ --> m₂
    (δ,γ), m₁ ↔ ∅
    (δ,γ), m₂ ↔ ∅
    β, m₁ --> m₁ + P₁
    β, m₂ --> m₂ + P₂
    μ, P₁ --> ∅
    μ, P₂ --> ∅
    (k₊,k₋), 2P₁ ↔ D₁
    (k₊,k₋), 2P₂ ↔ D₂
    (k₊,k₋), P₁+P₂ ↔ T
end α K n δ γ β μ k₊ k₋;

@test latexify(rn; cdot=false) ==
raw"\begin{align}
\frac{dm_1}{dt} =& \frac{\alpha K^{n}}{K^{n} + D_2^{n}} - \delta m_1 + \gamma \\
\frac{dm_2}{dt} =& \frac{\alpha K^{n}}{K^{n} + D_1^{n}} - \delta m_2 + \gamma \\
\frac{dP_1}{dt} =& \beta m_1 - \mu P_1 -2 k_+ \frac{P_1^{2}}{2} + 2 k_- D_1 - k_+ P_1 P_2 + k_- T \\
\frac{dP_2}{dt} =& \beta m_2 - \mu P_2 -2 k_+ \frac{P_2^{2}}{2} + 2 k_- D_2 - k_+ P_1 P_2 + k_- T \\
\frac{dD_1}{dt} =& k_+ \frac{P_1^{2}}{2} - k_- D_1 \\
\frac{dD_2}{dt} =& k_+ \frac{P_2^{2}}{2} - k_- D_2 \\
\frac{dT}{dt} =& k_+ P_1 P_2 - k_- T
\end{align}
"

@test latexify(rn; cdot=true) ==
raw"\begin{align}
\frac{dm_1}{dt} =& \frac{\alpha \cdot K^{n}}{K^{n} + D_2^{n}} - \delta \cdot m_1 + \gamma \\
\frac{dm_2}{dt} =& \frac{\alpha \cdot K^{n}}{K^{n} + D_1^{n}} - \delta \cdot m_2 + \gamma \\
\frac{dP_1}{dt} =& \beta \cdot m_1 - \mu \cdot P_1 -2 \cdot k_+ \cdot \frac{P_1^{2}}{2} + 2 \cdot k_- \cdot D_1 - k_+ \cdot P_1 \cdot P_2 + k_- \cdot T \\
\frac{dP_2}{dt} =& \beta \cdot m_2 - \mu \cdot P_2 -2 \cdot k_+ \cdot \frac{P_2^{2}}{2} + 2 \cdot k_- \cdot D_2 - k_+ \cdot P_1 \cdot P_2 + k_- \cdot T \\
\frac{dD_1}{dt} =& k_+ \cdot \frac{P_1^{2}}{2} - k_- \cdot D_1 \\
\frac{dD_2}{dt} =& k_+ \cdot \frac{P_2^{2}}{2} - k_- \cdot D_2 \\
\frac{dT}{dt} =& k_+ \cdot P_1 \cdot P_2 - k_- \cdot T
\end{align}
"

@test latexify(rn; env=:chemical, cdot=false) ==
raw"\begin{align}
\require{mhchem}
\ce{ \varnothing &->[\frac{\alpha K^{n}}{K^{n} + D_2^{n}}] m_{1}}\\
\ce{ \varnothing &->[\frac{\alpha K^{n}}{K^{n} + D_1^{n}}] m_{2}}\\
\ce{ m_{1} &<=>[\delta][\gamma] \varnothing}\\
\ce{ m_{2} &<=>[\delta][\gamma] \varnothing}\\
\ce{ m_{1} &->[\beta] m_{1} + P_{1}}\\
\ce{ m_{2} &->[\beta] m_{2} + P_{2}}\\
\ce{ P_{1} &->[\mu] \varnothing}\\
\ce{ P_{2} &->[\mu] \varnothing}\\
\ce{ 2 P_1 &<=>[k_{+}][k_{-}] D_{1}}\\
\ce{ 2 P_2 &<=>[k_{+}][k_{-}] D_{2}}\\
\ce{ P_{1} + P_{2} &<=>[k_{+}][k_{-}] T}
\end{align}
"

@test latexify(rn; env=:chemical, cdot=true) ==
raw"\begin{align}
\require{mhchem}
\ce{ \varnothing &->[\frac{\alpha \cdot K^{n}}{K^{n} + D_2^{n}}] m_{1}}\\
\ce{ \varnothing &->[\frac{\alpha \cdot K^{n}}{K^{n} + D_1^{n}}] m_{2}}\\
\ce{ m_{1} &<=>[\delta][\gamma] \varnothing}\\
\ce{ m_{2} &<=>[\delta][\gamma] \varnothing}\\
\ce{ m_{1} &->[\beta] m_{1} + P_{1}}\\
\ce{ m_{2} &->[\beta] m_{2} + P_{2}}\\
\ce{ P_{1} &->[\mu] \varnothing}\\
\ce{ P_{2} &->[\mu] \varnothing}\\
\ce{ 2 \cdot P_1 &<=>[k_{+}][k_{-}] D_{1}}\\
\ce{ 2 \cdot P_2 &<=>[k_{+}][k_{-}] D_{2}}\\
\ce{ P_{1} + P_{2} &<=>[k_{+}][k_{-}] T}
\end{align}
"

# tabular
@test latexify(rn.symjac; env=:tabular, cdot=false) ==
raw"\begin{tabular}{ccccccc}
$ - \delta$ & $0$ & $0$ & $0$ & $0$ & $\frac{ - K^{n} n \alpha D_2^{-1 + n}}{\left( K^{n} + D_2^{n} \right)^{2}}$ & $0$\\
$0$ & $ - \delta$ & $0$ & $0$ & $\frac{ - K^{n} n \alpha D_1^{-1 + n}}{\left( K^{n} + D_1^{n} \right)^{2}}$ & $0$ & $0$\\
$\beta$ & $0$ & $ - \mu - 2 k_+ P_1 - k_+ P_2$ & $ - k_+ P_1$ & $2 k_-$ & $0$ & $k_{-}$\\
$0$ & $\beta$ & $ - k_+ P_2$ & $ - \mu - k_+ P_1 - 2 k_+ P_2$ & $0$ & $2 k_-$ & $k_{-}$\\
$0$ & $0$ & $k_+ P_1$ & $0$ & $ - k_-$ & $0$ & $0$\\
$0$ & $0$ & $0$ & $k_+ P_2$ & $0$ & $ - k_-$ & $0$\\
$0$ & $0$ & $k_+ P_2$ & $k_+ P_1$ & $0$ & $0$ & $ - k_-$\\
\end{tabular}
"

@test latexify(rn.symjac; env=:tabular, cdot=true) ==
raw"\begin{tabular}{ccccccc}
$ - \delta$ & $0$ & $0$ & $0$ & $0$ & $\frac{ - K^{n} \cdot n \cdot \alpha \cdot D_2^{-1 + n}}{\left( K^{n} + D_2^{n} \right)^{2}}$ & $0$\\
$0$ & $ - \delta$ & $0$ & $0$ & $\frac{ - K^{n} \cdot n \cdot \alpha \cdot D_1^{-1 + n}}{\left( K^{n} + D_1^{n} \right)^{2}}$ & $0$ & $0$\\
$\beta$ & $0$ & $ - \mu - 2 \cdot k_+ \cdot P_1 - k_+ \cdot P_2$ & $ - k_+ \cdot P_1$ & $2 \cdot k_-$ & $0$ & $k_{-}$\\
$0$ & $\beta$ & $ - k_+ \cdot P_2$ & $ - \mu - k_+ \cdot P_1 - 2 \cdot k_+ \cdot P_2$ & $0$ & $2 \cdot k_-$ & $k_{-}$\\
$0$ & $0$ & $k_+ \cdot P_1$ & $0$ & $ - k_-$ & $0$ & $0$\\
$0$ & $0$ & $0$ & $k_+ \cdot P_2$ & $0$ & $ - k_-$ & $0$\\
$0$ & $0$ & $k_+ \cdot P_2$ & $k_+ \cdot P_1$ & $0$ & $0$ & $ - k_-$\\
\end{tabular}
"

f = @ode_def feedback begin
    dx = y/c_1 - x
    dy = x^c_2 - y
    end c_1 c_2

@test latexify(f; starred=true) ==
raw"\begin{align*}
\frac{dx}{dt} =& \frac{y}{c_{1}} - x \\
\frac{dy}{dt} =& x^{c_{2}} - y
\end{align*}
"

f = @ode_def TestRaw begin
    dx = y/c_1 - x
    dy = x^c_2 - y
end c_1 c_2
@test latexraw(f) == ["\\frac{dx}{dt} = \\frac{y}{c_{1}} - x", "\\frac{dy}{dt} = x^{c_{2}} - y"]

#==============================================================================#
#================================  Test align  ================================#
#==============================================================================#

f = @ode_def TestAlignFeedback begin
    dx = y/c_1 - x
    dy = x^c_2 - y
end c_1 c_2

g = @ode_def TestAlign2 begin
    dx = p_x - d_x*x - d_x_y*y*x
    dy = p_y - d_y*y
    dz = p_z
end p_x d_x d_x_y p_y d_y p_z

@test latexify(f) ==
raw"\begin{align}
\frac{dx}{dt} =& \frac{y}{c_{1}} - x \\
\frac{dy}{dt} =& x^{c_{2}} - y
\end{align}
"

@test latexify(f; rows=1) ==
raw"\begin{align}
\frac{dx}{dt} =& \frac{y}{c_{1}} - x
\end{align}
"

@test latexify(f; env=:align, field=:symjac) ==
raw"\begin{align}
\frac{dx}{dt} =& -1 \cdot 1 =& c_{1}^{-1} \\
\frac{dy}{dt} =& x^{-1 + c_{2}} \cdot c_{2} =& -1 \cdot 1
\end{align}
"

@test latexalign(["d$(x)/dt" for x = f.syms], f.funcs) ==
raw"\begin{align}
\frac{dx}{dt} =& \frac{y}{c_{1}} - x \\
\frac{dy}{dt} =& x^{c_{2}} - y
\end{align}
"

@test latexalign([f,g]) == "\\begin{align}\n\\frac{dx}{dt}  &=  \\frac{y}{c_{1}} - x  &  \\frac{dx}{dt}  &=  p_{x} - d_{x} \\cdot x - d_{x\\_y} \\cdot y \\cdot x  &  \\\\\n\\frac{dy}{dt}  &=  x^{c_{2}} - y  &  \\frac{dy}{dt}  &=  p_{y} - d_{y} \\cdot y  &  \\\\\n  &    &  \\frac{dz}{dt}  &=  p_{z} \\cdot 1  & \n\\end{align}\n"

@reaction_func hill2(x, v, k) = v*x^2/(k^2 + x^2)

rn = @reaction_network TestAlignMyRnType begin
  hill2(y, v_x, k_x), 0 --> x
  p_y, 0 --> y
  (d_x, d_y), (x, y) --> 0
  (r_b, r_u), x ↔ y
end v_x k_x p_y d_x d_y r_b r_u

@test latexalign(rn; clean=true).s ==
raw"\begin{align}
\frac{dx}{dt} =& \frac{v_{x} \cdot y^{2}}{k_{x}^{2} + y^{2}} - d_{x} \cdot x - r_{b} \cdot x + r_{u} \cdot y \\
\frac{dy}{dt} =& p_{y} - d_{y} \cdot y + r_{b} \cdot x - r_{u} \cdot y
\end{align}
"


using LinearAlgebra
using ForneyLab
import ForneyLab: unsafeMean, unsafeCov, unsafePrecision
using NARMAX

function generate_data(ϕ; θ_scale=0.1, τ_true=1e3, degree=1, M1=1, M2=1, M3=1, fMin=0.8, fMax=1.0, fs=1.0, uStd=1., T=100, split_index=50, start_index=10, num_periods=num_periods, points_period=points_period)

    # Generate signal
    input, output = gensignalNARMAX(ϕ, θ_scale, τ_true; M1=M1, M2=M2, M3=M3, degree=degree, fMin=fMin, fMax=fMax, fs=fs, uStd=uStd, T=T, num_periods=num_periods, points_period=points_period)

    # Split data
    ix_trn, ix_val = split_data(T, split_index, start_index=start_index)

    return input, output, ix_trn, ix_val
end

function model_specification(ϕ; M1=M1, M2=M2, M3=M3, N=N)

    graph = FactorGraph()

    # Observed variables
    @RV u_kmin1; placeholder(u_kmin1, :u_kmin1, dims=(M1,))
    @RV y_kmin1; placeholder(y_kmin1, :y_kmin1, dims=(M2,))
    @RV e_kmin1; placeholder(e_kmin1, :e_kmin1, dims=(M3,))
    @RV u_k; placeholder(u_k, :u_k)

    # Time-invariant parameters
    @RV τ ~ Gamma(placeholder(:a_τ), placeholder(:b_τ))
    @RV θ ~ GaussianMeanPrecision(placeholder(:m_θ, dims=(N,)), placeholder(:w_θ, dims=(N,N)))

    # Likelihood
    @RV y_k ~ NAutoRegressiveMovingAverageX(θ, u_k, u_kmin1, y_kmin1, e_kmin1, τ, ϕ=ϕ)
    placeholder(y_k, :y_k)

    q = PosteriorFactorization(θ, τ, ids=[:θ :τ])
    algorithm = messagePassingAlgorithm([θ; τ], q, free_energy=true)
    return algorithmSourceCode(algorithm, free_energy=true)

end

function experiment_FEM(input_trn, output_trn, input_tst, output_tst, ϕ, priors; M1=1, M2=1, M3=1, N=3, num_iters=5, computeFE=false)

    # Maximum delay
    maxM = maximum([M1,M2,M3])

    # Zero-padding of signals
    input_trn = [zeros(maxM,); input_trn]
    input_tst = [zeros(maxM,); input_tst]
    output_trn = [zeros(maxM,); output_trn]
    output_tst = [zeros(maxM,); output_tst]

    # Signal splits
    T_trn = length(input_trn)
    T_tst = length(input_tst)

    "Inference execution"

    # Extract prior parameters
    θ_k = priors["θ"]
    τ_k = priors["τ"]

    # Initialize marginals
    marginals = Dict(:θ => ProbabilityDistribution(Multivariate, GaussianMeanPrecision, m=θ_k[1], w=θ_k[2]),
                     :τ => ProbabilityDistribution(Univariate, Gamma, a=τ_k[1], b=τ_k[2]))

    # Preallocate free energy array
    Fq = zeros(T_trn-maxM, num_iters)

    # Preallocate parameter arrays
    params_θ = (zeros(T_trn-maxM,N), zeros(T_trn-maxM,N,N))
    params_τ = (zeros(T_trn-maxM,1), zeros(T_trn-maxM,1))

    # Preallocate prediction arrays
    predictions = (zeros(T_trn,), zeros(T_trn,))
    errors = zeros(T_trn,)

    for (ii,k) in enumerate(maxM+1:T_trn)

        # Update history vectors
        u_kmin1 = input_trn[k-1:-1:k-M1]
        y_kmin1 = output_trn[k-1:-1:k-M2]
        e_kmin1 = errors[k-1:-1:k-M3]

        ϕx = ϕ([input_trn[k]; u_kmin1; y_kmin1; e_kmin1])
        predictions[1][k] = θ_k[1]'*ϕx
        # predictions[2][k] = ϕx'*inv(θ_k[2])*ϕx + inv(τ_k[1]/τ_k[2])

        # Compute prediction error
        errors[k] = output_trn[k] - predictions[1][k]

        # Set data
        data = Dict(:y_k => output_trn[k],
                    :u_k => input_trn[k],
                    :u_kmin1 => u_kmin1,
                    :y_kmin1 => y_kmin1,
                    :e_kmin1 => e_kmin1,
                    :m_θ => θ_k[1],
                    :w_θ => θ_k[2],
                    :a_τ => τ_k[1],
                    :b_τ => τ_k[2])

        # Iterate updates
        for n = 1:num_iters

            # Compute free energy
            if computeFE
                Fq[ii,n] = freeEnergy(data,marginals)
            end

            # Update parameters
            stepθ!(data, marginals)
            stepτ!(data, marginals)

        end

        # Update params
        θ_k = (unsafeMean(marginals[:θ]), unsafePrecision(marginals[:θ]))
        τ_k = (marginals[:τ].params[:a], marginals[:τ].params[:b])

        # Store params
        params_θ[1][ii,:] = θ_k[1]
        params_θ[2][ii,:,:] = θ_k[2]
        params_τ[1][ii] = τ_k[1]
        params_τ[2][ii] = τ_k[2]

    end

    "1-step ahead prediction"

    # Prepare array
    predictions = (zeros(T_tst,), zeros(T_tst,))
    errors = zeros(T_tst,)

    for k in maxM+1:T_tst

        # Update history vectors
        u_kmin1 = input_tst[k-1:-1:k-M1]
        y_kmin1 = output_tst[k-1:-1:k-M2]
        e_kmin1 = errors[k-1:-1:k-M3]

        # Posterior predictive
        ϕx = ϕ([input_tst[k]; u_kmin1; y_kmin1; e_kmin1])
        predictions[1][k] = θ_k[1]'*ϕx
        # predictions[2][k] = ϕx'*inv(θ_k[2])*ϕx + inv(τ_k[1] / τ_k[2])

        # Update error
        errors[k] = output_tst[k] - predictions[1][k]

    end

    "Simulation"

    # Prepare array
    simulations = (zeros(T_tst,), zeros(T_tst,))

    for k in maxM+1:T_tst

        # Update history vectors
        u_kmin1 = input_tst[k-1:-1:k-M1]
        y_kmin1 = simulations[1][k-1:-1:k-M2]
        e_kmin1 = zeros(M3,)

        # Posterior predictive
        ϕx = ϕ([input_tst[k]; u_kmin1; y_kmin1; e_kmin1])
        simulations[1][k] = θ_k[1]'*ϕx
        # simulations[2][k] = ϕx'*inv(θ_k[2])*ϕx + inv(τ_k[1] / τ_k[2])

    end

    "Evaluation"

    # Compute root mean square error
    RMS_prd = sqrt(mean((predictions[1][maxM+1:end] - output_tst[maxM+1:end]).^2))
    RMS_sim = sqrt(mean((simulations[1][maxM+1:end] - output_tst[maxM+1:end]).^2))

    if computeFE
        return RMS_sim, RMS_prd, Fq
    else
        return RMS_sim, RMS_prd
    end
end

function experiment_RLS(input_trn, output_trn, input_tst, output_tst, ϕ; M1=1, M2=1, M3=1, N=1, λ=1.00)

    # Maximum delay
    maxM = maximum([M1,M2,M3])

    # Zero-padding of signals
    input_trn = [zeros(maxM,); input_trn]
    input_tst = [zeros(maxM,); input_tst]
    output_trn = [zeros(maxM,); output_trn]
    output_tst = [zeros(maxM,); output_tst]

    # Signal splits
    T_trn = length(input_trn)
    T_tst = length(input_tst)

    # Parameters
    P = λ.*Matrix{Float64}(I,N,N)
    w_k = zeros(N,)

    # Preallocate prediction array
    predictions = zeros(T_trn,)
    errors = zeros(T_trn,)

    for (ii,k) in enumerate(maxM+1:T_trn)

        # Update history vectors
        u_kmin1 = input_trn[k-1:-1:k-M1]
        y_kmin1 = output_trn[k-1:-1:k-M2]
        e_kmin1 = errors[k-1:-1:k-M3]

        # Update data vector
        ϕx = ϕ([input_trn[k]; u_kmin1; y_kmin1; e_kmin1])

        # Update weights
        α = output_trn[k] - w_k'*ϕx
        g = P*ϕx*inv(λ + ϕx'*P*ϕx)
        P = inv(λ)*P - g*ϕx'*inv(λ)*P
        w_k = w_k + α*g

        # Prediction
        predictions[k] = w_k'*ϕx
        errors[k] = output_trn[k] - predictions[k]

    end

    "1-step ahead prediction"

    # Prepare array
    predictions = zeros(T_tst,)
    errors = zeros(T_tst,)

    for k in maxM+1:T_tst

        # Update history vectors
        u_kmin1 = input_tst[k-1:-1:k-M1]
        y_kmin1 = output_tst[k-1:-1:k-M2]
        e_kmin1 = errors[k-1:-1:k-M3]

        # Posterior predictive
        ϕx = ϕ([input_tst[k]; u_kmin1; y_kmin1; e_kmin1])
        predictions[k] = w_k'*ϕx

        # Update error
        errors[k] = output_tst[k] - predictions[k]

    end

    "Simulation"

    # Prepare array
    simulations = zeros(T_tst,)

    for k in maxM+1:T_tst

        # Update history vectors
        u_kmin1 = input_tst[k-1:-1:k-M1]
        y_kmin1 = simulations[k-1:-1:k-M2]
        e_kmin1 = zeros(M3,)

        # Posterior predictive
        ϕx = ϕ([input_tst[k]; u_kmin1; y_kmin1; e_kmin1])
        simulations[k] = w_k'*ϕx

    end

    "Evaluation"

    # Compute root mean square error
    RMS_prd = sqrt(mean((predictions[maxM+1:end] - output_tst[maxM+1:end]).^2))
    RMS_sim = sqrt(mean((simulations[maxM+1:end] - output_tst[maxM+1:end]).^2))

    return RMS_sim, RMS_prd
end

function experiment_SYS(input_tst, output_tst, ϕ, θ; M1=1, M2=1, M3=1, N=1)

    # Maximum delay
    maxM = maximum([M1,M2,M3])

    # Zero-padding of signals
    input_tst = [zeros(maxM,); input_tst]
    output_tst = [zeros(maxM,); output_tst]

    # Length of validation signal
    T_tst = length(input_tst)

    "1-step ahead prediction"

    # Prepare array
    predictions = zeros(T_tst,)
    errors = zeros(T_tst,)

    for k in maxM+1:T_tst

        # Update history vectors
        u_kmin1 = input_tst[k-1:-1:k-M1]
        y_kmin1 = output_tst[k-1:-1:k-M2]
        e_kmin1 = errors[k-1:-1:k-M3]

        # Posterior predictive
        ϕx = ϕ([input_tst[k]; u_kmin1; y_kmin1; e_kmin1])
        predictions[k] = θ'*ϕx

        # Update error
        errors[k] = output_tst[k] - predictions[k]

    end

    "Simulation"

    # Prepare array
    simulations = zeros(T_tst,)

    for k in maxM+1:T_tst

        # Update history vectors
        u_kmin1 = input_tst[k-1:-1:k-M1]
        y_kmin1 = simulations[k-1:-1:k-M2]
        e_kmin1 = zeros(M3,)

        # Posterior predictive
        ϕx = ϕ([input_tst[k]; u_kmin1; y_kmin1; e_kmin1])
        simulations[k] = θ'*ϕx

    end

    "Evaluation"

    # Compute root mean square error
    RMS_prd = sqrt(mean((predictions[maxM+1:end] - output_tst[maxM+1:end]).^2))
    RMS_sim = sqrt(mean((simulations[maxM+1:end] - output_tst[maxM+1:end]).^2))

    return RMS_sim, RMS_prd
end