print("Last updated: $(read(`date`, String))")

# Load environment
import Pkg; Pkg.activate("../../../")

# Import Libraries
using Turing
using Turing: Variational
using Distributions
using AbstractGPs, KernelFunctions
using PyPlot
using StatsFuns
using JSON3
using Flux
using ProgressBars
import Random
import LinearAlgebra

# Define a kernel.
function sekernel(alpha, rho, jitter=0)
  kernel = alpha^2 * transform(SEKernel(), invsqrt2/rho)
  jitter <= 0 || (kernel += jitter * EyeKernel())
  return kernel
end

function compute_f(kernel, X, eta, beta=0, jitter=0)
  K = kernelmatrix(kernel, X, obsdim=1)
  jitter <= 0 || (K += LinearAlgebra.I * jitter)
  return LinearAlgebra.cholesky(K).L * eta .+ beta
end

@model function GPClassify(y, X, jitter=1e-6)
    # Priors.
    alpha ~ LogNormal(0, 1)
    rho ~ LogNormal(0, 1)
    beta ~ Normal(0, 1)  # intercept.
    eta ~ filldist(Normal(0, 1), length(y))

    # Latent GP
    kernel = sekernel(alpha, rho)
    f = compute_f(sekernel(alpha, rho), X, eta, beta, jitter)

    # Sampling Distribution.
    y ~ Product(Bernoulli.(logistic.(f)))
end;

# For getting quantiles along array dimensions
quantiles(X, q; dims) = mapslices(x -> quantile(x, q), X, dims=dims)

# To extract parameters from ADVI model.
function make_extractor(m, q; nsamples=300)
    qsamples = rand(q, nsamples)
    _, sym2range = Variational.bijector(m; sym_to_ranges=Val(true))
    return sym -> qsamples[collect(sym2range[sym][1]), :]
end;

function plot_data(X, y; edgecolors=nothing)
    colors = ["blue", "red"]
    plt.scatter(X[:, 1], X[:, 2], color=colors[y .+ 1], edgecolors=edgecolors)
end;

# Function for plotting parameter posterior.
function plot_post(samples, name; bins=nothing, suffix="")
    param = samples[name]
    plt.hist(param, density=true, bins=bins)
    plt.xlabel(name)
    plt.ylabel("density")
    if suffix !== ""
        suffix = "($(suffix))"
    end
    plt.title("Histogram of $(name) $(suffix)");
end;

function makegrid(xmin, xmax, ymin, ymax, ngrid; return_all=false)
  ngrid = floor(Int, sqrt(ngrid))
  grid = [[i, j]
          for i in range(xmin, xmax, length=ngrid),
          j in range(ymin, ymax, length=ngrid)]
  out =  Matrix(hcat(vec(grid)...)')

  if return_all
    return out, grid
  else
    return out
  end
end;

function gp_predict(postsamps, X, y, nnew; eps=0, prob=true, jitter=1e-6)
    # Make grid of prediction locations.
    xmin = minimum(X, dims=1)
    xmax = maximum(X, dims=1)
    Xnew, grid = makegrid(xmin[1]-eps, xmax[1]+eps,
                          xmin[2]-eps, xmax[2]+eps,
                          nnew, return_all=true)

    # Posterior samples of model parameters.
    alpha = postsamps[:alpha]
    rho = postsamps[:rho]
    beta = postsamps[:beta]
    nsamps = length(alpha)
    eta = postsamps[:eta]

    # Predict classification probabilities.
    ps = [let
            kernel = sekernel(alpha[i], rho[i], jitter)
            gp = GP(beta[i], kernel)
            f = compute_f(kernel, X, eta[:, i], beta[i])
            pfx = posterior(gp(X'), f)
            logit_p = rand(pfx(Xnew'))
            prob ? logistic.(logit_p) : logit_p
         end for i in ProgressBar(1:nsamps)]

    return hcat(ps...), Xnew, grid
end;

function plot_kernel_params(post_samples, kernel_params; bins=nothing, figsize=(8,3))
    plt.figure(figsize=figsize)
    nparams = length(kernel_params)
    for i in 1:nparams
        param = kernel_params[i]
        plt.subplot(1, nparams, i)
        plot_post(post_samples, param, bins=bins);
    end
    plt.tight_layout()
end;

function plot_uq(postsamps, X, y, algo;
                 nnew=100, eps=0.2, figsize=(10, 8),
                 kernel_params=nothing, jitter=1e-6,
                 color_res=100, color_res_sd=100, return_stuff=false,
                 data_edgecolors=nothing, vmin_sd=0, vmax_sd=nothing, prob=true)
    nsamps = postsamps[:alpha]
    ps, Xnew, grid = gp_predict(postsamps, X, y, nnew, eps=eps, prob=prob, jitter=jitter)

    plt.figure(figsize=figsize)
    gs = [[g[i] for g in grid] for i in 1:2]

    # Plot mean function.
    plt.subplot(2, 2, 1)
    if prob
        plt.contourf(gs[1], gs[2], reshape(mean(ps, dims=2), size(gs[1])...),
                     color_res, cmap=plt.get_cmap("bwr"), vmin=0, vmax=1)
    else
        plt.contourf(gs[1], gs[2], reshape(mean(ps, dims=2), size(gs[1])...),
                     color_res, cmap=plt.get_cmap("bwr"))
    end
    plt.colorbar();
    plot_data(X, y, edgecolors=data_edgecolors);
    plt.xticks([]); plt.yticks([]);
    plt.title("Posterior Mean Function ($(algo))")

    # Plot function sd.
    plt.subplot(2, 2, 2)
    plt.contourf(gs[1], gs[2], reshape(std(ps, dims=2), size(gs[1])...),
                 color_res_sd, cmap=plt.get_cmap("Oranges"),
                 vmin=vmin_sd, vmax=vmax_sd)
    plt.colorbar();
    plot_data(X, y)
    plt.xticks([]);
    plt.yticks([]);
    plt.title("Posterior Standard Deviation Function ($algo)")

    if kernel_params != nothing
        plot_kernel_params(postsamps, kernel_params, bins=30)
    end

    if return_stuff
        return ps, Xnew, grid
    end
end;

# Read data.

# Path to data.
data_path = joinpath(@__DIR__, "../data/gp-classify-data-N50.json")

# Load data in JSON format.
data = let
    x = open(f -> read(f, String), data_path)
    JSON3.read(x)
end

# Store data.
X = [data["x1"] data["x2"]]
y = Int64.(data["y"])

# Plot data.
plot_data(X, y);

# Create model.
m =  GPClassify(Float64.(y), X);
kernel_params = [:alpha, :rho, :beta];

# Fit via ADVI.
Random.seed!(7)

# NOTE: ADVI(num_elbo_samples, max_iters)

# Compile
@time q = vi(m, ADVI(1, 1))

# RUN
@time q = vi(m, ADVI(1, 1000))

# Get posterior samples
extract_gp = make_extractor(m, q, nsamples=500)
advi_samples = Dict{Symbol, Any}(sym => vec(extract_gp(sym)) for sym in kernel_params)
advi_samples[:eta] = extract_gp(:eta);

plot_uq(advi_samples, X, y, "ADVI", eps=0.5,
        nnew=500, data_edgecolors="gold",
        kernel_params=kernel_params,
        color_res=range(0, 1, length=101),
        color_res_sd=range(0, .5, length=101));

# Fit via HMC.
Random.seed!(0)
burn = 500
nsamples = 500

# Compile
@time _ = sample(m, HMC(0.01, 1), 1)

# Run
@time hmc_chain = sample(m, HMC(0.05, 20), burn + nsamples);

# Get posterior samples
hmc_samples = Dict{Symbol, Any}([
    sym => vec(group(hmc_chain, sym).value.data)[end-nsamples+1:end]
for sym in kernel_params])
hmc_samples[:eta] = Matrix(group(hmc_chain, :eta).value.data[end-nsamples+1:end, :, 1]');

# Plot HMC
ps, Xnew, grid = plot_uq(hmc_samples, X, y, "HMC", eps=0.5,
                         nnew=500, data_edgecolors="gold",
                         kernel_params=kernel_params,
                         color_res=range(0, 1, length=101),
                         return_stuff=true);

plt.plot(get(hmc_chain, :lp)[1].data[burn+1:end]);
plt.xlabel("HMC iterations");
plt.ylabel("log unnormalized joint density");

# Fit via NUTS.
Random.seed!(0)
burn = 500
nsamples = 500

# Compile
@time _ = sample(m, NUTS(1, 0.8, max_depth=2), 2)

# Run
@time nuts_chain = sample(m, NUTS(burn, 0.8, max_depth=10), burn + nsamples);

# Get posterior samples
nuts_samples = Dict{Symbol, Any}([
    sym => vec(group(nuts_chain, sym).value.data)
for sym in kernel_params])
nuts_samples[:eta] = Matrix(group(nuts_chain, :eta).value.data[:, :, 1]');

# Plot NUTS
plot_uq(nuts_samples, X, y, "NUTS", eps=0.5,
        nnew=500, data_edgecolors="gold",
        kernel_params=kernel_params,
        color_res=range(0, 1, length=101),
        color_res_sd=range(0, .5, length=101),
        return_stuff=true);

plt.plot(get(nuts_chain, :lp)[1].data);
plt.xlabel("NUTS iterations");
plt.ylabel("log unnormalized joint density");


using Interpolations
#using DataFrames, Gadfly

f(x) = sin(2pi * (x-1) / 10)
dfs = Array(DataFrame,0)
xrange = .5:.1:11.5

for (i,it) in enumerate((
    BSpline(Constant()),
    BSpline(Linear()),
    BSpline(Quadratic(Free()))
    ))
    itp = interpolate(f(1:11), it, OnCell())
    push!(dfs, DataFrame(x=xrange,y=[itp[x]+(i-1)*3 for x in xrange],t="$it"))
end

push!(dfs, DataFrame(x=xrange, y=f(xrange),t="Data"))
df = vcat(dfs)
#display(plot(df,x=:x,y=:y,color=:t,Geom.path,Guide.colorkey("Dataset")))


# # 120: Differing species sets in regions, 1D
# ([source code](SOURCE_URL))

module Example120_ThreeRegions1D

using Printf
using VoronoiFVM
using ExtendableGrids
using GridVisualize

function main(;n=30,Plotter=nothing,plot_grid=false, verbose=false,unknown_storage=:sparse,tend=10)
    h=3.0/(n-1)
    X=collect(0:h:3.0)
    grid=VoronoiFVM.Grid(X)
    cellmask!(grid,[0.0],[1.0],1)
    cellmask!(grid,[1.0],[2.1],2)
    cellmask!(grid,[1.9],[3.0],3)

    subgrid1=subgrid(grid,[1])
    subgrid2=subgrid(grid,[1,2,3])
    subgrid3=subgrid(grid,[3])

    if plot_grid
        plotgrid(grid,Plotter=Plotter)
        return
    end

    eps=[1,1,1]
    k=[1,1,1]

    physics=VoronoiFVM.Physics(
    num_species=3,
    reaction=function(f,u,node)
        if node.region==1
            f[1]=k[1]*u[1]
            f[2]=-k[1]*u[1]
        elseif node.region==3
            f[2]=k[3]*u[2]
            f[3]=-k[3]*u[2]
        else
            f[1]=0
        end
    end,

        flux=function(f,u0,edge)
            u=unknowns(edge,u0)

        if edge.region==1
            f[1]=eps[1]*(u[1,1]-u[1,2])
            f[2]=eps[2]*(u[2,1]-u[2,2])
        elseif edge.region==2
            f[2]=eps[2]*(u[2,1]-u[2,2])
        elseif edge.region==3
            f[2]=eps[2]*(u[2,1]-u[2,2])
            f[3]=eps[3]*(u[3,1]-u[3,2])
        end
    end,

    source=function(f,node)
        if node.region==1
           f[1]=1.0e-4*(3.0-node[1])
        end
    end,

    storage=function(f,u,node)
        f.=u
    end
    )

    sys=VoronoiFVM.System(grid,physics,unknown_storage=unknown_storage)

    enable_species!(sys,1,[1])
    enable_species!(sys,2,[1,2,3])
    enable_species!(sys,3,[3])

    boundary_dirichlet!(sys,3,2,0.0)

    inival=unknowns(sys)
    U=unknowns(sys)
    inival.=0

    control=VoronoiFVM.NewtonControl()
    control.verbose=verbose
    tstep=0.01
    time=0.0
    istep=0
    testval=0
    p=GridVisualizer(Plotter=Plotter,layout=(1,1))
    while time<tend
        time=time+tstep
        solve!(U,inival,sys,control=control,tstep=tstep)
        inival.=U
        if verbose
            @printf("time=%g\n",time)
        end
        tstep*=1.1
        istep=istep+1
        testval=U[2,5]

        U1=view(U[1,:],subgrid1)
        U2=view(U[2,:],subgrid2)
        U3=view(U[3,:],subgrid3)

        scalarplot!(p[1,1],subgrid1, U1,label="spec1", color=(0.5,0,0),xlimits=(0,3),flimits=(0,1e-3),
                  title=@sprintf("three regions t=%.3g",time))
        scalarplot!(p[1,1],subgrid2, U2,label="spec2", color=(0.0,0.5,0),clear=false)
        scalarplot!(p[1,1],subgrid3, U3,label="spec3", color=(0.0,0.0,0.5),clear=false,show=true)
        sleep(1.0e-2)
    end
    return testval
end

function test()
    testval=0.0005954993329548969
    main(unknown_storage=:sparse) ≈ testval &&
        main(unknown_storage=:dense) ≈ testval
end

end


# Test of vbroadcast

using NumericExtensions
using Base.Test

#### 2D ####

x = randn(5, 6)
yr = rand(6) + 0.5
yrm = reshape(yr, 1, 6)
yc = rand(5) + 0.5

# badd

@test_approx_eq badd(x, yc, 1) x .+ yc
@test_approx_eq badd(x, yr, 2) x .+ yrm

r = copy(x); badd!(r, yc, 1)
@test_approx_eq r x .+ yc

r = copy(x); badd!(r, yr, 2)
@test_approx_eq r x .+ yrm

# bsubtract

@test_approx_eq bsubtract(x, yc, 1) x .- yc
@test_approx_eq bsubtract(x, yr, 2) x .- yrm

r = copy(x); bsubtract!(r, yc, 1)
@test_approx_eq r x .- yc

r = copy(x); bsubtract!(r, yr, 2)
@test_approx_eq r x .- yrm

# bmultiply

@test_approx_eq bmultiply(x, yc, 1) x .* yc
@test_approx_eq bmultiply(x, yr, 2) x .* yrm

r = copy(x); bmultiply!(r, yc, 1)
@test_approx_eq r x .* yc

r = copy(x); bmultiply!(r, yr, 2)
@test_approx_eq r x .* yrm

# bdivide

@test_approx_eq bdivide(x, yc, 1) x ./ yc
@test_approx_eq bdivide(x, yr, 2) x ./ yrm

r = copy(x); bdivide!(r, yc, 1)
@test_approx_eq r x ./ yc

r = copy(x); bdivide!(r, yr, 2)
@test_approx_eq r x ./ yrm

# #### 3D ####

x = randn(4, 5, 6)

y1 = rand(4) + 0.5; y1m = reshape(y1, 4, 1, 1)
y2 = rand(5) + 0.5; y2m = reshape(y2, 1, 5, 1)
y3 = rand(6) + 0.5; y3m = reshape(y3, 1, 1, 6)

# badd

@test_approx_eq badd(x, y1, 1) x .+ y1m
@test_approx_eq badd(x, y2, 2) x .+ y2m
@test_approx_eq badd(x, y3, 3) x .+ y3m

# bsubtract

@test_approx_eq bsubtract(x, y1, 1) x .- y1m
@test_approx_eq bsubtract(x, y2, 2) x .- y2m
@test_approx_eq bsubtract(x, y3, 3) x .- y3m

# bmultiply

@test_approx_eq bmultiply(x, y1, 1) x .* y1m
@test_approx_eq bmultiply(x, y2, 2) x .* y2m
@test_approx_eq bmultiply(x, y3, 3) x .* y3m

# bdivide

@test_approx_eq bdivide(x, y1, 1) x ./ y1m
@test_approx_eq bdivide(x, y2, 2) x ./ y2m
@test_approx_eq bdivide(x, y3, 3) x ./ y3m


"""
	within(obj::AbstractMatrix, groups::Vector{Vector{Vector{Int64}}})

	This function performs the within transformation given a model matrix and
	fixed effects using the method of alternating projections.
"""
function within(obj::AbstractMatrix, D::Vector{Vector{Vector{Int64}}})
	output = mapslices(col -> within(col, D), obj, 1)
	return output
end
function within(obj::AbstractVector, D::Vector{Vector{Vector{Int64}}})
	μ = mean(obj)
	current = copy(obj)
	output = copy(obj)
	er = Inf
	while er > 1e-8
		for dimension ∈ D
			current = copy(output)
			for group ∈ dimension
				output[group] .-= mean(current[group])
			end
		end
		er = norm(output - current)
	end
	output .+= μ
	return output
end

function partialwithin(obj::AbstractMatrix, D::Vector{Vector{Int64}},
	θ::Vector{T}) where T <: Real
	means = mean(obj, 1)
	output = copy(obj)
	for idx ∈ eachindex(θ)
		output[D[idx],:] .-= θ[idx] * mean(obj[D[idx],:], 1)
	end
	return output
end
function partialwithin(obj::AbstractVector, D::Vector{Vector{Int64}},
	θ::Vector{T}) where T <: Real
	output = copy(obj)
	for idx ∈ eachindex(θ)
		output[D[idx]] .-= θ[idx] * mean(obj[D[idx]], 1)
	end
	return output
end


## --- Test bilinear exponential function

    @test bilinear_exponential(66.02, [1.219384666893631, 66.0846112095295, 0.10822726922099106, -1.277645745450562, 0.6971298010153942]) ≈ 0.15736531960405353
    @test bilinear_exponential([66.02, 66.08, 66.15], [1.219384666893631, 66.0846112095295, 0.10822726922099106, -1.277645745450562, 0.6971298010153942]) ≈ [0.1573653196040532, 4.614683898246965, 8.99724709626313e-8]
    @test bilinear_exponential_ll(66.02, [1.219384666893631, 66.0846112095295, 0.10822726922099106, -1.277645745450562, 0.6971298010153942]) ≈ -1.8491853001597212

## --


# write some simple stuff to the map
function gen_spin0!(m, powerlaw_exponent=2)
    for i in axes(m, 1)
        for j in axes(m, 2)
            m[i,j] = ( ((j-1) * size(m, 1)) + i )^powerlaw_exponent
        end
    end
end

@testset "spin-0 SHTs" begin
    shape, wcs = fullsky_geometry(10.0 * Pixell.degree)
    m = Enmap(zeros(shape), wcs)
    gen_spin0!(m)

    alms = map2alm(m; lmax=18)
    data = readdlm("data/simple_analytic_sht.txt")
    ref_alms = data[:,1] + 1im .* data[:,2]
    @test (alms.alm ≈ ref_alms)

    @test length(map2alm(m).alm) == 190

    alms = map2alm(m[6:end-2, 5:end-3]; lmax=18)
    data = readdlm("data/simple_analytic_sht_sliced.txt")
    ref_alms = data[:,1] + 1im .* data[:,2]
    @test (alms.alm ≈ ref_alms)

    box = [10   -10;           # RA
           -5     5] * Pixell.degree  # DEC
    shape, wcs = geometry(CarClenshawCurtis, box, 1.0 * Pixell.degree)
    m = Enmap(zeros(shape), wcs)
    gen_spin0!(m, 2.5)
    alms = map2alm(m; lmax=100)
    data = readdlm("data/simple_box_analytic_sht.txt")
    ref_alms = data[:,1] + 1im .* data[:,2]
    @test (alms.alm ≈ ref_alms)
end

##
function gen_spin2!(m)
    for i in axes(m, 1)
        for j in axes(m, 2)
            m[i,j,1] = i^2 * j
            m[i,j,2] = i * j
        end
    end
end

@testset "spin-2 SHTs" begin
    shape, wcs = fullsky_geometry(10.0 * Pixell.degree; dims=(2,))
    m_QU = Enmap(zeros(shape), wcs)
    gen_spin2!(m_QU)
    alm_e, alm_b = map2alm(m_QU; lmax=3 * 36)
    data = readdlm("data/simple_pol_analytic_sht.txt")
    ref_alm_e = data[:,1] + 1im .* data[:,2]
    ref_alm_b = data[:,3] + 1im .* data[:,4]
    @test (ref_alm_e ≈ alm_e.alm)
    @test (ref_alm_b ≈ alm_b.alm)

    alm_e, alm_b = map2alm((m_QU[:,:,1], m_QU[:,:,2]); lmax=3 * 36)
    @test (ref_alm_e ≈ alm_e.alm)
    @test (ref_alm_b ≈ alm_b.alm)

    shape, wcs = fullsky_geometry(10.0 * Pixell.degree; dims=(3,))
    m_IQU = Enmap(zeros(shape), wcs)
    gen_spin0!(@view(m_IQU[:,:,1]))
    gen_spin2!(@view(m_IQU[:,:,2:3]))
    alm_t, alm_e, alm_b = map2alm(m_IQU; lmax=3 * 36)

    data = readdlm("data/simple_analytic_sht_fullalm.txt")
    ref_alm_t = data[:,1] + 1im .* data[:,2]
    @test (ref_alm_t ≈ alm_t.alm)
    @test (ref_alm_e ≈ alm_e.alm)
    @test (ref_alm_b ≈ alm_b.alm)

    alm_t, alm_e, alm_b = map2alm((m_IQU[:,:,1], m_IQU[:,:,2], m_IQU[:,:,3]); lmax=3 * 36)
end
##

@testset "alm2map" begin
    alm_r = readdlm("data/TEB_alms_real.dat")
    alm_i = readdlm("data/TEB_alms_imag.dat")
    lmax = 540
    alms = [Alm(lmax,lmax), Alm(lmax,lmax), Alm(lmax,lmax)]
    for i in 1:3
        alms[i].alm .+= alm_r[i,:] .+ 1im .* alm_i[i,:]
    end

    test_IQU = read_map("data/IQU.fits")
    I, Q, U = alm2map(alms, size(test_IQU), test_IQU.wcs)

    pyI = read_map("data/pyI.fits")[:,:,1]
    pyQU = read_map("data/pyQU.fits")

    @test maximum(abs.(I .- pyI)) < 1e-9
    @test maximum(abs.(Q .- pyQU[:,:,1])) < 1e-9
    @test maximum(abs.(U .- pyQU[:,:,2])) < 1e-9
end

@testset "map2alm CMB" begin
    test_IQU = read_map("data/IQU.fits")
    aT, aE, aB = map2alm(test_IQU; lmax=540)
    data = readdlm("data/test_cls_IQU.txt")
    @test maximum(abs.(alm2cl(aT) .- data[:,1])) < 1e-10
    @test maximum(abs.(alm2cl(aT, aE) .- data[:,2])) < 1e-10
    @test maximum(abs.(alm2cl(aE) .- data[:,3])) < 1e-10
    @test maximum(abs.(alm2cl(aB) .- data[:,4])) < 1e-10
end


## Define a particle consisting of all cluster variables for a Binomial mixture
## Assumes N is 25
mutable struct BinomCluster
  n::Int                          # count of cluster members
  α::Array{Int}     # count of occurrence of levels
  β::Array{Int}           # The denominator in logprob calculation
  BinomCluster(dataFile::Matrix{Int}) = new(0,
                                              ones(Int, size(dataFile, 2)),
                                              ones(Int, size(dataFile, 2)))
end

function copy_particle(particle::BinomCluster, dataFile)
    new_particle = BinomCluster(dataFile)
    new_particle.n = particle.n
    for i in eachindex(particle.α)
      new_particle.α[i] = particle.α[i]
      new_particle.β[i] = particle.β[i]
    end
    return new_particle
end

function calc_logprob(obs::Array{Int}, cl::BinomCluster)
  out = 0.0
  for q in 1:length(obs)
    @inbounds out += - lgamma(obs[q] + 1) - lgamma(25 - obs[q] + 1) +
                    lgamma(cl.α[q] + obs[q]) + lgamma(25 - obs[q] + cl.β[q]) +
                    lgamma(cl.α[q] + cl.β[q]) -
                    lgamma(cl.α[q] + 25 + cl.β[q]) - lgamma(cl.α[q]) - lgamma(cl.β[q])
  end
  return out
end

function cluster_add!(cl::BinomCluster, obs::Array{Int})
  @inbounds cl.n  += Int(1)
  @inbounds cl.α  += obs
  @inbounds cl.β  += 25 .- obs
end


# complex polya

# makie

using CairoMakie
let
        x = y = -2:0.005:2
        f(z) = 1 / (z * (z^2 - z - 1 - 3im))
        fvals = [f(u + 1im * v) for u in x, v in y]
        fvalues = abs.(fvals)
        fargs = angle.(fvals)
        polya(x, y) = Point2f(real(f(x + 1im * y)), -imag(f(x + 1im * y)))

        fig = Figure(resolution = (900, 400))
        axs = [Axis(fig[1, i], aspect = 1) for i in 1:2]
        cmap = :roma
        streamplot!(axs[1], polya, -2 .. 2, -2 .. 2, colormap = (:black, :red),
                gridsize = (40, 40), arrow_size = 6, linewidth = 1)
        pltobj2 = CairoMakie.heatmap!(axs[2], x, y, fargs, colorrange = (-π, π), colormap = cmap)
        CairoMakie.streamplot!(axs[2], polya, -2 .. 2, -2 .. 2, colormap = (:black, :black),
                gridsize = (40, 40), arrow_size = 6, linewidth = 1)
        Colorbar(fig[1, 3], pltobj2, ticks = ([-π, -π / 2, 0, π / 2, π],
                [L"-\pi", L"-\pi/2", L"0", L"\pi/2", L"\pi"]))
        limits!(axs[1], -2, 2, -2, 2)
        limits!(axs[2], -2, 2, -2, 2)
        colsize!(fig.layout, 1, Aspect(1, 1.0))
        colsize!(fig.layout, 2, Aspect(1, 1.0))
        display(fig)
end;


using Plexifications
using Base.Test: @testset

@testset "Cmplex" begin include("cmplex_test.jl") end
@testset "Prplex" begin include("prplex_test.jl") end
@testset "Nlplex" begin include("nlplex_test.jl") end

@testset "BiComplex" begin include("bicomplex_test.jl") end
@testset "BiPerplex" begin include("biperplex_test.jl") end
@testset "BiNilplex" begin include("binilplex_test.jl") end

@testset "DualComplex" begin include("dualcomplex_test.jl") end
@testset "DualPerplex" begin include("dualperplex_test.jl") end

@testset "TriComplex" begin include("tricomplex_test.jl") end
@testset "TriPerplex" begin include("triperplex_test.jl") end
@testset "TriNilplex" begin include("trinilplex_test.jl") end

@testset "HyperComplex" begin include("hypercomplex_test.jl") end
@testset "HyperPerplex" begin include("hyperperplex_test.jl") end


# A class of Matern isotropic functions including the Matrern 3/2 and 5/2, where d= 3 or 5. Also the exponential function, where d=1
abstract type MaternIso <: Isotropic{Euclidean} end
abstract type MaternARD <: StationaryARD{WeightedEuclidean} end

@inline function dKij_dθp(mat::MaternARD, X1::AbstractMatrix, X2::AbstractMatrix, i::Int, j::Int, p::Int, dim::Int)
    r=distij(metric(mat),X1, X2,i,j,dim)
    if p <= dim
        wdiffp=dist2ijk(metric(mat),X1, X2,i,j,p)
        if wdiffp > 0
            return dk_dll(mat,r,wdiffp)
        else
            return zero(dk_dll(mat,r,wdiffp))
        end
    elseif p==dim+1
        return dk_dlσ(mat, r)
    else
        return NaN
    end
end
@inline function dKij_dθp(mat::MaternARD, X1::AbstractMatrix, X2::AbstractMatrix, data::StationaryARDData, i::Int, j::Int, p::Int, dim::Int)
    return dKij_dθp(mat,X1,X2,i,j,p,dim)
end

@inline function dk_dθp(mat::MaternIso, r::Real, p::Int)
    if p==1
        return r == 0.0 ? 0.0 : dk_dll(mat, r)
    elseif p==2
        return dk_dlσ(mat, r)
    else
        return NaN
    end
end

include("mat12_iso.jl")
include("mat12_ard.jl")

include("mat32_iso.jl")
include("mat32_ard.jl")

include("mat52_iso.jl")
include("mat52_ard.jl")

"""
    Matern(ν::Real, ll::Union{Real,Vector{Real}}, lσ::Real)

Create Matérn kernel of type `ν` (i.e. `ν = 1/2`, `ν = 3/2`, or `ν = 5/2`) with length scale
`exp.(ll)` and signal standard deviation `exp(σ)`.

See also [`Mat12Iso`](@ref), [`Mat12Ard`](@ref), [`Mat32Iso`](@ref), [`Mat32Ard`](@ref),
[`Mat52Iso`](@ref), and [`Mat52Ard`](@ref).
"""
function Matern(ν::Real, ll::Real, lσ::Real)
    if ν==1/2
        kern = Mat12Iso(ll, lσ)
    elseif ν==3/2
        kern = Mat32Iso(ll, lσ)
    elseif ν==5/2
        kern = Mat52Iso(ll, lσ)
    else throw(ArgumentError("Only Matern 1/2, 3/2 and 5/2 are implementable"))
    end
    return kern
end

function Matern(ν::Real, ll::Vector{<:Real}, lσ::Real)
    if ν==1/2
        kern = Mat12Ard(ll, lσ)
    elseif ν==3/2
        kern = Mat32Ard(ll, lσ)
    elseif ν==5/2
        kern = Mat52Ard(ll, lσ)
    else throw(ArgumentError("Only Matern 1/2, 3/2 and 5/2 are implementable"))
    end
    return kern
end

@deprecate Mat Matern


export periodize

periodize(shp::Shape{N}, A::AbstractMatrix{<:Real}, ∆range::Shape{N}) where {N} = periodize(shp, SMatrix{N,N}(A), ∆range)

function periodize(shp::S,  # shape to periodize
                   A::SMatrix{N,N,<:Real},  # columns: primitive vectors of Bravais lattice
                   ∆range::Shape{N}  # range of translation vectors; boundaries included
				   ) where {N,S<:Shape{N}}
    # R = n₁a₁ + n₂a₂ + n₃a₃ is a translation vector for A = [a₁ a₂ a₃].  Find the ranges of
    # n₁, n₂, n₃ to test the inclusion of R in the Shape ∆range.
    #
    # The minimum and maximum nᵢ's to examine can be found by findfirstg nᵢ's at the corners of
    # the bounding box of ∆range.  This can be easily seen by unitary transformation of aᵢ's
    # into the Cartesian directions.  The transformation transforms the lattice planes into
    # the x-, y-, z-normal planes, and the bounding box into a parallelpiped.  The nᵢ's at
    # the corners of the original bounding box corresponds to the Cartesian indices of the
    # corners of the parallelpiped after transformation, and the minimum and maximum nᵢ's
    # are the Cartesian indices of the corners of the bounding box of this post-transform
    # parallelpiped.  Because the post-transform parallelpiped is enclosed by its bounding
    # box, all the lattice points within the post-transform parallelpiped are also within
    # the bounding box of the parallelpiped, whose corner indices are the minimum and
    # maximum nᵢ's.
    ∆bound = bounds(∆range)  # (SVector{3}, SVector{3}): bounds of translation range
    nmax_fl = SVector(ntuple(k->-Inf, Val(N)))  # [-Inf, -Inf, -Inf] for N = 3
    nmin_fl = SVector(ntuple(k->Inf, Val(N)))  # [Inf, Inf, Inf] for N = 3
    rcart = CartesianIndices(ntuple(k->2, Val(N)))  # (2,2,2) for N = 3
    for icart = rcart  # icart: CartesianIndex (see, e.g., https://julialang.org/blog/2016/02/iteration)
        cnr = map((∆b1,∆b2,i)-> i==1 ? ∆b1 : ∆b2, ∆bound[1], ∆bound[2], SVector(icart.I))  # SVector: corner of ∆bound (note icart.I[k] = 1 or 2)
        n_fl = A \ cnr  # SVector: A * n_fl = corner
        nmax_fl = max.(nmax_fl, n_fl)  # SVector
        nmin_fl = min.(nmin_fl, n_fl)  # SVector
    end

    # Find the integral nᵢ's.
    nmax = ceil.(Int, nmax_fl)  # SVector
    nmin = floor.(Int, nmin_fl)  # SVector

    # For nᵢ's within the range calculated above, check if R = n₁a₁ + n₂a₂ + n₃a₃ is within
    # the Shape ∆range.  If it is, then create a shape transformed by R.
    nshape = prod((nmax.-nmin).+1)  # tentative number of shapes
    shp_array = Vector{S}(undef, nshape)  # preallocation (will be truncated later)
    ishape = 0
    nrange = map((n1,n2)->n1:n2, nmin.data, nmax.data)  # e.g., (1:10, 1:10, 1:10) for N = 3
    for n = CartesianIndices(nrange)  # n: CartesianIndex
		∆ = A * SVector(n.I)  # lattice vector R
		if ∆ ∈ ∆range
			shape = translate(shp, ∆)
			ishape += 1
			shp_array[ishape] = shape
		end
    end

    resize!(shp_array, ishape)  # return resized shp_array
end


function dilate!( mask::Array{T,N}, dil::Array{Bool,N}, rad; kern=nothing, th=nothing ) where {T,N}

    if kern == nothing
        size = ones( Int64, N ) .* rad;
        kern = ones( UInt8, size... )
    end

    conv = FFTConvolution_crop( mask, kern, typ=Float32 );

    th = ( th == nothing ) ? 1 - 0.1 : th;

    @inbounds @simd for e in 1:length(conv)
        dil[e] = conv[e] > th;
    end

    return dil
end

function dilate( mask::Array{T,N}, rad; kern=nothing, th=nothing ) where {T,N}

    return dilate!( mask, zeros(Bool,size(mask)), rad, kern=kern, th=th );
end

function erode!( mask::Array{T,N}, ero::Array{Bool,N}, rad; kern=nothing, th=nothing ) where {T,N}

    if kern == nothing
        kern = ones( UInt8, (ones( Int64, N ) .*  rad)... ) # ones( rad, rad ) or ones( rad, rad, rad )
    end

    th = ( th == nothing ) ? sum( kern ) - 0.1 : th;

    conv = FFTConvolution_crop( mask, kern, typ=Float32 );

    for e in 1:length(conv)
        ero[e] = ( conv[e] > th )
    end

    return ero

end

function erode( mask::Array{T,N}, rad; kern=nothing, th=nothing ) where {T,N}

    return erode!( mask, zeros(Bool,size(mask)), rad, kern=kern, th=th );
end

function closing( mask, size; kern=nothing )
    s1 = dilate( mask, size, kern=kern  )
    s2 = erode( s1, size, kern=kern )
    return s2
end

function opening( mask, size; kern=nothing )
    s1 = erode( mask, size, kern=kern )
    s2 = dilate( s1, size, kern=kern )
    return s2
end

function distanceTransform( mask )

    distances = zeros( UInt16, size(mask) )

    kern_size = 3;
    keepgoing = true
    while keepgoing

        kern  = ones( UInt8, kern_size, kern_size );
        th    = sum( kern );

        mask_ = convolution( UInt8.( mask ), kern );
        h, w  = size( mask_ );
        off   = div( kern_size, 2 );

        trues = 0;
        for col in off:w-off
            for row in off:h-off
                if mask_[row,col] >= length(kern) - 0.1
                    distances[ row-off+1, col-off+1 ] += 1;
                    trues += 1;
                end
            end
        end

        if trues == 0
            keepgoing = false;
        end

        kern_size += 2;
    end

    println( "maximum kern_size: ", kern_size )

    return distances
end


# Basic constructors of descriptor systems
"""
    sys = dss(A, E, B, C, D; Ts = 0, check_reg = false,
              atol = 0, atol1 = atol, atol2 = atol, rtol = n*ϵ )

Create for `Ts = 0` a descriptor system model `sys = (A-λE,B,C,D)` for a continuous-time state space system of the form

    Edx(t)/dt = Ax(t) + Bu(t) ,
    y(t)      = Cx(t) + Du(t) ,

where `x(t)`, `u(t)` and `y(t)` are the system state vector, system input vector and system output vector, respectively,
for the continuous time variable `t`.

For a nonzero positive sampling time `Ts = ΔT`, the descriptor system model specifies a discrete-time state space system of the form

    Ex(t+ΔT) = Ax(t) + Bu(t)
    y(t)     = Cx(t) + Du(t)

for the discrete values of the time variable `t = 0, ΔT, 2ΔT, ...`.
Use `Ts = -1` if the sampling time is not specified. In this case, by convention  `ΔT = 1`.

For a system with zero feedthrough matrix `D`, it is possible to set `D = 0` (the scalar zero).

For a standard state space system, `E` is the identity matrix. In this case, it is possible to set `E = I` (the boolean uniform scaling).
Alternatively, use

    sys = dss(A, B, C, D; Ts = 0)

to create a standard system.

For a system corresponding to a static gain `D`, use

    sys = dss(D; Ts = 0)

It is possible to specify a descriptor system via all or part of its matrices using the form

    sys = dss(A = mat1, E = mat2, B = mat3, C = mat4, D = mat5; Ts = 0, check_reg = false,
              atol = 0, atol1 = atol, atol2 = atol, rtol = n*ϵ)

where `A`, `E`, `B`, `C`, and `D` are keyword parameters set to appropriate matrix values
`mat1`, `mat2`, `mat3`, `mat4`, and `mat5`, respectively. If
some of the system matrices are omited, then zero matrices of appropriate sizes are employed instead.

It is assumed that the pencil `A-λE` is regular (i.e., `det(A-λE) ̸≡ 0`), and therefore, in the interest of efficiency,
the regularity of `A-λE` is by default _not_ tested. If `check_reg = true`, the regularity of `A-λE` is
additionally checked. In this case, the keyword arguments `atol1`, `atol2` and `rtol` specify, respectively,
the absolute tolerance for the nonzero elements of `A`, the absolute tolerance for the nonzero elements of `E`,
and the relative tolerance for the nonzero elements of `A` and `E`.
The default relative tolerance is `n*ϵ`, where `n` is the order of the square matrices `A` and `E`, and  `ϵ` is the working machine epsilon.
The keyword argument `atol` can be used to simultaneously set `atol1 = atol` and `atol2 = atol`.
"""
function dss(A::AbstractNumOrArray, E::Union{AbstractNumOrArray,UniformScaling}, B::AbstractNumOrArray, C::AbstractNumOrArray, D::AbstractNumOrArray;
             Ts::Real = 0, check_reg::Bool = false,
             atol::Real = zero(real(eltype(A))), atol1::Real = atol, atol2::Real = atol,
             rtol::Real = (typeof(A) <: Number ? 1 : min(size(A)...))*eps(real(float(one(eltype(A)))))*iszero(min(atol1,atol2)))

    T = promote_type(eltype(A), E == I ? Bool : eltype(E), eltype(B), eltype(C), eltype(D))
    check_reg && E != I && !MatrixPencils.isregular(to_matrix(T,A), to_matrix(T,E), atol1 = atol1, atol2 = atol2, rtol = rtol ) &&
                            error("The pencil A-λE is not regular")
    p = typeof(C) <: Union{Vector,Number} ? (size(A,1) <= 1 ? size(C,1) : 1) : size(C,1)
    m = typeof(B) <: Union{Vector,Number} ? 1 : size(B,2)
    return DescriptorStateSpace{T}(to_matrix(T,A), E == I ? I : to_matrix(T,E), to_matrix(T,B), to_matrix(T,C,p <=1),
                                   typeof(D) <: Number && iszero(D) ? zeros(T,p,m) : p <=1 ? to_matrix(T,D,m > p) : to_matrix(T,D), Ts)
end
function dss(A::AbstractNumOrArray, B::AbstractNumOrArray, C::AbstractNumOrArray, D::AbstractNumOrArray; Ts::Real = 0)
    T = promote_type(eltype(A),eltype(B),eltype(C),eltype(D))
    p = typeof(C) <: Union{Vector,Number} ? (size(A,1) <= 1 ? size(C,1) : 1) : size(C,1)
    m = size(B,2)
    return DescriptorStateSpace{T}(to_matrix(T,A), I, to_matrix(T,B), to_matrix(T,C,p <=1),
                                   typeof(D) <: Number && iszero(D) ? zeros(T,p,m) : p <=1 ? to_matrix(T,D,m > p) : to_matrix(T,D), Ts)
end
function dss(D::AbstractNumOrArray; Ts::Real = 0)
    D == [] && (T = Int64; return DescriptorStateSpace{T}(zeros(T,0,0),I,zeros(T,0,0),zeros(T,0,0),zeros(T,0,0),Ts))
    p, m = size(D,1),size(D,2)
    T = eltype(D)
    return DescriptorStateSpace{T}(zeros(T,0,0),I,zeros(T,0,m),zeros(T,p,0),to_matrix(T,D),Ts)
end
function dss(;A::Union{AbstractNumOrArray} = zeros(Bool,0,0), E::Union{AbstractNumOrArray,UniformScaling} = I, B::Union{AbstractNumOrArray,Missing} = missing,
              C::Union{AbstractNumOrArray,Missing} = missing, D::Union{AbstractNumOrArray,Missing} = missing,
              Ts::Real = 0, check_reg::Bool = false,
              atol::Real = zero(real(eltype(A))), atol1::Real = atol, atol2::Real = atol,
              rtol::Real = (typeof(A) <: Number ? 1 : min(size(A)...))*eps(real(float(one(eltype(A)))))*iszero(min(atol1,atol2)))
    T = Bool
    T = promote_type(T, ismissing(A) ? T : eltype(A), E == I ? Bool : eltype(E), ismissing(B) ? T : eltype(B),
                     ismissing(C) ? T : eltype(C), ismissing(D) ? T : eltype(D))
    A = to_matrix(T,A)
    n = size(A,1)
    E == I || (E = to_matrix(T,E))
    check_reg && E != I && !isregular(A, E, atol1 = atol1, atol2 = atol2, rtol = rtol ) &&
                            error("The pencil A-λE is not regular")

    ismissing(D) || (D1 = to_matrix(T,D))
    if ismissing(B)
       ismissing(D) ? m = 0 : m = size(D,2)
       B = zeros(T,n,m)
    else
       B = to_matrix(T,B)
       m = size(B,2)
    end
    if ismissing(C)
        ismissing(D) ? p = 0 : p = size(D,1)
        C = zeros(T,p,n)
    else
        C = to_matrix(T,C)
        p = size(C,1)
    end
    ismissing(D) && (D1 = zeros(T,p,m))
    return DescriptorStateSpace{T}(A, E == I ? I : E, B, C, D1, Ts)
end
"""
     sys = dss(A, E, B, F, C, G, D, H; compacted = false,
               atol = 0, atol1 = atol, atol2 = atol, atol3 = atol, rtol = min(atol1,atol2,atol3)>0 ? 0 : n*ϵ)

Construct an input-output equivalent descriptor system representation `sys = (Ad-λdE,Bd,Cd,Dd)` to a pencil based linearization
`(A-λE,B-λF,C-λG,D-λH)` satisfying

                -1                        -1
     Cd*(λEd-Ad)  *Bd + Dd = (C-λG)*(λE-A)  *(B-λF) + D-λH .

If `compacted = true`, a compacted descriptor system realization is determined by exploiting possible rank defficiencies of the
matrices `F`, `G`, and `H`. Any of the matrices `F`, `G`, and `H` can be set to `missing`.

The keyword arguments `atol1`, `atol2`, `atol3`, and `rtol`, specify, respectively, the absolute tolerance for the
nonzero elements of `F`, the absolute tolerance for the nonzero elements of `G`,
the absolute tolerance for the nonzero elements of `H`  and the relative tolerance
for the nonzero elements of `F`, `G` and `H`. The default relative tolerance is `n*ϵ`, where `n` is the size of
of `A`, and `ϵ` is the machine epsilon of the element type of `A`.
The keyword argument `atol` can be used to simultaneously set `atol1 = atol`, `atol2 = atol` and `atol3 = atol`.
"""
function dss(A::Matrix, E::Union{Matrix,UniformScaling}, B::Matrix, F::Union{Matrix,Missing},
             C::Matrix, G::Union{Matrix,Missing}, D::Matrix, H::Union{Matrix,Missing};
             Ts::Real = 0, compacted::Bool = false,
             atol::Real = zero(real(eltype(A))), atol1::Real = atol, atol2::Real = atol, atol3::Real = atol,
             rtol::Real = (min(size(A)...)*eps(real(float(one(eltype(A))))))*iszero(min(atol1,atol2,atol3)))

    T = promote_type(eltype(A), E == I ? Bool : eltype(E), eltype(B), eltype(C), eltype(D))
    T = promote_type(T, ismissing(F) ? T : eltype(F), ismissing(G) ? T : eltype(G), ismissing(H) ? T :  eltype(H))

    return DescriptorStateSpace{T}(lps2ls(A, E, B, F, C, G, D, H; compacted = compacted, atol1 = atol1, atol2 = atol2, atol3 = atol2, rtol = rtol)..., Ts)

end
# function dss(sys::PencilStateSpace; compacted::Bool = false,
#              atol1::Real = zero(real(eltype(sys.A))), atol2::Real = zero(real(eltype(sys.A))),
#              rtol::Real = (min(size(sys.A)...)*eps(real(float(one(eltype(sys.A))))))*iszero(min(atol1,atol2)))
#     return DescriptorStateSpace{eltype(sys.A)}(lps2ls(sys.A,sys.E,sys.B,sys.F,sys.C,sys.G,sys.D,sys.H; compacted = compacted, atol1 = atol1, atol2 = atol2, rtol = rtol)..., sys.Ts)
# end
# function dss(D::VecOrMat, H::VecOrMat; Ts::Real = 0)
#     typeof(D) <: Vector ? (p, m) = (length(D), 1) : (p,m) = size(D)
#     T = promote_type(eltype(D), eltype(H))
#     D1 = reshape(copy_oftype(D,T),p,m)
#     iszero(H) &&  (return DescriptorStateSpace{T}(zeros(T,0,0),I,zeros(T,0,m),zeros(T,p,0),D1,Ts))
#     if m <= p
#         A = Matrix{T}(I,2*m,2*m)
#         E = [zeros(T,m,m) I; zeros(T,m,2*m)]
#         B = [zeros(T,m,m); I]
#         C = [H zeros(T,p,m)]
#      else
#         A = Matrix{T}(I,2*p,2*p)
#         E = [zeros(T,p,p) I; zeros(T,p,2*p)]
#         B = [zeros(T,p,m); -H]
#         C = [-I zeros(T,p,p)]
#      end
#      return DescriptorStateSpace{T}(A,E,B,C,D1,Ts)
# end
"""
    sys = dss(NUM, DEN; contr = false, obs = false, noseig = false, minimal = false, fast = true, atol = 0, rtol)

Convert the rational matrix `R(λ) = NUM(λ) ./ DEN(λ)` to a descriptor system representation `sys = (A-λE,B,C,D)` such that
the transfer function matrix of `sys` is `R(λ)`.

`NUM(λ)` is a polynomial matrix of the form `NUM(λ) = N_1 + λ N_2 + ... + λ**k N_(k+1)`, for which
the coefficient matrices `N_i`, `i = 1, ..., k+1` are stored in the 3-dimensional matrix `NUM`,
where `NUM[:,:,i]` contains the `i`-th coefficient matrix `N_i` (multiplying `λ**(i-1)`).

`DEN(λ)` is a polynomial matrix of the form `DEN(λ) = D_1 + λ D_2 + ... + λ**l D_(l+1)`, for which
the coefficient matrices `D_i`, `i = 1, ..., l+1`, are stored in the 3-dimensional matrix `DEN`,
where `DEN[:,:,i]` contain the `i`-th coefficient matrix `D_i` (multiplying `λ**(i-1)`).

Alternatively, `NUM(λ)` and `DEN(λ)` can be specified as matrices of elements of the `Polynomial` type
provided by the [Polynomials](https://github.com/JuliaMath/Polynomials.jl) package.

If `n` is the order of `A-λE`, then the computed linearization satisfies:

(1) `A-λE` is regular and `R(λ) = C*inv(λE-A)*B+D`;

(2) `rank[B A-λE] = n` (controllability) if `minimal = true` or `contr = true`;

(3) `rank[A-λE; C] = n` (observability)  if `minimal = true` or `obs = true`;

(4) `A-λE` has no non-dynamic modes if `minimal = true` or `noseig = true`.

If conditions (1)-(4) are satisfied, the realization is called `minimal` and the resulting order `n`
is the least achievable order. If conditions (1)-(3) are satisfied, the realization is called `irreducible`
and the resulting order `n` is the least achievable order using orthogonal similarity transformations.
An irreducible realization preserves the pole-zero and singular structures of `R(λ)`.

The descriptor system based realization is built using the methods described in [1] in conjunction with
pencil manipulation algorithms [2] and [3] to compute reduced order realization. These algorithms
employ rank determinations based on either the use of
rank revealing QR-decomposition with column pivoting, if `fast = true`, or the SVD-decomposition, if `fast = false`.
The rank decision based on the SVD-decomposition is generally more reliable, but the involved computational effort is higher.

The keyword arguments `atol` and `rtol`, specify the absolute and relative tolerances, respectively, for the
nonzero coefficients of `NUM(λ)` and `DEN(λ)`.

[1] A. Varga, On computing the Kronecker structure of polynomial and rational matrices using Julia, 2020,
[arXiv:2006.06825](https://arxiv.org/pdf/2006.06825).

[2] P. Van Dooreen, The generalized eigenstructure problem in linear system theory,
IEEE Transactions on Automatic Control, vol. AC-26, pp. 111-129, 1981.

[3] A. Varga, Solving Fault Diagnosis Problems - Linear Synthesis Techniques, Springer Verlag, 2017.
"""
function dss(NUM::Array{T1,3},DEN::Array{T2,3};
    Ts::Real = 0, minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
    atol::Real = zero(real(T1)), rtol::Real = (min(size(NUM)...)*eps(real(float(one(T1)))))*iszero(atol)) where {T1,T2}
    sysdata = rm2ls(NUM, DEN, contr = contr, obs = obs, noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
    return DescriptorStateSpace{eltype(sysdata[1])}(sysdata[1:5]...,Ts)
end
dss(NUM::Union{VecOrMat{Polynomial{T1,X}},Polynomial{T1,X},Number,VecOrMat},
    DEN::Union{VecOrMat{Polynomial{T2,X}},Polynomial{T2,X},Number,VecOrMat}; kwargs...) where {T1,T2,X} =
    dss(poly2pm(NUM),poly2pm(DEN); kwargs...)
"""
    sys = dss(R; Ts=missing, contr = false, obs = false, noseig = false, minimal = false, fast = true, atol = 0, rtol)

Convert the rational transfer function matrix `R(λ)` to a descriptor system representation `sys = (A-λE,B,C,D)` such that
the transfer function matrix of `sys` is `R(λ)`. The resulting `sys` is a continuous-time system if `Ts = 0` or
discrete-time system if `Ts = -1` or `Ts > 0`.
If `Ts = missing`, the sampling time of `sys` is inherited from the sampling time `TRs` of the elements of `R`, unless `TRs = nothing`,
in which case `Ts = 0` is used (by default).

`R(λ)` is a matrix with rational transfer function entries (see [`RationalTransferFunction`](@ref) ) corresponding to a
multiple-input multiple-outputs system or a rational transfer function corresponding to a
single-input single-output system. The numerators and denominators of the elements of R are of type
 `Polynomial` as provided by the [Polynomials](https://github.com/JuliaMath/Polynomials.jl) package.

If `n` is the order of `A-λE`, then the computed realization satisfies:

(1) `A-λE` is regular and `R(λ) = C*inv(λE-A)*B+D`;

(2) `rank[B A-λE] = n` (controllability) if `minimal = true` or `contr = true`;

(3) `rank[A-λE; C] = n` (observability)  if `minimal = true` or `contr = true`;

(4) `A-λE` has no non-dynamic modes if `minimal = true` or `noseig = true`.

If conditions (1)-(4) are satisfied, the realization is called `minimal` and the resulting order `n`
is the least achievable order. If conditions (1)-(3) are satisfied, the realization is called `irreducible`
and the resulting order `n` is the least achievable order using orthogonal similarity transformations.
An irreducible realization preserves the pole-zero and singular structures of `R(λ)`.

The underlying pencil manipulation algorithms [1] and [2] to compute reduced order realizations
employ rank determinations based on either the use of
rank revealing QR-decomposition with column pivoting, if `fast = true`, or the SVD-decomposition, if `fast = false`.
The rank decision based on the SVD-decomposition is generally more reliable, but the involved computational effort is higher.

The keyword arguments `atol` and `rtol`, specify the absolute and relative tolerances for the
nonzero coefficients of `R(λ)`.

[1] P. Van Dooreen, The generalized eigenstructure problem in linear system theory,
IEEE Transactions on Automatic Control, vol. AC-26, pp. 111-129, 1981.

[2] A. Varga, Solving Fault Diagnosis Problems - Linear Synthesis Techniques, Springer Verlag, 2017.
"""
# function dss(R::Union{VecOrMat{<:RationalTransferFunction},RationalTransferFunction}; Ts::Union{Real,Missing} = missing,
#              minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
#              atol::Real = zero(float(real(_eltype(R)))), rtol::Real = 100*eps(one(atol))*iszero(atol))
#     ismissing(Ts) && (eltype(R) <: RationalTransferFunction ? Ts = R[1].Ts : Ts = R.Ts )
#     isnothing(Ts) && (Ts = 0)
#     dss(poly2pm(numpoly.(R)), poly2pm(denpoly.(R)); Ts = Ts, contr = contr, obs = obs,
#             noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
# end
# function dss(R::VecOrMat{RationalTransferFunction{T,X}}; Ts::Union{Real,Missing} = missing,
#              minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
#              atol::Real = zero(float(real(T))), rtol::Real = 100*eps(one(atol))*iszero(atol)) where T where X
#     length(R) == 0 && (return dss(zeros(_eltype(R),size(R)...),Ts = Ts))
#     ismissing(Ts) && (Ts = R[1].Ts)
#     isnothing(Ts) && (Ts = 0)
#     dss(poly2pm(numpoly.(R)), poly2pm(denpoly.(R)); Ts = Ts, contr = contr, obs = obs,
#             noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
# end
function dss(R::VecOrMat{<:RationalTransferFunction}; Ts::Union{Real,Missing,Nothing} = missing,
             minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
             atol::Real = zero(float(real(_eltype(R)))), rtol::Real = 100*eps(one(atol))*iszero(atol))
    length(R) == 0 && (return dss(zeros(_eltype(R),size(R)...),Ts = Ts))
    ismissing(Ts) && (Ts = R[1].Ts)
    isnothing(Ts) && (Ts = 0)
    dss(poly2pm(numpoly.(R)), poly2pm(denpoly.(R)); Ts = Ts, contr = contr, obs = obs,
            noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
end
function dss(R::VecOrMat{<:RationalFunction}; Ts::Real = 0,
             minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
             atol::Real = zero(float(real(eltype(eltype(eltype(R)))))), rtol::Real = 100*eps(one(atol))*iszero(atol))
    length(R) == 0 && (return dss(zeros(eltype(eltype(eltype(G))),size(R)...),Ts = Ts))
    dss(poly2pm(numerator.(R)), poly2pm(denominator.(R)); Ts = Ts, contr = contr, obs = obs,
            noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
end
function dss(R::RationalTransferFunction; Ts::Union{Real,Missing,Nothing} = missing,
             minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
             atol::Real = zero(float(real(_eltype(R)))), rtol::Real = 100*eps(one(atol))*iszero(atol))
    ismissing(Ts) && (Ts = R.Ts)
    isnothing(Ts) && (Ts = 0)
    dss(poly2pm(numpoly(R)), poly2pm(denpoly(R)); Ts = Ts, contr = contr, obs = obs,
            noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
end
function dss(R::RationalFunction; Ts::Real = 0,
             minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
             atol::Real = zero(float(real(eltype(eltype(R))))), rtol::Real = 100*eps(one(atol))*iszero(atol))
    dss(poly2pm(numerator(R)), poly2pm(denominator(R)); Ts = Ts, contr = contr, obs = obs,
            noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
end
"""
    sys = dss(P; Ts = 0, contr = false, obs = false, noseig = false, minimal = false, fast = true, atol = 0, rtol)

Convert the polynomial matrix `P(λ)` to a descriptor system representation `sys = (A-λE,B,C,D)` such that
the transfer function matrix of `sys` is `P(λ)`. The resulting `sys` is a continuous-time system if `Ts = 0` or
discrete-time system if `Ts = -1` or `Ts > 0`.

`P(λ)` can be specified as a grade `k` polynomial matrix of the form `P(λ) = P_1 + λ P_2 + ... + λ**k P_(k+1)`,
for which the coefficient matrices `P_i`, `i = 1, ..., k+1`, are stored in the 3-dimensional matrix `P`,
where `P[:,:,i]` contains the `i`-th coefficient matrix `P_i` (multiplying `λ**(i-1)`).

`P(λ)` can also be specified as a matrix, vector or scalar of elements of the `Polynomial` type
provided by the [Polynomials](https://github.com/JuliaMath/Polynomials.jl) package.

If `d` is the degree of `P(λ)` and `n` is the order of `A-λE`, then the computed realization satisfies:

(1) `A-λE` is regular and `P(λ) = C*inv(λE-A)*B+D`;

(2) `rank[B A-λE] = n` (controllability) if `minimal = true` or `contr = true`;

(3) `rank[A-λE; C] = n` (observability)  if `minimal = true` or `contr = true`;

(4) `A-λE` has no non-dynamic modes if `minimal = true` or `noseig = true`.

If conditions (1)-(4) are satisfied, the realization is called `minimal` and the resulting order `n`
is the least achievable order. If conditions (1)-(3) are satisfied, the realization is called `irreducible`
and the resulting order `n` is the least achievable order using orthogonal similarity transformations.
An irreducible realization preserves the pole-zero and singular structures of `P(λ)`.

The underlying pencil manipulation algorithms [1] and [2] to compute reduced order realizations
employ rank determinations based on either the use of
rank revealing QR-decomposition with column pivoting, if `fast = true`, or the SVD-decomposition, if `fast = false`.
The rank decision based on the SVD-decomposition is generally more reliable, but the involved computational effort is higher.

The keyword arguments `atol` and `rtol`, specify the absolute and relative tolerances for the
nonzero coefficients of `P(λ)`, respectively.

[1] P. Van Dooreen, The generalized eigenstructure problem in linear system theory,
IEEE Transactions on Automatic Control, vol. AC-26, pp. 111-129, 1981.

[2] A. Varga, Solving Fault Diagnosis Problems - Linear Synthesis Techniques, Springer Verlag, 2017.
"""
function dss(P::Array{T,3};
    Ts::Real = 0, minimal::Bool = false, contr::Bool = false, obs::Bool = false, noseig::Bool = false,
    atol::Real = zero(real(T)), rtol::Real = (min(size(P)...)*eps(real(float(one(T)))))*iszero(atol)) where T
    sysdata = pm2ls(P, contr = contr, obs = obs, noseig = noseig, minimal = minimal, atol = atol, rtol = rtol)
    return DescriptorStateSpace{eltype(sysdata[1])}(sysdata..., Ts)
end
dss(P::Union{VecOrMat{<:Polynomial},Polynomial}; kwargs...)  = dss(poly2pm(P); kwargs...)
function dss(P::LaurentPolynomial; kwargs...)
    n = degree(P)
    m = length(P)-1-n
    T = eltype(P)
    t = Polynomial{T,Polynomials.indeterminate(P)}([zeros(T,m);one(T)][:])
    dss(poly2pm(Polynomial(P*t)),poly2pm(t); kwargs...)
end
"""
    sys = dss(T, U, V, W; fast = true, contr = false, obs = false, minimal = false, atol = 0, rtol)

Construct an input-output equivalent descriptor system representation `sys = (A-λE,B,C,D)`
to a polynomial model specified by the polynomial matrices `T(λ)`, `U(λ)`, `V(λ)`, and `W(λ)`
such that

      V(λ)*inv(T(λ))*U(λ)+W(λ) = C*inv(λE-A)*B+D.

If `minimal = true`, the resulting realization `(A-λE,B,C,D)` has the least possible order `n` of `A-λE`.

`T(λ)`, `U(λ)`, `V(λ)`, and `W(λ)` can be specified as polynomial matrices of the form `X(λ) = X_1 + λ X_2 + ... + λ**k X_(k+1)`,
for `X = T`, `U`, `V`, and `W`, for which the coefficient matrices `X_i`, `i = 1, ..., k+1`, are stored in
the 3-dimensional matrices `X`, where `X[:,:,i]` contains the `i`-th coefficient matrix `X_i` (multiplying `λ**(i-1)`).

`T(λ)`, `U(λ)`, `V(λ)`, and `W(λ)` can also be specified as matrices, vectors or scalars of elements of the `Polynomial` type
provided by the [Polynomials](https://github.com/JuliaMath/Polynomials.jl) package.
In this case, no check is performed that `T(λ)`, `U(λ)`, `V(λ)`  and `W(λ)` have the same indeterminates.

The computed descriptor realization satisfies:

(1) `A-λE` is regular;

(2) `rank[B A-λE] = n` (controllability) if `minimal = true` or `contr = true`;

(3) `rank[A-λE; C] = n` (observability)  if `minimal = true` or `obs = true`;

(4) `A-λE` has no simple infinite eigenvalues if `minimal = true`.

The keyword arguments `atol` and `rtol`, specify, respectively, the absolute and relative tolerance for the
nonzero coefficients of the matrices `T(λ)`, `U(λ)`, `V(λ)` and `W(λ)`. The default relative tolerance is `nt*ϵ`,
where `nt` is the size of the square matrix `T(λ)` and `ϵ` is the machine epsilon of the element type of its coefficients.

The descriptor realization is built using the methods described in [1].

[1] A. Varga, On computing the Kronecker structure of polynomial and rational matrices using Julia, 2020,
[arXiv:2006.06825](https://arxiv.org/pdf/2006.06825).
"""
function dss(T::Array{T1,3}, U::Array{T2,3}, V::Array{T3,3}, W::Array{T4,3};
             Ts::Real = 0, contr::Bool = false, obs::Bool = false, minimal::Bool = false,
             fast::Bool = true, atol::Real = zero(real(T1)),
             rtol::Real = size(T,1)*eps(real(float(one(T1))))*iszero(atol)) where {T1, T2, T3, T4}
    sysdata = spm2ls(T, U, V, W, contr = contr, obs = obs, minimal = minimal, atol = atol, rtol = rtol)
    return DescriptorStateSpace{eltype(sysdata[1])}(sysdata..., Ts)
end
# function dss(T::Union{VecOrMat{Polynomial{T1}},VecOrMat{Polynomial{T1,X}},Polynomial{T1},Polynomial{T1,X}},
#              U::Union{VecOrMat{Polynomial{T2}},VecOrMat{Polynomial{T2,X}},Polynomial{T2},Polynomial{T2,X}},
#              V::Union{VecOrMat{Polynomial{T3}},VecOrMat{Polynomial{T3,X}},Polynomial{T3},Polynomial{T3,X}},
#              W::Union{VecOrMat{Polynomial{T4}},VecOrMat{Polynomial{T4,X}},Polynomial{T4},Polynomial{T4,X}}; kwargs...) where {T1, T2, T3, T4, X}
#              println("T = $T")
#        dss(poly2pm(T),poly2pm(U),poly2pm(V),poly2pm(W); kwargs...)
# end
# function dss(T::Union{VecOrMat{TP},TP},
#              U::Union{VecOrMat{TP},TP,Number,VecOrMat{<:Number}},
#              V::Union{VecOrMat{TP},TP,Number,VecOrMat{<:Number}},
#              W::Union{VecOrMat{TP},TP,Number,VecOrMat{<:Number}};
#              Ts::Real = 0, contr::Bool = false, obs::Bool = false, minimal::Bool = false,
#              fast::Bool = true, atol::Real = zero(float(real(eltype(eltype(T))))),
#              rtol::Real = size(T,1)*eps(one(eltype(atol)))*iszero(atol)) where {TP <: Polynomial}
#     return dss(poly2pm(T),poly2pm(U),poly2pm(V),poly2pm(W); Ts = Ts, contr = contr, obs = obs, minimal = minimal, fast = fast, atol = atol, rtol = rtol)
# end
function dss(T::Union{VecOrMat{<:Polynomial},Polynomial},
             U::Union{VecOrMat{<:Polynomial},Polynomial,Number,VecOrMat{<:Number}},
             V::Union{VecOrMat{<:Polynomial},Polynomial,Number,VecOrMat{<:Number}},
             W::Union{VecOrMat{<:Polynomial},Polynomial,Number,VecOrMat{<:Number}};
             Ts::Real = 0, contr::Bool = false, obs::Bool = false, minimal::Bool = false,
             fast::Bool = true, atol::Real = zero(float(real(eltype(eltype(T))))),
             rtol::Real = size(T,1)*eps(one(eltype(atol)))*iszero(atol))
return dss(poly2pm(T),poly2pm(U),poly2pm(V),poly2pm(W); Ts = Ts, contr = contr, obs = obs, minimal = minimal, fast = fast, atol = atol, rtol = rtol)
end

"""
    A, E, B, C, D  = dssdata([T,] sys)

Extract the matrices `A`, `E`, `B`, `C`, `D` of a descriptor system model `sys = (A-λE,B,C,D)`.
If the type `T` is specified, the resulting matrices are converted to this type.
"""
function dssdata(sys::DescriptorStateSpace)
    return copy(sys.A), copy(sys.E), copy(sys.B), copy(sys.C), copy(sys.D)
end
function dssdata(T::Type,sys::DescriptorStateSpace)
    #eltype(sys) == T ? (return sys.A, sys.E, sys.B, sys.C, sys.D) :
    return copy_oftype(sys.A,T), sys.E == I ? I : copy_oftype(sys.E,T), copy_oftype(sys.B,T), copy_oftype(sys.C,T), copy_oftype(sys.D,T)
end


# ---
# title: 122. Best Time to Buy and Sell Stock II
# id: problem122
# author: AquaIndigo
# date: 2020-11-08
# difficulty: Easy
# categories: Array, Greedy
# link: <https://leetcode.com/problems/best-time-to-buy-and-sell-stock-ii/description/>
# hidden: true
# ---
#
# Say you have an array `prices` for which the _i_ th element is the price of a
# given stock on day _i_.
#
# Design an algorithm to find the maximum profit. You may complete as many
# transactions as you like (i.e., buy one and sell one share of the stock
# multiple times).
#
# **Note:** You may not engage in multiple transactions at the same time (i.e.,
# you must sell the stock before you buy again).
#
# **Example 1:**
#
#
#
#     Input: [7,1,5,3,6,4]
#     Output: 7
#     Explanation: Buy on day 2 (price = 1) and sell on day 3 (price = 5), profit = 5-1 = 4.
#                  Then buy on day 4 (price = 3) and sell on day 5 (price = 6), profit = 6-3 = 3.
#
#
# **Example 2:**
#
#
#
#     Input: [1,2,3,4,5]
#     Output: 4
#     Explanation: Buy on day 1 (price = 1) and sell on day 5 (price = 5), profit = 5-1 = 4.
#                  Note that you cannot buy on day 1, buy on day 2 and sell them later, as you are
#                  engaging multiple transactions at the same time. You must sell before buying again.
#
#
# **Example 3:**
#
#
#
#     Input: [7,6,4,3,1]
#     Output: 0
#     Explanation: In this case, no transaction is done, i.e. max profit = 0.
#
#
#
# **Constraints:**
#
#   * `1 <= prices.length <= 3 * 10 ^ 4`
#   * `0 <= prices[i] <= 10 ^ 4`
#
#
## @lc code=start
using LeetCode

function max_profit(prices::Vector{Int})::Int
    res = 0
    for i in 2:length(prices)
        res += (prices[i] > prices[i - 1]) ? (prices[i] - prices[i - 1]) : 0
    end
    return res
end
## @lc code=end


using Test
using PerronFrobenius

pts = rand(3, 10000)
μF = [4, 4, 4]

μ_avg = PerronFrobenius.average_measure(pts, μF , [μF, μF])
μ_induced = PerronFrobenius.induced_measure(pts, μF, μF)

@testset "Average measure" begin
	@testset "Average measure sums to 1" begin
		@test sum(μ_avg) ≈ 1
	end

	@testset "Measure induced from multiple same resolutions matches original" begin
		μ_induced = induced_measure(pts, μF, μF)
		@test all(μ_avg .≈ μ_induced)
	end
end


"""
    AbstractHMM{F<:VariateForm}

A custom HMM type must at-least implement the following interface:
```julia
struct CustomHMM{F,T} <: AbstractHMM{F}
    a::AbstractVector{T}               # Initial state distribution
    A::AbstractMatrix{T}               # Transition matrix
    B::AbstractVector{Distribution{F}} # Observations distributions
    # Optional, custom, fields ....
end
```
"""
abstract type AbstractHMM{F<:VariateForm} end

"""
    HMM([a, ]A, B) -> HMM

Build an HMM with transition matrix `A` and observation distributions `B`.
If the initial state distribution `a` is not specified, a uniform distribution is assumed.

Observations distributions can be of different types (for example `Normal` and `Exponential`),
but they must be of the same dimension.

**Arguments**
- `a::AbstractVector{T}`: initial probabilities vector.
- `A::AbstractMatrix{T}`: transition matrix.
- `B::AbstractVector{<:Distribution{F}}`: observations distributions.

**Example**
```julia
using Distributions, HMMBase
hmm = HMM([0.9 0.1; 0.1 0.9], [Normal(0,1), Normal(10,1)])
```
"""
struct HMM{F,T} <: AbstractHMM{F}
    a::Vector{T}
    A::Matrix{T}
    B::Vector{Distribution{F}}
    HMM{F,T}(a, A, B) where {F,T} = assert_hmm(a, A, B) && new(a, A, B)
end

HMM(a::AbstractVector{T}, A::AbstractMatrix{T}, B::AbstractVector{<:Distribution{F}}) where {F,T} = HMM{F,T}(a, A, B)
HMM(A::AbstractMatrix{T}, B::AbstractVector{<:Distribution{F}}) where {F,T} = HMM{F,T}(ones(size(A)[1])/size(A)[1], A, B)

"""
    assert_hmm(a, A, B)

Throw an `ArgumentError` if the initial state distribution and the transition matrix rows does not sum to 1,
and if the observation distributions do not have the same dimensions.
"""
function assert_hmm(a::AbstractVector,
                    A::AbstractMatrix,
                    B::AbstractVector{<:Distribution})
    @argcheck isprobvec(a)
    @argcheck istransmat(A)
    @argcheck all(length.(B) .== length(B[1])) ArgumentError("All distributions must have the same dimensions")
    @argcheck length(a) == size(A,1) == length(B)
    return true
end

"""
    issquare(A) -> Bool

Return true if `A` is a square matrix.
"""
issquare(A::AbstractMatrix) = size(A,1) == size(A,2)

"""
    istransmat(A) -> Bool

Return true if `A` is square and its rows sums to 1.
"""
istransmat(A::AbstractMatrix) = issquare(A) && all([isprobvec(A[i,:]) for i in 1:size(A,1)])

==(h1::AbstractHMM, h2::AbstractHMM) = (h1.a == h2.a) && (h1.A == h2.A) && (h1.B == h2.B)

"""
    rand([rng, ]hmm, T; init, seq) -> Array | (Vector, Array)

Sample a trajectory of `T` timesteps from `hmm`.

**Keyword Arguments**
- `init::Integer = rand(Categorical(hmm.a))`: initial state.
- `seq::Bool = false`: whether to return the hidden state sequence or not.

**Output**
- `Vector{Int}` (if `seq == true`): hidden state sequence.
- `Vector{Float64}` (for `Univariate` HMMs): observations (`T`).
- `Matrix{Float64}` (for `Multivariate` HMMs): observations (`T x dim(obs)`).

**Examples**
```julia
using Distributions, HMMBase
hmm = HMM([0.9 0.1; 0.1 0.9], [Normal(0,1), Normal(10,1)])
y = rand(hmm, 1000) # or
z, y = rand(hmm, 1000, seq = true)
size(y) # (1000,)
```

```julia
using Distributions, HMMBase
hmm = HMM([0.9 0.1; 0.1 0.9], [MvNormal(ones(2)), MvNormal(ones(2))])
y = rand(hmm, 1000) # or
z, y = rand(hmm, 1000, seq = true)
size(y) # (1000, 2)
```
"""
function rand(rng::AbstractRNG, hmm::AbstractHMM, T::Integer; init = rand(rng, Categorical(hmm.a)), seq = false)
    z = Vector{Int}(undef, T)
    (T >= 1) && (z[1] = init)
    for t in 2:T
        z[t] = rand(rng, Categorical(hmm.A[z[t-1],:]))
    end
    y = rand(rng, hmm, z)
    seq ? (z, y) : y
end

"""
    rand([rng, ]hmm, z) -> Array

Sample observations from `hmm` according to trajectory `z`.

**Output**
- `Vector{Float64}` (for `Univariate` HMMs): observations (`T`).
- `Matrix{Float64}` (for `Multivariate` HMMs): observations (`T x dim(obs)`).

**Example**
```julia
using Distributions, HMMBase
hmm = HMM([0.9 0.1; 0.1 0.9], [Normal(0,1), Normal(10,1)])
y = rand(hmm, [1, 1, 2, 2, 1])
```
"""
function rand(rng::AbstractRNG, hmm::AbstractHMM{Univariate}, z::AbstractVector{<:Integer})
    y = Vector{Float64}(undef, length(z))
    for t in eachindex(z)
        y[t] = rand(rng, hmm.B[z[t]])
    end
    y
end

function rand(rng::AbstractRNG, hmm::AbstractHMM{Multivariate}, z::AbstractVector{<:Integer})
    y = Matrix{Float64}(undef, length(z), size(hmm, 1))
    for t in eachindex(z)
        y[t,:] = rand(rng, hmm.B[z[t]])
    end
    y
end

rand(hmm::AbstractHMM, T::Integer; kwargs...) = rand(GLOBAL_RNG, hmm, T; kwargs...)

rand(hmm::AbstractHMM, z::AbstractVector{<:Integer}) = rand(GLOBAL_RNG, hmm, z)

"""
    size(hmm, [dim]) -> Int | Tuple

Return the number of states in `hmm` and the dimension of the observations.

**Example**
```jldoctest
using Distributions, HMMBase
hmm = HMM([0.9 0.1; 0.1 0.9], [Normal(0,1), Normal(10,1)])
size(hmm)
# output
(2, 1)
```
"""
size(hmm::AbstractHMM, dim=:) = (length(hmm.B), length(hmm.B[1]))[dim]

"""

    copy(hmm) -> HMM

Return a copy of `hmm`.
"""
copy(hmm::HMM) = HMM(copy(hmm.a), copy(hmm.A), copy(hmm.B))

"""
    permute(hmm, perm) -> HMM

Permute the states of `hmm` according to `perm`.

**Arguments**

- `perm::Vector{<:Integer}`: permutation of the states.

**Example**
```julia
using Distributions, HMMBase
hmm = HMM([0.8 0.2; 0.1 0.9], [Normal(0,1), Normal(10,1)])
hmm = permute(hmm, [2, 1])
hmm.A # [0.9 0.1; 0.2 0.8]
hmm.B # [Normal(10,1), Normal(0,1)]
```
"""
function permute(hmm::AbstractHMM, perm::Vector{<:Integer})
    a = hmm.a[perm]
    B = hmm.B[perm]
    A = zeros(size(hmm.A))
    for i in 1:size(A,1), j in 1:size(A,2)
        A[i,j] = hmm.A[perm[i],perm[j]]
    end
    HMM(a, A, B)
end

"""
    nparams(hmm) -> Int

Return the number of _free_ parameters in `hmm`, without counting the
observation distributions parameters.

**Example**
```jldoctest
using Distributions, HMMBase
hmm = HMM([0.9 0.1; 0.1 0.9], [Normal(0,1), Normal(10,1)])
nparams(hmm)
# output
3
```
"""
function nparams(hmm::AbstractHMM)
    (length(hmm.a) - 1) + (length(hmm.A) - size(hmm.A, 1))
end

"""
    statdists(hmm) -> Vector{Vector}

Return the stationnary distribution(s) of `hmm`.
That is, the eigenvectors of transpose(hmm.A) with eigenvalues 1.
"""
function statdists(hmm::AbstractHMM)
    eig = eigen(collect(transpose(hmm.A)))
    dists = []
    for (i, val) in enumerate(eig.values)
        if val == 1.0
            dist = eig.vectors[:,i]
            dist /= sum(dist)
            push!(dists, dist)
        end
    end
    dists
end


"""
    RootSolvers

Module containing functions for solving roots of non-linear
equations. See [`find_zero`](@ref).

## Example

```
using CLIMA.Utilities.RootSolvers

sol = find_zero(x -> x^2 - 100^2, 0.0, 1000.0, SecantMethod(), CompactSolution())
x_root = sol.root
converged = sol.converged
```

"""
module RootSolvers

export find_zero, SecantMethod, RegulaFalsiMethod, NewtonsMethodAD, NewtonsMethod
export CompactSolution, VerboseSolution

import ForwardDiff

abstract type RootSolvingMethod end
Base.broadcastable(method::RootSolvingMethod) = Ref(method)

struct SecantMethod <: RootSolvingMethod end
struct RegulaFalsiMethod <: RootSolvingMethod end
struct NewtonsMethodAD <: RootSolvingMethod end
struct NewtonsMethod <: RootSolvingMethod end

abstract type SolutionType end
Base.broadcastable(soltype::SolutionType) = Ref(soltype)

"""
    VerboseSolution <: SolutionType

Used to return a [`VerboseSolutionResults`](@ref)
"""
struct VerboseSolution <: SolutionType end

abstract type AbstractSolutionResults{AbstractFloat} end

struct VerboseSolutionResults{FT} <: AbstractSolutionResults{FT}
  "solution ``x^*`` of the root of the equation ``f(x^*) = 0``"
  root::FT
  "indicates convergence"
  converged::Bool
  "error of the root of the equation ``f(x^*) = 0``"
  err::FT
  "number of iterations performed"
  iter_performed::Int
  "solution per iteration"
  root_history::Vector{FT}
  "error of the root of the equation ``f(x^*) = 0`` per iteration"
  err_history::Vector{FT}
end
SolutionResults(soltype::VerboseSolution, args...) = VerboseSolutionResults(args...)

"""
    CompactSolution <: SolutionType

Used to return a [`CompactSolutionResults`](@ref)
"""
struct CompactSolution <: SolutionType end

struct CompactSolutionResults{FT} <: AbstractSolutionResults{FT}
  "solution ``x^*`` of the root of the equation ``f(x^*) = 0``"
  root::FT
  "indicates convergence"
  converged::Bool
end
SolutionResults(soltype::CompactSolution, root, converged, args...) = CompactSolutionResults(root, converged)

init_history(::VerboseSolution, x::FT) where {FT<:AbstractFloat} = FT[x]
init_history(::CompactSolution, x) = nothing
init_history(::VerboseSolution, ::Type{FT}) where {FT<:AbstractFloat} = FT[]
init_history(::CompactSolution, ::Type{FT}) where {FT<:AbstractFloat} = nothing

push_history!(history::Vector{FT}, x::FT, ::VerboseSolution) where {FT<:AbstractFloat} =
  push!(history, x)
push_history!(history::Nothing, x::FT, ::CompactSolution) where {FT<:AbstractFloat} =
  nothing

# TODO: CuArrays.jl has trouble with isapprox on 1.1
# we use simple checks for now, will switch to relative checks later.

"""
    sol = find_zero(f[, f′], x0[, x1], method, solutiontype,
                    xatol=1e-3,
                    maxiters=10_000)

Finds the nearest root of `f` to `x0` and `x1`. Returns a the value of the root `x` such
that `f(x) ≈ 0`, and a Boolean value `converged` indicating convergence.

`method` can be one of:
- `SecantMethod()`: [Secant method](https://en.wikipedia.org/wiki/Secant_method)
- `RegulaFalsiMethod()`: [Regula Falsi method](https://en.wikipedia.org/wiki/False_position_method#The_regula_falsi_(false_position)_method).
- `NewtonsMethodAD()`: [Newton's method](https://en.wikipedia.org/wiki/Newton%27s_method) using Automatic Differentiation
  - The `x1` argument is omitted for Newton's method.
- `NewtonsMethod()`: [Newton's method](https://en.wikipedia.org/wiki/Newton%27s_method)
  - The `x1` argument is omitted for Newton's method.
  - `f′`: derivative of function `f` whose zero is sought

The keyword arguments:
- `xatol` is the absolute tolerance of the input.
- `maxiters` is the maximum number of iterations.
"""
function find_zero end

function find_zero(f::F, x0::FT, x1::FT, ::SecantMethod, soltype::SolutionType,
                   xatol=FT(1e-3),
                   maxiters=10_000) where {F, FT<:AbstractFloat}
  y0 = f(x0)
  y1 = f(x1)
  x_history = init_history(soltype, x0)
  y_history = init_history(soltype, y0)
  for i in 1:maxiters
    Δx = x1 - x0
    Δy = y1 - y0
    x0, y0 = x1, y1
    push_history!(x_history, x0, soltype)
    push_history!(y_history, y0, soltype)
    x1 -= y1 * Δx / Δy
    y1 = f(x1)
    if abs(x0-x1) <= xatol
      return SolutionResults(soltype, x1, true, y1, i, x_history, y_history)
    end
  end
  return SolutionResults(soltype, x1, false, y1, maxiters, x_history, y_history)
end

function find_zero(f::F, x0::FT, x1::FT, ::RegulaFalsiMethod, soltype::SolutionType,
                   xatol=FT(1e-3),
                   maxiters=10_000) where {F, FT<:AbstractFloat}
  y0 = f(x0)
  y1 = f(x1)
  @assert y0 * y1 < 0
  x_history = init_history(soltype, x0)
  y_history = init_history(soltype, y0)
  lastside = 0
  local x, y
  for i in 1:maxiters
    x = (x0 * y1 - x1 * y0)/ (y1 - y0)
    y = f(x)
    push_history!(x_history, x, soltype)
    push_history!(y_history, y, soltype)
    if y * y0 < 0
      if abs(x-x1) <= xatol
        return SolutionResults(soltype, x, true, y, i, x_history, y_history)
      end
      x1, y1 = x, y
      if lastside == +1
        y0 /= 2
      end
      lastside = +1
    else
      if abs(x0-x) <= xatol
        return SolutionResults(soltype, x, true, y, i, x_history, y_history)
      end
      x0, y0 = x, y
      if lastside == -1
        y1 /= 2
      end
      lastside = -1
    end
  end
  return SolutionResults(soltype, x, false, y, maxiters, x_history, y_history)
end

"""
    value_deriv(f, x)

Compute the value and derivative `f(x)` using ForwardDiff.jl.
"""
function value_deriv(f, x::FT) where {FT}
    tag = typeof(ForwardDiff.Tag(f, FT))
    y = f(ForwardDiff.Dual{tag}(x,one(x)))
    ForwardDiff.value(tag, y), ForwardDiff.partials(tag, y, 1)
end

function find_zero(f::F, x0::FT, ::NewtonsMethodAD, soltype::SolutionType,
                   xatol=FT(1e-3),
                   maxiters=10_000) where {F, FT<:AbstractFloat}
  local y
  x_history = init_history(soltype, FT)
  y_history = init_history(soltype, FT)
  if soltype isa VerboseSolution
    y,y′ = value_deriv(f, x0)
    push_history!(x_history, x0, soltype)
    push_history!(y_history, y , soltype)
  end
  for i in 1:maxiters
    y,y′ = value_deriv(f, x0)
    x1 = x0 - y/y′
    push_history!(x_history, x1, soltype)
    push_history!(y_history, y,  soltype)
    if abs(x0-x1) <= xatol
      return SolutionResults(soltype, x1, true, y, i, x_history, y_history)
    end
    x0 = x1
  end
  return SolutionResults(soltype, x0, false, y, maxiters, x_history, y_history)
end

function find_zero(f::F, f′::F′, x0::FT, ::NewtonsMethod, soltype::SolutionType,
                   xatol=1e-3,
                   maxiters=10_000) where {F, F′, FT<:AbstractFloat}
  x_history = init_history(soltype, FT)
  y_history = init_history(soltype, FT)
  if soltype isa VerboseSolution
    y,y′ = f(x0), f′(x0)
    push_history!(x_history, x0, soltype)
    push_history!(y_history, y , soltype)
  end
  for i in 1:maxiters
    y,y′ = f(x0), f′(x0)
    x1 = x0 - y/y′
    push_history!(x_history, x1, soltype)
    push_history!(y_history, y,  soltype)
    if abs(x0-x1) <= xatol
      return SolutionResults(soltype, x1, true, y, i, x_history, y_history)
    end
    x0 = x1
  end
  return SolutionResults(soltype, x0, false, y, maxiters, x_history, y_history)
end

end


using Klara

### Define the log-target as a function (generic or anonymous):

plogtarget(z::Vector{Float64}) = -dot(z, z)

### Define the parameter via BasicContMuvParameter (it is a continuous multivariate variable)
### The input arguments for BasicContMuvParameter are:
### 1) the variable key,
### 2) the log-target

p = BasicContMuvParameter(:p, logtarget=plogtarget)

### Define the model using the likelihood_model generator
### The second argument informs the likelihood_model generator that p.index has not been set

model = likelihood_model(p, false)

### Define a Metropolis-Hastings sampler with an identity covariance matrix

sampler = MH(ones(2))

### Set MCMC sampling range

mcrange = BasicMCRange(nsteps=10000, burnin=1000)

### Set initial values for simulation

v0 = Dict(:p=>[5.1, -0.9])

### Specify job to be run

job = BasicMCJob(model, sampler, mcrange, v0)

### Run the simulation

run(job)

### Get simulated values

chain = output(job)

chain.value

### Check that the simulated values are close to the zero-mean target

mean(chain)

reset(job, [3.2, 9.4])

run(job)

chain = output(job)

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true))

run(job)

chain = output(job)

outopts = Dict{Symbol, Any}(:monitor=>[:value, :logtarget])

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts)

run(job)

chain = output(job)

chain.logtarget

outopts = Dict{Symbol, Any}(:monitor=>[:value, :logtarget], :diagnostics=>[:accept])

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts)

filepath = dirname(@__FILE__)
filesuffix = "csv"

outopts = Dict{Symbol, Any}(
  :monitor=>[:value, :logtarget],
  :diagnostics=>[:accept],
  :destination=>:iostream,
  :filepath=>filepath
)

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts)

run(job)

for fname in (:value, :logtarget, :diagnosticvalues)
  rm(joinpath(filepath, string(fname)*"."*filesuffix))
end

outopts = Dict{Symbol, Any}(:monitor=>[:value, :logtarget], :diagnostics=>[:accept])

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts, plain=false)

run(job)

chain = output(job)

reset(job, [-2.8, 3.4])

run(job)

chain = output(job)

plogtarget(z::Vector{Float64}) = -dot(z, z)

pgradlogtarget(z::Vector{Float64}) = -2*z

p = BasicContMuvParameter(:p, logtarget=plogtarget, gradlogtarget=pgradlogtarget)

model = likelihood_model(p, false)

### Set driftstep to 0.9

sampler = MALA(0.9)

mcrange = BasicMCRange(nsteps=10000, burnin=1000)

v0 = Dict(:p=>[5.1, -0.9])

### Save grad-log-target along with the chain (value and log-target)

outopts = Dict{Symbol, Any}(:monitor=>[:value, :logtarget, :gradlogtarget], :diagnostics=>[:accept])

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts)

run(job)

chain = output(job)

chain.gradlogtarget

mean(chain)

job = BasicMCJob(
  model,
  sampler,
  mcrange,
  v0,
  tuner=AcceptanceRateMCTuner(0.6, verbose=true),
  outopts=outopts
)

run(job)

chain = output(job)

plogtarget(z::Vector) = -dot(z, z)

p = BasicContMuvParameter(:p, logtarget=plogtarget, autodiff=:forward)

model = likelihood_model(p, false)

sampler = MALA(0.9)

mcrange = BasicMCRange(nsteps=10000, burnin=1000)

v0 = Dict(:p=>[5.1, -0.9])

outopts = Dict{Symbol, Any}(:monitor=>[:value, :logtarget, :gradlogtarget], :diagnostics=>[:accept])

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts)

run(job)

chain = output(job)

plogtarget(z::Vector) = -dot(z, z)

p = BasicContMuvParameter(:p, logtarget=plogtarget, autodiff=:reverse, init=[(:z, ones(2))])

model = likelihood_model(p, false)

sampler = MALA(0.9)

mcrange = BasicMCRange(nsteps=10000, burnin=1000)

v0 = Dict(:p=>[5.1, -0.9])

outopts = Dict{Symbol, Any}(:monitor=>[:value, :logtarget, :gradlogtarget], :diagnostics=>[:accept])

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts)

run(job)

chain = output(job)

p = BasicContMuvParameter(:p, logtarget=:(-dot(z, z)), autodiff=:reverse, init=[(:z, ones(2))])

model = likelihood_model(p, false)

sampler = MALA(0.9)

mcrange = BasicMCRange(nsteps=10000, burnin=1000)

v0 = Dict(:p=>[5.1, -0.9])

outopts = Dict{Symbol, Any}(:monitor=>[:value, :logtarget, :gradlogtarget], :diagnostics=>[:accept])

job = BasicMCJob(model, sampler, mcrange, v0, tuner=VanillaMCTuner(verbose=true), outopts=outopts)

run(job)

chain = output(job)


#=
Computes quantities related to the Gaussian linear state space model

    x_{t+1} = A x_t + C w_{t+1}

        y_t = G x_t

The shocks {w_t} are iid and N(0, I)

@author : Spencer Lyon <spencer.lyon@nyu.edu>

@date : 2014-07-28

References
----------

TODO: Come back and update to match `LinearStateSpace` type from py side
TODO: Add docstrings

http://quant-econ.net/jl/linear_models.html

=#

"""
A type that describes the Gaussian Linear State Space Model
of the form:

    x_{t+1} = A x_t + C w_{t+1}

        y_t = G x_t

where {w_t} and {v_t} are independent and standard normal with dimensions
k and l respectively.  The initial conditions are mu_0 and Sigma_0 for x_0
~ N(mu_0, Sigma_0).  When Sigma_0=0, the draw of x_0 is exactly mu_0.

#### Fields

- `A::Matrix` Part of the state transition equation.  It should be `n x n`
- `C::Matrix` Part of the state transition equation.  It should be `n x m`
- `G::Matrix` Part of the observation equation.  It should be `k x n`
- `k::Int` Dimension
- `n::Int` Dimension
- `m::Int` Dimension
- `mu_0::Vector` This is the mean of initial draw and is of length `n`
- `Sigma_0::Matrix` This is the variance of the initial draw and is `n x n` and
                    also should be positive definite and symmetric

"""
type LSS{TSampler<:MVNSampler}
    A::Matrix
    C::Matrix
    G::Matrix
    k::Int
    n::Int
    m::Int
    mu_0::Vector
    Sigma_0::Matrix
    dist::TSampler
end

function LSS(A::ScalarOrArray, C::ScalarOrArray, G::ScalarOrArray,
             mu_0::ScalarOrArray,
             Sigma_0::Matrix=zeros(size(G, 2), size(G, 2)))
    k = size(G, 1)
    n = size(G, 2)
    m = size(C, 2)

    # coerce shapes
    A = reshape(vcat(A), n, n)
    C = reshape(vcat(C), n, m)
    G = reshape(vcat(G), k, n)

    mu_0 = reshape([mu_0;], n)

    dist = MVNSampler(mu_0,Sigma_0)
    LSS(A, C, G, k, n, m, mu_0, Sigma_0, dist)
end

# make kwarg version
function LSS(A::ScalarOrArray, C::ScalarOrArray, G::ScalarOrArray;
             mu_0::Vector=zeros(size(G, 2)),
             Sigma_0::Matrix=zeros(size(G, 2), size(G, 2)))
    return LSS(A, C, G, mu_0, Sigma_0)
end

function simulate(lss::LSS, ts_length=100)
    x = Array{Float64}(lss.n, ts_length)
    x[:, 1] = rand(lss.dist)
    w = randn(lss.m, ts_length - 1)
    for t=1:ts_length-1
        x[:, t+1] = lss.A * x[:, t] .+ lss.C * w[:, t]
    end
    y = lss.G * x

    return x, y
end

"""
Simulate num_reps observations of x_T and y_T given x_0 ~ N(mu_0, Sigma_0).

#### Arguments

- `lss::LSS` An instance of the Gaussian linear state space model.
- `t::Int = 10` The period that we want to replicate values for.
- `num_reps::Int = 100` The number of replications we want

#### Returns

- `x::Matrix` An n x num_reps matrix, where the j-th column is the j_th
              observation of x_T
- `y::Matrix` An k x num_reps matrix, where the j-th column is the j_th
              observation of y_T

"""
function replicate(lss::LSS, t::Integer, num_reps::Integer=100)
    x = Array{Float64}(lss.n, num_reps)
    for j=1:num_reps
        x_t, _ = simulate(lss, t+1)
        x[:, j] = x_t[:, end]
    end

    y = lss.G * x
    return x, y
end

replicate(lss::LSS; t::Integer=10, num_reps::Integer=100) =
    replicate(lss, t, num_reps)

immutable LSSMoments
    lss::LSS
end

Base.start(L::LSSMoments) = (copy(L.lss.mu_0), copy(L.lss.Sigma_0))
Base.done(L::LSSMoments, x) = false
function Base.next(L::LSSMoments, moms)
    A, C, G = L.lss.A, L.lss.C, L.lss.G
    mu_x, Sigma_x = moms

    mu_y, Sigma_y = G * mu_x, G * Sigma_x * G'

    # Update moments of x
    mu_x2 = A * mu_x
    Sigma_x2 = A * Sigma_x * A' + C * C'

    (mu_x, mu_y, Sigma_x, Sigma_y), (mu_x2, Sigma_x2)
end

"""
Create an iterator to calculate the population mean and
variance-convariance matrix for both x_t and y_t, starting at
the initial condition (self.mu_0, self.Sigma_0).  Each iteration
produces a 4-tuple of items (mu_x, mu_y, Sigma_x, Sigma_y) for
the next period.

#### Arguments

- `lss::LSS` An instance of the Gaussian linear state space model

"""
moment_sequence(lss::LSS) = LSSMoments(lss)

"""
Compute the moments of the stationary distributions of x_t and
y_t if possible.  Computation is by iteration, starting from the
initial conditions lss.mu_0 and lss.Sigma_0

#### Arguments

- `lss::LSS` An instance of the Guassian linear state space model
- `;max_iter::Int = 200` The maximum number of iterations allowed
- `;tol::Float64 = 1e-5` The tolerance level one wishes to achieve

#### Returns

- `mu_x::Vector` Represents the stationary mean of x_t
- `mu_y::Vector`Represents the stationary mean of y_t
- `Sigma_x::Matrix` Represents the var-cov matrix
- `Sigma_y::Matrix` Represents the var-cov matrix

"""
function stationary_distributions(lss::LSS; max_iter=200, tol=1e-5)
    # Initialize iteration
    m = moment_sequence(lss)
    mu_x, mu_y, Sigma_x, Sigma_y = first(m)

    i = 0
    err = tol + 1.0

    for (mu_x1, mu_y, Sigma_x1, Sigma_y) in m
        i > max_iter && error("Convergence failed after $i iterations")
        i += 1
        err_mu = maximum(abs, mu_x1 - mu_x)
        err_Sigma = maximum(abs, Sigma_x1 - Sigma_x)
        err = max(err_Sigma, err_mu)
        mu_x, Sigma_x = mu_x1, Sigma_x1

        if err < tol && i > 1
            break
        end
    end

    return mu_x, mu_y, Sigma_x, Sigma_y
end

function geometric_sums(lss::LSS, bet, x_t)
    I = eye(lss.n)
    S_x = (I - bet .* lss.A) \ x_t
    S_y = lss.G * S_x
    return S_x, S_y
end


# ---
# title: 639. Decode Ways II
# id: problem639
# author: Tian Jun
# date: 2020-10-31
# difficulty: Hard
# categories: Dynamic Programming
# link: <https://leetcode.com/problems/decode-ways-ii/description/>
# hidden: true
# ---
#
# A message containing letters from `A-Z` is being encoded to numbers using the
# following mapping way:
#
#
#
#     'A' -> 1
#     'B' -> 2
#     ...
#     'Z' -> 26
#
#
# Beyond that, now the encoded string can also contain the character '*', which
# can be treated as one of the numbers from 1 to 9.
#
# Given the encoded message containing digits and the character '*', return the
# total number of ways to decode it.
#
# Also, since the answer may be very large, you should return the output mod 109
# \+ 7.
#
# **Example 1:**
#
#
#
#     Input: "*"
#     Output: 9
#     Explanation: The encoded message can be decoded to the string: "A", "B", "C", "D", "E", "F", "G", "H", "I".
#
#
# **Example 2:**
#
#
#
#     Input: "1*"
#     Output: 9 + 9 = 18
#
#
# **Note:**
#
#   1. The length of the input string will fit in range [1, 105].
#   2. The input string will only contain the character '*' and digits '0' - '9'.
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


# ---
# title: 84. Largest Rectangle in Histogram
# id: problem84
# author: Tian Jun
# date: 2020-10-31
# difficulty: Hard
# categories: Array, Stack
# link: <https://leetcode.com/problems/largest-rectangle-in-histogram/description/>
# hidden: true
# ---
#
# Given _n_ non-negative integers representing the histogram's bar height where
# the width of each bar is 1, find the area of largest rectangle in the
# histogram.
#
#
#
# ![](https://assets.leetcode.com/uploads/2018/10/12/histogram.png)
# Above is a histogram where width of each bar is 1, given height =
# `[2,1,5,6,2,3]`.
#
#
#
# ![](https://assets.leetcode.com/uploads/2018/10/12/histogram_area.png)
# The largest rectangle is shown in the shaded area, which has area = `10` unit.
#
#
#
# **Example:**
#
#
#
#     Input: [2,1,5,6,2,3]
#     Output: 10
#
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


#    Problem : GROUP C
#    *********
#    This is a Power problem by Oren, the corrected SIF version
# 	 of the POWER problem to reflect Oren's formula.
#
#    Origonal SIF Source:
#    S.S. Oren,
#    Self-scaling variable metric algorithms,
#    Part II: implementation and experiments"
#    Management Science 20(5):863-874, 1974.
#
#    POWER.SIF classification OUR2-AN-V-0
#
# Daniel Henderson, 08/2021
@warn "TODO: verify power_corrected.jl formula matches Oren's specification"

f = (x) -> begin
    fx = 0.0
    for i in firstindex(x):lastindex(x)
        fx += (x[i]i)^2
    end
    return fx
end

g! = (g, x) -> begin
    for i in firstindex(x):lastindex(x)
        g[i] = 2x[i]i^2
    end
    return g
end

fg! = (g, x) -> begin
    α = 0.0
    for i in firstindex(x):lastindex(x)
        α += x[i]^2*i
    end
    for i in firstindex(x):lastindex(x)
        g[i] = 4α*i*x[i]
    end
    return fx, g
end

init = (n::Int=10000) -> begin
    return n, ones(n)
end

@warn "power_corrected: Currently excluded, but maybe include after testing?"
#TestSet["power_corrected"] = UncProgram("power_corrected", f, g!, fg!, init)


using StatsBase
using Test

@testset "ECDF" begin
    x = randn(10000000)
    fnecdf = ecdf(x)
    y = [-1.96, -1.644854, -1.281552, -0.6744898, 0, 0.6744898, 1.281552, 1.644854, 1.96]
    @test isapprox(fnecdf(y), [0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.975], atol=1e-3)
    @test isapprox(fnecdf(1.96), 0.975, atol=1e-3)
    @test fnecdf(y) ≈ map(fnecdf, y)
    @test extrema(fnecdf) == (minimum(fnecdf), maximum(fnecdf)) == extrema(x)
    fnecdf = ecdf([0.5])
    @test fnecdf([zeros(5000); ones(5000)]) == [zeros(5000); ones(5000)]
    @test extrema(fnecdf) == (minimum(fnecdf), maximum(fnecdf)) == (0.5, 0.5)
end

@testset "Weighted ECDF" begin
    x = randn(10000000)
    w1 = rand(10000000)
    w2 = weights(w1)
    fnecdf = ecdf(x, weights=w1)
    fnecdfalt = ecdf(x, weights=w2)
    @test fnecdf.sorted_values == fnecdfalt.sorted_values
    @test fnecdf.weights == fnecdfalt.weights
    @test fnecdf.weights != w1  #  check that w wasn't accidently modified in place
    @test fnecdfalt.weights != w2
    y = [-1.96, -1.644854, -1.281552, -0.6744898, 0, 0.6744898, 1.281552, 1.644854, 1.96]
    @test isapprox(fnecdf(y), [0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.975], atol=1e-3)
    @test isapprox(fnecdf(1.96), 0.975, atol=1e-3)
    @test fnecdf(y) ≈ map(fnecdf, y)
    @test extrema(fnecdf) == (minimum(fnecdf), maximum(fnecdf)) == extrema(x)
    fnecdf = ecdf([1.0, 0.5], weights=weights([3, 1]))
    @test fnecdf(0.75) == 0.25
    @test extrema(fnecdf) == (minimum(fnecdf), maximum(fnecdf)) == (0.5, 1.0)
    @test_throws ArgumentError ecdf(rand(8), weights=weights(rand(10)))
    #  Check frequency weights
    v = randn(100)
    r = rand(1:100, 100)
    vv = vcat(fill.(v, r)...)  #  repeat elements of v according to r
    fw = fweights(r)
    frecdf1 = ecdf(v, weights=fw)
    frecdf2 = ecdf(vv)
    @test frecdf1(y) ≈ frecdf2(y)
    #  Check probability weights
    a = randn(100)
    b = rand(100)
    b̃ = abs(10randn()) * b
    bw1 = pweights(b)
    bw2 = pweights(b̃)
    precdf1 = ecdf(a, weights=bw1)
    precdf2 = ecdf(a, weights=bw2)
    @test precdf1(y) ≈ precdf2(y)
end


import AlgebraicMultigrid: scale_cols_by_largest_entry!,
            SymmetricStrength, poisson
function symmetric_soc(A::SparseMatrixCSC{T,V}, θ) where {T,V}
    D = abs.(diag(A))
    i,j,v = findnz(A)
    mask = i .!= j
    DD = D[i] .* D[j]
    mask = mask .& (abs.(v.^2) .>= (θ * θ * DD))

    i = i[mask]
    j = j[mask]
    v = v[mask]

    S = sparse(i,j,v, size(A)...) + spdiagm(0=>D)

    for i = 1:size(S.nzval,1)
        S.nzval[i] = abs(S.nzval[i])
    end

    scale_cols_by_largest_entry!(S)

    S
end

# Set up tests
function test_symmetric_soc()

    cases = generate_matrices()

    for matrix in cases
        for θ in (0.0, 0.1, 0.5, 1., 10.)
            ref_matrix = symmetric_soc(matrix, θ)
            calc_matrix, _ = SymmetricStrength(θ)(matrix)

            @test sum(abs2, ref_matrix - calc_matrix) < 1e-6
        end
    end
end

function generate_matrices()

    cases = []

    # Random matrices
    seed!(0)
    for T in (Float32, Float64)

        for s in [2, 3, 5]
            push!(cases, sprand(T, s, s, 1.))
        end

        for s in [2, 3, 5, 7, 10, 11, 19]
            push!(cases, poisson(T, s))
        end
    end

    cases
end

function stand_agg(C)
    n = size(C, 1)

    R = Set(1:n)
    j = 0
    Cpts = Int[]

    aggregates = -ones(Int, n)

    # Pass 1
    for i = 1:n
        Ni = union!(Set(C.rowval[nzrange(C, i)]), Set(i))
        if issubset(Ni, R)
            push!(Cpts, i)
            setdiff!(R, Ni)
            for x in Ni
                aggregates[x] = j
            end
            j += 1
        end
    end

    # Pass 2
    old_R = copy(R)
    for i = 1:n
        if ! (i in R)
            continue
        end

        for x in C.rowval[nzrange(C, i)]
            if !(x in old_R)
                aggregates[i] = aggregates[x]
                setdiff!(R, i)
                break
            end
        end
    end

    # Pass 3
    for i = 1:n
        if !(i in R)
            continue
        end
        Ni = union(Set(C.rowval[nzrange(C,i)]), Set(i))
        push!(Cpts, i)

        for x in Ni
            if x in R
                aggregates[x] = j
            end
            j += 1
        end
    end

    @assert length(R) == 0

    Pj = aggregates .+ 1
    Pp = collect(1:n+1)
    Px = ones(eltype(C), n)

    SparseMatrixCSC(maximum(aggregates .+ 1), n, Pp, Pj, Px)
end

# Standard aggregation tests
function test_standard_aggregation()

    cases = generate_matrices()

    for matrix in cases
        for θ in (0.0, 0.1, 0.5, 1., 10.)
            C = symmetric_soc(matrix, θ)
            calc_matrix = StandardAggregation()(matrix)
            ref_matrix = stand_agg(matrix)
            @test sum(abs2, ref_matrix - calc_matrix) < 1e-6
        end
    end

end

# Test fit_candidates
function test_fit_candidates()

    cases = generate_fit_candidates_cases()

    for (i, (AggOp, fine_candidates)) in enumerate(cases)

        mask_candidates!(AggOp, fine_candidates)

        Q, coarse_candidates = fit_candidates(AggOp, fine_candidates)

        @test isapprox(fine_candidates, Q * coarse_candidates)
        @test isapprox(Q * (Q' * fine_candidates), fine_candidates)
    end
end
function mask_candidates!(A,B)
    B[(diff(A.colptr) .== 0)] .= 0
end

function generate_fit_candidates_cases()
    cases = []

    for T in (Float32, Float64)

        # One candidate
        AggOp = SparseMatrixCSC(2, 5, collect(1:6),
                        [1,1,1,2,2], ones(T,5))
        B =  ones(T,5)
        push!(cases, (AggOp, B))

        AggOp = SparseMatrixCSC(2, 5, collect(1:6),
                        [2,2,1,1,1], ones(T,5))
        B = ones(T, 5)
        push!(cases, (AggOp, B))

        AggOp = SparseMatrixCSC(3, 9, collect(1:10),
                        [1,1,1,2,2,2,3,3,3], ones(T, 9))
        B = ones(T, 9)
        push!(cases, (AggOp, B))

        AggOp = SparseMatrixCSC(3, 9, collect(1:10),
                        [3,2,1,1,2,3,2,1,3], ones(T,9))
        B = T.(collect(1:9))
        push!(cases, (AggOp, B))
    end

    cases
end

# Test approximate spectral radius
function test_approximate_spectral_radius()

    cases = []
    seed!(0)

    push!(cases, [2. 0.
                  0. 1.])

    push!(cases, [-2. 0.
                   0  1])

    push!(cases, [100.   0.  0.
                    0. 101.  0.
                    0.   0. 99.])

    for i in 2:5
        push!(cases, rand(i,i))
    end

    for A in cases
        @static if VERSION < v"0.7-"
            E,V = eig(A)
        else
            E,V = (eigen(A)...,)
        end
        E = abs.(E)
        largest_eig = findall(E .== maximum(E))[1]
        expected_eig = E[largest_eig]

        @test isapprox(approximate_spectral_radius(A), expected_eig)
    end

    # Symmetric matrices
    for A in cases
        A = A + A'
        @static if VERSION < v"0.7-"
            E,V = eig(A)
        else
            E,V = (eigen(A)...,)
        end
        E = abs.(E)
        largest_eig = findall(E .== maximum(E))[1]
        expected_eig = E[largest_eig]

        @test isapprox(approximate_spectral_radius(A), expected_eig)
    end
end

# Test Gauss Seidel
import AlgebraicMultigrid: gs!
function test_gauss_seidel()
    N = 1
    A = spdiagm(0 => 2 * ones(N), -1 => -ones(N-1), 1 => -ones(N-1))
    x = eltype(A).(collect(0:N-1))
    b = zeros(N)
    s = GaussSeidel(ForwardSweep())
    s(A, x, b)
    @test sum(abs2, x - zeros(N)) < 1e-8

    N = 3
    A = spdiagm(0 => 2 * ones(N), -1 => -ones(N-1), 1 => -ones(N-1))
    x = eltype(A).(collect(0:N-1))
    b = zeros(N)
    s = GaussSeidel(ForwardSweep())
    s(A, x, b)
    @test sum(abs2, x - [1.0/2.0, 5.0/4.0, 5.0/8.0]) < 1e-8

    N = 1
    A = spdiagm(0 => 2 * ones(N), -1 => -ones(N-1), 1 => -ones(N-1))
    x = eltype(A).(collect(0:N-1))
    b = zeros(N)
    s = GaussSeidel(BackwardSweep())
    s(A, x, b)
    @test sum(abs2, x - zeros(N)) < 1e-8

    N = 3
    A = spdiagm(0 => 2 * ones(N), -1 => -ones(N-1), 1 => -ones(N-1))
    x = eltype(A).(collect(0:N-1))
    b = zeros(N)
    s = GaussSeidel(BackwardSweep())
    s(A, x, b)
    @test sum(abs2, x - [1.0/8.0, 1.0/4.0, 1.0/2.0]) < 1e-8

    N = 1
    A = spdiagm(0 => 2 * ones(N), -1 => -ones(N-1), 1 => -ones(N-1))
    x = eltype(A).(collect(0:N-1))
    b = eltype(A).([10.])
    s = GaussSeidel(ForwardSweep())
    s(A, x, b)
    @test sum(abs2, x - [5.]) < 1e-8

    N = 3
    A = spdiagm(0 => 2 * ones(N), -1 => -ones(N-1), 1 => -ones(N-1))
    x = eltype(A).(collect(0:N-1))
    b = eltype(A).([10., 20., 30.])
    s = GaussSeidel(ForwardSweep())
    s(A, x, b)
    @test sum(abs2, x - [11.0/2.0, 55.0/4, 175.0/8.0]) < 1e-8

    N = 100
    A = spdiagm(0 => 2 * ones(N), -1 => -ones(N-1), 1 => -ones(N-1))
    x = ones(eltype(A), N)
    b = zeros(eltype(A), N)
    s1 = GaussSeidel(ForwardSweep(), 200)
    s1(A, x, b)
    resid1 = norm(A*x,2)
    x = ones(eltype(A), N)
    s2 = GaussSeidel(BackwardSweep(), 200)
    s2(A, x, b)
    resid2 = norm(A*x,2)
    @test resid1 < 0.01 && resid2 < 0.01
    @test isapprox(resid1, resid2)

end

# Test jacobi smooth prolongator with local weighting
function test_jacobi_prolongator()
    A = poisson(100)
    T = poisson(100)
    x = JacobiProlongation(4/3)(A, T, 1, 1)
    ref = include("ref_R.jl")
    @test sum(abs2, x - ref) < 1e-6
end

# Issue #24
function nodes_not_agg()
    A = include("onetoall.jl")
    ml = smoothed_aggregation(A)
    @test size(ml.levels[2].A) == (11,11)
    @test size(ml.final_A) == (2,2)
end

# Issue 26
function test_symmetric_sweep()
    A = poisson(10)
    s = GaussSeidel(SymmetricSweep(), 4)
    x = ones(size(A,1))
    b = zeros(size(A,1))
    s(A, x, b)
    @test sum(abs2, x - [0.176765; 0.353529; 0.497517; 0.598914;
                            0.653311; 0.659104; 0.615597; 0.52275;
                            0.382787; 0.203251]) < 1e-6
end


module TestCategoricalAlgebra

using Test

@testset "FreeDiagrams" begin
  include("FreeDiagrams.jl")
end

@testset "Limits" begin
  include("Limits.jl")
end

@testset "Matrices" begin
  include("Matrices.jl")
end

@testset "FinSets" begin
  include("FinSets.jl")
end

@testset "FinRelations" begin
  include("FinRelations.jl")
end

@testset "Permutations" begin
  include("Permutations.jl")
end

@testset "PredicatedSets" begin
  include("PredicatedSets.jl")
end

@testset "CSets" begin
  include("CSetDataStructures.jl")
  include("CSets.jl")
end

@testset "StructuredCospans" begin
  include("StructuredCospans.jl")
end

@testset "Squares" begin
  include("Squares.jl")
end

end


module LocalHillResponses

using ForwardDiff

export local_sensitivity, respose_coefficient, local_hills

"""
    local_sensitivity(f, s)

Compute the local sensitivity (`df/ds`) of the function `f` for the singular input `s`.
"""
function local_sensitivity(f, s)
    return ForwardDiff.derivative(f, s)
end

"""
    respose_coefficient(f, s)

Compute the response coefficient (a.k.a. relative local sensitivity):

```math
R^{x}_{s} = \frac{dx}{ds} \frac{s}{x}
```

where ``R^{x}_{s}`` is the response coefficient of the input ``s`` and the output ``x = f(s)``.

# Warning

The result may be fluctuated due to the singularity point.
"""
function respose_coefficient(f, s, y = f(s))
    return local_sensitivity(f, s) * s / y
end

const relative_local_sensitivity = respose_coefficient

"""
Check if function `f(s)` is monotonous between `smin` and `smax`. A positive `direction` checks increasing function; A negative `direction` checks decreasing function. If no `direction` is given, checks either increasing or decreasing.
"""
function is_monotonous(f, smin, smax, direction; Δs = (smax - smin) * 1e-3)
    for s in smin:Δs:smax
        if ForwardDiff.derivative(f, s) * direction < 0
            return false
        end
    end
    return true
end

function is_monotonous(f, smin, smax; Δs = (smax - smin) * 1e-3)
    is_monotonous(f, smin, smax, +1; Δs = Δs) || is_monotonous(f, smin, smax, -1; Δs = Δs)
end

"""
Activated fraction for reponse `y` bounded by minimal response `ymin` and maximal response `ymax`.
"""
activated_fraction(y, ymin, ymax) = (y - ymin) / (ymax - ymin)

"""
Local Hill coefficient.
"""
function local_hills(f, s, ymin, ymax)
    responseCoef = respose_coefficient(f, s)
    actFrac = activated_fraction(y, ymin, ymax)
    nh = responseCoef / (one(actFrac) - actFrac)
    return nh
end

end # module


__precompile__(true)

module SparseDistributions

export BinarySparse, Sparse

using Distributions

"""
    BinarySparse(n, k)

Create k-sparse vector. Each value is 0 or 1.
"""
function BinarySparse(n::Int64, k::Int64)
    x = cat(1, zeros(n - k), ones(k))
    shuffle(x)
end

"""
    BinarySparse(n, ρ)

Create k-sparse vector. k = trunc(ρ*n). Each value is 0 or 1.
"""
function BinarySparse(n::Int64, ρ::Float64)
    k = trunc(ρ*n)
    x = cat(1, zeros(n - k), ones(k))
    shuffle(x)
end

"""
    Sparse(n, ρ, distribution)

Create sparse vector (has type Array{Float64}, not type SparseArray) from distribution.
"""
function Sparse(n::Int64, ρ::Float64, distribution)
    k = trunc(ρ*n)
    x = cat(1, zeros(n - k), rand(distribution, k))
    shuffle(x)
end

end # module


# Reconstruct a digit from an array
function nd(a)
  multiplicand,result = 1,0
  for i in reverse(1:length(a))
    result += multiplicand * a[i]
    multiplicand *= 10
  end
  result
end

# Check if one array is a permutation of another
function ispermutation(a, b)
  sort(a) == sort(b)
end