using AlgebraicPetri
using Catlab.CategoricalAlgebra
using Catalyst
using OrdinaryDiffEq
using DiffEqFlux #=, Flux=#
using Plots
#import Catalyst: ReactionSystem

println("Starting")

#=SIR = LabelledPetriNet([:S, :I, :R],
  :inf => ((:S, :I)=>(:I, :I)),
  :rec => (:I=>:R),
  :id => (:S => :S),
  :id => (:I => :I),
  :id => (:R => :R)
)=#

get_infected_states(g::AbstractLabelledPetriNet) =
   [i for (i,s) in enumerate(g[:sname]) if occursin("I", string(s))]

get_susceptible_states(g::AbstractLabelledPetriNet) =
   [i for (i,s) in enumerate(g[:sname]) if occursin("S", string(s))]

get_recovered_states(g::AbstractLabelledPetriNet) =
   [i for (i,s) in enumerate(g[:sname]) if occursin("R", string(s)) || occursin("V", string(s))]

counter(a) = [count(==(i),a) for i in unique(a)]
function MakeReactionSystem(pn::AbstractPetriNet)
    @parameters t k[1:(nt(pn) + ns(pn))]
    @variables S[1:ns(pn)](t)

    rxs = map(1:nt(pn)) do t
      inpts = pn[incident(pn, t, :it),:is]
      otpts = pn[incident(pn, t, :ot),:os]
      in_count = collect(counter(inpts))
      ot_count = collect(counter(otpts))
      Reaction(k[t], [S[i] for i in unique(inpts)],
                     [S[o] for o in unique(otpts)],
                     in_count, ot_count)
    end

    ReactionSystem(rxs, t, S, k; name=:model)
end

#=SIR_rxn = MakeReactionSystem(SIR)
#AlgebraicPetri.Graph(SIR)
#Catalyst.Graph(SIR_rxn)

p_real = [.1/1000, .01]
tspan = (0.0,250.0)
u0 = [999.0, 1.0, 0.0]=#
#=sample_times = range(tspan[1],stop=tspan[2],length=100)

op = ODEProblem(SIR_rxn, u0, tspan, p_real)
sol_real = solve(op, Tsit5(), tstops=sample_times)

sample_vals = [sol_real.u[findfirst(sol_real.t .>= ts)][var] * (1+(0.1rand()-0.05))
                for var in 1:3, ts in sample_times]

plt = plot(sol_real, lw=2, label=reshape(map(string, SIR[:, :sname]), 1, ns(SIR)))
plot!(sample_times, sample_vals', seriestype=:scatter, label="")
display(plt)=#

"""
Generate infection data from a petri net model given:
  - rate parameters p
  - initial concentrations u0
  - timespan
  - number of samples
Returns sample infectious data, problem, and solution
"""
function generate_data(model::AbstractLabelledPetriNet, p, u0, tspan, num_samples)
    sample_times = range(tspan[1], stop=tspan[2], length=num_samples)

    prob = ODEProblem(MakeReactionSystem(model), u0, tspan, p)
    sol = solve(prob, Tsit5(), tstops=sample_times)

    inf_sample_vals = [sol.u[findfirst(sol.t .>= ts)][var] * (1+(0.1rand()-0.05))
                for var in get_infected_states(model), ts in sample_times]

    total_inf_samples = map(sum, eachcol(inf_sample_vals))

    susc_sample_vals = [sol.u[findfirst(sol.t .>= ts)][var] * (1+(0.1rand()-0.05))
                for var in get_susceptible_states(model), ts in sample_times]

    total_susc_samples = map(sum, eachcol(susc_sample_vals))

    rec_sample_vals = [sol.u[findfirst(sol.t .>= ts)][var] * (1+(0.1rand()-0.05))
                for var in get_recovered_states(model), ts in sample_times]

    total_rec_samples = map(sum, eachcol(rec_sample_vals))

    return hcat(total_inf_samples, total_rec_samples, total_susc_samples), sample_times, prob, sol
end

function optimise_p(model::AbstractLabelledPetriNet, op, p_init, tend, sample_data, sample_times)
    # Loss on all populations
    #=loss = function (p)
        sol = solve(remake(op,tspan=(0.0,tend),p=p), Tsit5(), tstops=sample_times)
        vals = hcat(map(ts -> sol.u[findfirst(sol.t .>= ts)], sample_times[1:findlast(sample_times .<= tend)])...)
        loss = sum(abs2, vals .- sample_vals[:,1:size(vals)[2]])
        return loss, sol
    end=#

    # Loss on just infected population (requires closer initial values)
    loss = function (p)
        sol = solve(remake(op,tspan=(0.0,tend),p=p), Tsit5(), tstops=sample_times)
        vals = hcat(map(ts -> sol.u[findfirst(sol.t .>= ts)], sample_times[1:findlast(sample_times .<= tend)])...)
        inf_vals = map(sum, collect(zip([vals[i,:] for i in get_infected_states(model)]...)))

        #loss = sum(abs2, vals[2,:] .- sample_vals[:,1:size(vals)[2]][2,:])
        inf_samples = sample_data[:,1]
        rec_samples = sample_data[:,2]
        susc_samples = sample_data[:,3]

        inf_loss = sum(abs2, inf_vals .- inf_samples[1:size(inf_vals)[1]])

        susc_vals = map(sum, collect(zip([vals[i,:] for i in get_susceptible_states(model)]...)))
        susc_loss = sum(abs2, susc_vals .- susc_samples[1:size(susc_vals)[1]])

        rec_vals = map(sum, collect(zip([vals[i,:] for i in get_recovered_states(model)]...)))
        rec_loss = sum(abs2, susc_vals .- rec_samples[1:size(rec_vals)[1]])
        return susc_loss + inf_loss, sol
    end

    callback = function (p, l, pred)
        display(l)
        plt = plot(pred, lw=2, linestyle=:dash, color=[:orange :green :blue])
        plot!(sample_times, sample_vals', seriestype=:scatter, label="")
        display(plt)
        return false
    end

    return DiffEqFlux.sciml_train(loss, p_init, #=ADAM(0.05), cb=callback,=# maxiters=500, abstol=1e-4, reltol=1e-4,
            lower_bounds=repeat([1e-6], length(p_init)), upper_bounds=ones(length(p_init)))
end

function full_train(model, u0, tspan, training_data, sample_times, param_guess)
    rxn = MakeReactionSystem(model)
    #p_estimate = zeros(numreactionparams(rxn))
    #p_estimate = repeat([1e-6], numreactionparams(rxn)-ns(model))
    p_estimate = param_guess
    prob = ODEProblem(rxn, u0, tspan, p_estimate)
    loss = 0
    sol_estimate = Nothing
    # TODO: algorithmically determine this range
    for i in 50:50:250
        res_ode = optimise_p(model, prob, p_estimate, i, training_data, sample_times)
        p_estimate = res_ode.minimizer
        sol_estimate = solve(remake(prob,tspan=tspan,p=p_estimate), Tsit5())

        susc_vals = map(sum, collect(zip([sol_estimate[i,:] for i in get_susceptible_states(model)]...)))
        rec_vals = map(sum, collect(zip([sol_estimate[i,:] for i in get_recovered_states(model)]...)))
        inf_vals = map(sum, collect(zip([sol_estimate[i,:] for i in get_infected_states(model)]...)))

        #=plt = plot(sample_times, training_data, seriestype=:scatter, label="")
        plot!(susc_vals, lw=2, label="S")
        plot!(rec_vals, lw=2, label="R")
        plot!(inf_vals, lw=2, label="I")
        display(plt)=#
        #=plt = plot(sample_times, training_data, seriestype=:scatter, label="")
        plot!(sol_estimate, lw=2, label=reshape(map(string, model[:, :sname]), 1, ns(model)))
        display(plt)=#

        println("Estimated params: $p_estimate")
        loss = res_ode.minimum
        println("Loss: $loss")
    end
    return  sol_estimate, loss
end

function stupid()
    return "dumb"
end

function full_train(model, u0, tspan, training_data, sample_times)
    rxn = MakeReactionSystem(model)
    #p_estimate = zeros(numreactionparams(rxn))
    p_estimate = repeat([1e-5], numreactionparams(rxn)-ns(model))
    #p_estimate = param_guess
    prob = ODEProblem(rxn, u0, tspan, p_estimate)
    loss = 0
    sol_estimate = Nothing
    # TODO: algorithmically determine this range
    for i in 50:50:250
        res_ode = optimise_p(model, prob, p_estimate, i, training_data, sample_times)
        p_estimate = res_ode.minimizer
        sol_estimate = solve(remake(prob,tspan=tspan,p=p_estimate), Tsit5())

        susc_vals = map(sum, collect(zip([sol_estimate[i,:] for i in get_susceptible_states(model)]...)))
        rec_vals = map(sum, collect(zip([sol_estimate[i,:] for i in get_recovered_states(model)]...)))
        inf_vals = map(sum, collect(zip([sol_estimate[i,:] for i in get_infected_states(model)]...)))

        #=plt = plot(sample_times, training_data, seriestype=:scatter, label="")
        plot!(susc_vals, lw=2, label="S")
        plot!(rec_vals, lw=2, label="R")
        plot!(inf_vals, lw=2, label="I")
        display(plt)=#
        #=plt = plot(sample_times, training_data, seriestype=:scatter, label="")
        plot!(sol_estimate, lw=2, label=reshape(map(string, model[:, :sname]), 1, ns(model)))
        display(plt)=#

        println("Estimated params: $p_estimate")
        loss = res_ode.minimum
        println("Loss: $loss")
    end
    return  sol_estimate, loss
end

function run_param_est()
    println("Generating data...")

    data, sample_times, op, sol_real = generate_data(SIR, p_real, u0, tspan, 100)
    plt = plot(sol_real, lw=2, label=reshape(map(string, SIR[:, :sname]), 1, ns(SIR)))
    plot!(sample_times, data, seriestype=:scatter, label="")
    display(plt)

    println("Running param estimation...")
    p_estimate = [0.0,0.0]
    for i in 50.0:50.0:250.0
        res_ode = optimise_p(SIR, op, p_estimate, i, data, sample_times)
        p_estimate = res_ode.minimizer
        sol_estimate = solve(remake(op,tspan=(0.0,250.0),p=p_estimate), Tsit5())

        plt = plot(sample_times, sample_vals', seriestype=:scatter, label="")
        plot!(sol_estimate, lw=2, label=reshape(map(string, SIR[:, :sname]), 1, ns(SIR)))
        display(plt)

        println("Estimated params: $p_estimate")
        loss = res_ode.minimum
        println("Loss: $loss")
    end

    sol_estimate = solve(remake(op,tspan=(0.0,250.0),p=p_estimate), Tsit5())
    return sol_estimate, p_estimate
end

#sol_estimate, p_estimate = run_param_est()

#=plot(sample_times, sample_vals', seriestype=:scatter, label="")
plot!(sol_estimate, lw=2)=#
#println("Final param estimate: $p_estimate")


abstract type BBsearch end

abstract type BrFS <: BBsearch end
abstract type DFS <: BBsearch end
abstract type BFS <: BBsearch end

Priority(::BrFS, f::Function, x::IntervalBox) = diam(x)
Priority(::DFS, f::Function, x::IntervalBox) = minimum(diam.(x))
Priority(::BFS, f::Function, x::IntervalBox) = f(x).lo

function optimize(f::Function, X::IntervalBox; eps::Float64, bisect, ) where {N,T}

  LB = f(X).lo

  init_list = PriorityQueue(X => LB)

 final_list = typeof(X)[ ]

  m = Inf

  iterations = 0

 while length(init_list) > 0

      x = dequeue!(init_list)

     gradient = ForwardDiff.gradient(f, x.v)

      if any(0 .∉ gradient)

          iterations = iterations + 1

          continue

      end

      x_random = random(x, 10)

      x_min, arg_min = findmin([f(a) for a in x_random])

      x_midbox = x_random[arg_min]

      mini = localoptim(f, x_midbox)

      m = min(m, mini)

      if f(x).lo > m

          iterations = iterations + 1

          continue

      end

      if diam(x) < eps

          push!(final_list, x)

          iterations = iterations + 1

          continue

      end

      x1, x2 = bisect(x)

      enqueue!(init_list, x1, f(x1).lo)

      enqueue!(init_list, x2, f(x2).lo)

      iterations = iterations + 1

  end

  final_list = clean_list(f, final_list, m)

  return m, length(final_list), iterations

end


"""
    solve(A::AbstractMatrix{T},
          b::AbstractVector{T},
          solver::AbstractIterativeSolver,
          [precondition]::AbstractPrecondition=_default_precondition(A, solver),
          [X]::AbstractVector{T}=enclose(A, b)) where {T<:Interval}

Solves the square interval system ``Ax=b`` using the given algorithm, preconditioner
and initial enclosure

### Input

- `A` -- square interval matrix
- `b` -- interval vector
- `solver` -- algorithm used to solve the linear system
- `precondition` -- preconditioner used. If not given, it is automatically computed based on
                    the matrix `A` and the solver.
- `X` -- initial enclosure.
         if not given, it is automatically computed using [`enclose`](@ref)

### Examples

```jldoctest
julia> A = [2..4 -1..1;-1..1 2..4]
2×2 Matrix{Interval{Float64}}:
  [2, 4]  [-1, 1]
 [-1, 1]   [2, 4]

julia> b = [-2..2, -1..1]
2-element Vector{Interval{Float64}}:
 [-2, 2]
 [-1, 1]

julia> solve(A, b, GaussSeidel(), NoPrecondition(), [-10..10, -10..10])
2-element Vector{Interval{Float64}}:
 [-1.66668, 1.66668]
 [-1.33334, 1.33334]

julia> solve(A, b, GaussSeidel())
2-element Vector{Interval{Float64}}:
 [-1.66667, 1.66667]
 [-1.33334, 1.33334]
```
"""
function solve(A::AbstractMatrix{T},
               b::AbstractVector{T},
               solver::AbstractIterativeSolver,
               precondition::AbstractPrecondition=_default_precondition(A, solver),
               X::AbstractVector{T}=enclose(A, b)) where {T<:Interval}

    checksquare(A) == length(b) == length(X) || throw(DimensionMismatch())
    A, b = precondition(A, b)
    return solver(A, b, X)

end

"""
    solve(A::AbstractMatrix{T},
          b::AbstractVector{T},
          solver::AbstractDirectSolver,
          [precondition]::AbstractPrecondition=_default_precondition(A, solver)) where
          {T<:Interval}

Solves the square interval system ``Ax=b`` using the given algorithm, preconditioner
and initial enclosure

### Input

- `A` -- square interval matrix
- `b` -- interval vector
- `solver` -- algorithm used to solve the linear system
- `precondition` -- preconditioner used. If not given, it is automatically computed based on
                    the matrix `A` and the solver.

### Examples

```jldoctest
julia> A = [2..4 -1..1;-1..1 2..4]
2×2 Matrix{Interval{Float64}}:
  [2, 4]  [-1, 1]
 [-1, 1]   [2, 4]

julia> b = [-2..2, -1..1]
2-element Vector{Interval{Float64}}:
 [-2, 2]
 [-1, 1]

julia> solve(A, b, HansenBliekRohn(), InverseMidpoint())
2-element Vector{Interval{Float64}}:
 [-1.66667, 1.66667]
 [-1.33334, 1.33334]

julia> solve(A, b, HansenBliekRohn())
2-element Vector{Interval{Float64}}:
 [-1.66667, 1.66667]
 [-1.33334, 1.33334]
```
"""
function solve(A::AbstractMatrix{T},
               b::AbstractVector{T},
               solver::AbstractDirectSolver,
               precondition::AbstractPrecondition=_default_precondition(A, solver)) where
               {T<:Interval}

    checksquare(A) == length(b) || throw(DimensionMismatch())

    A, b = precondition(A, b)
    return solver(A, b)

end

# fallback
"""
    solve(A::AbstractMatrix{T},
          b::AbstractVector{T},
          [solver]::AbstractLinearSolver,
          [precondition]::AbstractPrecondition=_default_precondition(A, solver)) where
          {T<:Interval}

Solves the square interval system ``Ax=b`` using the given algorithm, preconditioner
and initial enclosure

### Input

- `A` -- square interval matrix
- `b` -- interval vector
- `solver` -- algorithm used to solve the linear system. If not given,
              [`GaussianElimination`](@ref) is used.
- `precondition` -- preconditioner used. If not given, it is automatically computed based on
                    the matrix `A` and the solver.

### Examples

```jldoctest
julia> A = [2..4 -1..1;-1..1 2..4]
2×2 Matrix{Interval{Float64}}:
  [2, 4]  [-1, 1]
 [-1, 1]   [2, 4]

julia> b = [-2..2, -1..1]
2-element Vector{Interval{Float64}}:
 [-2, 2]
 [-1, 1]

julia> solve(A, b)
2-element Vector{Interval{Float64}}:
 [-1.66667, 1.66667]
 [-1.33334, 1.33334]
```
"""
function solve(A::AbstractMatrix{T},
               b::AbstractVector{T},
               solver::AbstractLinearSolver=_default_solver(),
               precondition::AbstractPrecondition=_default_precondition(A, solver)) where {T<:Interval}

    checksquare(A) == length(b) || throw(DimensionMismatch())

    A, b = precondition(A, b)
    return solver(A, b)

end

## Default settings
_default_solver() = GaussianElimination()

function _default_precondition(A, ::AbstractDirectSolver)
    if is_strictly_diagonally_dominant(A) || is_M_matrix(A)
        return NoPrecondition()
    else
        return InverseMidpoint()
    end
end

# fallback
_default_precondition(_, ::AbstractLinearSolver) = InverseMidpoint()


using Compat.Test
import Roots.fzero

## Test `fzero` interface to `find_zero`
## test `fzeros` interface for functions

orders = [0,1,2,5,8,16]

### derivative free
fn, xstar, x0, br = x -> sin(x) - x/2, 1.89549426703398094714, 2.0, [pi/2, pi]
@test fzero(fn, x0)  ≈ xstar
for o in orders
    @test fzero(fn, x0, order=o)  ≈ xstar
end
### bisection
@test fzero(fn, br)  ≈ xstar
@test fzero(x->5-x,5,10)==5

### test tolerances
fn, xstar, x0, br = x -> x^5 - x - 1, 1.1673039782614187, 1.0, [1.0, 2.0]
@test fzero(fn, x0, order=1)  ≈ xstar

@test_throws Roots.ConvergenceFailed fzero(fn, x0, order=1, maxevals=2)
#@test abs(fzero(fn, x0, order=1, ftol=1e-2) - xstar) > 1e-5
#@test abs(fzero(fn, x0, order=1, xtol=1e-2) - xstar) > 1e-10

@test abs(find_zero(fn, br, Roots.A42(), xatol=1e-1) - xstar) > 1e-5

## various tests
## issue #29, basically f(a) or f(b) so big we get NaN for guess from bracket.
f =  x -> x + exp(x)
@test fzero(f, [-1e6, 1e6])  ≈ -0.5671432904097838

f =  x -> 1/x - 1
@test fzero(f, [0, 2])  ≈ 1.0

## test infinite range
@test fzero(x -> x, [-Inf, Inf])  ≈ 0.0

##################################################
## fzeros function
rts = 1:5
@test all((abs.(fzeros(x -> prod([x-r for r in rts]),0,10)) .- collect(1:5)) .<= 1e-15)

fn = x -> cos(10*pi*x)
@test length(fzeros(fn, 0, 1)) == 10

### issue with fzeros and roots near 'b'
@test 0 <  maximum(fzeros(x -> sin(x) - 1/1000*x, 0, pi)) < pi


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,jl:hydrogen
#     text_representation:
#       extension: .jl
#       format_name: hydrogen
#       format_version: '1.3'
#       jupytext_version: 1.10.3
#   kernelspec:
#     display_name: Julia 1.6.3
#     language: julia
#     name: julia-1.6
# ---

# %%
using SymPy
Poly = sympy.Poly
@vars a b p q x α β

# %%
λ = symbols("λ[0:9]")

# %%
f = x^3 + a*x + b
Poly(f, x)

# %%
g = x^3 + p*x + q
Poly(g, x)

# %%
H = sum(λ[begin+k]*x^k for k in 0:8) + x^9
Poly(H, x)

# %%
h = H(x => α + β).expand()
for k in 6:-1:0
    h = h(α^(k+3) => α^(k+3) - α^k * f(x=>α), β^(k+3) => β^(k+3) - β^k * g(x=>β)).expand()
end
P = Poly(h, α, β)

# %%
C = P.coeffs()

# %%
length(C)

# %%
sol = @time solve(C, λ)

# %% [markdown]
# Assume that
#
# * the minimal polynomial of $\alpha$ is equal to $x^3 + ax + b$,
# * the minimal polynomial of $\beta$ is equal to $x^3 + px + q$, and
# * the minimal polynomial of $\alpha + \beta$ is of degree $9$.
#
# Then the minimal polynomial of $\alpha + \beta$ is equal to
#
# $x^{9} + \left(3 a + 3 p\right) x^{7} + \left(3 b + 3 q\right) x^{6} + \left(3 a^{2} + 3 a p + 3 p^{2}\right) x^{5} + \left(6 a b - 3 a q - 3 b p + 6 p q\right) x^{4} \\+ \left(a^{3} + a^{2} p + a p^{2} + 3 b^{2} - 21 b q + p^{3} + 3 q^{2}\right) x^{3} + \left(3 a^{2} b + 3 a^{2} q - 6 a b p - 6 a p q + 3 b p^{2} + 3 p^{2} q\right) x^{2} \\+ \left(a^{3} p - 2 a^{2} p^{2} + 3 a b^{2} - 3 a b q + a p^{3} - 6 a q^{2} - 6 b^{2} p - 3 b p q + 3 p q^{2}\right) x \\+ a^{3} q + a^{2} b p - 2 a^{2} p q - 2 a b p^{2} + a p^{2} q + b^{3} + 3 b^{2} q + b p^{3} + 3 b q^{2} + q^{3}$.

# %%
h = sum(sol[λ[begin+k]]*x^k for k in 0:8) + x^9
Poly(h, x)

# %% [markdown]
# Assume that
#
# * the minimal polynomial of $\alpha$ is equal to $x^3 + ax + b$,
# * the minimal polynomial of $\beta$ is equal to $x^3 - p $, and
# * the minimal polynomial of $\alpha + \beta$ is of degree $9$.
#
# Then the minimal polynomial of $\alpha + \beta$ is equal to
#
# $x^{9} + 3 a x^{7} + \left(3 b - 3 p\right) x^{6} + 3 a^{2} x^{5} + \left(6 a b + 3 a p\right) x^{4} + \left(a^{3} + 3 b^{2} + 21 b p + 3 p^{2}\right) x^{3} \\+ \left(3 a^{2} b - 3 a^{2} p\right) x^{2} + \left(3 a b^{2} + 3 a b p - 6 a p^{2}\right) x -  a^{3} p + b^{3} - 3 b^{2} p + 3 b p^{2} - p^{3}$.

# %%
h1 = h(p=>0)(q=>-p)
Poly(h1, x)

# %% [markdown]
# Assume that
#
# * the minimal polynomial of $\alpha$ is equal to $x^3 - a$,
# * the minimal polynomial of $\beta$ is equal to $x^3 - b$, and
# * the minimal polynomial of $\alpha + \beta$ is of degree $9$.
#
# Then the minimal polynomial of $\alpha + \beta$ is equal to
#
# $x^{9} + \left(- 3 a - 3 b\right) x^{6} + \left(3 a^{2} - 21 a b + 3 b^{2}\right) x^{3} -  a^{3} - 3 a^{2} b - 3 a b^{2} - b^{3}$.

# %%
h2 = h(a=>0, p=>0)(b=>-a)(q=>-b)
Poly(h2, x)

# %% [markdown]
# The minimal polynomial of $\sqrt[3]{2} + \sqrt[3]{3}$ is equal to $x^{9} - 15 x^{6} - 87 x^{3} - 125$.

# %%
h3 = h2(a=>2, b=>3)
Poly(h3, x)

# %%

0023/minimal polynomials of sums of two algebraic elements of degree 3.jl


using Pkg # hideall
Pkg.activate("_literate/A-learning-networks/Project.toml")
Pkg.update();

# ## Preliminary steps
#
# Let's generate a `DataFrame` with some dummy regression data, let's also load the good old ridge regressor.

using MLJ, StableRNGs
import DataFrames
MLJ.color_off() # hide
Ridge = @load RidgeRegressor pkg=MultivariateStats

rng = StableRNG(551234) # for reproducibility

x1 = rand(rng, 300)
x2 = rand(rng, 300)
x3 = rand(rng, 300)
y = exp.(x1 - x2 -2x3 + 0.1*rand(rng, 300))

X = DataFrames.DataFrame(x1=x1, x2=x2, x3=x3)
first(X, 3) |> pretty

# Let's also prepare the train and test split which will be useful later on.

test, train = partition(eachindex(y), 0.8);

# ## Defining a learning network
#
# In MLJ, a *learning network* is a directed acyclic graph (DAG) whose *nodes* apply trained or untrained operations such as a `predict` or `transform` (trained) or `+`, `vcat` etc. (untrained).
# Learning networks can be seen as pipelines on steroids.
#
# Let's consider the following simple DAG:
#
# ![Operation DAG](/assets/diagrams/composite1.svg)
#
# It corresponds to a fairly standard regression workflow: the data is standardized, the target is transformed using a Box-Cox transformation, a ridge regression is applied and the result is converted back by inverting the transform.
#
# **Note**: actually  this DAG is simple enough that it could also have been done with a pipeline.
#
# ### Sources and nodes
#
# In MLJ a learning network starts at **source** nodes and flows through nodes (`X` and `y`) defining operations/transformations (`W`, `z`, `ẑ`, `ŷ`).
# To define the source nodes, use the `source` function, you should specify whether it's a target:

Xs = source(X)
ys = source(y)

# To define an "trained-operation" node, you must simply create a machine wrapping a model and another node (the data) and indicate which operation should be performed (e.g. `transform`):

stand = machine(Standardizer(), Xs)
W = transform(stand, Xs)

# You can `fit!` a trained-operation node at any point, MLJ will fit whatever it needs that is upstream of that node.
# In this case, there is just a source node upstream of `W` so fitting `W` will just fit the standardizer:

fit!(W, rows=train);

# If you want to get the transformed data, you can then call the node speciying on which part of the data the operation should be performed:

W()             # transforms all data
W(rows=test, )  # transforms only test data
W(X[3:4, :])    # transforms specific data

# Let's now define the other nodes:

box_model = UnivariateBoxCoxTransformer()
box = machine(box_model, ys)
z = transform(box, ys)

ridge_model = Ridge(lambda=0.1)
ridge = machine(ridge_model, W, z)
ẑ = predict(ridge, W)

ŷ = inverse_transform(box, ẑ)

# Note that we have not yet done any training, but if we now call `fit!` on `ŷ`, it will fit all nodes upstream of `ŷ` that need to be re-trained:

fit!(ŷ, rows=train);

# Now that `ŷ` has been fitted, you can apply the full graph on test data (or any compatible data). For instance, let's get the `rms` between the ground truth and the predicted values:

rms(y[test], ŷ(rows=test))

# ### Modifying hyperparameters
#
# Hyperparameters can be accessed using the dot syntax as usual.
# Let's modify the regularisation parameter of the ridge regression:

ridge_model.lambda = 5.0;

# Since the node `ẑ` corresponds to a machine that wraps `ridge_model`, that node has effectively changed and will be retrained:

fit!(ŷ, rows=train)
rms(y[test], ŷ(rows=test))


"""
    struct GeneralSpace{𝕜} <: ElementarySpace{𝕜}

A finite-dimensional space over an arbitrary field `𝕜` without additional structure. It is
thus characterized by its dimension, and whether or not it is the dual and/or conjugate
space. For a real field `𝕜`, the space and its conjugate are the same.
"""
struct GeneralSpace{𝕜} <: ElementarySpace{𝕜}
    d::Int
    dual::Bool
    conj::Bool
    function GeneralSpace{𝕜}(d::Int, dual::Bool, conj::Bool) where {𝕜}
        d >= 0 ||
            throw(ArgumentError("Dimension of a vector space should be bigger than zero"))
        if 𝕜 isa Field
            new{𝕜}(Int(d), dual, (𝕜 ⊆ ℝ) ? false : conj)
        else
            throw(ArgumentError("Unrecognised scalar field: $𝕜"))
        end
    end
end
GeneralSpace{𝕜}(d::Int; dual::Bool = false, conj::Bool = false) where {𝕜} =
    GeneralSpace{𝕜}(d, dual, conj)

dim(V::GeneralSpace) = V.d
isdual(V::GeneralSpace) = V.dual
isconj(V::GeneralSpace) = V.conj

Base.axes(V::GeneralSpace) = Base.OneTo(dim(V))

dual(V::GeneralSpace{𝕜}) where {𝕜} =
    GeneralSpace{𝕜}(dim(V), !isdual(V), isconj(V))
Base.conj(V::GeneralSpace{𝕜}) where {𝕜} =
    GeneralSpace{𝕜}(dim(V), isdual(V), !isconj(V))

function Base.show(io::IO, V::GeneralSpace{𝕜}) where {𝕜}
    if isconj(V)
        print(io, "conj(")
    end
    print(io, "GeneralSpace{", 𝕜, "}(", dim(V), ")")
    if isdual(V)
        print(io, "'")
    end
    if isconj(V)
        print(io, ")")
    end
end
Base.show(io::IO, ::Type{GeneralSpace}) = print(io, "GeneralSpace")


module constrsparsereg_test

using Base.Test, ConstrainedLasso, ECOS

info("Test lsq_constrsparsereg: sum-to-zero constraint")

# set up
srand(123)
n, p = 100, 20
# truth with sum constraint sum(β) = 0
β = zeros(p)
β[1:round(Int, p / 4)] = 0
β[(round(Int, p / 4) + 1):round(Int, p / 2)] = 1
β[(round(Int, p / 2) + 1):round(Int, 3p / 4)] = 0
β[(round(Int, 3p / 4) + 1):p] = -1
# generate data
X = randn(n, p)
y = X * β + randn(n)
Aeq = ones(1, p)
beq = [0.0]
penwt  = ones(p)
solver = ECOSSolver(maxit=10e8, verbose=0)

info("Optimize at a single tuning parameter value")
ρ = 10.0
β̂, = lsq_constrsparsereg(X, y, ρ; Aeq = Aeq, beq = beq,
    penwt = penwt, solver = solver)
@test sum(β̂)≈0.0 atol=1e-6

info("Optimize at multiple tuning parameter values")
ρlist = [0.0:10.0; Inf]
β̂, = lsq_constrsparsereg(X, y, ρlist; Aeq = Aeq, beq = beq,
    penwt = penwt, solver = solver)
@testset "zero-sum for multiple param values" begin
for si in sum(β̂, 1)
    @test si≈0.0 atol=1e-6
end
end

info("Optimize at multiple tuning parameter values (warm start)")
β̂ws, = lsq_constrsparsereg(X, y, ρlist; Aeq = Aeq, beq = beq,
    penwt = penwt, solver = solver, warmstart = true)
#@show sum(β̂ws, 1)
@testset "zero-sum for multiple param values" begin
for si in sum(β̂ws, 1)
    @test si≈0.0 atol=1e-6
end
end

end # end of module


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ ec1fe200-8d73-11eb-0ba3-e593aa79dab2
begin
	using PlutoUI
	PlutoUI.TableOfContents(aside=true)
end

# ╔═╡ 973abf88-9125-41c7-8372-61efb2976fc2
using SymPy

# ╔═╡ da340240-f845-4591-94c3-9ffce6a33d8a
begin
	using Random
	Random.seed!(123)
	A₄=rand(-4:4,4,4)
end

# ╔═╡ 60d351a5-0ef2-47cd-a5a5-5ecf18c4e898
begin
	using SpecialMatrices
	using Polynomials
end

# ╔═╡ bb8416e3-b682-4733-af6d-36fd48f0a7b8
using Plots

# ╔═╡ 10910df9-240f-4e6b-8076-1a9be4d66ba1
md"""
# Eigenvalue Decomposition - Definitions and Facts

__Prerequisites__

The reader should be familiar with basic linear algebra concepts.

__Competences__

The reader should be able to understand and check the facts about eigenvalue decomposition.

__Selected references__

There are many excellent books on the subject. Here we list a few:

* J.W. Demmel, _Applied Numerical Linear Algebra_, SIAM, Philadelphia, 1997.
* G. H. Golub and C. F. Van Loan, _Matrix Computations_, 4th ed., The John Hopkins University Press, Baltimore, MD, 2013.
* N. Higham, _Accuracy and Stability of Numerical Algorithms_, SIAM, Philadelphia, 2nd ed., 2002.
* L. Hogben, ed., _Handbook of Linear Algebra_, CRC Press, Boca Raton, 2014.
* B. N. Parlett, _The Symmetric Eigenvalue Problem_, Prentice-Hall, Englewood Cliffs, NJ, 1980, also SIAM, Philadelphia, 1998.
* G. W. Stewart, _Matrix Algorithms, Vol. II: Eigensystems_, SIAM, Philadelphia, 2001.
* L. N. Trefethen and D. Bau, III, _Numerical Linear Algebra_, SIAM, Philadelphia, 1997.
* J. H. Wilkinson, _The Algebraic Eigenvalue Problem_, Clarendon Press, Oxford, U.K.,  1965.

"""

# ╔═╡ b87354b9-ee7b-4ce0-bf26-b4e71c6be6a9
md"""
# General matrices

For more details and the proofs of the Facts below, see
[L. M. DeAlba, Determinants and Eigenvalues, pp. 4.1-1.15](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and the references therein.

## Definitions

We state the basic definitions:

Let $\mathbb{F}=\mathbb{R}$ or $F=\mathbb{C}$ and let $A\in \mathbb{F}^{n\times n}$ with elements $a_{ij}\in \mathbb{F}$.

An element $\lambda \in \mathbb{F}$ is an __eigenvalue__ of $A$ if
$\exists x\in \mathbb{F}^n$, $x\neq 0$ such that

$$Ax=\lambda x,$$

and $x$ is an __eigenvector__ of $\lambda$.

__Characteristic polynomial__ of $A$ is $p_A(x)=\det(A-xI)$.

__Algebraic multiplicty__, $\alpha(\lambda)$, is the multiplicity of $\lambda$ as a root of $p_A(x)$.

__Spectrum of $A$__, $\sigma(A)$, is the multiset of all eigenvalues of $A$, with each eigenvalue appearing $\alpha(A)$ times.

__Spectral radius__ of $A$ is

$$\rho(A)=\max\{|\lambda|, \lambda \in \sigma(A)\}.$$

__Eigenspace__ of $\lambda$ is

$$
E_{\lambda}(A)=\ker(A-\lambda I).$$

__Geometric multiplicity__ of $\lambda$ is

$$\gamma(\lambda)=\dim(E_{\lambda}(A)).$$

 $\lambda$ is __simple__ if $\alpha(\lambda)=1$.

 $\lambda$ is __semisimple__ if $\alpha(\lambda)=\gamma(\lambda)$.

 $A$ is __nonderogatory__ if $\gamma(\lambda)=1$ for all $\lambda$.

 $A$ is __nondefective__ if every $\lambda$ is semisimple.

 $A$ is __diagonalizable__ if there exists nonsingular $B$ matrix and diagonal matrix $D$ such that $A=BDB^{-1}$.

__Trace__ of $A$ is

$$\mathop{\mathrm{tr}}(A)=\sum_i a_{ii}.$$

 $Q\in\mathbb{C}^{n\times n}$ is __unitary__ if $Q^*Q=QQ^*=I$,
where $Q^*=(\bar Q)^T$.

__Schur decomposition__ of $A$ is $A=QTQ^*$,
where $Q$ is unitary and $T$ is upper triangular.

 $A$ and $B$ are __similar__ if $B=QAQ^{-1}$ for some nonsingular matrix $Q$.

 $A$ is __normal__ if $AA^*=A^*A$.
"""

# ╔═╡ 31a71b9e-1dba-4a69-b87e-9202292eb3ed
md"""
## Facts

There are many facts related to the eigenvalue problem for general matrices. We state some basic ones:

1.  $\lambda\in\sigma(A) \Leftrightarrow p_A(\lambda)=0$.

1. __Cayley-Hamilton Theorem.__ $p_A(A)=0$.

2. A simple eigenvalue is semisimple.

3.  $\mathop{\mathrm{tr}}(A)=\sum_{i=1}^n \lambda_i$.

4.  $\det(A)=\prod_{i=1}^n \lambda_i$.

5.  $A$ is singular $\Leftrightarrow$ $\det(A)=0$ $\Leftrightarrow$ $0\in\sigma(A)$.

7. If $A$ is triangular, $\sigma(A)=\{a_{11},a_{22},\ldots,a_{nn}\}$.

6. For $A\in\mathbb{C}^{n\times n}$, $\lambda\in\sigma(A)$ $\Leftrightarrow$ $\bar\lambda\in\sigma(A^*)$.

8. __Corollary of the Fundamental theorem of algebra.__ For $A\in\mathbb{R}^{n\times n}$, $\lambda\in\sigma(A)$ $\Leftrightarrow$ $\bar\lambda\in\sigma(A)$.

9. If $(\lambda,x)$ is an eigenpair of a nonsingular $A$, then $(1/\lambda,x)$ is an eigenpair of $A^{-1}$.

10. Eigenvectors corresponding to distinct eigenvalues are linearly independent.

11.  $A$ is diagonalizable $\Leftrightarrow$ $A$ is nondefective $\Leftrightarrow$ $A$ has $n$ linearly independent eigenvectors.

12. Every $A$ has Schur decomposition. Moreover, $T_{ii}=\lambda_i$.  _(For proof see GVL p.351 (375).)_

13. If $A$ is normal, matrix $T$ from its Schur decomposition is normal. Consequently:

    *  $T$ is diagonal, and has eigenvalues of $A$ on diagonal,
    * matrix $Q$ of the Schur decomposition is the unitary matrix of eigenvectors,
    * all eigenvalues of $A$ are semisimple and $A$ is nondefective.

14. Real matrix has a __real Schur decomposition__, $Q^TAQ=T$, where $Q$ is real orthogonal and $T$ is upper block-triangular with $1\times 1$ and $2 \times 2$ blocks on the diagonal. The $1\times 1$ blocks correspond to real eigenvalues, and the $2\times 2$ blocks correspond to pairs of complex conjugate eigenvalues.

15. If $A$ and $B$ are similar, $\sigma(A)=\sigma(B)$. Consequently, $\mathop{\mathrm{tr}}(A)=\mathop{\mathrm{tr}}(B)$ and $\det(A)=\det(B)$.

16. Eigenvalues and eigenvectors are continous and differentiable: if $\lambda$ is a simple eigenvalue of $A$ and $A(\varepsilon)=A+\varepsilon E$ for some $E\in F^{n\times n}$, for small $\varepsilon$ there exist differentiable functions $\lambda(\varepsilon)$ and $x(\varepsilon)$ such that $A(\varepsilon) x(\varepsilon) = \lambda(\varepsilon) x(\varepsilon)$.

17. Classical motivation for the eigenvalue problem is the following: consider the system of linear differential equations with constant coefficients, $\dot y(t)=Ay(t)$. If the solution is $y=e^{\lambda t} x$ for some constant vector $x$, then

$$\lambda e^{\lambda t} x=Ae^{\lambda t} x \quad \textrm{or} \quad Ax=\lambda x.$$

"""

# ╔═╡ 2f5aee03-3c2e-4920-9d10-c509b06be627
md"""
## Examples

We shall illustrate above Definitions and Facts on several small examples, using symbolic computation.
"""

# ╔═╡ 37d24090-8d78-11eb-059d-737014bb6588
md"
### Defective eigenvalue
"

# ╔═╡ 184bc4ec-c566-4f21-99b3-b6e8cd328b47
A=[-3 7 -1; 6 8 -2; 72 -28 19]

# ╔═╡ 5d0aac9e-4f48-4f09-956f-5033a1fb8f11
x=symbols("x")

# ╔═╡ 73ece9ec-97dd-46bc-8e17-10ce7bfc18bc
begin
	using LinearAlgebra
	eye(n)=Matrix{Int}(I,n,n)
	A-x*eye(3)
end

# ╔═╡ ce83f04c-e915-4599-beeb-d4a0a681bc17
# Characteristic polynomial p_A(λ)
p(x)=det(A-x*eye(3))

# ╔═╡ ff5044c0-8d76-11eb-3951-6bb4b5caa7d0
p(x)

# ╔═╡ 55171a69-3557-4ba5-8a4a-813f38c03dd3
# Characteristic polynomial in nicer form
factor(p(x))

# ╔═╡ 6df42190-a29e-4c68-8684-07d8d0c9d200
λ=solve(p(x),x)

# ╔═╡ b94f7fcd-c6f4-4f4f-924a-20f00e82d94e
md"""
The eigenvalues are $\lambda_1=-6$ and $\lambda_2=15$ with algebraic multiplicities
$\alpha(\lambda_1)=1$ and $\alpha(\lambda_2)=2$.
"""

# ╔═╡ 1b90d50a-bf97-4b1a-9580-d729b03c914e
nullspace(float.(A-λ[1]*I))

# ╔═╡ 15b60cd5-eb30-43bb-bbcf-dd533e29c35f
nullspace(float.(A-λ[2]*I))

# ╔═╡ 3619fca4-1374-4adc-b29c-8058b2e147e0
md"""
The geometric multiplicities are $\gamma(\lambda_1)=1$ and $\gamma(\lambda_2)=1$. Thus, $\lambda_2$ is not semisimple, therefore $A$ is defective and not diagonalizable.
"""

# ╔═╡ d46018c0-a5f8-4dd0-8f3d-73322be792e9
# Trace and determinant
tr(A), λ[1]+λ[2]+λ[2]

# ╔═╡ d387b26f-bec1-47a9-b1c6-f20c43f0bde8
det(A), λ[1]*λ[2]*λ[2]

# ╔═╡ 703da5c4-cd09-4832-b64e-5fda75099db5
# Schur decomposition
S=schur(A)

# ╔═╡ e26f113c-dd98-4e97-b495-581de1b431af
fieldnames(typeof(S))

# ╔═╡ 62779bfc-ea5d-4eec-94ad-2641e11fc44d
S.T

# ╔═╡ bdf079a6-beee-44f5-8200-613e54b3a8dc
S.Z

# ╔═╡ e1f5027c-1080-4c95-81cc-a75dde641b88
# Orthogonality
S.Z'*S.Z

# ╔═╡ c71a7760-b4c6-404e-b5ef-f4879cb787cd
begin
	# Similar matrices
	M=rand(-5:5,3,3)
	eigvals(M*A*inv(M)), tr(M*A*inv(M)), det(M*A*inv(M))
end

# ╔═╡ 343c8f45-6226-4361-a149-5313a47c355c
md"
### Real Schur decomposition
"

# ╔═╡ 41a05c50-f6df-4b93-bd8a-50412829105e
begin
	# Fact 14
	Random.seed!(326)
	A₀=rand(-9:9,6,6)
end

# ╔═╡ cd8fb666-d2f2-423c-ac39-405c842e97ba
schur(A₀)

# ╔═╡ 7aa7ac89-837c-4ee8-b9f1-8ad0256f25e1
md"""
### Diagonalizable matrix

This matrix is nondefective and diagonalizable.
"""

# ╔═╡ c3724c4f-142b-4863-8c04-4a0c15411f40
A₁=[57 -21 21; -14 22 -7; -140 70 -55]

# ╔═╡ 792f632e-fdc2-4016-baaa-b366704f02fd
p₁(x)=factor(det(A₁-x*eye(3)))

# ╔═╡ a8664a40-8d78-11eb-325e-41c6497be282
factor(p₁(x))

# ╔═╡ 1d42245a-2ed6-4cc7-8cbd-2b6aa4e64861
λ₁=solve(p₁(x),x)

# ╔═╡ f7aae752-70c4-4a10-b96e-206c13a6ced1
nullspace(float.(A₁-λ₁[2]*I))

# ╔═╡ 60525ba5-2c76-406e-8eb3-63ced2e00bdd
# F=schur(A)
schur(A₁).T

# ╔═╡ 189544ef-122c-4e30-bfd6-cfc126fc0320
md"""
### Symbolic computation for $n=4$

Let us try some random examples of dimension $n=4$ (the largest $n$ for which we can compute eigevalues symbolically).
"""

# ╔═╡ e72bf173-7e55-4476-91ff-789847dc53cf
p₄(x)=factor(det(A₄-x*eye(4)))

# ╔═╡ da4a5d70-92d8-11eb-0635-11c530ff7d21
p₄(x)

# ╔═╡ ec7fc5c0-92d8-11eb-16df-af1133ccb42a
f(x)=x^4-5*x^3+2*x^2-30*x-30*x^0

# ╔═╡ 29de5fd0-92d9-11eb-09c0-0356001caaef
# Cayley-Hamilton teorem
f(A₄)

# ╔═╡ d43480a3-9247-48a4-8a3b-e006ab58539e
λ₄=solve(p₄(x),x)

# ╔═╡ 5995cd31-a75d-4718-8808-4ada01920344
λ₄[1]

# ╔═╡ 57e2742e-8d79-11eb-0a33-f7250ac02840
λ₄[2]

# ╔═╡ 5e069120-8d79-11eb-3c00-0b5249d1f2dc
λ₄[3]

# ╔═╡ 63167eee-8d79-11eb-29c2-2febbd7cbd93
λ₄[4]

# ╔═╡ e051d2eb-3972-4c97-b7eb-8a18629f6171
md"""
Since all eigenvalues are distinct, they are all simple and the matrix is diagonalizable.
With high probability, all eigenvalues of a random matrix are simple.

Do not try to use `nullspace()` here.

Symbolic computation does not work well with floating-point numbers - in the following example, the determinant of $A-xI$ is a rational function:
"""

# ╔═╡ 22f0a627-beb2-4b06-aa94-ee1e90fe8a6c
begin
	A₂=rand(4,4)
    factor(det(A₂-x*eye(4)))
end

# ╔═╡ da6baba0-92d9-11eb-1603-7961ad1f49c8
A₂

# ╔═╡ c6ab93ab-b336-4bb4-9dda-b725108f7827
md"""
Let us try `Rational` numbers:
"""

# ╔═╡ ffcb919f-9754-4650-92f1-7dd163c5df25
Aᵣ=map(Rational,A₂)

# ╔═╡ 32ce58b9-783b-4739-9001-0fe581b2953d
begin
	pᵣ(x)=factor(det(Aᵣ-x*eye(4)))
	pᵣ(x)
end

# ╔═╡ a7e8d711-57a6-477a-a641-b929789dbadf
λᵣ=solve(pᵣ(x),x)

# ╔═╡ 5781508c-388a-4df0-a6f9-0fe312a81335
# For example:
λᵣ[1]

# ╔═╡ 90b4341d-7964-4e41-a268-8a047a058286
md"""
### Circulant matrix

For more details, see
[A. Böttcher and I. Spitkovsky, Special Types of Matrices, pp. 22.1-22.20](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and the references therein.

Given $a_0,a_1,\ldots,a_{n-1}\in \mathbb{C}$, the __circulant matrix__ is

$$C(a_0,a_1,\ldots,a_{n-1})=\begin{bmatrix}
a_0 & a_{n-1} & a_{n-2} & \cdots & a_{1} \\
a_1 & a_0 & a_{n-1} & \cdots & a_{2} \\
a_2 & a_1 & a_{0} & \cdots & a_{3} \\
\vdots & \vdots & \vdots & \ddots & \vdots \\
a_{n-1} & a_{n-2} & a_{n-3} & \cdots & a_{0}
\end{bmatrix}.$$

Let $a(z)=a_0+a_1z+a_2z^2+\cdots +a_{n-1}z^{n-1}$ be the associated complex polynomial.

Let $\omega_n=\exp\big(\displaystyle\frac{2\pi i}{n}\big)$. The __Fourier matrix__ of order $n$ is

$$F_n=\frac{1}{\sqrt{n}} \bigg[ \omega_n^{(j-1)(k-1)}\bigg]_{j,k=1}^n=
\frac{1}{\sqrt{n}} \begin{bmatrix}
1 & 1 & \cdots & 1 \\
1& \omega_n & \omega_n^2 & \cdots \omega_n^{n-1} \\
1& \omega_n^2 & \omega_n^4 & \cdots \omega_n^{2(n-1)} \\
\vdots & \vdots & \ddots & \vdots \\
1& \omega_n^{n-1} & \omega_n^{2(n-1)} & \cdots \omega_n^{(n-1)(n-1)}
\end{bmatrix}.$$

Fourier matrix is unitary.
Fourier matrix is a Vandermonde matrix,

$$F_n=\displaystyle\frac{1}{\sqrt{n}} V(1,\omega_n,\omega_n^2,\ldots, \omega_n^{n-1}).$$

Circulant matrix is normal and, thus, unitarily diagonalizable, with the eigenvalue decomposition

$$
C(a_0,a_1,\ldots,a_{n-1})=F_n^*\mathop{\mathrm{diag}}[(a(1),a(\omega_n),a(\omega_n^2),\ldots,
a(\omega_n^{n-1})]F_n.$$

We shall use the package [SpecialMatrices.jl](https://github.com/JuliaMatrices/SpecialMatrices.jl).
"""

# ╔═╡ 16795c64-2994-4fca-89f3-f035fb6a2e45
# using Pkg; Pkg.add(PackageSpec(name="SpecialMatrices",rev="master"))

# ╔═╡ 76ffa073-59b8-4118-aa32-522f4ff05979
varinfo(SpecialMatrices)

# ╔═╡ f85ce269-7555-48ed-a2fd-1193d3f33f40
begin
	Random.seed!(124)
	n=6
	a=rand(-9:9,n)
end

# ╔═╡ e6544939-c8c2-464c-83a0-a6bf1751efa1
C=Circulant(a)

# ╔═╡ b559d56b-02bb-47d3-8f0f-8dae09d918d8
# Check for normality
C*C'-C'*C

# ╔═╡ 5e0adf59-29d3-47ac-8f3f-52929cc4e1c1
pᵪ=Polynomial(a)

# ╔═╡ 83ba2d1b-66de-41bf-8ded-7070bed15587
# Polynomials are easy to manipulate
pᵪ*pᵪ

# ╔═╡ b0255c92-3242-442a-a670-1bfef38e955c
# n-th complex root of unity
ω=exp(2*π*im/n)

# ╔═╡ c6116576-5b25-45b2-bc71-a889f302b5ac
begin
	v=[ω^k for k=0:n-1]
	Fₙ=Vandermonde(v)/√n
end

# ╔═╡ 3de2dc38-ba95-479e-89bc-77cb520daf2f
Λᵪ=Fₙ*C*Fₙ'

# ╔═╡ ad32c51c-7cbb-42f9-abc8-561828df1379
# Check
[diag(Λᵪ) pᵪ.(v) eigvals(Matrix(C))]

# ╔═╡ a7999464-7060-4345-b396-e2e15fcbcaac
md"""

# Hermitian and real symmetric matrices

For more details and the proofs of the Facts below, see
[W. Barrett, Hermitian and Positive Definite Matrices, pp. 9.1-9.13](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and the references therein.

## Definitions

Matrix $A\in \mathbb{C}^{n\times n}$ is __Hermitian__ or __self-adjoint__ if $A^*=A$,
or element-wise, $\bar a_{ij}=a_{ji}$. We say $A\in\mathcal{H}_n$.

Matrix $A\in \mathbb{R}^{n\times n}$ is __symmetric__ if $A^T=A$, or element-wise, $a_{ij}=a_{ji}$.
We say $A\in\mathcal{S}_n$.

__Rayleigh qoutient__ of $A\in\mathcal{H}_n$ and nonzero vector $x\in\mathbb{C}^n$ is

$$
R_A(x)=\frac{x^*Ax}{x^*x}.$$

Matrices $A,B \in\mathcal{H}_n$ are __congruent__ if there exists nonsingular matrix $C$ such that
$B=C^*AC$.

__Inertia__ of $A\in\mathcal{H}_n$ is the ordered triple

$$\mathop{\mathrm{in}}(A)=(\pi(A),\nu(A),\zeta(A)),$$

where $\pi(A)$ is the number of positive eigenvalues of $A$,  $\nu(A)$ is the number of negative eigenvalues of $A$,
and $\zeta(A)$ is the number of zero eigenvalues of $A$.

__Gram matrix__ of a set of vectors $x_1,x_2,\ldots,x_k\in\mathbb{C}^{n}$ is the matrix $G$ with entries $G_{ij}=x_i^*x_j$.
"""

# ╔═╡ 01e2e27b-506e-439a-ba9a-bef034328482
md"""
## Facts

Assume $A$ is Hermitian and $x\in\mathbb{C}^n$ is nonzero. Then

1. Real symmetric matrix is Hermitian, and real Hermitian matrix is symmetric.

2. Hermitian and real symmetric matrices are normal.

3.  $A+A^*$, $A^*A$, and $AA^*$ are Hermitian.

4. If $A$ is nonsingular, $A^{-1}$ is Hermitian.

5. Main diagonal entries of $A$ are real.

6. Matrix $T$ from the Schur decomposition of $A$ is Hermitian. Consequently:

    *  $T$ is diagonal and real, and has eigenvalues of $A$ on diagonal,
    * matrix $Q$ of the Schur decomposition is the unitary matrix of eigenvectors,
    * all eigenvalues of $A$ are semisimple and $A$ is nondefective,
    * eigenvectors corresponding to distinct eigenvalues are orthogonal.

7. __Spectral Theorem.__ To summarize:

    * if $A\in\mathcal{H}_n$, there is a unitary matrix $U$ and real diagonal matrix $\Lambda$ such that $A=U\Lambda U^*$. The diagonal entries of $\Lambda$ are the eigenvalues of $A$, and the columns of $U$ are the corresponding eigenvectors.
    * if $A\in\mathcal{S}_n$, the same holds with orthogonal matrix $U$, $A=U\Lambda U^T$.
    * if $A\in\mathcal{H}_n$ with eigenpairs $(\lambda_i,u_i)$, then $A=\lambda_1u_1u_1^*+\lambda_2 u_2u_2^* +\cdots +\lambda_n u_nu_n^*$.

    * similarly, if  $A\in\mathcal{S}_n$, then $A=\lambda_1u_1u_1^T+\lambda_2 u_2u_2^T +\cdots +\lambda_n u_nu_n^T$.

8. Since all eigenvalues of $A$ are real, they can be ordered: $\lambda_1\geq \lambda_2\geq \cdots \geq \lambda_n$.

9. __Rayleigh-Ritz Theorem.__ It holds:

$$\begin{aligned}
\lambda_n &\leq \frac{x^*Ax}{x^*x} \leq \lambda_1, \\
\lambda_1&=\max_x\frac{x^*Ax}{x^*x} =\max_{\|x\|_2=1} x^*Ax, \\
\lambda_n&=\min_x\frac{x^*Ax}{x^*x} =\min_{\|x\|_2=1} x^*Ax.
\end{aligned}$$

10. __Courant-Fischer Theorem.__ It holds:

$$\begin{aligned}
\lambda_k& =\max_{\dim(W)=k}\min_{x\in W} \frac{x^*Ax}{x^*x}\\
& =\min_{\dim(W)=n-k+1}\max_{x\in W} \frac{x^*Ax}{x^*x}.
\end{aligned}$$

11. __Cauchy Interlace Theorem.__ Let $B$ be the principal submatrix of $A$ obtained by deleting the $i$-th row and the $i$-th column of $A$. Let $\mu_1\geq \mu_2\geq \cdots \geq \mu_{n-1}$ be the eignvalues of $B$. Then

$$\lambda_1\geq \mu_1\geq \lambda_2\geq \mu_2\geq \lambda_3\geq\cdots\geq \lambda_{n-1}\geq\mu_{n-1}\geq\lambda_n.$$

12. __Weyl Inequalities.__ For $A,B\in\mathcal{H}_n$, it holds:

$$\begin{aligned}
\lambda_{j+k-1}(A+B)& \leq \lambda_j(A)+\lambda_k(B), & \textrm{for} \ j+k\leq n+1,\\
\lambda_{j+k-n}(A+B)& \geq \lambda_j(A)+\lambda_k(B), & \textrm{for} \ j+k\geq n+1,\\
\lambda_j(A)+\lambda_n(B) & \leq \lambda_j(A+B) \leq \lambda_j(A)+\lambda_1(B), & \textrm{for} \ j=1,2,\ldots,n.
\end{aligned}$$

13.  $\pi(A)+\nu(A)+\zeta(A)=n$.

14.  $\mathop{\mathrm{rank}}(A)=\pi(A)+\nu(A)$.

15. If $A$ is nonsingular, $\mathop{\mathrm{in}}(A)=\mathop{\mathrm{in}}(A^{-1})$.

16. If $A,B \in\mathcal{H}_n$ are similar,  $\mathop{\mathrm{in}}(A)=\mathop{\mathrm{in}}(B)$.

17. __Sylvester's Law of Inertia.__ $A,B \in\mathcal{H}_n$ are congruent if and only if $\mathop{\mathrm{in}}(A)=\mathop{\mathrm{in}}(B)$.

18. __Subadditivity of Inertia.__ For $A,B \in\mathcal{H}_n$,

$$\pi(A+B)\leq \pi(A)+\pi(B), \qquad \nu(A+B)\leq \nu(A)+\nu(B).$$

19. Gram matrix is Hermitian.
"""

# ╔═╡ 40c67883-7c5a-4033-96df-090c58b72f65
md"""
## Examples
### Hermitian matrix
"""

# ╔═╡ 43ee6302-cd9e-472e-8c25-9c079d90bd1c
begin
	# Generating Hermitian matrix
	Random.seed!(432)
	nₕ=6
	Aₕ=rand(ComplexF64,nₕ,nₕ)
	Aₕ=Aₕ+adjoint(Aₕ)
end

# ╔═╡ f2b96941-4a21-4c1a-8c91-cd97786ff285
ishermitian(Aₕ)

# ╔═╡ 7a4e98d5-40ba-4ff1-9644-df99163e250d
# Diagonal entries
Diagonal(Aₕ)

# ╔═╡ 669c86fe-c5ac-4b6e-ac38-be628d2d38f3
# Schur decomposition
schur(Aₕ)

# ╔═╡ c19945f2-1fcc-4a33-acdf-615cc9621b12
λₕ,Uₕ=eigen(Aₕ)

# ╔═╡ e98a08fc-58af-4e07-ad77-80492f28490e
# Spectral theorem
norm(Aₕ-Uₕ*Diagonal(λₕ)*Uₕ')

# ╔═╡ 96f6750c-422d-4c2f-9719-32a3c8042181
# Spectral theorem
Aₕ-sum([λₕ[i]*Uₕ[:,i]*Uₕ[:,i]' for i=1:nₕ])

# ╔═╡ 189b6d8f-a080-473c-a9a7-0507707707c8
λₕ

# ╔═╡ a6339443-0249-4b12-b94b-1f4b2b1b6d10
begin
	# Cauchy Interlace Theorem (repeat several times)
	@show i=rand(1:nₕ)
	eigvals(Aₕ[[1:i-1;i+1:end],[1:i-1;i+1:end]])
end

# ╔═╡ b42042c0-5592-4793-b3ff-72ff269dcf6e
# Inertia
inertia(A)=[sum(eigvals(A).>0), sum(eigvals(A).<0), sum(eigvals(A).==0)]

# ╔═╡ 73eb2ab0-92f7-4d74-8960-0b11e8773f4b
inertia(Aₕ)

# ╔═╡ b6adec3b-d812-4268-9d13-9e82dcf30f22
begin
	# Similar matrices
	Mₕ=rand(ComplexF64,nₕ,nₕ)
	eigvals(Mₕ*Aₕ*inv(Mₕ))
end

# ╔═╡ 3181a27c-8360-4a08-bcb0-f6669a9ce062
md"""
This did not work numerically due to rounding errors!
"""

# ╔═╡ e439d2b6-b581-4b87-9f07-ee0e93338a5f
# Congruent matrices - this does not work either, without some preparation
inertia(A)==inertia(Mₕ'*Aₕ*Mₕ)

# ╔═╡ e7a79e8d-d7d7-4280-88b7-58d6d2f228b8
# However,
inertia(Aₕ)==inertia(Hermitian(Mₕ'*Aₕ*Mₕ))

# ╔═╡ 1549b463-2849-4994-a2a3-45c71efda75e
# or,
inertia((Mₕ'*Aₕ*Mₕ+(Mₕ'*Aₕ*Mₕ)')/2)

# ╔═╡ de537b01-876b-40a3-a941-4b5d8bcd2e0f
begin
	# Weyl's Inequalities
	B=rand(ComplexF64,nₕ,nₕ)
	Bₕ=(B+B')/10
	μₕ=eigvals(Bₕ)
	γₕ=eigvals(Aₕ+Bₕ)
	μₕ,γₕ
end

# ╔═╡ 5e49f18c-d633-469e-af8a-1e3877ebf07a
begin
	# Theorem uses different order!
	j=rand(1:nₕ)
	k=rand(1:nₕ)
	@show j,k
	if j+k<=n+1
	    @show sort(γₕ,rev=true)[j+k-1], sort(λₕ,rev=true)[j]+sort(μₕ,rev=true)[k]
	end
	if j+k>=n+1
	    @show sort(γₕ,rev=true)[j+k-n], sort(λₕ,rev=true)[j]+sort(μₕ,rev=true)[k]
	end
end

# ╔═╡ 1866e2e0-8766-4ebd-b045-1f7dcae30b24
sort(λₕ,rev=true)

# ╔═╡ 80c270bc-e313-4b78-94d0-2fcd9aa29a89
md"""
### Real symmetric matrix
"""

# ╔═╡ da3207e5-1cdb-474f-a5f0-11b7c7ea5640
begin
	# Generating real symmetric matrix
	Random.seed!(531)
	nₛ=5
	Aₛ=rand(-9:9,nₛ,nₛ)
	Aₛ=Aₛ+Aₛ'
end

# ╔═╡ e44e5b0c-83e0-4dec-be0d-c5d9bebc7d55
issymmetric(Aₛ)

# ╔═╡ f804922d-b411-4d6b-a973-0e9486756279
schur(Aₛ)

# ╔═╡ 157fd857-d4c8-4433-9e41-9aba559ce0ea
cond(A)

# ╔═╡ dc690338-6ba8-463d-b35b-e57b4e8b0159
inertia(Aₛ)

# ╔═╡ f8334f36-4e46-423f-8028-8a51caff6fc4
begin
	Cₛ=rand(nₛ,nₛ)
	inertia(Cₛ'*Aₛ*Cₛ)
end

# ╔═╡ 53637040-356d-4310-970e-6a9b9c1260c9
md"""
# Positive definite matrices

These matrices are an important subset of Hermitian or real symmteric matrices.

## Definitions

Matrix $A\in\mathcal{H}_n$ is __positive definite__ (PD) if $x^*Ax>0$ for all nonzero $x\in\mathbb{C}^n$.

Matrix $A\in\mathcal{H}_n$ is __positive semidefinite__ (PSD) if $x^*Ax\geq 0$ for all nonzero $x\in\mathbb{C}^n$.

## Facts

1.  $A\in\mathcal{S}_n$ is PD if $x^TAx>0$ for all nonzero $x\in \mathbb{R}^n$, and is PSD if $x^TAx\geq 0$ for all $x\in \mathbb{R}^n$.

2. If $A,B\in \mathrm{PSD}_n$, then $A+B\in \mathrm{PSD}_n$. If, in addition, $A\in \mathrm{PD}_n$, then $A+B\in \mathrm{PD}_n$.

3. If $A\in \mathrm{PD}_n$, then $\mathop{\mathrm{tr}}(A)>0$ and $\det(A)>0$.

3. If $A\in \mathrm{PSD}_n$, then $\mathop{\mathrm{tr}}(A)\geq 0$ and $\det(A)\geq 0$.

4. Any principal submatrix of a PD matrix is PD. Any principal submatrix of a PSD matrix is PSD. Consequently, all minors are positive or nonnegative, respectively.

5.  $A\in\mathcal{H}_n$ is PD iff __every leading__ principal minor of $A$ is positive. $A\in\mathcal{H}_n$ is PSD iff __every__ principal minor is nonnegative.

6. For $A\in \mathrm{PSD}_n$, there exists unique PSD $k$-th __root__, $A^{1/k}=U\Lambda^{1/k} U^*$.

7. __Cholesky Factorization.__ $A\in\mathcal{H}_n$ if PD iff there is an invertible lower triangular matrix $L$ with positive diagonal entries such that $A=LL^*$.

8. Gram matrix is PSD. If the vectors are linearly independent, Gram matrix is PD.
"""

# ╔═╡ 5082d501-8e0f-45ec-8927-ad1a0be51ae3
md"""
## Examples
### Positive definite matrix
"""

# ╔═╡ e00702a4-2ad5-4dee-8d0c-7d15c4ffdeb8
begin
	# Generating positive definite matrix as a Gram matrix
	nₚ=5
	Aₚ=rand(ComplexF64,nₚ,nₚ)
	Aₚ=Aₚ*Aₚ'
end

# ╔═╡ fb844ff8-1696-45a8-97a9-a08ae0ace55b
ishermitian(Aₚ)

# ╔═╡ c6d47f33-1c78-4568-a0f6-e9d1deb669ca
eigvals(Aₚ)

# ╔═╡ ada85c66-6268-4664-a458-4213d7d7111f
# Positivity of principal leading minors
[det(Aₚ[1:i,1:i]) for i=1:nₚ]

# ╔═╡ a43a9532-f865-4166-b0c6-124045030e64
begin
	# Matrix function - square root
	λₚ,Uₚ=eigen(Aₚ)
	A2=Uₚ*√Diagonal(λₚ)*Uₚ'
	Aₚ-A2^2
end

# ╔═╡ 8a4c37e0-a442-46da-929c-24cdc8280313
# Cholesky factorization - lower triangular factor
Lₚ=cholesky(Aₚ).L

# ╔═╡ 1ad5826e-f849-4cef-82a1-2bd97cf38731
norm(Aₚ-Lₚ*Lₚ')

# ╔═╡ 8b3db768-e5d1-49cf-8320-7cb36a400ce1
md"
### Positive semidefinite matrix
"

# ╔═╡ 18457042-90d2-4180-a999-8d3454134528
begin
	# Generate positive semidefinite matrix as a Gram matrix, try it several times
	A₅=rand(-9:9,6,4)
	A₅=A₅*A₅'
end

# ╔═╡ 853b9c26-a15b-4109-81ee-5ce6a2dd42bd
# There are rounding errors!
eigvals(A₅)

# ╔═╡ ce5c03dd-f8a8-453d-a3b8-d618aa33be9e
rank(A₅)

# ╔═╡ 710b83d0-8d8c-11eb-39c2-e97b1880a16c
md"
How does function `rank()` work?
"

# ╔═╡ 854dfa30-8d8c-11eb-0d82-7d17e7d40d98
@which rank(A₅)

# ╔═╡ 5a229e53-a1e0-4d0f-b479-22bbd4dfaa06
s₅=svdvals(A₅)

# ╔═╡ 7056a7af-f3cf-4fa6-aa4e-b891b6444aa6
s₅[1]*eps()

# ╔═╡ f4b34f27-082b-4bbe-8ee0-c1066de09a9b
# Cholesky factorization can fail
cholesky(A₅)

# ╔═╡ 0528ccb5-04a8-4b8e-a0c8-d845ca716c64
md"
### Covariance and correlation matrices

[Covariance and correlation matrices](https://en.wikipedia.org/wiki/Covariance_matrix) are PSD.

Correlation matrix is diagonally scaled covariance matrix.
"

# ╔═╡ 9daeb277-5cbb-406e-b42f-c21ddaf4366f
begin
	Random.seed!(651)
	y=rand(10,5)
end

# ╔═╡ 69346220-2310-4015-b29b-5b7186791fa8
begin
	using Statistics
	Cov=cov(y)
end

# ╔═╡ 688efeb3-07da-4d75-a35b-a79f9440da76
# Covariance matrix is a Gram matrix
(y.-mean(y,dims=1))'*(y.-mean(y,dims=1))/(size(y,1)-1)-Cov

# ╔═╡ f28e0e9a-23c5-4114-a853-2da285a24641
# Correlation matrix
Cor=cor(y)

# ╔═╡ 867926b1-3d3e-4c7f-95cf-54ed6b54c112
# Diagonal scaling of ovariance matrix
D=inv(√Diagonal(Cov))

# ╔═╡ c5fe8288-4650-4a38-a322-38780978095d
D*Cov*D

# ╔═╡ f6be3df8-6368-4535-8085-537a80dd3ae0
eigvals(Cov)

# ╔═╡ 808904ca-b1d9-4eca-ae40-bcd53451fa83
eigvals(Cor)

# ╔═╡ 34001f70-e2db-4529-a315-caf38ecb87b0
Covₜ=cov(y')

# ╔═╡ 80a7d977-99e7-4d51-9cc9-4e6b7c410a6a
eigvals(Covₜ)

# ╔═╡ c74eb635-2948-4db3-bd53-c7d7a5b2a454
inertia(Covₜ)

# ╔═╡ cc0422bf-2e9b-463a-9b1a-468b09bb370e
rank(Covₜ)

# ╔═╡ 3e29a10b-0e07-4ddf-ad6d-848100804262
md"
# Eigenvalues of random matrices
"

# ╔═╡ 846047e6-56c7-4ce5-b7f6-6d037c9918bb
md"""
k = $(@bind kₘ Slider(10:30,show_value=true))

n = $(@bind nₘ Slider(10:30,show_value=true))

Matrix type = $(@bind mt Select(["Uniform", "Normal", "Uniform symmetric", "Normal Symmetric"]))
"""

# ╔═╡ c36a856f-a726-4ccb-bee9-691f01147b97
begin
	E=Array{Any}(undef,nₘ,kₘ)
	for i=1:kₘ
		if mt=="Uniform"
			# Unsymmetric uniform distribution
	    	A=rand(nₘ,nₘ)
		elseif mt=="Normal"
			# Unsymmetric normal distribution
			A=randn(nₘ,nₘ)
		elseif mt=="Uniform symmetric"
			# Symmetric uniform distribution
			A=Symmetric(rand(nₘ,nₘ))
		else
			# Symmetric normal distribution
			A=Symmetric(randn(nₘ,nₘ))
		end
	    E[:,i]=eigvals(A)
	end
	# We need this since plot cannot handle `Any`
	E=map(eltype(E[1,1]),E)
	scatter(E,legend=false)
end

# ╔═╡ Cell order:
# ╟─10910df9-240f-4e6b-8076-1a9be4d66ba1
# ╟─ec1fe200-8d73-11eb-0ba3-e593aa79dab2
# ╟─b87354b9-ee7b-4ce0-bf26-b4e71c6be6a9
# ╟─31a71b9e-1dba-4a69-b87e-9202292eb3ed
# ╟─2f5aee03-3c2e-4920-9d10-c509b06be627
# ╠═973abf88-9125-41c7-8372-61efb2976fc2
# ╟─37d24090-8d78-11eb-059d-737014bb6588
# ╠═184bc4ec-c566-4f21-99b3-b6e8cd328b47
# ╠═5d0aac9e-4f48-4f09-956f-5033a1fb8f11
# ╠═73ece9ec-97dd-46bc-8e17-10ce7bfc18bc
# ╠═ce83f04c-e915-4599-beeb-d4a0a681bc17
# ╠═ff5044c0-8d76-11eb-3951-6bb4b5caa7d0
# ╠═55171a69-3557-4ba5-8a4a-813f38c03dd3
# ╠═6df42190-a29e-4c68-8684-07d8d0c9d200
# ╟─b94f7fcd-c6f4-4f4f-924a-20f00e82d94e
# ╠═1b90d50a-bf97-4b1a-9580-d729b03c914e
# ╠═15b60cd5-eb30-43bb-bbcf-dd533e29c35f
# ╟─3619fca4-1374-4adc-b29c-8058b2e147e0
# ╠═d46018c0-a5f8-4dd0-8f3d-73322be792e9
# ╠═d387b26f-bec1-47a9-b1c6-f20c43f0bde8
# ╠═703da5c4-cd09-4832-b64e-5fda75099db5
# ╠═e26f113c-dd98-4e97-b495-581de1b431af
# ╠═62779bfc-ea5d-4eec-94ad-2641e11fc44d
# ╠═bdf079a6-beee-44f5-8200-613e54b3a8dc
# ╠═e1f5027c-1080-4c95-81cc-a75dde641b88
# ╠═c71a7760-b4c6-404e-b5ef-f4879cb787cd
# ╟─343c8f45-6226-4361-a149-5313a47c355c
# ╠═41a05c50-f6df-4b93-bd8a-50412829105e
# ╠═cd8fb666-d2f2-423c-ac39-405c842e97ba
# ╟─7aa7ac89-837c-4ee8-b9f1-8ad0256f25e1
# ╠═c3724c4f-142b-4863-8c04-4a0c15411f40
# ╠═792f632e-fdc2-4016-baaa-b366704f02fd
# ╠═a8664a40-8d78-11eb-325e-41c6497be282
# ╠═1d42245a-2ed6-4cc7-8cbd-2b6aa4e64861
# ╠═f7aae752-70c4-4a10-b96e-206c13a6ced1
# ╠═60525ba5-2c76-406e-8eb3-63ced2e00bdd
# ╟─189544ef-122c-4e30-bfd6-cfc126fc0320
# ╠═da340240-f845-4591-94c3-9ffce6a33d8a
# ╠═e72bf173-7e55-4476-91ff-789847dc53cf
# ╠═da4a5d70-92d8-11eb-0635-11c530ff7d21
# ╠═ec7fc5c0-92d8-11eb-16df-af1133ccb42a
# ╠═29de5fd0-92d9-11eb-09c0-0356001caaef
# ╠═d43480a3-9247-48a4-8a3b-e006ab58539e
# ╠═5995cd31-a75d-4718-8808-4ada01920344
# ╠═57e2742e-8d79-11eb-0a33-f7250ac02840
# ╠═5e069120-8d79-11eb-3c00-0b5249d1f2dc
# ╠═63167eee-8d79-11eb-29c2-2febbd7cbd93
# ╟─e051d2eb-3972-4c97-b7eb-8a18629f6171
# ╠═22f0a627-beb2-4b06-aa94-ee1e90fe8a6c
# ╠═da6baba0-92d9-11eb-1603-7961ad1f49c8
# ╟─c6ab93ab-b336-4bb4-9dda-b725108f7827
# ╠═ffcb919f-9754-4650-92f1-7dd163c5df25
# ╠═32ce58b9-783b-4739-9001-0fe581b2953d
# ╠═a7e8d711-57a6-477a-a641-b929789dbadf
# ╠═5781508c-388a-4df0-a6f9-0fe312a81335
# ╟─90b4341d-7964-4e41-a268-8a047a058286
# ╠═16795c64-2994-4fca-89f3-f035fb6a2e45
# ╠═60d351a5-0ef2-47cd-a5a5-5ecf18c4e898
# ╠═76ffa073-59b8-4118-aa32-522f4ff05979
# ╠═f85ce269-7555-48ed-a2fd-1193d3f33f40
# ╠═e6544939-c8c2-464c-83a0-a6bf1751efa1
# ╠═b559d56b-02bb-47d3-8f0f-8dae09d918d8
# ╠═5e0adf59-29d3-47ac-8f3f-52929cc4e1c1
# ╠═83ba2d1b-66de-41bf-8ded-7070bed15587
# ╠═b0255c92-3242-442a-a670-1bfef38e955c
# ╠═c6116576-5b25-45b2-bc71-a889f302b5ac
# ╠═3de2dc38-ba95-479e-89bc-77cb520daf2f
# ╠═ad32c51c-7cbb-42f9-abc8-561828df1379
# ╟─a7999464-7060-4345-b396-e2e15fcbcaac
# ╟─01e2e27b-506e-439a-ba9a-bef034328482
# ╟─40c67883-7c5a-4033-96df-090c58b72f65
# ╠═43ee6302-cd9e-472e-8c25-9c079d90bd1c
# ╠═f2b96941-4a21-4c1a-8c91-cd97786ff285
# ╠═7a4e98d5-40ba-4ff1-9644-df99163e250d
# ╠═669c86fe-c5ac-4b6e-ac38-be628d2d38f3
# ╠═c19945f2-1fcc-4a33-acdf-615cc9621b12
# ╠═e98a08fc-58af-4e07-ad77-80492f28490e
# ╠═96f6750c-422d-4c2f-9719-32a3c8042181
# ╠═189b6d8f-a080-473c-a9a7-0507707707c8
# ╠═a6339443-0249-4b12-b94b-1f4b2b1b6d10
# ╠═b42042c0-5592-4793-b3ff-72ff269dcf6e
# ╠═73eb2ab0-92f7-4d74-8960-0b11e8773f4b
# ╠═b6adec3b-d812-4268-9d13-9e82dcf30f22
# ╟─3181a27c-8360-4a08-bcb0-f6669a9ce062
# ╠═e439d2b6-b581-4b87-9f07-ee0e93338a5f
# ╠═e7a79e8d-d7d7-4280-88b7-58d6d2f228b8
# ╠═1549b463-2849-4994-a2a3-45c71efda75e
# ╠═de537b01-876b-40a3-a941-4b5d8bcd2e0f
# ╠═5e49f18c-d633-469e-af8a-1e3877ebf07a
# ╠═1866e2e0-8766-4ebd-b045-1f7dcae30b24
# ╟─80c270bc-e313-4b78-94d0-2fcd9aa29a89
# ╠═da3207e5-1cdb-474f-a5f0-11b7c7ea5640
# ╠═e44e5b0c-83e0-4dec-be0d-c5d9bebc7d55
# ╠═f804922d-b411-4d6b-a973-0e9486756279
# ╠═157fd857-d4c8-4433-9e41-9aba559ce0ea
# ╠═dc690338-6ba8-463d-b35b-e57b4e8b0159
# ╠═f8334f36-4e46-423f-8028-8a51caff6fc4
# ╟─53637040-356d-4310-970e-6a9b9c1260c9
# ╟─5082d501-8e0f-45ec-8927-ad1a0be51ae3
# ╠═e00702a4-2ad5-4dee-8d0c-7d15c4ffdeb8
# ╠═fb844ff8-1696-45a8-97a9-a08ae0ace55b
# ╠═c6d47f33-1c78-4568-a0f6-e9d1deb669ca
# ╠═ada85c66-6268-4664-a458-4213d7d7111f
# ╠═a43a9532-f865-4166-b0c6-124045030e64
# ╠═8a4c37e0-a442-46da-929c-24cdc8280313
# ╠═1ad5826e-f849-4cef-82a1-2bd97cf38731
# ╟─8b3db768-e5d1-49cf-8320-7cb36a400ce1
# ╠═18457042-90d2-4180-a999-8d3454134528
# ╠═853b9c26-a15b-4109-81ee-5ce6a2dd42bd
# ╠═ce5c03dd-f8a8-453d-a3b8-d618aa33be9e
# ╟─710b83d0-8d8c-11eb-39c2-e97b1880a16c
# ╠═854dfa30-8d8c-11eb-0d82-7d17e7d40d98
# ╠═5a229e53-a1e0-4d0f-b479-22bbd4dfaa06
# ╠═7056a7af-f3cf-4fa6-aa4e-b891b6444aa6
# ╠═f4b34f27-082b-4bbe-8ee0-c1066de09a9b
# ╟─0528ccb5-04a8-4b8e-a0c8-d845ca716c64
# ╠═9daeb277-5cbb-406e-b42f-c21ddaf4366f
# ╠═69346220-2310-4015-b29b-5b7186791fa8
# ╠═688efeb3-07da-4d75-a35b-a79f9440da76
# ╠═f28e0e9a-23c5-4114-a853-2da285a24641
# ╠═867926b1-3d3e-4c7f-95cf-54ed6b54c112
# ╠═c5fe8288-4650-4a38-a322-38780978095d
# ╠═f6be3df8-6368-4535-8085-537a80dd3ae0
# ╠═808904ca-b1d9-4eca-ae40-bcd53451fa83
# ╠═34001f70-e2db-4529-a315-caf38ecb87b0
# ╠═80a7d977-99e7-4d51-9cc9-4e6b7c410a6a
# ╠═c74eb635-2948-4db3-bd53-c7d7a5b2a454
# ╠═cc0422bf-2e9b-463a-9b1a-468b09bb370e
# ╟─3e29a10b-0e07-4ddf-ad6d-848100804262
# ╠═bb8416e3-b682-4733-af6d-36fd48f0a7b8
# ╟─846047e6-56c7-4ce5-b7f6-6d037c9918bb
# ╠═c36a856f-a726-4ccb-bee9-691f01147b97

src/Module B - Eigenvalue and Singular Value Decompositions/L3a Eigenvalue Decomposition - Definitions and Facts.jl


random_point(M::Manifold) = random_point(M, Val(:Gaussian))

@doc raw"""
    random_point(M, :Uniform)

return a random point on the [Circle](https://juliamanifolds.github.io/Manifolds.jl/stable/interface.html#ManifoldsBase.Manifold) $\mathbb S^1$ by
picking a random element from $[-\pi,\pi)$ uniformly.
"""
random_point(M::Circle, ::Val{:Uniform}) = sym_rem(rand()*2*π)
random_point(M::Circle) = random_point(M,Val(:Uniform)) # introduce different default

@doc raw"""
    random_point(M::Euclidean[,T=Float64])

generate a random point on the `Euclidean` manifold `M`, where the
optional parameter determines the type of the entries of the
resulting point on the Euclidean space d.
"""
random_point(M::Euclidean) = randn(manifold_dimension(M))

@doc raw"""
    random_point(M::Grassmannian [,type=:Gaussian, σ=1.0])

return a random point `x` on `Grassmannian` manifold `M` by
generating a random (Gaussian) matrix with standard deviation `σ` in matching
size, which is orthonormal.
"""
function random_point(M::Grassmann{n,k,𝔽}, ::Val{:Gaussian}, σ::Float64=1.0) where {n,k,𝔽}
  	V = σ * randn(𝔽===ℝ ? Float64 : ComplexF64, (n, k))
  	A = qr(V).Q[:,1:k]
    return A
end

function random_point(M::AbstractPowerManifold{𝔽,Mt,NestedPowerRepresentation}, options...) where {𝔽,Mt}
    return [ random_point(M.manifold, options...) for i in get_iterator(M) ]
end
function random_point(M::AbstractPowerManifold{𝔽,Mt,ArrayPowerRepresentation}, options...) where {𝔽,Mt}
    return cat(
        [ random_point(M.manifold, options...) for i in get_iterator(M) ]...,
        dims=length(representation_size(M.manifold))+1
    )
end

@doc raw"""
    random_point(M::ProductManifold [,type=:Gaussian, σ=1.0])

return a random point `x` on `Grassmannian` manifold `M` by
generating a random (Gaussian) matrix with standard deviation `σ` in matching
size, which is orthonormal.
"""
function random_point(M::ProductManifold, o...)
    return ProductRepr([ random_point(N,o...) for N in M.manifolds ]...)
end
@doc raw"""
    randomMPoint(M::Rotations [,type=:Gaussian, σ=1.0])

return a random point `p` on the manifold `Rotations`
by generating a (Gaussian) random orthogonal matrix with determinant $+1$. Let $QR = A$
be the QR decomposition of a random matrix $A$, then the formula reads $p = QD$
where $D$ is a diagonal matrix with the signs of the diagonal entries of $R$, i.e.
````math
D_{ij}=\begin{cases}
\operatorname{sgn}(R_{ij}) & \text{if} \; i=j \\
0 & \, \text{otherwise.}
\end{cases}$
````
It can happen that the matrix gets -1 as a determinant. In this case, the first
and second columns are swapped.
"""
function random_point(M::Rotations, ::Val{:Gaussian}, σ::Real=1.0)
  d = manifold_dimension(M)
  if d == 1
    return ones(1,1)
  else
    A=randn(Float64, d, d)
    s=diag(sign.(qr(A).R))
    D=Diagonal(s)
    C = qr(A).Q*D
    if det(C)<0
      C[:,[1,2]] = C[:,[2,1]]
    end
    return C
  end
end

@doc raw"""
    random_point(M::SymmetricPositiveDefinite, :Gaussian[, σ=1.0])

gerenate a random symmetric positive definite matrix on the
`SymmetricPositiveDefinite` manifold `M`.
"""
function random_point(M::SymmetricPositiveDefinite{N},::Val{:Gaussian},σ::Float64=1.0) where N
    D = Diagonal( 1 .+ randn(N) ) # random diagonal matrix
    s = qr(σ * randn(N,N)) # random q
    return s.Q * D * transpose(s.Q)
end

@doc raw"""
    random_point(M::Stiefel, :Gaussian, σ=1.0])

return a random (Gaussian) point `x` on the `Stiefel` manifold `M` by generating a (Gaussian)
matrix with standard deviation `σ` and return the orthogonalized version, i.e. return the Q
component of the QR decomposition of the random matrix of size $n×k$.
"""
function random_point(M::Stiefel{n,k,𝔽}, ::Val{:Gaussian}, σ::Float64=1.0) where {n,k,𝔽}
    A = σ*randn(𝔽===ℝ ? Float64 : ComplexF64, n, k)
    return Matrix(qr(A).Q)
end

@doc raw"""
    random_point(M::Sphere, :Gaussian, σ=1.0])
return a random point on the Sphere by projecting a normal distirbuted vector
from within the embedding to the sphere.
"""
function random_point(M::Sphere, ::Val{:Gaussian}, σ::Float64=1.0)
	return project(M, σ * randn(manifold_dimension(M)+1))
end

@doc raw"""
    random_tangent(M,p)

generate a random tangent vector in the tangent space of `p` on `M`. By default
this is a `:Gaussian` distribution.
"""
random_tangent(M::Manifold, p) = random_tangent(M,p,Val(:Gaussian))

@doc raw"""
    random_tangent(M::Circle, x[, :Gaussian, σ=1.0])

return a random tangent vector from the tangent space of the point `x` on the
[Circle](https://juliamanifolds.github.io/Manifolds.jl/stable/interface.html#ManifoldsBase.Manifold) $\mathbb S^1$ by using a normal distribution with
mean 0 and standard deviation 1.
"""
random_tangent(M::Circle, p, ::Val{:Gaussian}, σ::Real=1.0) = σ*randn()

doc"""
    random_tangent(M,x,:Gaussian[,σ=1.0])

generate a Gaussian random vector on the `Euclidean` manifold `M` with
standard deviation `σ`.
"""
random_tangent(M::Euclidean, p, ::Val{:Gaussian}, σ::Float64=1.0) = σ * randn(manifold_dimension(M))

@doc raw"""
    random_tangent(M::GRassmann,x[,type=:Gaussian, σ=1.0])

return a (Gaussian) random vector from the tangent space $T_x\mathrm{Gr}(n,k)$ with mean
zero and standard deviation `σ` by projecting a random Matrix onto the  `x`.
"""
function random_tangent(M::Grassmann, p, ::Val{:Gaussian}, σ::Float64=1.0)
    Z = σ * randn(eltype(p), size(p))
    X = project(M, p, Z)
    X = X/norm(X)
	return X
end

@doc raw"""
    random_tangent(M::Hyperpolic, p)

generate a random point on the Hyperbolic manifold by projecting a point from the embedding
with respect to the Minkowsky metric.
"""
function random_tangent(M::Hyperbolic, p, ::Val{:Gaussian})
    Y = randn(eltype(p), size(p))
    X = project(M, p, Y)
    return X
end

function random_tangent(M::PowerManifold, p, options...)
    rep_size = representation_size(M.manifold)
    X = zero_tangent_vector(M, p)
    for i in get_iterator(M)
        set_component!(
            M,
            X,
            random_tangent(
                M.manifold,
                get_component(M, p, i),
                options...
            ),
            i
        )
    end
    return X
end

@doc raw"""
    random_tangent(M::ProductManifold, x)

generate a random tangent vector in the tangent space of the point `p` on the
`ProductManifold` `M`.
"""
function random_tangent(M::ProductManifold, p, options...)
    X = map(
        (m,p) -> random_tangent(m, p, options...),
        M.manifolds,
        submanifold_components(M, p)
    )
    return ProductRepr(X...)
end

@doc raw"""
    random_tangent(M::Rotations, p[, type=:Gaussian, σ=1.0])

return a random tangent vector in the tangent space
$T_x\mathrm{SO}(n)$ of the point `x` on the `Rotations` manifold `M` by generating
a random skew-symmetric matrix. The function takes the real upper triangular matrix of a
(Gaussian) random matrix $A$ with dimension $n\times n$ and subtracts its transposed matrix.
Finally, the matrix is normalized.
"""
function random_tangent(M::Rotations, p, ::Val{:Gaussian}, σ::Real=1.0)
  d = manifold_dimension(M)
  if d == 1
    return zeros(1,1)
  else
    A = randn(Float64, d, d)
    A = triu(A,1) - transpose(triu(A,1))
    A = (1/norm(A))*A
    return A
  end
end

@doc raw"""
    random_tangent(M::Sphere, x[, :Gaussian, σ=1.0])

return a random tangent vector in the tangent space of `x` on the `Sphere` `M`.
"""
function random_tangent(M::Sphere, p, ::Val{:Gaussian}, σ::Float64=1.0)
    n = σ * randn( size(p) ) # Gaussian in embedding
    return n - dot(n, p)*p #project to TpM (keeps Gaussianness)
end

@doc raw"""
    random_tangent(M, p[, :Gaussian, σ = 1.0])

generate a random tangent vector in the tangent space of the point `p` on the
`SymmetricPositiveDefinite` manifold `M` by using a Gaussian distribution
with standard deviation `σ` on an ONB of the tangent space.
"""
function random_tangent(M::SymmetricPositiveDefinite, p, ::Val{:Gaussian}, σ::Float64 = 0.01)
    # generate ONB in TxM
    I = one(p)
    B = get_basis(M, p, DiagonalizingOrthonormalBasis(I))
    Ξ = get_vectors(M,p,B)
    Ξx = vector_transport_to.(Ref(M), Ref(I), Ξ, Ref(p), Ref(ParallelTransport()))
    return sum( randn(length(Ξx)) .* Ξx )
end

@doc raw"""
    random_tangent(M,x, Val(:Rician) [,σ = 0.01])
generate a random tangent vector in the tangent space of `x` on
the `SymmetricPositiveDefinite` manifold `M` by using a Rician distribution
with standard deviation `σ`.
"""
function random_tangent(M::SymmetricPositiveDefinite, p, ::Val{:Rician}, σ::Real = 0.01)
    # Rician
    C = cholesky( Hermitian(p) )
    R = C.L + sqrt(σ) * triu( randn(size(p,1), size(p,2)), 0)
    return R*R'
end


function test_triangle_istriangular()
    @test find([istriangular(i) for i in 1:400]) == [1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105, 120, 136, 153, 171, 190, 210, 231, 253, 276, 300, 325, 351, 378]
end

function test_triangle_nthtriangular()
    @test nthtriangular(1) == 1
    @test nthtriangular(2) == 3
    @test nthtriangular(10) == 55
end

function test_triangle_ntriangular()
    @test ntriangular(10) == [1, 3, 6, 10, 15, 21, 28, 36, 45, 55]
    @test eltype(ntriangular(10)) == Int
    @test eltype(ntriangular(10, BigInt)) == BigInt
end

function test_triangle_sometriangular()
    @test collect(sometriangular(35)) == [1, 3, 6, 10, 15, 21, 28]
    @test collect(sometriangular(36)) == [1, 3, 6, 10, 15, 21, 28, 36]
    @test eltype(sometriangular(36)) == Int
    @test eltype(sometriangular(big(36))) == BigInt
end

function test_triangle_exacttriangular()
    @test collect(exacttriangular(10)) == [1, 3, 6, 10, 15, 21, 28, 36, 45, 55]
    @test eltype(exacttriangular(10)) == Int
    @test eltype(exacttriangular(10, BigInt)) == BigInt
end

function test_triangle_primitive_pythagorean_triples()
    @test collect(take(primitive_pythagorean_triples(), 13)) == Array{Int,1}[
        [3,4,5],
        [5,12,13],
        [21,20,29],
        [15,8,17],
        [7,24,25],
        [55,48,73],
        [45,28,53],
        [39,80,89],
        [119,120,169],
        [77,36,85],
        [33,56,65],
        [65,72,97],
        [35,12,37]
    ]
    @test eltype(primitive_pythagorean_triples()) == Array{Int,1}
    @test eltype(primitive_pythagorean_triples(BigInt)) == Array{BigInt,1}

    @test collect(primitive_pythagorean_triples(11)) == Array{Int,1}[]
    @test collect(primitive_pythagorean_triples(12)) == Array{Int,1}[[3,4,5]]
    @test collect(primitive_pythagorean_triples(100)) == Array{Int,1}[
        [3,4,5],
        [5,12,13],
        [21,20,29],
        [15,8,17],
        [7,24,25],
        [35,12,37],
        [9,40,41]
    ]
    @test eltype(primitive_pythagorean_triples(100)) == Array{Int,1}
    @test eltype(primitive_pythagorean_triples(big(100))) == Array{BigInt,1}

    @test count(_->true, primitive_pythagorean_triples(10)) == 0
    @test count(_->true, primitive_pythagorean_triples(100)) == 7
    @test count(_->true, primitive_pythagorean_triples(1000)) == 70
    @test count(_->true, primitive_pythagorean_triples(10000)) == 703
    @test count(_->true, primitive_pythagorean_triples(100000)) == 7026
    @test count(_->true, primitive_pythagorean_triples(1000000)) == 70229
end

function test_triangle_all()
    print(rpad("Math.NumberTheory.Triangle...", 50, ' '))

    test_triangle_istriangular()
    test_triangle_nthtriangular()
    test_triangle_ntriangular()
    test_triangle_sometriangular()
    test_triangle_exacttriangular()

    test_triangle_primitive_pythagorean_triples()

    println("PASS")
end


# Parameters

L = 5 # m
v = 5 # m/s

# Dynamics model functions

function next_state(x, θ, ϕ; dt = 0.05)
    # x is in meters and θ is in degrees
	# ϕ is steering angle input (deg)

	# Dynamics model
	ẋ = v * sind(θ)
	θ̇ = (v / L) * tand(ϕ)

	x′ = x + ẋ * dt
	θ′ = θ + rad2deg(θ̇) * dt

	return x′, θ′
end

function next_state(x, θ, y, ϕ; dt = 0.05)
    # for plotting purposes, we might also want the y value
    # x is in meters and θ is in degrees
	# ϕ is steering angle input (deg)

	# Dynamics model
	ẋ = v * sind(θ)
	ẏ = v * cosd(θ)
	θ̇ = (v / L) * tand(ϕ)

	x′ = x + ẋ * dt
	y′ = y + ẏ * dt
	θ′ = θ + rad2deg(θ̇) * dt

	return x′, θ′, y′
end

function next_state(x, θ, y, ϕ, cosval; dt = 0.05)
    # for plotting purposes, we might also want the y value
    # x is in meters and θ is in degrees
	# ϕ is steering angle input (deg)

	# Dynamics model
	ẋ = v * sind(θ)
	ẏ = v * cosval
	θ̇ = (v / L) * tand(ϕ)

	x′ = x + ẋ * dt
	y′ = y + ẏ * dt
	θ′ = θ + rad2deg(θ̇) * dt

	return x′, θ′, y′
end

function sim_steps(x, θ, ϕ; dt = 0.05, num_steps = 20)
    for i = 1:num_steps
		x, θ = next_state(x, θ, ϕ)
	end
    return x, θ
end

function sim_steps(x, θ, y, ϕ; dt = 0.05, num_steps = 20)
    for i = 1:num_steps
		x, θ, y = next_state(x, θ, y, ϕ)
	end
    return x, θ, y
end

function sim_steps(x, θ, y, ϕ, cosval; dt = 0.05, num_steps = 20)
    for i = 1:num_steps
		x, θ, y = next_state(x, θ, y, ϕ)
	end
    return x, θ, y
end

# NOTE: this function is only valid for θs between -90 and 90 degrees
function reachable_cell(lbs, ubs, ϕ_min, ϕ_max)
	next_lb_x, next_lb_θ = sim_steps(lbs[1], lbs[2], ϕ_min)
	next_ub_x, next_ub_θ = sim_steps(ubs[1], ubs[2], ϕ_max)
	return [next_lb_x, next_lb_θ], [next_ub_x, next_ub_θ]
end

function cosdextrema(xmin, xmax)
	b1, b2 = cosd(xmin), cosd(xmax)
	curr_min, curr_max = min(b1, b2), max(b1, b2)
	if xmin < 0 && 0 < xmax
		curr_max = 1
	end
	curr_min, curr_max
end

function reachable_cell_dtp(lbs, ubs, ϕ_min, ϕ_max; num_steps = 20)
	cosdmin, cosdmax = cosdextrema(ϕ_min, ϕ_max)

	next_lb_x, next_lb_θ, next_lb_y = sim_steps(lbs[1], lbs[2], lbs[3], ϕ_min, cosdmin, num_steps = num_steps)
	next_ub_x, next_ub_θ, next_ub_y = sim_steps(ubs[1], ubs[2], ubs[3], ϕ_max, cosdmax, num_steps = num_steps)
	return [next_lb_x, next_lb_θ, next_lb_y], [next_ub_x, next_ub_θ, next_ub_y]
end


#---------- Calculators ----------
function generateERAdjMat(
	N::Integer,
	p::Real;
	directed::Bool,
	seed::Integer
)
	(p >= 1) && (return adjMat(GlobalNetwork(N)))
	(p <= 0) && (return adjMat(EmptyNetwork(N)))

	rng = Random.MersenneTwister(seed)
	mat = adjMat(EmptyNetwork(N), sparse=true)

	if directed
		for i in 1:N, j in 1:N
			(i == j) && continue
			(rand(rng) < p) && (mat[i,j] = 1)
		end
	else
		for i in 1:N, j in i+1:N
			if rand(rng) < p
				mat[i,j] = 1
				mat[j,i] = 1
			end
		end
	end

	return mat
end

function generateERAdjMat(
	N::Integer,
	numConnections::Int;
	directed::Bool,
	seed::Integer
)
	maxConnections = Int(N * (N - 1) / 2)
	directed || (maxConnections /= 2)

	(numConnections >= maxConnections) && (return adjMat(GlobalNetwork(N)))
	(numConnections <= 0) && (return adjMat(EmptyNetwork(N)))

	rng = Random.MersenneTwister(seed)
	mat = adjMat(EmptyNetwork(N), sparse=true)
	nc = 0

	while nc < numConnections
		# Choose a random non-existing connection
		origin = rand(rng, 1:N)
		dest   = rand(rng, 1:N)

		(origin == dest) && continue
		(mat[dest, origin] == 1) && continue

		mat[dest, origin] = 1
		directed || (mat[origin, dest] = 1)
		nc += 1
	end

	return mat
end


using DifferentialEquations
using IterableTables, DataFrames
using RCall

H_comps = 4
V_comps = 3

# in sub functions, du / u are the particular relevant slices only
function F1H(du,u,p,t,I_V,N_H)
    S_H, E_H, I_H, R_H = u

    # host dynamics
    host_infection = (p.β*S_H*I_V)/N_H
    host_mortality = p.μ_H .* u # include S_H, so easier to remove mortality
    host_births = sum(host_mortality)
    host_progression = p.σ_H*E_H
    recovery = p.λ*I_H

    du[1] = -host_infection + host_births
    du[2] = host_infection - host_progression
    du[3] = host_progression - recovery
    du[4] = recovery
    du[1:end] -= host_mortality
end

# in sub functions, du / u are the particular relevant slices only
function FV(du,u,p,t,sum_β_I_H)
    S_V, E_V, I_V = u
    vec_infection = sum_β_I_H*S_V/p.N_H
    vec_mortality = p.μ_V .* u # include S_V, so easier to remove mortality
    vec_births = sum(vec_mortality)
    vec_progression = p.σ_V*E_V

    du[1] = -vec_infection + vec_births
    du[2] = vec_infection - vec_progression
    du[3] = vec_progression
    du[1:end] -= vec_mortality
end

function F(du,u,p,t)
    uvec = @view(u[(p.nHosts*H_comps+1):end]) # grab the vector compartments
    S_V, E_V, I_V = uvec
    sum_β_I_H = 0.0
    for host in 0:(p.nHosts-1)
        slice = (1:H_comps).+(H_comps*host)
        F1H(@view(du[slice]), @view(u[slice]), p.host[host+1], t, I_V, p.vec.N_H)
        # must use @view here, so that these arrays can be modified in F1H
        sum_β_I_H += p.host[host+1].β * u[slice[3]] # this host's I compartment
    end
    FV(@view(du[(p.nHosts*4+1):end]), uvec, p.vec,t,sum_β_I_H)
end

nH = 5
srand(0)

S_Hs = ones(nH) .* 100.0
E_Hs = zeros(nH)
I_Hs = shuffle(vcat(zeros(nH-1),[1.0]))
R_Hs = zeros(nH)
host0 = reshape(hcat(S_Hs,E_Hs,I_Hs,R_Hs)', nH*4, 1)
vec0 = [10000.0, 0.0, 0.0]
u0 = vcat(host0, vec0)

srand(1)

μs = 1 ./ (rand(nH) .* 360)
σs = 1 ./ (rand(nH) .* 6)
λs = 1 ./ (rand(nH) .* 28)
βs = rand(nH) ./ 10.0

using NamedTuples
# nb: in >= Julia v0.7, can eliminate this import
#  and the @NT syntax
p = @NT(
  nHosts = nH,
  vec = @NT(μ_V=1/30, σ_V=1/7, N_H = sum(host0)),
  host = [@NT(μ_H=μs[i], σ_H=σs[i], λ=λs[i], β=βs[i]) for i in 1:nH]
  # just building up a random collection of params for demonstration
)

tspan = (0.0, 365.0)
prob = ODEProblem(F, u0, tspan, p)
sol = solve(prob,Tsit5(),reltol=1e-8,abstol=1e-8,saveat=linspace(0,365,365*10+1))

# rename!(df, Dict(:timestamp => :t,
#  :value1 => :S_H, :value2 => :E_H, :value3 => :I_H, :value4 => :R_H,
#  :value5 => :S_V, :value6 => :E_V, :value7 => :I_V
# ))
# mlt[:host] = contains.(string.(mlt[:variable]),"H"); # tag which entries are host vs vector
# df
df = DataFrame(sol)
mlt = melt(df,:timestamp) # convert results into long format for plotting
mlt[:index] = parse.(Int,replace.(string.(mlt[:variable]),r"[^\d]+"=>""))
mlt[:name] = [
    "S_H1","E_H1","I_H1","R_H1",
    "S_H2","E_H2","I_H2","R_H2",
    "S_H3","E_H3","I_H3","R_H3",
    "S_H4","E_H4","I_H4","R_H4",
    "S_H5","E_H5","I_H5","R_H5",
    "S_V","E_V","I_V"
][mlt[:index]]
mlt[:facet] = replace.(string.(mlt[:name]),r"\w+_"=>"")
mlt[:compartment] = replace.(string.(mlt[:name]),r"_\w+"=>"")
mlt

# current version RCall supports better transfers, which would simplify this mess
# but requires Julia v >= 0.7
vals = mlt[:value]
tstamps = mlt[:timestamp]
fcts = mlt[:facet]
comps = mlt[:compartment]
@rput vals tstamps fcts comps
R"
library(ggplot2)
suppressPackageStartupMessages(library(data.table))
dt <- data.table(t=tstamps, y=vals, species=fcts, compartment=comps)
ggplot(dt) + aes(x=t, y=y, color=compartment) + facet_grid(species ~ ., scale = 'free_y') +
  theme_minimal() +
  geom_line()
"


using Quaternions: argq
using LinearAlgebra

let # test accessors
    c = rand(4)
    @test c[1] == real(quat(c...))
    @test c[2:4] == imag(quat(c...))
end

# creating random examples
sample(QT::Type{Quaternion{T}}) where {T <: Integer} = QT(rand(-100:100, 4)..., false)
sample(QT::Type{Quaternion{T}}) where {T <: AbstractFloat} = QT(rand(Bool) ? quatrand() : nquatrand())
sample(CT::Type{Complex{T}}) where {T <: Integer} = CT(rand(-100:100, 2)...)
sample(CT::Type{Complex{T}}) where {T <: AbstractFloat} = CT(randn(2)...)
sample(T, n) = T[sample(T) for _ in 1:n]

# test algebraic properties of quaternions
for _ in 1:10, T in (Float32, Float64, Int32, Int64)
    q, q1, q2, q3 = sample(Quaternion{T}, 4)

    # skewfield
    test_group(q1, q2, q3, +, zero(q), -)
    test_group(q1, q2, q3, *, one(q), inv)
    test_multiplicative(q1, q2, *, norm)

    # complex embedding
    c1, c2 = sample(Complex{T}, 2)
    test_multiplicative(c1, c2, *, Quaternion)
    test_multiplicative(c1, c2, +, Quaternion)
end

let # test rotations
    qx = qrotation([1, 0, 0], pi / 4)
    @test qx * qx ≈ qrotation([1, 0, 0], pi / 2)
    @test qx^2 ≈ qrotation([1, 0, 0], pi / 2)
    theta = pi / 8
    qx = qrotation([1, 0, 0], theta)
    c = cos(theta); s = sin(theta)
    Rx = [1 0 0; 0 c -s; 0 s c]
    @test rotationmatrix(qx) ≈ Rx
    theta = pi / 6
    qy = qrotation([0, 1, 0], theta)
    c = cos(theta); s = sin(theta)
    Ry = [c 0 s; 0 1 0; -s 0 c]
    @test rotationmatrix(qy) ≈ Ry
    theta = 4pi / 3
    qz = qrotation([0, 0, 1], theta)
    c = cos(theta); s = sin(theta)
    Rz = [c -s 0; s c 0; 0 0 1]
    @test rotationmatrix(qz) ≈ Rz

    @test rotationmatrix(qx * qy * qz) ≈ Rx * Ry * Rz
    @test rotationmatrix(qy * qx * qz) ≈ Ry * Rx * Rz
    @test rotationmatrix(qz * qx * qy) ≈ Rz * Rx * Ry

    a, b = qrotation([0, 0, 1], deg2rad(0)), qrotation([0, 0, 1], deg2rad(180))
    @test slerp(a, b, 0.0) ≈ a
    @test slerp(a, b, 1.0) ≈ b
    @test slerp(a, b, 0.5) ≈ qrotation([0, 0, 1], deg2rad(90))

    @test angle(qrotation([1, 0, 0], 0)) ≈ 0
    @test angle(qrotation([0, 1, 0], pi / 4)) ≈ pi / 4
    @test angle(qrotation([0, 0, 1], pi / 2)) ≈ pi / 2

    let # test numerical stability of angle
        ax = randn(3)
        for θ in [1e-9, π - 1e-9]
            q = qrotation(ax, θ)
            @test angle(q) ≈ θ
        end
    end
end

for _ in 1:100
    let # test specialfunctions
        c = Complex(randn(2)...)
        q, q2 = sample(Quaternion{Float64}, 4)
        unary_funs = [exp, log, sin, cos, sqrt, inv, conj, abs2, norm]
        # since every quaternion is conjugate to a complex number,
        # one can establish correctness as follows:
        for fun in unary_funs
            @test fun(Quaternion(c)) ≈ Quaternion(fun(c))
            @test q2 * fun(q) * inv(q2) ≈ fun(q2 * q * inv(q2))
        end

        @test exp(log(q)) ≈ q
        @test exp(zero(q)) ≈ one(q)
    end

    let # test qrotation and angleaxis inverse
        ax = randn(3); ax = ax / norm(ax)
        Θ = π * rand()
        q = qrotation(ax, Θ)
        @test angle(q) ≈ Θ
        @test axis(q) ≈ ax
        @test angleaxis(q)[1] ≈ Θ
        @test angleaxis(q)[2] ≈ ax
    end

    let # test argq
        q, q2 = sample(Quaternion{Float64}, 2)
        @test q2 * argq(q) * inv(q2) ≈ argq(q2 * q * inv(q2))
        v = Quaternion(0, randn(3)...)
        @test argq(v) * norm(v) ≈ v
    end

    let # test normalize
        q = quatrand()
        @test norm(normalize(q)) ≈ 1
        @test normalize(q).norm
        @test q ≈ norm(q) * normalize(q)
        qn = nquatrand()
        @test qn.norm
        @test normalize(qn) === qn
    end

    let # test rotation <=> rotationmatrix
        q1 = nquatrand()
        q2 = qrotation(rotationmatrix(q1), q1)
        @test q1 ≈ q2
    end

    let # test slerp and linpol if q1 = 1
        q1 = quat(1, 0, 0, 0.)
        # there are numerical stability issues with slerp atm
        θ = clamp(rand() * 3.5, deg2rad(5e-1), π)
        ax = randn(3)
        q2 = qrotation(ax, θ)
        t = rand()
        slerp(q1, q2, 0.) ≈ q1
        @test slerp(q1, q2, 0.) ≈ q1
        @test slerp(q1, q2, 1.) ≈ q2
        @test slerp(q1, q2, t) ≈ qrotation(ax, t * θ)
        @test norm(slerp(q1, q2, t)) ≈ 1
        @test slerp(q1, q2, 0.5) ≈ qrotation(ax, 0.5 * θ)
        @test linpol(q1, q2, 0.5) ≈ qrotation(ax, 0.5 * θ)

    end
    let # test conjugation invariance
        q, q1, q2 = sample(Quaternion{Float64}, 3)
        ⊗(s, t) = s * t * inv(s)
        t = rand()
        @test q ⊗ slerp(q1, q2, t) ≈ slerp(q ⊗ q1, q ⊗ q2, t)
        @test q ⊗ linpol(q1, q2, t) ≈ linpol(q ⊗ q1, q ⊗ q2, t)
    end
end


using Singular
using SignatureGB
R, (x1, x2, x3, x4, x5, x6, x7, x8) = Singular.PolynomialRing(Fp(101), ["x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8"])
I = spoly{n_Zp}[x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8, x1*x2 + x2*x3 + x3*x4 + x4*x5 + x5*x6 + x6*x7 + x1*x8 + x7*x8, x1*x2*x3 + x2*x3*x4 + x3*x4*x5 + x4*x5*x6 + x5*x6*x7 + x1*x2*x8 + x1*x7*x8 + x6*x7*x8, x1*x2*x3*x4 + x2*x3*x4*x5 + x3*x4*x5*x6 + x4*x5*x6*x7 + x1*x2*x3*x8 + x1*x2*x7*x8 + x1*x6*x7*x8 + x5*x6*x7*x8, x1*x2*x3*x4*x5 + x2*x3*x4*x5*x6 + x3*x4*x5*x6*x7 + x1*x2*x3*x4*x8 + x1*x2*x3*x7*x8 + x1*x2*x6*x7*x8 + x1*x5*x6*x7*x8 + x4*x5*x6*x7*x8, x1*x2*x3*x4*x5*x6 + x2*x3*x4*x5*x6*x7 + x1*x2*x3*x4*x5*x8 + x1*x2*x3*x4*x7*x8 + x1*x2*x3*x6*x7*x8 + x1*x2*x5*x6*x7*x8 + x1*x4*x5*x6*x7*x8 + x3*x4*x5*x6*x7*x8, x1*x2*x3*x4*x5*x6*x7 + x1*x2*x3*x4*x5*x6*x8 + x1*x2*x3*x4*x5*x7*x8 + x1*x2*x3*x4*x6*x7*x8 + x1*x2*x3*x5*x6*x7*x8 + x1*x2*x4*x5*x6*x7*x8 + x1*x3*x4*x5*x6*x7*x8 + x2*x3*x4*x5*x6*x7*x8, x1*x2*x3*x4*x5*x6*x7*x8 - 1]
comp_ideal = gens(R)
SignatureGB.f5(comp_ideal)
gb = SignatureGB.f5(I, verbose = true)
println(SignatureGB.is_gb(gb))


    """
    Cartesian 3×3 matrix

        Matrix3()
        Matrix3(a11,a12,a13,a21,a23,a23,a31,a32,a33)*
        Matrix3(matrix)

    (*) definition is row major, overriding the column major definition of `SMatrix{}`

    """
    const Matrix3 = SMatrix{3, 3, Float64}

    Matrix3() = Matrix3(fill(0.0, 3,3))
    # Define matrix elements in row major order
    Matrix3(a11,a12,a13,a21,a22,a23,a31,a32,a33) = Matrix3([a11 a12 a13; a21 a22 a23; a31 a32 a33])

    export Ô, Î

    """
    Zero 3×3 matrix ``\\hat{O} = \\pmatrix{0 & 0 & 0 \\\\ 0 & 0 & 0 \\\\ 0 & 0 & 0}``

        const Ô = Matrix3(
            0,0,0,
            0,0,0,
            0,0,0)

    """
    const Ô = Matrix3()
    """
    Identity 3×3 matrix ``\\hat{I} = \\pmatrix{1 & 0 & 0 \\\\ 0 & 1 & 0 \\\\ 0 & 0 & 1}``

        const Î = Matrix3(
            1,0,0,
            0,1,0,
            0,0,1)

    """
    const Î = Matrix3(I)

    function Base.inv(m::Matrix3)::Matrix3
        t2 = m[1,1]*m[2,2]*m[3,3]
        t3 = m[1,2]*m[2,3]*m[3,1]
        t4 = m[1,3]*m[2,1]*m[3,2]
        t7 = m[1,1]*m[2,3]*m[3,2]
        t8 = m[1,2]*m[2,1]*m[3,3]
        t9 = m[1,3]*m[2,2]*m[3,1]
        t6 = 1.0/(t2+t3+t4-t7-t8-t9)
        return Matrix3(
             t6*(m[2,2]*m[3,3]-m[2,3]*m[3,2]),
            -t6*(m[1,2]*m[3,3]-m[1,3]*m[3,2]),
             t6*(m[1,2]*m[2,3]-m[1,3]*m[2,2]),
            -t6*(m[2,1]*m[3,3]-m[2,3]*m[3,1]),
             t6*(m[1,1]*m[3,3]-m[1,3]*m[3,1]),
            -t6*(m[1,1]*m[2,3]-m[1,3]*m[2,1]),
             t6*(m[2,1]*m[3,2]-m[2,2]*m[3,1]),
            -t6*(m[1,1]*m[3,2]-m[1,2]*m[3,1]),
             t6*(m[1,1]*m[2,2]-m[1,2]*m[2,1]))
    end

    function solve(m::Matrix3, v::Vector3)::Vector3
        t = (-m[1,2]*m[3,3]+m[1,3]*m[3,2])*m[2,1]+(-m[2,2]*m[1,3]+m[2,3]*m[1,2])*m[3,1]+(m[2,2]*m[3,3]-m[2,3]*m[3,2])*m[1,1]
        return Vector3(
            (v[1]*( m[2,2]*m[3,3] - m[2,3]*m[3,2]) + v[2]*(-m[1,2]*m[3,3] + m[1,3]*m[3,2]) + v[3]*(-m[2,2]*m[1,3] + m[2,3]*m[1,2]))/t,
            (v[1]*(-m[2,1]*m[3,3] + m[2,3]*m[3,1]) + v[2]*( m[1,1]*m[3,3] - m[1,3]*m[3,1]) + v[3]*( m[2,1]*m[1,3] - m[2,3]*m[1,1]))/t,
            (v[1]*( m[2,1]*m[3,2] - m[2,2]*m[3,1]) + v[2]*(-m[1,1]*m[3,2] + m[1,2]*m[3,1]) + v[3]*(-m[2,1]*m[1,2] + m[2,2]*m[1,1]))/t)
    end

    function Base.:\(A::Matrix3, b::Vector3)::Vector3
        return solve(A,b)
    end

    function Base.getproperty(m::Matrix3, name::Symbol)
        if name===:A11; return m[1,1]; end
        if name===:A12; return m[1,2]; end
        if name===:A13; return m[1,3]; end
        if name===:A21; return m[2,1]; end
        if name===:A22; return m[2,2]; end
        if name===:A23; return m[2,3]; end
        if name===:A31; return m[3,1]; end
        if name===:A32; return m[3,2]; end
        if name===:A33; return m[3,3]; end
        if name===:Trace; return m[1,1]+m[2,2]+m[3,3]; end
        if name===:Determinat; return m[1,1]*(m[2,2]*m[3,3]-m[2,3]*m[3,2])+m[1,2]*(m[2,3]*m[3,1]-m[2,1]*m[3,3])+m[1,3]*(m[2,1]*m[3,2]-m[2,2]*m[3,1]); end
        if name===:Row1; return Vector3(m[1,1], m[1,2], m[1,3]); end
        if name===:Row2; return Vector3(m[2,1], m[2,2], m[2,3]); end
        if name===:Row3; return Vector3(m[3,1], m[3,2], m[3,3]); end
        if name===:Column1; return Vector3(m[1,1], m[2,1], m[3,1]); end
        if name===:Column2; return Vector3(m[1,2], m[2,2], m[3,2]); end
        if name===:Column3; return Vector3(m[1,3], m[2,3], m[3,3]); end
        return getfield(m, name)
    end

    function Base.propertynames(m::Matrix3)
        return (:A11, :A12, :A13, :A21, :A22, :A23, :A31, :A32, :A33,
        :Row1, :Row2, :Row3, :Column1, :Column2, :Column3, :Trace, :Determinat)
    end


function round(x::FloatingPoint)
    isfinite(x) || return x+x
    t = trunc(x)
    if x > 0.0
        x-t >= half(x) ? t+one(t) : t
    else
        x-t <= -half(x) ? t-one(t) : t
    end
end

function floor(x::FloatingPoint)
    t = trunc(x)
    x < zero(x) && x!=t ? t-one(x) : t
end

function ceil(x::FloatingPoint)
    t = trunc(x)
    x > zero(x) && x!=t ? t+one(x) : t
end


using Test, YaoBlocks, YaoArrayRegister

function heisenberg(n::Int; periodic::Bool=true)
    Sx(i) = put(n, i=>X)
    Sy(i) = put(n, i=>Y)
    Sz(i) = put(n, i=>Z)

    return sum(1:(periodic ? n : n-1)) do i
        j = mod1(i, n)
        Sx(i) * Sx(j) + Sy(i) * Sy(j) + Sz(i) * Sz(j)
    end
end

const hm = heisenberg(4)

@testset "constructor:time evolution" begin
    te = TimeEvolution(hm, 0.2)
    # copy
    cte = copy(te)
    @test cte == te
    @test cte !== te

    # dispatch
    dispatch!(cte, [2.0])
    @test cte != te
    @test cte.dt == 2.0
    @test setiparams!(cte, 0.5).dt == 0.5
    @test setiparams!(cte, :random).dt != 0.5
    @test setiparams!(cte, :zero).dt == 0.0
end

@testset "test imaginary time evolution" begin
    tei = TimeEvolution(hm, 0.2im)
    r = rand_state(4)
    r1 = copy(r) |> tei
    @test exp(Matrix(mat(hm)) * 0.2) * r.state ≈ r1.state

    @test applymatrix(tei) ≈ mat(tei)
    @test applymatrix(adjoint(tei)) ≈ applymatrix(tei)'
    @test !isunitary(tei)
    @test !isunitary(tei |> mat)
end

@testset "test time evolution" begin
    te = TimeEvolution(hm, 0.2)

    r = rand_state(4)
    r1 = copy(r) |> te
    @test exp(Matrix(mat(hm)) * -0.2im) * r.state ≈ r1.state
    @test applymatrix(adjoint(te)) ≈ applymatrix(te)'

    @test applymatrix(te) ≈ mat(te)
    @test isunitary(te)
    @test isunitary(te |> mat)
    cte = copy(te)
    @test cte == te
end


using LinearAlgebra, FastSolversForWeightedTV, CUDA, Flux, Test, TestImages
CUDA.allowscalar(false)
include("./test_utils.jl")

## 2-D

# flag_gpu = true
flag_gpu = false

# Random data
T = Float64
CT = Complex{T}
step = T(1e-5)
rtol = T(1e-4)
n = (8,8)
y = CT.(T.(TestImages.shepp_logan(n...))); flag_gpu && (y = y |> gpu)
h = (T(1),T(1))

g = gradient_norm(2, 1, n, h; T=CT); flag_gpu && (g = g |> gpu)

# Optimization options
opt = opt_fista(T(1/8); niter=100000, tol_x=nothing, nesterov=true)

# Proxy
λ = T(0.5)*norm(y)^2/g(y)
x = proxy(y, λ, g, opt)

# Gradient test
fun = proxy_objfun(λ, g, opt)
test_grad(fun, y; step=step, rtol=rtol)

# Projection test
ε = T(0.1)*g(y)
x = project(y, ε, g, opt)
@test g(x) ≈ ε rtol=rtol

# Gradient test
fun = proj_objfun(ε, g, opt)
test_grad(fun, y; step=step, rtol=rtol)

## 3-D

# Random data
n = (8,8,8)
h = (T(1),T(1),T(1))
y = T.(TestImages.shepp_logan(n[1:2]...)); flag_gpu && (y = y |> gpu)
y = CT.(repeat(y; outer=(1,1,8)))

g = gradient_norm(2, 1, n, h; T=CT); flag_gpu && (g = g |> gpu)

# Optimization options
opt = opt_fista(T(1/12); niter=100000, tol_x=nothing, nesterov=true)

# Proxy
λ = T(0.5)*norm(y)^2/g(y)
x = proxy(y, λ, g, opt)

# Gradient test
fun = proxy_objfun(λ, g, opt)
test_grad(fun, y; step=step, rtol=rtol)

# Projection test
ε = T(0.1)*g(y)
x = project(y, ε, g, opt)
@test g(x) ≈ ε rtol=rtol

# Gradient test
fun = proj_objfun(ε, g, opt)
test_grad(fun, y; step=step, rtol=rtol)


### A Pluto.jl notebook ###
# v0.12.18

using Markdown
using InteractiveUtils

# ╔═╡ 0b8acfe4-5073-11eb-1630-33963891fa65
begin
	using Revise

	using MAT

	using Clustering
	using ISODATA
	using MLDataUtils
	using MultivariateStats
	using Plots
	using StatsBase
end

# ╔═╡ 9d50e9dc-5112-11eb-261e-fb54e7238993
gradient = cgrad([:black, :purple, :red, :yellow])

# ╔═╡ 24a82cb0-5073-11eb-198e-a9de05e0c7a3
begin
	dfile = matopen("data/SalinasA.mat")
	Xraw = read(dfile)["salinasA"]
	close(dfile)
	Xraw
end

# ╔═╡ 103c6df2-5077-11eb-1ee3-137d3269b6eb
heatmap(Xraw[:, :, 4])

# ╔═╡ a688fe58-5073-11eb-2968-bd2d796d1f2f
begin
	tfile = matopen("data/SalinasA_gt.mat")
	Yraw = read(tfile)["salinasA_gt"]
	close(tfile)
	Yraw
end

# ╔═╡ b58883cc-5075-11eb-0c36-87c4d7f584f3
heatmap(Yraw, c=gradient)

# ╔═╡ 02200e30-5074-11eb-18c1-bbbd91537f47
X = reshape(deepcopy(Xraw), 7138, 224)'

# ╔═╡ 97b50c02-5074-11eb-1974-47e0028a67bb
begin
	Y = reshape(deepcopy(Yraw), 7138)
	for i in 10:14
		Y[findall(x -> x == Float64(i), Y)] .= i - 8.0
	end
	Y = round.(Int, Y)
end

# ╔═╡ efe37d6e-50f6-11eb-1e0d-c3b097dac6d6
heatmap(reshape(Y, 83, 86), c=gradient)

# ╔═╡ 59fa6a82-510d-11eb-06a9-c729dc0b2920
struct ExperimentResult
	iter::Float64
	centers::Float64
	adj_idx::Float64
	rand_idx::Float64
	mirkin_idx::Float64
	hubert_idx::Float64
end

# ╔═╡ 32c3f282-5110-11eb-050b-e3dc9548a51e
function avg_result(results::Vector{ExperimentResult})
	n = size(results, 1)

	iters = 0.0
	centers = 0.0
	adjs = 0.0
	rands = 0.0
	mirkins = 0.0
	huberts = 0.0
	for i in 1:n
		iters += results[i].iter
		centers += results[i].centers
		adjs += results[i].adj_idx
		rands += results[i].rand_idx
		mirkins += results[i].mirkin_idx
		huberts += results[i].hubert_idx
	end

	avg_iters = iters/n
	avg_centers = centers/n
	avg_adjs = adjs/n
	avg_rands = rands/n
	avg_mirkins = mirkins/n
	avg_huberts = huberts/n
	return ExperimentResult(
		avg_iters, avg_centers, avg_adjs, avg_rands, avg_mirkins, avg_huberts)
end

# ╔═╡ 1db73a8c-510d-11eb-089b-7b92b54526e5
# kmeans on salinasA
function experiment4(
	data,
	k,
	num_trials,
	iseeds;
	normalize=true
)
	results = Vector{ExperimentResult}(undef, num_trials)

	best_score = 0.0
	best_centers = Array{Float64}
	best_idx = 0

	for i in 1:num_trials
		train, test = splitobs(shuffleobs(data), at=0.8)
		train_x, train_y = train[1:end-1, :], round.(Int, train[end, :])
		test_x, test_y = test[1:end-1, :], round.(Int, test[end, :])

		if normalize
			train_z = fit(ZScoreTransform, train_x, dims=2)
			train_x = StatsBase.transform(train_z, train_x)
			test_x = StatsBase.transform(train_z, test_x)
		end

		# run kmeans
		res = ISODATA.kmeans(train_x, 6, iseeds=iseeds)

		# fit centers to test data
		fit_y = clusterfit(test_x, res.centers)

		# cluster analysis
		adj_idx, rand_idx, mirkin_idx, hubert_idx = randindex(test_y, fit_y)
		num_centers = size(res.centers, 2)

		results[i] = ExperimentResult(
			res.iter, num_centers, adj_idx, rand_idx, mirkin_idx, hubert_idx)

		if adj_idx > best_score
			best_score = adj_idx
			best_centers = res.centers
			best_idx = i
		end
	end

	return results[best_idx], best_centers, avg_result(results)
end

# ╔═╡ 415ae730-50f5-11eb-03e3-6365247ed76b
begin
	inds = findall(x -> x != 0.0, Y)

	M = fit(PCA, X; maxoutdim=3)
	Xout = transform(M, X)

	full = transpose(cat(Xout', Y, dims=2))
	filtered = full[:, inds]
end

# ╔═╡ f65644f8-5114-11eb-2fd9-09e75a642aca
transpose(cat(X', Y, dims=2))

# ╔═╡ f9a6065e-510d-11eb-2a36-e72f2d507a66
rkmeans_res, rkmeans_centers, rkmeans_avg = experiment4(
	filtered, 6, 100, randomcenters)

# ╔═╡ 226e3d78-510f-11eb-2d8d-155aabe1918d
begin
	rkmeans_pred = clusterfit(filtered[1:end-1, :], rkmeans_centers)
	rkmeans_Y = deepcopy(Y)
	rkmeans_Y[inds] .= rkmeans_pred
	heatmap(reshape(rkmeans_Y, 83, 86), c=gradient)
end

# ╔═╡ afa88d40-510f-11eb-327f-ab765d0a3c8b
kkmeans_res, kkmeans_centers, kkmeans_avg = experiment4(
	filtered, 6, 100, kmppcenters)

# ╔═╡ baae817c-510f-11eb-372b-05a0b920d188
begin
	kkmeans_pred = clusterfit(filtered[1:end-1, :], kkmeans_centers)
	kkmeans_Y = deepcopy(Y)
	kkmeans_Y[inds] .= kkmeans_pred
	heatmap(reshape(kkmeans_Y, 83, 86), c=gradient)
end

# ╔═╡ e8d00128-510f-11eb-359e-9964c959ce74
ikmeans_res, ikmeans_centers, ikmeans_avg = experiment4(
	filtered, 6, 100, isodata_s, normalize=true)

# ╔═╡ f17da848-510f-11eb-04b6-7d129dec43fb
begin
	ikmeans_pred = clusterfit(filtered[1:end-1, :], ikmeans_centers)
	ikmeans_Y = deepcopy(Y)
	ikmeans_Y[inds] .= ikmeans_pred
	heatmap(reshape(ikmeans_Y, 83, 86), c=gradient)
end

# ╔═╡ b159bdf4-5110-11eb-00a0-1bece66bc22b
# isodata on salinasA
function experiment5(
	data,
	k,
	num_trials,
	iseeds;
	normalize=true,
	_θn_bias::Float64=0.20,
	_θe_bias::Float64=0.20,
	_θc_bias::Float64=0.20
)
	results = Vector{ExperimentResult}(undef, num_trials)

	best_score = 0.0
	best_centers = Array{Float64}
	best_idx = 0

	for i in 1:num_trials
		train, test = splitobs(shuffleobs(data), at=0.8)
		train_x, train_y = train[1:end-1, :], round.(Int, train[end, :])
		test_x, test_y = test[1:end-1, :], round.(Int, test[end, :])

		if normalize
			train_z = fit(ZScoreTransform, train_x, dims=2)
			train_x = StatsBase.transform(train_z, train_x)
			test_x = StatsBase.transform(train_z, test_x)
		end

		# calculate isodata params
		summaries = summarize(train)
		θn, θe, θc = iso_param(
			train_x,
			summaries,
			_θn_bias=_θn_bias,
			_θe_bias=_θe_bias,
			_θc_bias=_θc_bias)

		# run kmeans
		res = base_conv(train_x, k, θn=θn, θe=θe, θc=θc, iseeds=iseeds)

		# fit centers to test data
		fit_y = clusterfit(test_x, res.centers)

		# cluster analysis
		adj_idx, rand_idx, mirkin_idx, hubert_idx = randindex(test_y, fit_y)
		num_centers = size(res.centers, 2)

		results[i] = ExperimentResult(
			res.iter, num_centers, adj_idx, rand_idx, mirkin_idx, hubert_idx)

		if adj_idx > best_score
			best_score = adj_idx
			best_centers = res.centers
			best_idx = i
		end
	end

	return results[best_idx], best_centers, avg_result(results)
end

# ╔═╡ 5b33ed70-5111-11eb-0795-015a8472911b
risodata_res, risodata_centers, risodata_avg = experiment5(
	filtered, 6, 1000, randomcenters, normalize=true,
	_θn_bias=0.0,
	_θe_bias=0.0,
	_θc_bias=0.0)

# ╔═╡ 02b383c0-5116-11eb-2748-d371c5aa42b6
begin
	risodata_pred = clusterfit(filtered[1:end-1, :], risodata_centers)
	risodata_Y = deepcopy(Y)
	risodata_Y[inds] .= risodata_pred
	heatmap(reshape(risodata_Y, 83, 86), c=gradient)
end

# ╔═╡ 9bc5aa2e-5111-11eb-06d7-b3cb072d95cd
kisodata_res, kisodata_centers, kisodata_avg = experiment5(
	filtered, 6, 1000, kmppcenters, normalize=true,
	_θn_bias=0.0,
	_θe_bias=0.0,
	_θc_bias=0.0)

# ╔═╡ aae23512-5115-11eb-0b01-4764155fb2d1
begin
	kisodata_pred = clusterfit(filtered[1:end-1, :], kisodata_centers)
	kisodata_Y = deepcopy(Y)
	kisodata_Y[inds] .= kisodata_pred
	heatmap(reshape(kisodata_Y, 83, 86), c=gradient)
end

# ╔═╡ a66f7ffc-5111-11eb-12a4-91874657e647
iisodata_res, iisodata_centers, iisodata_avg = experiment5(
	filtered, 6, 1000, isodata_s, normalize=true,
	_θn_bias=0.0,
	_θe_bias=0.0,
	_θc_bias=0.0)

# ╔═╡ d1d323d6-5113-11eb-3835-c13bc51b8e13
begin
	iisodata_pred = clusterfit(filtered[1:end-1, :], iisodata_centers)
	iisodata_Y = deepcopy(Y)
	iisodata_Y[inds] .= iisodata_pred
	heatmap(reshape(iisodata_Y, 83, 86), c=gradient)
end

# ╔═╡ Cell order:
# ╠═0b8acfe4-5073-11eb-1630-33963891fa65
# ╠═9d50e9dc-5112-11eb-261e-fb54e7238993
# ╠═24a82cb0-5073-11eb-198e-a9de05e0c7a3
# ╠═103c6df2-5077-11eb-1ee3-137d3269b6eb
# ╠═a688fe58-5073-11eb-2968-bd2d796d1f2f
# ╠═b58883cc-5075-11eb-0c36-87c4d7f584f3
# ╠═02200e30-5074-11eb-18c1-bbbd91537f47
# ╠═97b50c02-5074-11eb-1974-47e0028a67bb
# ╠═efe37d6e-50f6-11eb-1e0d-c3b097dac6d6
# ╠═59fa6a82-510d-11eb-06a9-c729dc0b2920
# ╠═32c3f282-5110-11eb-050b-e3dc9548a51e
# ╠═1db73a8c-510d-11eb-089b-7b92b54526e5
# ╠═415ae730-50f5-11eb-03e3-6365247ed76b
# ╠═f65644f8-5114-11eb-2fd9-09e75a642aca
# ╠═f9a6065e-510d-11eb-2a36-e72f2d507a66
# ╠═226e3d78-510f-11eb-2d8d-155aabe1918d
# ╠═afa88d40-510f-11eb-327f-ab765d0a3c8b
# ╠═baae817c-510f-11eb-372b-05a0b920d188
# ╠═e8d00128-510f-11eb-359e-9964c959ce74
# ╠═f17da848-510f-11eb-04b6-7d129dec43fb
# ╠═b159bdf4-5110-11eb-00a0-1bece66bc22b
# ╠═5b33ed70-5111-11eb-0795-015a8472911b
# ╠═02b383c0-5116-11eb-2748-d371c5aa42b6
# ╠═9bc5aa2e-5111-11eb-06d7-b3cb072d95cd
# ╠═aae23512-5115-11eb-0b01-4764155fb2d1
# ╠═a66f7ffc-5111-11eb-12a4-91874657e647
# ╠═d1d323d6-5113-11eb-3835-c13bc51b8e13


using PyPlot

function ts(t, alpha)
  if t > 0
    a = ts(t-1, alpha)
    last_element = a -> alpha * a[end] + randn()
    return push!(a, last_element(a))
  else
    return [0.0]
  end
end

plot(ts(200, 0.98))
plot(ts(200, 0.8))
plot(ts(200, 0))
legend()