## Elliptical copulas

"""
    gausscopulagen(t::Int, Σ::Matrix{Float64} = [1. 0.5; 0.5 1.])

Returns: t x n matrix of t realisations of multivariate data generated
using gaussian copula with Σ - correlation matrix. If the symmetric covariance
matrix is imputed, it will be converted into a correlation matrix automatically.

```jldoctest

julia> Random.seed!(43);

julia> gausscopulagen(10)
10×2 Array{Float64,2}:
 0.589188  0.815308
 0.708285  0.924962
 0.747341  0.156994
 0.227634  0.183116
 0.227575  0.957376
 0.271558  0.364803
 0.445691  0.52792
 0.585362  0.23135
 0.498593  0.48266
 0.190283  0.594451
```
"""
function gausscopulagen(t::Int, Σ::Matrix{Float64} = [1. 0.5; 0.5 1.])
  z = transpose(rand(MvNormal(Σ),t))
  for i in 1:size(Σ, 1)
    d = Normal(0, sqrt.(Σ[i,i]))
    z[:,i] = cdf.(d, z[:,i])
  end
  Array(z)
end

"""
  tstudentcopulagen(t::Int, Σ::Matrix{Float64} = [1. 0.5; 0.5 1.], ν::Int=10)

Generates data using t-student Copula given Σ - correlation matrix, ν - degrees of freedom.
If the symmetric covariance matrix is imputed, it will be converted into a
correlation matrix automatically.

```jldoctest
julia> Random.seed!(43);

julia> tstudentcopulagen(10)
10×2 Array{Float64,2}:
 0.658199  0.937148
 0.718244  0.92602
 0.809521  0.0980325
 0.263068  0.222589
 0.187187  0.971109
 0.245373  0.346428
 0.452336  0.524498
 0.57113   0.272525
 0.498443  0.48082
 0.113788  0.633349
 ```
"""
function tstudentcopulagen(t::Int, Σ::Matrix{Float64} = [1. 0.5; 0.5 1.], ν::Int=10)
  z = transpose(rand(MvNormal(Σ),t))
  U = rand(Chisq(ν), size(z, 1))
  for i in 1:size(Σ, 1)
    x = z[:,i].*sqrt.(ν./U)./sqrt(Σ[i,i])
    z[:,i] = cdf.(TDist(ν), x)
  end
  Array(z)
end

### Frechet familly

"""
  frechetcopulagen(t::Int, n::Int, α::Union{Int, Float64})

Returns t realisation of n variate data generated from one parameter frechet multidimentional copula,
a combination of maximal copla with  weight α and independent copula with  weight 1-α

```jldoctest
julia> Random.seed!(43);

julia> frechetcopulagen(10, 2, 0.5)
10×2 Array{Float64,2}:
 0.180975  0.661781
 0.775377  0.775377
 0.888934  0.125437
 0.924876  0.924876
 0.408278  0.408278
 0.912603  0.740184
 0.828727  0.00463791
 0.400537  0.0288987
 0.429437  0.429437
 0.955881  0.851275
```
"""
function frechetcopulagen(t::Int, n::Int, α::Union{Int, Float64})
  0 <= α <= 1 || throw(DomainError("generaton not supported for α ∉ [0,1]"))
  u = rand(t, n)
  for j in 1:t
    if (α >= rand())
      for i in 1:n
        u[j,i] = u[j, 1]
      end
    end
  end
  u
end

"""
  frechetcopulagen(t::Int, n::Int, α::Union{Int, Float64}, β::Union{Int, Float64})

Two parameters Frechet copula C = α C_{max} + β C_{min} + (1- α - β) C_{⟂}, supported
only for n == 2

```jldoctest
julia> Random.seed!(43);

julia> frechetcopulagen(10, 2, 0.4, 0.2)
10×2 Array{Float64,2}:
 0.180975  0.661781
 0.775377  0.775377
 0.888934  0.125437
 0.924876  0.924876
 0.408278  0.591722
 0.912603  0.740184
 0.828727  0.171273
 0.400537  0.0288987
 0.429437  0.429437
 0.955881  0.851275
```
"""
function frechetcopulagen(t::Int, n::Int, α::Union{Int, Float64}, β::Union{Int, Float64})
  n == 2 || throw(AssertionError("two parameters Frechet copula supported only for n = 2"))
  0 <= α+β <= 1 || throw(DomainError("α+β must be in range [0,1]"))
  u = rand(t,2)
  for j in 1:t
    v = rand()
    if (α >= v)
      u[j,2] = u[j, 1]
    elseif (α < v <= α+β)
      u[j,2] = 1-u[j, 1]
    end
  end
  u
end

### Marshall - Olkin familly

"""
  marshallolkincopulagen(t::Int, λ::Vector{Float64})

Returns: t x n Matrix{Float}, t realisations of n-variate data generated from Marshall-Olkin
copula with parameter vector λ of non-negative elements λₛ.
Number of marginals is n = ceil(Int, log(2, length(λ)-1)).
Parameters are ordered as follow:
λ = [λ₁, λ₂, ..., λₙ, λ₁₂, λ₁₃, ..., λ₁ₙ, λ₂₃, ..., λₙ₋₁ₙ, λ₁₂₃, ..., λ₁₂...ₙ]
If reversed = true, returns data from reversed Marshall-Olkin copula.

```jldoctest

julia> Random.seed!(43)

julia> marshallolkincopulagen(10, [0.2, 1.2, 1.6])
10×2 Array{Float64,2}:
 0.99636   0.994344
 0.167268  0.0619408
 0.977418  0.965093
 0.495167  0.0247053
 0.410336  0.250159
 0.778989  0.678064
 0.50927   0.350059
 0.925875  0.887095
 0.353646  0.219006
 0.782477  0.686799
 ```
"""
function marshallolkincopulagen(t::Int, λ::Vector{Float64} = rand(7); reverse::Bool = false)
  minimum(λ) >= 0 || throw(AssertionError("all parameters must by >= 0 "))
  n = floor(Int, log(2, length(λ)+1))
  U = mocopula(rand(t,2^n-1), n, λ)
  reverse ? 1 .-U : U
end

"""
  mocopula(u::Matrix{Float64}, n::Int, λ::Vector{Float64})

  Returns: t x n Matrix{Float}, t realisations of n-variate data generated from Marshall-Olkin
  copula with parameter vector λ of non-negative elements λₛ, given [0,1]ᵗˣˡ ∋ u, where
  l = 2ⁿ-1

```jldoctest

  julia> mocopula([0.2 0.3 0.4; 0.3 0.4 0.6; 0.4 0.5 0.7], 2, [1., 1.5, 2.])
  3×2 Array{Float64,2}:
   0.252982  0.201189
   0.464758  0.409039
   0.585662  0.5357

```
"""
function mocopula(u::Matrix{Float64}, n::Int, λ::Vector{Float64})
  s = collect(combinations(1:n))
  t,l = size(u)
  U = zeros(t, n)
    for j in 1:t
      for i in 1:n
        inds = findall([i in s[k] for k in 1:l])
        x = minimum([-log(u[j,k])./(λ[k]) for k in inds])
        Λ = sum(λ[inds])
        U[j,i] = exp.(-Λ*x)
      end
    end
    U
end

# transforms univariate distributions
"""
  convertmarg!(X::Matrix, d::UnionAll, p::Vector{Vector})

Takes matrix X of realisations of size(X,2) = n dimensional random variable, with
uniform marginals numbered by i, and convert those marginals to common distribution
d with parameters p[i].
If `testunif = true` each marginal is tested for uniformity.

```jldoctest
julia> Random.seed!(43);

julia> x = rand(10,2);

julia> convertmarg!(x, Normal, [[0, 1],[0, 10]])

julia> x
10×2 Array{Float64,2}:
 -0.911655    4.17328
  0.756673  -14.4472
  1.22088   -11.4823
  1.43866   -13.1053
 -0.231978  -11.2415
  1.35696     6.43914
  0.949145  -26.0172
 -0.251957  -18.9723
 -0.177808    0.54172
  1.70477    10.4192
```
"""
VecVec = Union{Vector{Vector{Int64}}, Vector{Vector{Float64}}}

function convertmarg!(U::Matrix{T}, d::UnionAll, p::VecVec = [fill([0,1], size(U, 2))...];
                                                testunif::Bool = true) where T <: AbstractFloat
  for i = 1:size(U, 2)
    if testunif
      pvalue(ExactOneSampleKSTest(U[:,i],Uniform(0,1)))>0.0001 || throw(AssertionError("$i marg. not unif."))
    end
    @inbounds U[:,i] = quantile.(d(p[i]...), U[:,i])
  end
end

  # generates covariance matrix

"""
    cormatgen(n::Int, ρ::Float64 = 0.5, ordered::Bool = false, altersing::Bool = true)

Returns symmetric correlation matrix Σ of size n x n, with reference correlation 0 < ρ < 1.
If ordered = false, Σ elements varies arround ρ, i.e. σᵢⱼ ≈ ρ+δ else they drop
as indices differences rise, i.e. σᵢⱼ ≳ σᵢₖ as |i-j| < |i-k|.
If altersing = true, some σ are positive and some negative, else ∀ᵢⱼ σᵢⱼ ≥ 0.

```jldoctest
julia> Random.seed!(43);

julia> julia> cormatgen(2)
2×2 Array{Float64,2}:
 1.0       0.660768
 0.660768  1.0
```
"""
function cormatgen(n::Int = 20)
  a = rand(n,n)
  b = a*a'
  c = b./maximum(b)
  c .- Matrix(Diagonal(c)) .+ Matrix(1.0I, size(c)...)

end

function cormatgen_rand(n::Int = 20)
  a = rand(n,n)
  b = a*a'
  diagb = Matrix(Diagonal(1 ./sqrt.(LinearAlgebra.diag(b))))
  b = diagb*b*diagb
  (b+b')/2
end

function cormatgen_constant(n::Int, α::Float64)
  @assert 0 <= α <= 1 "α should satisfy 0 <= α <= 1"
  α .*ones(n, n) .+(1-α) .*Matrix(1.0I, n,n)
end

function random_unit_vector(dim::Int)
  result = rand(Normal(), dim, 1)
  result /= norm(result)
end

function cormatgen_constant_noised(n::Int, α::Float64; ϵ::Float64 = (1 .-α)/2.)
  @assert 0 <= ϵ <= 1-α "ϵ must satisfy 0 <= ϵ <= 1-α"
  result = cormatgen_constant(n, α)
  u = hcat([random_unit_vector(n) for i=1:n]...)
  result += ϵ .*(u'*u)
  result - ϵ .*Matrix(1.0I, size(result)...)
end

function cormatgen_two_constant(n::Int, α::Float64, β::Float64)
  @assert α > β "First argument must be greater"
  result = fill(β, (n,n))
  result[1:div(n,2),1:div(n,2)] = fill(α, (div(n,2),div(n,2)))
  result += Matrix(1.0I, size(result)...) - Matrix(Diagonal(result))
  result
end

function cormatgen_two_constant_noised(n::Int, α::Float64, β::Float64; ϵ::Float64= (1-α)/2)
  @assert ϵ <= 1-α
  result = cormatgen_two_constant(n, α, β)
  u = hcat([random_unit_vector(n) for i=1:n]...)
  result += ϵ .*(u'*u)
  result - ϵ .*Matrix(1.0I, size(result)...)
end

function cormatgen_toeplitz(n::Int, ρ::Float64)
  @assert 0 <= ρ <= 1 "ρ needs to satisfy 0 <= ρ <= 1"
  [ρ^(abs(i-j)) for i=0:n-1, j=0:n-1]
end

function cormatgen_toeplitz_noised(n::Int, ρ::Float64; ϵ=(1-ρ)/(1+ρ)/2)
  @assert 0 <= ϵ <= (1-ρ)/(1+ρ) "ϵ must satisfy 0 <= ϵ <= (1-ρ)/(1+ρ)"

  result = cormatgen_toeplitz(n, ρ)
  u = hcat([random_unit_vector(n) for i=1:n]...)
  result += ϵ .*(u'*u)
  result - ϵ .*Matrix(1.0I, size(result)...)
end


MeshCat.LinearMap(q::Quaternion) = MeshCat.LinearMap(rotation_matrix(q))
MeshCat.js_quaternion(q::Quaternion) = [q.v1, q.v2, q.v3, q.s]

RotX(θ) = Quaternion(cos(θ/2), sin(θ/2), 0, 0)
RotY(θ) = Quaternion(cos(θ/2), 0, sin(θ/2), 0)
RotZ(θ) = Quaternion(cos(θ/2), 0, 0, sin(θ/2))

quateltype(x) = eltype(x) # TODO not super elegant
quateltype(::Quaternion{T}) where T = T

vector(q::Quaternion) = SA[q.s, q.v1, q.v2, q.v3]
vector(q::AbstractVector) = q

function Lmat(q::Quaternion)
    SA[
        q.s  -q.v1 -q.v2 -q.v3;
        q.v1  q.s  -q.v3  q.v2;
        q.v2  q.v3  q.s  -q.v1;
        q.v3 -q.v2  q.v1  q.s;
    ]
end

function Rmat(q::Quaternion)
    SA[
        q.s  -q.v1 -q.v2 -q.v3;
        q.v1  q.s   q.v3 -q.v2;
        q.v2 -q.v3  q.s   q.v1;
        q.v3  q.v2 -q.v1  q.s;
    ]
end

Lᵀmat(q) = Lmat(q)'
Rᵀmat(q) = Rmat(q)'

function Tmat(::Type{T}=Float64) where T
    SA{T}[
        1  0  0  0;
        0 -1  0  0;
        0  0 -1  0;
        0  0  0 -1;
    ]
end

function Vmat(::Type{T}=Float64) where T
    SA{T}[
        0 1 0 0;
        0 0 1 0;
        0 0 0 1;
    ]
end

function Vᵀmat(::Type{T}=Float64) where T
    SA{T}[
        0 0 0;
        1 0 0;
        0 1 0;
        0 0 1;
    ]
end

Vmat(q::Quaternion) = SA[q.v1, q.v2, q.v3]

function VLmat(q::Quaternion)
    SA[
        q.v1  q.s -q.v3  q.v2;
        q.v2  q.v3  q.s -q.v1;
        q.v3 -q.v2  q.v1  q.s;
    ]
end

function VLᵀmat(q::Quaternion)
    SA[
        -q.v1  q.s   q.v3 -q.v2;
        -q.v2 -q.v3  q.s   q.v1;
        -q.v3  q.v2 -q.v1  q.s;
    ]
end

function VRmat(q::Quaternion)
    SA[
        q.v1  q.s  q.v3 -q.v2;
        q.v2 -q.v3  q.s  q.v1;
        q.v3  q.v2 -q.v1  q.s;
    ]
end

function VRᵀmat(q::Quaternion)
    SA[
        -q.v1  q.s -q.v3  q.v2;
        -q.v2  q.v3  q.s -q.v1;
        -q.v3 -q.v2  q.v1  q.s;
    ]
end

function LVᵀmat(q::Quaternion)
    SA[
        -q.v1 -q.v2 -q.v3;
         q.s -q.v3  q.v2;
         q.v3  q.s -q.v1;
        -q.v2  q.v1  q.s;
    ]
end

function LᵀVᵀmat(q::Quaternion)
    SA[
         q.v1  q.v2  q.v3;
         q.s  q.v3 -q.v2;
        -q.v3  q.s  q.v1;
         q.v2 -q.v1  q.s;
    ]
end

function RVᵀmat(q::Quaternion)
    SA[
        -q.v1 -q.v2 -q.v3;
         q.s  q.v3 -q.v2;
        -q.v3  q.s  q.v1;
         q.v2 -q.v1  q.s;
    ]
end

function RᵀVᵀmat(q::Quaternion)
    SA[
         q.v1  q.v2  q.v3;
         q.s -q.v3  q.v2;
         q.v3  q.s -q.v1;
        -q.v2  q.v1  q.s;
    ]
end

################################################################################
# Matrix-Vector Product Jacobian
################################################################################
function ∂VLmat∂q(p::AbstractVector) # 𝞉(VLmat(q)*p)/∂q
	SA[
    	0     p[1]  p[2]  p[3];
    	p[1]  0     p[3] -p[2];
    	p[2] -p[3]  0     p[1];
    	p[3]  p[2] -p[1]  0;
    ]
end

function ∂LVᵀmat∂q(p::AbstractVector) # 𝞉(∂LVᵀmat∂q(q)*p)/∂q
	SA[
    	0    -p[1] -p[2] -p[3];
    	p[1]  0     p[3] -p[2];
    	p[2] -p[3]  0     p[1];
    	p[3]  p[2] -p[1]  0;
    ]
end

function ∂VLᵀmat∂q(p::AbstractVector) # 𝞉(VLᵀmat(q)*p)/∂q
	SA[
		p[2] -p[1] -p[4]  p[3];
		p[3]  p[4] -p[1] -p[2];
		p[4] -p[3]  p[2] -p[1];
    ]
end

function ∂LᵀVᵀmat∂q(p::AbstractVector) # 𝞉(LᵀVᵀmat(q)*p)/∂q
	SA[
    	0     p[1]  p[2]  p[3];
    	p[1]  0    -p[3]  p[2];
    	p[2]  p[3]  0    -p[1];
    	p[3] -p[2]  p[1]  0;
    ]
end

function ∂VRmat∂q(p::AbstractVector) # 𝞉(VRmat(q)*p)/∂q
	SA[
		p[2]  p[1] -p[4]  p[3];
		p[3]  p[4]  p[1] -p[2];
		p[4] -p[3]  p[2]  p[1];
    ]
end

function ∂RᵀVᵀmat∂q(p::AbstractVector) # 𝞉(RᵀVᵀmat(q)*p)/∂q
	SA[
    	p[2]  p[1]  p[4] -p[3];
    	p[3] -p[4]  p[1]  p[2];
    	p[4]  p[3] -p[2]  p[1];
    ]
end

function ∂VRᵀmat∂q(p::AbstractVector) # 𝞉(RᵀVᵀmat(q)*p)/∂q
	SA[
    	p[2] -p[1]  p[4] -p[3];
    	p[3] -p[4] -p[1]  p[2];
    	p[4]  p[3] -p[2] -p[1];
    ]
end

function ∂Rᵀmat∂q(p::AbstractVector) # 𝞉(Rᵀmat(q)*p)/∂q
	SA[
    	p[1]  p[2]  p[3]  p[4];
    	p[2] -p[1]  p[4] -p[3];
    	p[3] -p[4] -p[1]  p[2];
    	p[4]  p[3] -p[2] -p[1];
    ]
end

function ∂Lmat∂q(p::AbstractVector) # 𝞉(Lmat(q)*p)/∂q
	SA[
    	p[1] -p[2] -p[3] -p[4];
    	p[2]  p[1]  p[4] -p[3];
    	p[3] -p[4]  p[1]  p[2];
    	p[4]  p[3] -p[2]  p[1];
    ]
end

function skew(p)
    SA[
    	 0    -p[3]  p[2];
    	 p[3]  0    -p[1];
    	-p[2]  p[1]  0;
    ]
end

function ∂skew∂p(λ) # 𝞉(skew(p)*λ)/∂p
	skew(-λ)
end


###############################################################################
#
# Pre-defined parameter sets
#
###############################################################################
"""
    create_soil_VC(
                vc::AbstractSoilVC{FT},
                name::String,
                α::Number,
                n::Number,
                Θs::Number,
                Θr::Number
    ) where {FT<:AbstractFloat}
    create_soil_VC(
                vc::AbstractSoilVC{FT},
                name::String
    ) where {FT<:AbstractFloat}

Create a [`AbstractSoilVC`](@ref) type of soil VC, given
- `vc` [`AbstractSoilVC`](@ref) type identifier
- `name` Soil type name. Supported names include
  - "Sand"
  - "Loamy Sand"
  - "Sandy Loam"
  - "Loam"
  - "Sandy Clay Loam"
  - "Silt Loam"
  - "Silt"
  - "Clay Loam"
  - "Silty Clay Loam"
  - "Sandy Clay"
  - "Silty Clay"
  - "Clay"
- `α` Soil α
- `n` Soil n
- `Θs` SWC at saturation
- `Θr` Residual SWC
"""
function create_soil_VC(
            vc::BrooksCorey{FT},
            name::String,
            α::Number,
            n::Number,
            Θs::Number,
            Θr::Number
) where {FT<:AbstractFloat}
    _vc_vG = VanGenuchten{FT}(stype = name,
                              α     = α   ,
                              n     = n   ,
                              Θs    = Θs  ,
                              Θr    = Θr  )
    _vc_BC = BrooksCorey{FT}(stype = name        ,
                             b     = (2n-1)/(n-1),
                             ϕs    = 1/α         ,
                             Θs    = Θs          ,
                             Θr    = Θr          )
    fit_soil_VC!(_vc_vG, _vc_BC);

    return _vc_BC
end

function create_soil_VC(
            vc::VanGenuchten{FT},
            name::String,
            α::Number,
            n::Number,
            Θs::Number,
            Θr::Number
) where {FT<:AbstractFloat}
    return VanGenuchten{FT}(stype = name,
                            α     = α   ,
                            n     = n   ,
                            Θs    = Θs  ,
                            Θr    = Θr  )
end

function create_soil_VC(
            vc::AbstractSoilVC{FT},
            name::String
) where {FT<:AbstractFloat}
    # Parameters from Silt soil
    paras = [ 163.2656, 1.37, 0.46, 0.034];

    if name=="Sand"
        paras = [1479.5945, 2.68, 0.43, 0.045];
    elseif name=="Loamy Sand"
        paras = [1265.3084, 2.28, 0.41, 0.057];
    elseif name=="Sandy Loam"
        paras = [ 765.3075, 1.89, 0.41, 0.065];
    elseif name=="Loam"
        paras = [ 367.3476, 1.56, 0.43, 0.078];
    elseif name=="Sandy Clay Loam"
        paras = [ 602.0419, 1.48, 0.39, 0.100];
    elseif name=="Silt Loam"
        paras = [ 204.0820, 1.41, 0.45, 0.067];
    elseif name=="Silt"
        paras = [ 163.2656, 1.37, 0.46, 0.034];
    elseif name=="Clay Loam"
        paras = [ 193.8779, 1.31, 0.41, 0.095];
    elseif name=="Silty Clay Loam"
        paras = [ 102.0410, 1.23, 0.43, 0.089];
    elseif name== "Sandy Clay"
        paras = [ 275.5107, 1.23, 0.38, 0.100];
    elseif name=="Silty Clay"
        paras = [  51.0205, 1.09, 0.36, 0.070];
    elseif name=="Clay"
        paras = [  81.6328, 1.09, 0.38, 0.068];
    else
        @warn "Soil type $(name) not recognized, use Silt soil instead.";
    end

    return create_soil_VC(vc, name, paras...)
end


using Projections, CSV, DataFrames, Plots, Distributions, LinearAlgebra, Random

path     = @__DIR__
savepath = path

include(joinpath(path, "experiments.jl"))

## initialization
save = false

l1 = unique(vcat(2:0.1:3, 3:0.05:6));
l2 = unique(vcat(2:0.1:3, 3:0.05:4));
N1 = @. ceil(Int64, 10^l1)
N2 = @. ceil(Int64, 10^l2)

## benchmarks
comparison_solver(Our(), N1; save = false, maxevals = 1);
@time table1 = comparison_solver(Our(), N1; save = save,  savepath = savepath, maxevals = 100);

comparison_solver(General(), N2; save = false, maxevals = 1);
@time table2 = comparison_solver(General(), N2; save = save,  savepath = savepath, maxevals = 100);

comparison_philpott(N2; save = false, maxevals = 1);
@time table3 = comparison_philpott(N2; save = save, savepath = savepath, maxevals = 100);

## tables
savetable(vcat(table1, table2, table3), save = save, savepath = savepath, savename = "comparison")

## Kullback-Leibler divergence
m = model_DRO(10, KullbackLeibler(), 0.1)
λ = 1:0.01:3
h(λ,m; save = save, savepath = savepath)

## Burg entropy
m = model_DRO(10, Burg(), 0.1)
λ = 1.9:0.01:3
h(λ, m; save = save, savepath = savepath)

## Hellinger distance
m = model_DRO(10, Hellinger(), 0.1)
λ = 1.9:0.01:5
h(λ, m; save = save, savepath = savepath)

## χ²-distance distance
m = model_DRO(10, ChiSquare(), 0.1)
λ = 1.9:0.01:3
h(λ, m; save = save, savepath = savepath)

## Modified χ²-distance distance
m = model_DRO(10, ModifiedChiSquare(), 0.1)
λ = -1.8:0.01:3
h(λ, m; save = save, savepath = savepath)

## l-2 norm
m = model_DRO(10, Ltwo(), 0.1)
λ = 0:0.01:30
h(λ, m; save = save, savepath = savepath)


@testset "Geometries" begin
  @testset "Rectangle" begin
    r = Rectangle((1.,1.), (2.,3.))
    @test extrema(r) == ([1.,1.], [2.,3.])
    @test sides(r) == [1.,2.]
    @test GeoStatsBase.center(r) == [3/2, 4/2]
    @test GeoStatsBase.diagonal(r) == sqrt(1^2 + 2^2)
    @test volume(r) == 1*2
  end
end


# Map projections

export UNSEEN,
    lat2colat, colat2lat,
    project,
    equiprojinv, mollweideprojinv, orthoinv,
    equirectangular, mollweide, orthographic,
    orthographic2, gnomonic

import RecipesBase

const UNSEEN = -1.6375e+30

"""
    project(invprojfn, m::Map{T, O}, bmpwidth, bmpheight; kwargs...) where {T <: Number, O <: Order}

Return a 2D bitmap (array) containing a cartographic projection of the
map and a 2D bitmap containing a boolean mask. The size of the bitmap
is `bmpwidth`×`bmpheight` pixels. The function `projfn` must be a
function which accepts as input two parameters `x` and `y` (numbers
between -1 and 1).

The following keywords can be used in the call:

- `center`: 2-tuple specifying the location (colatitude, longitude) of the sky
  point that is to be placed in the middle of the image (in radians)
- `unseen`: by default, Healpix maps use the value -1.6375e+30 to mark
  unseen pixels. You can specify a different value using this
  keyword. This should not be used in common applications.

Return a `Array{Union{Missing, Float32}}` containing the intensity of
each pixel. Pixels falling outside the projection are marked as NaN,
and unseen pixels are marked as `missing`.
"""
function project(invprojfn, m::Map{T,O}, bmpwidth, bmpheight,
                 projparams = Dict()) where {T <: Number,O <: Healpix.Order}

    center = get(projparams, :center, (0, 0))
    unseen = get(projparams, :unseen, UNSEEN)
    desttype = get(projparams, :desttype, Float32)

    img = Array{desttype}(undef, bmpheight, bmpwidth)
    masked = zeros(Bool, bmpheight, bmpwidth)

    anymasked = false
    for j in 1:bmpheight
        y = 2 * (j - 1) / (bmpheight - 1) - 1
        for i in 1:bmpwidth
            x = 2 * (i - 1) / (bmpwidth - 1) - 1
            visible, lat, long = invprojfn(x, y)
            if visible
                value = m.pixels[Healpix.ang2pix(m, lat2colat(lat), long)]
                if ismissing(value) || isnan(value) || (
                    !ismissing(unseen) && unseen == value)
                    img[j, i] = NaN
                    masked[j, i] = true
                    anymasked = true
                else
                    img[j, i] = value
                end
            else
                img[j, i] = NaN
            end
        end
    end

    img, masked, anymasked
end

################################################################################

lat2colat(x) = π / 2 - x
colat2lat(x) = π / 2 - x

@doc raw"""
   lat2colat(x)
   colat2lat(x)

Convert colatitude into latitude and vice versa. Both `x` and the
result are expressed in radians.
"""
lat2colat, colat2lat

################################################################################

"""
    equiproj(lat, lon)

Equirectangular projection. Given the latitude `lat` (in radians) and the
longitude (in radians), return a tuple (Bool, Number, Number) where the
first Boolean is a flag telling if the point falls within the projection (true)
or not (false), and the two numbers are the x and y coordinates of the point
on the projection plane (both are in the range [−1, 1]).
"""
function equiproj(lat, lon)
    # We use `rem2pi` because we need angles in the range [-π, +π]
    x, y = ((rem2pi(lon + π, RoundNearest) - π) / π, 2 * (rem2pi(lat + π, RoundNearest) - π) / π)
    (true, x, y)
end

"""
    equiprojinv(x, y)

Inverse equirectangular projection. Given a point (x, y)
on the plane [-1, 1] × [-1, 1], return a tuple (Bool, Number, Number)
where the first Boolean is a flag telling if the point falls
within the projection (true) or not (false), and the two numbers
are the latitude and longitude in radians.
"""
function equiprojinv(x, y)
    ((-1 ≤ x ≤ 1) && (-1 ≤ y ≤ 1)) || return (false, 0, 0)

    (true, π / 2 * y, π * x)
end

"""
    mollweideproj(lat, lon)

Mollweide projection. Given the latitude `lat` (in radians) and the
longitude (in radians), return a tuple (Bool, Number, Number) where the
first Boolean is a flag telling if the point falls within the projection (true)
or not (false), and the two numbers are the x and y coordinates of the point
on the projection plane (both are in the range [−1, 1]).
"""
function mollweideproj(lat, lon)
    sinlat, coslat = sincos(lat)
    (true, sinlat, 2 / π * lon * coslat)
end

"""
    mollweideprojinv(x, y)

Inverse Mollweide projection. Given a point (x, y) on the plane,
with x ∈ [-1, 1], y ∈ [-1, 1], return a 3-tuple of type
(Bool, Number, Number). The boolean specifies if (x, y) falls within
the map (true) or not (false), the second and third arguments are
the latitude and longitude in radians.
"""
function mollweideprojinv(x, y)
    # See https://en.wikipedia.org/wiki/Mollweide_projection, we set
    #
    #     R = 1/√2
    #
    # x ∈ [-1, 1], y ∈ [-1, 1]

    x^2 + y^2 ≥ 1 && return (false, 0.0, 0.0)

    sinθ = y
    cosθ = sqrt(1 - sinθ^2)
    θ = asin(sinθ)

    lat = asin((2θ + 2 * sinθ * cosθ) / π)
    long = -2 * π * x / (2cosθ)
    (true, lat, long)

end

"""
    orthoinv(x, y, ϕ1, λ0)

Inverse orthographic projection centered on (ϕ1, λ0). Given a
point (x, y) on the plane, with x ∈ [-1, 1], y ∈ [-1, 1], return
a 3-tuple of type (Bool, Number, Number). The boolean specifies
if (x, y) falls within the map (true) or not (false), the second
and third arguments are the latitude and longitude in radians.
"""
function orthoinv(x, y, ϕ1, λ0)
    # Assume R = 1/√2. The notation ϕ1, λ0 closely follows
    # the book "Map projections — A working manual" by
    # John P. Snyder (page 145 and ff.)

    R = 1
    ρ = √(x^2 + y^2)
        if ρ > R
        return (false, zero(ϕ1), zero(λ0))
    end

    c = asin(ρ / R)
    sinc, cosc = sin(c), cos(c)
    if cosc < 0
        return (false, zero(ϕ1), zero(λ0))
    end

    if ρ ≈ 0
        return (true, ϕ1, λ0)
    end

    ϕ = asin(cosc * sin(ϕ1) + y * sinc * cos(ϕ1) / ρ)
    if ϕ1 ≈ π / 2
        λ = λ0 + atan(x, -y)
    elseif ϕ1 ≈ -π / 2
        λ = λ0 + atan(x, y)
    else
        λ = λ0 + atan(x * sinc, (ρ * cos(ϕ1) * cosc - y * sin(ϕ1) * sinc))
    end

    (true, ϕ, λ)
end

"""
    function ortho2inv(x, y, ϕ1, λ0)

Inverse stereo orthographic projection centered on (ϕ1, λ0). Given
a point (x, y) on the plane, with x ∈ [-1, 1], y ∈ [-1, 1], return
a 3-tuple of type (Bool, Number, Number). The boolean specifies
if (x, y) falls within the map (true) or not (false), the second
and third arguments are the latitude and longitude in radians.
"""
function ortho2inv(x, y, ϕ1, λ0)
    x ≤ 0 && return orthoinv(2x + 1, y, ϕ1, λ0)
    orthoinv(2x - 1, y, ϕ1, λ0 + π)
end

"""
    function gnomonicinv(x, y, ϕ1, λ0, fov_rad)

Gnomonic projection centered on (ϕ1, λ0), with a field of view
equal to `fov_rad` (in radians).  Given a point (x, y) on the plane,
with x ∈ [-1, 1], y ∈ [-1, 1], return a 3-tuple of type (Bool,
Number, Number). The boolean specifies if (x, y) falls within
the map (true) or not (false), the second and third arguments
are the latitude and longitude in radians.
"""
function gnominv(x, y, ϕ1, λ0, ψ0, fov_rad)
    # We fix a Earth radius such that the field of view of the
    # projection is the one expected. Note that the formula
    # diverges if `fov_rad` is 90° (as expected).
    R = 1. / tan(fov_rad)

    # We use basic geometry to cast a 3D ray from the center of
    # the sphere to the tangent plane, and then we compute the
    # intersection between the ray and the sphere.
    gamma = 1 / sqrt(1 + (x^2 + y^2) / R^2)
    vecx, vecy, vecz = gamma * R, -gamma * x, -gamma * y

    # We implement the rotations in the following order:
    # 1. Rotation by -ψ0 (orientation) around the x axis
    # 2. Rotation by λ0 (longitude) around the z axis
    # 3. Rotation by ϕ1 (latitude) around the y axis

    sin_ϕ1, cos_ϕ1 = sincos(ϕ1)
    sin_λ0, cos_λ0 = sincos(λ0)
    sin_ψ0, cos_ψ0 = sincos(-ψ0) # Change the sign to match Healpy conventions

    vecxrot = (cos_ϕ1 * cos_λ0 * vecx +
               (-cos_ψ0 * cos_ϕ1 * sin_λ0 + sin_ψ0 * sin_ϕ1) * vecy +
               (-sin_ψ0 * cos_ϕ1 * sin_λ0 + sin_ϕ1 * cos_ψ0) * vecz)
    vecyrot = ((sin_λ0) * vecx +
               (cos_ψ0 * cos_λ0 + sin_ψ0 * sin_ϕ1 * sin_λ0) * vecy +
               (-sin_ψ0 * cos_λ0) * vecz)
    veczrot = ((-sin_ϕ1 * cos_λ0) * vecx +
               (sin_ψ0 * sin_ϕ1 * sin_λ0 + cos_ϕ1 * sin_ψ0) * vecy +
               (-sin_ψ0 * sin_ϕ1 * sin_λ0 + cos_ϕ1 * cos_ψ0) * vecz)

    theta, phi = vec2ang(vecxrot, vecyrot, veczrot)
    (true, colat2lat(theta), phi)
end

################################################################################

"""
    equirectangular(m::Map{T,O}; kwargs...) where {T <: Number, O <: Order}

High-level wrapper around `project` for equirectangular projections.
"""
function equirectangular(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}
    width = get(projparams, :width, 720)
    height = get(projparams, :height, width)
    project(equiprojinv, m, width, height, projparams)
end

"""
    mollweide(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}

High-level wrapper around `project` for Mollweide projections.

The following parameters can be set in the `projparams` dictionary:

- `width`: width of the image, in pixels (default: 720 pixels)
- `height`: height of the image, in pixels; if not specified, it will be assumed
  to be equal to `width`
"""
function mollweide(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}
    width = get(projparams, :width, 720)
    height = get(projparams, :height, width ÷ 2)
    project(mollweideprojinv, m, width, height, projparams)
end

"""
    orthographic(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}

High-level wrapper around `project` for orthographic projections.

The following parameters can be set in the `projparams` dictionary:

- `width`: width of the image, in pixels (default: 720 pixels)
- `height`: height of the image, in pixels; if not specified, it will be assumed
  to be equal to `width`
- `center`: position of the pixel in the middle of the left globe (*latitude* and
  longitude).
"""
function orthographic(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}
    width = get(projparams, :width, 720)
    height = get(projparams, :height, width)
    ϕ0, λ0 = get(projparams, :center, (0, 0))
    project(m, width, height, projparams) do x, y
        orthoinv(x, y, ϕ0, λ0)
    end
end

"""
    orthographic2(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}

High-level wrapper around `project` for stereo orthographic projections.

The following parameters can be set in the `projparams` dictionary:

- `width`: width of the image, in pixels (default: 720 pixels)
- `height`: height of the image, in pixels; if not specified, it will be assumed
  to be equal to `width`
- `center`: position of the pixel in the middle of the left globe (*latitude* and
  longitude). Default is (0, 0).
"""
function orthographic2(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}
    width = get(projparams, :width, 720)
    height = get(projparams, :height, width ÷ 2)
    ϕ0, λ0 = get(projparams, :center, (0, 0))
    project(m, width, height, projparams) do x, y
ortho2inv(x, y, ϕ0, λ0)
    end
end

"""
    gnomonic(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}

High-level wrapper around `project` for gnomonic projections.

The following parameters can be set in the `projparams` dictionary:

- `width`: width of the image, in pixels (default: 720 pixels)
- `height`: height of the image, in pixels; if not specified, it will be assumed
  to be equal to `width`
- `center`: position and orientation of the pixel in the middle. It is a 3-element
  tuple containing:

  1. The *latitude* of the pixel, in radians
  2. The longitude of the pixel, in radians
  3. The rotation to be applied to the image, in radians

- `fov_rad`: size of the image along the x and y axes, in radians (default: 15°)

# Example

````julia
plot(m, gnomonic, Dict(:fov_rad = deg2rad(1.5), :center = (0, 0, deg2rad(45))))
````
"""
function gnomonic(m::Map{T,O}, projparams = Dict()) where {T <: Number,O <: Order}
    width = get(projparams, :width, 720)
    height = get(projparams, :height, width)
    ϕ0, λ0, ψ0 = get(projparams, :center, (0, 0, 0))
    fov_rad = get(projparams, :fov_rad, deg2rad(15 / 60))
    project(m, width, height, projparams) do x, y
        gnominv(x, y, ϕ0, λ0, ψ0, fov_rad)
    end
end

################################################################################

RecipesBase.@recipe function plot(m::Map{T,O},
                                  projection = mollweide,
                                  projparams = Dict()) where {T <: Number,O <: Order}

    img, mask, anymasked = projection(m, projparams)

    if anymasked
        RecipesBase.@series begin
            seriestype --> :shape
            primary --> false
            c --> :grey
            line --> :grey

            width, height = size(mask)
            xm = Float64[]
            ym = Float64[]
            for y in 1:height
                curx = 1
                # Instead of drawing each single masked pixel as a
                # square, squash together long runs of masked pixels
                # into one rectangle
                while curx ≤ width
                    if mask[curx, y]
                        startx = curx
                        while curx ≤ width && mask[curx, y]
                            curx += 1
                        end

                        append!(xm, [
                            startx - 0.5,
                            startx - 0.5,
                            curx + 0.5,
                            curx + 0.5, NaN
                        ])
                        append!(ym, [
                            y - 0.5,
                            y + 0.5,
                            y + 0.5,
                    y - 0.5, NaN])
                    else
                    curx += 1
                    end
                end
            end

            ym, xm
        end
    end

    seriestype --> :heatmap
    aspect_ratio --> 1
    colorbar --> :bottom
    framestyle --> :none

    img
end

@doc raw"""
    plot(m::Map{T,O}, projection = mollweide, projparams = Dict())

Draw a representation of the map, using some specific projection. The
parameter `projection` must be a function returning the
bitmap. Possible values for `projection` are the following:

- `equirectangular`
- `mollweide`
- `orthographic`
- `orthographic2`
- `gnomonic`

You can define your own projections.

The dictionary `projparams` allows to hack a number of parameters used
in the projection.

# References

See also [`equirectangular`](@ref), [`mollweide`](@ref),
[`orthographic`](@ref), [`orthographic2`](@ref), and
[`gnomonic`](@ref).
"""
plot


### A Pluto.jl notebook ###
# v0.14.0

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 4c1ebac8-81b7-11eb-19fa-f704b4d84a21
html"""

Section 2.1

 Principal Component Analysis

Lecture Video

"""

# ╔═╡ cf82077a-81c2-11eb-1de2-09ed6c35d810
begin
	import Pkg
	Pkg.activate(mktempdir())
	Pkg.add([
			Pkg.PackageSpec(name="Images", version="0.22.4"),
			Pkg.PackageSpec(name="ImageMagick", version="0.7"),
			Pkg.PackageSpec(name="PlutoUI", version="0.7"),
			Pkg.PackageSpec(name="Plots", version="1.10"),
			Pkg.PackageSpec(name="Colors", version="0.12"),
			Pkg.PackageSpec(name="ColorSchemes", version="3.10"),
			Pkg.PackageSpec(name="ForwardDiff"),
			Pkg.PackageSpec(name="LaTeXStrings"),

			])

	using PlutoUI
	using Colors, ColorSchemes, Images
	using Plots
	using LaTeXStrings

	using Statistics, LinearAlgebra  # standard libraries
end

# ╔═╡ c593a748-81b6-11eb-295a-a9800f9dec6d
PlutoUI.TableOfContents(aside=true)

# ╔═╡ deb2af50-8524-11eb-0dd4-9d799ff6d3e2
md"""
# Introduction: Understanding data

In this notebook we will start looking at more general kinds of **data**, not only images, and we'll try to extract some information from the image using statistical methods, namely [**principal component analysis**](https://en.wikipedia.org/wiki/Principal_component_analysis).

This method tries to answer the questions "which 'directions' are the most important  in the data" and "can we [reduce the dimensionality](https://en.wikipedia.org/wiki/Dimensionality_reduction) (number of useful variables) of the data"?

This can be viewed as another take on the question of finding and exploiting **structure** in data. It also leads towards ideas of **machine learning**.
"""

# ╔═╡ 2e50a070-853f-11eb-2045-b1cc43c29768
md"""
# Rank of a matrix
"""

# ╔═╡ ed7ff6b2-f863-11ea-1a59-eb242a8674e3
md"## Flags"

# ╔═╡ fed5845e-f863-11ea-2f95-c331d3c62647
md"Let's start off by recalling ideas about  **multiplication tables** or **outer products**:"

# ╔═╡ 0e1a6d80-f864-11ea-074a-5f7890180114
outer(v, w) = [x * y for x in v, y in w]

# ╔═╡ 2e497e30-f895-11ea-09f1-d7f2c1f61193
outer(1:10, 1:12)

# ╔═╡ cfdd04f8-815a-11eb-0409-79a2599c29ab
md"""
Each column is a multiple (in general not an integer multiple) of any other column; and each row is a multiple of any other row.

This is an example of a **structured matrix**, i.e. one in which we need to store *less information* to store the matrix than the full $(m \times n)$ table of numbers (even though no element is 0). For example, for an outer product we only need $m + n$ numbers, which is usually *much* less.
"""

# ╔═╡ ab3d55cc-f905-11ea-2f22-5398f3aca803
md"Some flags give simple examples of outer products, for example"

# ╔═╡ 13b6c108-f864-11ea-2447-2b0741f15c7b
flag = outer([1, 0.1, 2], ones(6))

# ╔═╡ e66b30a6-f914-11ea-2c0f-35282d45a30a
ones(6)

# ╔═╡ 43bff19e-f864-11ea-2315-0f85b532a325
show_image(flag)

# ╔═╡ 71d1b12e-f895-11ea-39df-f5c18a7766c3
flag2 = outer([1, 0.1, 2], [1, 1, 1, 3, 3, 3])

# ╔═╡ 79d2c6f4-f895-11ea-30c4-9d1102c99482
show_image(flag2)

# ╔═╡ 356267fa-815b-11eb-1c57-ad14fd6e91a7
md"""
Note that outer products are not always immediate to recognise just by looking at an image! But you should be able to recognise that there is some kind of structure.
"""

# ╔═╡ cdbe1d8e-f905-11ea-3884-efeeef386dda
md"## Matrix rank"

# ╔═╡ d9aa9af0-f865-11ea-379e-f16b452bd94c
md"""
If a matrix can be written exactly as a *single* multiplication table / outer product, we say that its **rank** is 1, and we call it a **rank-1** matrix. Similarly, if it can be written as the *sum* of *two* outer products, it has **rank 2**, etc.
"""

# ╔═╡ 2e8ae92a-f867-11ea-0219-1bdd9627c1ea
md"Let's see what a random rank-1 matrix looks like:"

# ╔═╡ 9ad13804-815c-11eb-0253-8f8baf15eee3
w = 300

# ╔═╡ 38adc490-f867-11ea-1de5-3b633aff7c97
image = outer([1; 0.4; rand(50)], rand(w));

# ╔═╡ b183b6ca-f864-11ea-0b34-4dd3f4f5e69d
show_image(image)

# ╔═╡ 946fde3c-815b-11eb-3039-db4105bc43ab
md"""
It has a characteristic checkerboard or patchwork look.
"""

# ╔═╡ ab924210-815b-11eb-07fe-411db58fbc3a
md"""
Here's a random rank-2 matrix:
"""

# ╔═╡ 74c04322-815b-11eb-2308-7b3d571cf613
begin

	image2 = outer([1; 0.4; rand(50)], rand(w)) +
	         outer(rand(52), rand(w))

	show_image(image2)
end

# ╔═╡ b5094384-815b-11eb-06fd-1f40134c6fd8
md"""
We see that it starts to look less regular.
"""

# ╔═╡ cc4f3fee-815b-11eb-2982-9b797b806b45
md"""
#### Exercise:

Make an interactive visualisation of a random rank-$n$ matrix where you can vary $n$.
"""

# ╔═╡ dc55775a-815b-11eb-15b7-7993190bffab
md"""
## Effect of noise
"""

# ╔═╡ 9cf23f9a-f864-11ea-3a08-af448aceefd8
md"""
Now what happens if we add a bit of **noise**, i.e. randomness, to a rank-1 matrix?
"""

# ╔═╡ a5b62530-f864-11ea-21e8-71ccfed487f8
noisy_image = image .+ 0.03 .* randn.();

# ╔═╡ f6713bec-815b-11eb-2fc4-6b0326a64b16
show_image(image)

# ╔═╡ 5471ddce-f867-11ea-2519-21981f5ea68b
show_image(noisy_image)

# ╔═╡ c41df86c-f865-11ea-1253-4942bbdbe9d2
md"""The noisy image now has a rank larger than 1. But visually we can see that it is "close to" the original rank-1 matrix.

Given this matrix, how could we discover that it is close to a structured, rank-1 matrix? We would like to be able to find this out and say that the matrix is close to a simple one."""

# ╔═╡ 7fca33ac-f864-11ea-2a8b-933eb382c172
md"## Images as data"

# ╔═╡ 283f5da4-f866-11ea-27d4-957ca2551b92
md"""

Now let's treat the image as a **data matrix**, so that each column of the image / matrix is a **vector** representing one observation of data. (In data science it is often the rows that correspond to observations.)

Let's try to visualize those vectors, taking just the first two rows of the image as the $x$ and $y$ coordinates of our data points:
"""

# ╔═╡ 1957f71c-f8eb-11ea-0dcf-339bfa7f96fc
show_image(image[1:2, 1:20])

# ╔═╡ 54977286-f908-11ea-166d-d1df33f38454
image[1:2, 1:20]

# ╔═╡ 7b4e90b4-f866-11ea-26b3-95efde6c650b
begin
	xx = image[1, :]
	yy = image[2, :]
end

# ╔═╡ f574ad7c-f866-11ea-0efa-d9d0602aa63b
md"# From images to data"

# ╔═╡ 8775b3fe-f866-11ea-3e6f-9732e39a3525
md"""
We would like to **visualise** this data with the `Plots.jl` package, passing it the $x$ and $y$ coordinates as arguments and plotting a point at each $(x_i, y_i)$ pair.
We obtain the following plot of the original rank-1 matrix and the noisy version:
"""

# ╔═╡ cb7e85e4-815c-11eb-3923-7b9537801bae
gr()

# ╔═╡ 1147cbda-f867-11ea-08fa-ef6ed2ae1e93
begin
	xs = noisy_image[1, :]
	ys = noisy_image[2, :]

	scatter(xs, ys, label="noisy", m=:., alpha=0.3, ms=4, ratio=1)

	scatter!(xx, yy,
			leg=:topleft, label="rank-1", ms=3, alpha=0.3,
			size=(500, 400), m=:square, c=:red,
			framestyle=:origin)

	title!("Plotting a rank-1 matrix gives a straight line!")
end

# ╔═╡ 8a611e36-f867-11ea-121f-317b7c145fe3
md"We see that the exact rank-1 matrix has columns that **lie along a line** through the origin in this representation, since they are just multiples of one another.

E.g. If $(x_1, y_1)$ and $(x_2, y_2)$ are two columns with $x_2 = cx_1$ and $y_2 = cy_1$, then $y_2 / x_2 = y_1 / x_1$, so they lie along the same line through the origin.

The approximate rank-1 matrix has columns that **lie *close to* the line**!"

# ╔═╡ f7371934-f867-11ea-3b53-d1566684585c
md"So, given the data, we want to look at it do see if it lies close to a line or not.
How can we do so in an *automatic* way?
"

# ╔═╡ 987c1f2e-f868-11ea-1125-0d8c02843ae4
md"""
## Measuring data cloud "size" -- using statistics
"""

# ╔═╡ 9e78b048-f868-11ea-192e-d903265d1eb5
md"Looking at this cloud of data points, a natural thing to do is to try to *measure* it: How wide is it, and how tall?"

# ╔═╡ 24df1f32-ec90-11ea-1f6d-03c1bfa5df8e
md"""For example, let's think about calculating the width of the cloud, i.e. the range of possible $x$-values of the data. For this purpose the $y$-values are actually irrelevant.
"""

# ╔═╡ b264f724-81bf-11eb-1052-295b81cde5fb
md"""
A natural idea would be to just scan the data and take the maximum and minimum values. However, real data often contains anomalously large values called **outliers**, which would dramatically affect this calculation. Instead we need to use a **statistical** method where we weight the data and average over all the data points. This process will hopefully be affected less by outliers, and will give a more representative idea of the size of the **bulk** of the data.
"""

# ╔═╡ 09991230-8526-11eb-04aa-fb904bd2036c
md"""
A first step in analysing data is often to **centre** it data around 0 by subtracting the mean, sometimes called "de-meaning":
"""

# ╔═╡ aec46a9b-f743-4cbd-97a7-3ef3cac78b12
begin
	xs_centered = xs .- mean(xs)
	ys_centered = ys .- mean(ys)
end

# ╔═╡ 1b8c743e-ec90-11ea-10aa-e3b94f768f82
scatter(xs_centered, ys_centered, ms=5, alpha=0.5, ratio=1, leg=false, framestyle=:origin)

# ╔═╡ eb867f18-852e-11eb-005f-e15b6d0d0d95
md"""
## Measuring a "width" of a data set
"""

# ╔═╡ f7079016-852e-11eb-3bc9-53fa0846276f
md"""
A natural way to measure the width of a data set could be to measure some kind of width *separately* in both the $x$ and $y$ directions, in other words by **projecting** the data onto one of the axes, while ignoring the other one. Let's start with the $x$ coordinates of the centred data and try to average them.

If we literally average them we will get $0$ (since we have subtracted the mean). We could ask "on average how far away from the origin is the data". This would give the [mean absolute deviation](https://en.wikipedia.org/wiki/Average_absolute_deviation):
"""

# ╔═╡ 870d3efa-f8fc-11ea-1593-1552511dcf86
begin
	scatter(xs_centered, ys_centered, ms=5, alpha=0.5, ratio=1, leg=false, framestyle=:origin)

	scatter!(xs_centered, zeros(size(xs_centered)), ms=5, alpha=0.1, ratio=1, leg=false, framestyle=:origin)

	for i in 1:length(xs_centered)
		plot!([(xs_centered[i], ys_centered[i]), (xs_centered[i], 0)], ls=:dash, c=:black, alpha=0.1)
	end

	plot!()

end

# ╔═╡ 4fb82f18-852f-11eb-278d-cf93571f4adc
mean(abs.(xs_centered))

# ╔═╡ 5fcf832c-852f-11eb-1354-792933a891a5
md"""
This is a perfectly good computational measure of distance. However, there is another measure which is easier to reason about theoretically / analytically:
"""

# ╔═╡ ef4a2a54-81bf-11eb-358b-0da2072f20c8
md"""
### Root-mean-square distance: Standard deviation
"""

# ╔═╡ f5358ce4-f86a-11ea-2989-b1f37be89183
md"""
The **standard deviation** is the **root-mean-square** distance of the centered data from the origin.

In other words, we first *square* the distances (or displacements) from the origin, then take the mean of those, giving the **variance**. However, since we have squared the original distances, this gives a quantity with units "distance$^2$", so we need to take the square root to get back to a measurable length:
"""

# ╔═╡ 2c3721da-f86b-11ea-36cf-3fe4c6622dc6
begin
	σ_x = √(mean(xs_centered.^2))   # root-mean-square distance from 0
	σ_y = √(mean(ys_centered.^2))
end

# ╔═╡ 03ab44c0-f8fd-11ea-2243-1f3580f98a65
md"This gives the following approximate extents (standard deviations) of the cloud:"

# ╔═╡ 6dec0db8-ec93-11ea-24ad-e17870ee64c2
begin
	scatter(xs_centered, ys_centered, ms=5, alpha=0.5, ratio=1, leg=false,
			framestyle=:origin)

	vline!([-2*σ_x, 2*σ_x], ls=:dash, lw=2, c=:green)
	hline!([-2*σ_y, 2*σ_y], ls=:dash, lw=2, c=:blue)

	annotate!( 2σ_x * 0.93, 0.03, text(L"2\sigma_x",  14, :green))
	annotate!(-2σ_x * 0.88, 0.03, text(L"-2\sigma_x", 14, :green))

	annotate!(0.05,  2σ_y * 1.13, text(L"2\sigma_y",  14, :blue))
	annotate!(0.06, -2σ_y * 1.14, text(L"-2\sigma_y", 14, :blue))

end

# ╔═╡ 5fab2c32-f86b-11ea-2f27-ed5feaac1fa5
md"""
We expect most (around 95%) of the data to be contained within the interval $\mu \pm 2 \sigma$, where $\mu$ is the mean and $\sigma$ is the standard deviation. (This assumes that the data is **normally distributed**, which is not actually the case for the data generated above.)
"""

# ╔═╡ ae9a2900-ec93-11ea-1ae5-0748221328fc
md"## Correlated data"

# ╔═╡ b81c9db2-ec93-11ea-0dbd-4bd0951cb2cc
md"""
However, from the figure it is clear that $x$ and $y$ are not the "correct directions" to use for this data set. It would be more natural to think about other directions: the direction in which the data set is mainly pointing (roughly, the direction in which it's longest), together with the approximately perpendicular direction in which it is  narrowest.

We need to find *from the data* which directions these are, and the extent (width) of the data cloud in those directions.

However, we cannot obtain any information about those directions by looking *separately* at $x$-coordinates and $y$-coordinates, since within the same bounding boxes that we just calculated the data can be distributed in many different ways.

Rather, the information that we need is encoded in the *relationship* between the values of $x_i$ and $y_i$ *for the points in the data set*.

For our data set, when $x$ is large and negative, $y$ is also rather negative; when $x$ is 0, $y$ is near $0$, and when $x$ is large and positive, so is $y$. We say that $x$ and $y$ are **correlated** -- literally they are mutually ("co") related, such that knowing some information about one of them allows us to predict something about the other.

For example, if I measure a new data point from the same process and I find that $x$ is around $0.25$ then I would expect $y$ to be within the range $0.05$ to $0.2$, and it would be very surprising if $y$ were -0.5.

"""

# ╔═╡ 80722856-f86d-11ea-363d-53fc5f6b8152
md"""
Although there are standard methods to calculate this correlation, we prefer to hone our **intuition** using computational thinking instead!
"""

# ╔═╡ b8fa6a1c-f86d-11ea-3d6b-2959d737254b
md"""
We want to think about different *directions*, so let's introduce an angle $\theta$ to describe the direction along which we are looking. We want to calculate the width of the cloud *along that direction*.

Effectively we are *changing coordinates* to a new coordinate, oriented along the line. To do this  requires more linear algebra than we are assuming in this course, but let's see what it looks like:
"""

# ╔═╡ 3547f296-f86f-11ea-1698-53d3c1a0bc30
md"## Rotating the axes"

# ╔═╡ 7a83101e-f871-11ea-1d87-4946162777b5
md"""By rotating the axes we can "look in different directions" and calculate the width of the data set "along that direction". What we are really doing is a perpendicular **projection** of the data onto that direction."""

# ╔═╡ e8276b4e-f86f-11ea-38be-218a72452b10
M = [xs_centered ys_centered]'

# ╔═╡ 1f373bd0-853f-11eb-0f8e-19cb7f376182
eigvals(cov(M')) .* 199

# ╔═╡ 31e4b138-84e8-11eb-36a8-8b90746fbb0f
variances = σs.^2 ./ 199

# ╔═╡ d71fdaea-f86f-11ea-1a1f-45e4d50926d3
imax = argmax(M[1, :])

# ╔═╡ 757c6808-f8fe-11ea-39bb-47e4da65113a
svdvals(M)

# ╔═╡ 1232e848-8540-11eb-089b-2185cc06f23a
M

# ╔═╡ 3b71142c-f86f-11ea-0d43-47011d00786c
p1 = begin

	scatter(M[1, :], M[2, :], ratio=1, leg=false, ms=2.5, alpha=0.5,
			framestyle=:origin)

	projected = ([cos(θ) sin(θ)] * M) .* [cos(θ) sin(θ)]'
	scatter!(projected[1, :], projected[2, :], m=:3, alpha=0.1, c=:green)

	lines_x = reduce(vcat, [M[1, i], projected[1, i], NaN] for i in 1:size(M, 2))
	lines_y = reduce(vcat, [M[2, i], projected[2, i], NaN] for i in 1:size(M, 2))

# 	for i in 1:size(M, 2)
# 		plot!([M[1, i], projected[1, i]], [M[2, i], projected[2, i]], ls=:dash, c=:black, alpha=0.1)
# 	end

	plot!(lines_x, lines_y, ls=:dash, c=:black, alpha=0.1)

	plot!([0.7 .* (-cos(θ), -sin(θ)), 0.7 .* (cos(θ), sin(θ))], lw=1, arrow=true, c=:red, alpha=0.3)
	xlims!(-0.7, 0.7)
	ylims!(-0.7, 0.7)

	scatter!([M[1, imax]], [M[2, imax]],  ms=3, alpha=1, c=:yellow)

	annotate!(0, 1.2, text("align arrow with cloud", :red, 10))
end;

# ╔═╡ 88bbe1bc-f86f-11ea-3b6b-29175ddbea04
p2 = begin
	M2 = R(θ) * M

	scatter(M2[1, :], M2[2, :],ratio=1, leg=false, ms=2.5, alpha=0.3, framestyle=:origin, size=(500, 500))

	#plot!([(-0.6, 0), (0.6, 0)], lw=3, arrow=true, c=:red, xaxis=false, yaxis=false, xticks=[], yticks=[])

	#scatter!([M2[1, imax]], [M2[2, imax]], ms=3, alpha=1, c=:yellow)

	xlims!(-0.7, 0.7)
	ylims!(-0.7, 0.7)

	scatter!(M2[1, :], zeros(size(xs_centered)), ms=3, alpha=0.1, ratio=1, leg=false, framestyle=:origin, c=:green)

	lines2_x = reduce(vcat, [M2[1, i], M2[1, i], NaN] for i in 1:size(M2, 2))
	lines2_y = reduce(vcat, [M2[2, i], 0, NaN] for i in 1:size(M2, 2))

	# for i in 1:size(M2, 2)
	# 	plot!([(M2[1, i], M2[2, i]), (M2[1, i], 0)], ls=:dash, c=:black, alpha=0.1)
	# end

	plot!(lines2_x, lines2_y, ls=:dash, c=:black, alpha=0.1)

	σ = std(M2[1, :])
	vline!([-2σ, 2σ], ls=:dash, lw=2)

# 	plot!(0.5 * [-cos(θ), cos(θ)], 0.5 * [sin(θ), -sin(θ)], c=:black, alpha=0.5, lw=1, arrow=true)
# 	plot!(0.5 * [-sin(θ), sin(θ)], 0.5 * [-cos(θ), cos(θ)], c=:black, alpha=0.5, lw=1, arrow=true)

# 	plot!(0.5 * [cos(θ), -cos(θ)], 0.5 * [-sin(θ), sin(θ)], c=:black, alpha=0.5, lw=1, arrow=true)
# 	plot!(0.5 * [sin(θ), -sin(θ)], 0.5 * [cos(θ), -cos(θ)], c=:black, alpha=0.5, lw=1, arrow=true)

	title!("σ = $(round(σ, digits=4))")

	annotate!(2σ+0.05, 0.05, text("2σ", 10, :green))
	annotate!(-2σ-0.05, 0.05, text("-2σ", 10, :green))

end;

# ╔═╡ 7cb04c9a-8358-11eb-1255-8d8c90916c37
gr()

# ╔═╡ cd9e05ee-f86f-11ea-0422-25f8329c7ef2
R(θ)= [cos(θ) sin(θ)
	  -sin(θ) cos(θ)]

# ╔═╡ 7eb51908-f906-11ea-19d2-e947d81cb743
md"In the following figure, we are rotating the axis (red arrow) around in the left panel. In the right panel we are viewing the data from the point of view of that new coordinate direction, in other words projecting onto that direction, effectively as if we rotated our head so the red vector was horizontal:"

# ╔═╡ 8b8e6b2e-8531-11eb-1ea6-637db25b28d5
p1

# ╔═╡ 4f1980ea-f86f-11ea-3df2-35cca6c961f3
md"""
degrees = $(@bind degrees Slider(0:360, default=28, show_value=true))
"""

# ╔═╡ c9da6e64-8540-11eb-3984-47fdf8be0dac
md"""
## Rotating the data
"""

# ╔═╡ f70065aa-835a-11eb-00cb-ffa27bcb486e
θ = π * degrees / 180   # radians

# ╔═╡ 2ffe7ed0-f870-11ea-06aa-390581500ca1
plot(p2)

# ╔═╡ a5cdad52-f906-11ea-0486-755a6403a367
md"""
We see that the extent of the data in the direction $\theta$ varies as $\theta$ changes. Let's plot the variance in direction $\theta$ as a function of $\theta$:
"""

# ╔═╡ 0115c974-f871-11ea-1204-054510848849
begin
	variance(θ) = var( (R(θ) * M)[1, :] )
	variance(θ::AbstractArray) = variance(θ[1])
end

# ╔═╡ 0935c870-f871-11ea-2a0b-b1b824379350
p3 = begin
	plot(0:360, variance.(range(0, 2π, length=361)), leg=false, size=(400, 200))
	scatter!([degrees], [σ^2])
	xlabel!("θ")
	ylabel!("variance in direction θ")
end

# ╔═╡ 6646abe0-835b-11eb-328a-55ca22f89c7d
σs = svdvals(M)

# ╔═╡ ef850e8e-84e7-11eb-1cb0-870c3000841d
1 ./ σs

# ╔═╡ 031a2894-84e8-11eb-1381-9b9e86f2fa0a
M

# ╔═╡ e4af4d26-f877-11ea-1de3-a9f8d389138e
md"""The direction in which the variance is **maximised** gives the most important direction: It is the direction along which the data "points", or the direction which best distinguishes different data points. This is often called the first **principal component** in statistics, or the first **singular vector** in linear algebra.

We can also now quantify *how close* the data is to lying along that single line, using the width in the *perpendicular* direction: if that width is "much smaller" than  the width in the first principal direction then the data is close to being rank 1.
"""

# ╔═╡ bf57f674-f906-11ea-08eb-9b50818a025b
md"The simplest way to maximise this function is to evaluate it everywhere and find one of the places where it takes the maximum value:"

# ╔═╡ 17e015fe-f8ff-11ea-17b4-a3aa072cd7b3
begin
	θs = 0:0.01:2π
	fs = variance.(θs)

	θmax = θs[argmax(fs)]
	θmin = θs[argmin(fs)]

	fmax = variance(θmax)
	fmin = variance(θmin)
end

# ╔═╡ 045b9b98-f8ff-11ea-0d49-5b209319e951
begin
	scatter(xs_centered, ys_centered, ms=5, alpha=0.3, ratio=1, leg=false,
			framestyle=:origin)

	plot!([(0, 0), 2*sqrt(fmax) .* (cos(θmax), sin(θmax))], arrow=true, lw=3, c=:red)
	plot!([(0, 0), 2*sqrt(fmin) .* (cos(θmin), sin(θmin))], arrow=true, lw=3, c=:red)

end

# ╔═╡ cfec1ec4-f8ff-11ea-265d-ab4844f0f739
md"""
Note that the directions that maximise and minimise variance are perpendicular. This is always the case, as shown using the Singular-Value Decomposition (SVD) in linear algebra.

There are different ways to think about this procedure. We can think of it as effectively "fitting an ellipse" to the data, where the widths of the ellipse axes show the relative importance of each direction in the data.

Alternatively we can think of it as fitting a multivariate normal distribution by finding a suitable covariance matrix.
"""

# ╔═╡ e6e900b8-f904-11ea-2a0d-953b99785553
begin
	circle = [cos.(θs) sin.(θs)]'
	stretch = [2 * sqrt(fmax) 0
				0   		2 * sqrt(fmin)]
	ellipse = R(-θmax) * stretch * circle

	plot!(ellipse[1, :], ellipse[2, :], series=:shape, alpha=0.4, fill=true, c=:orange)
end

# ╔═╡ 301f4d06-8162-11eb-1cd6-31dd8da164b6
md"""
Note also that an ellipse is the image of the unit circle under a *linear* transformation. We are effectively learning what the best linear transformation is that transforms the unit circle into our data cloud.
"""

# ╔═╡ aaff88e8-f877-11ea-1527-ff4d3db663db
md"## Higher dimensions"

# ╔═╡ aefa84de-f877-11ea-3e26-678008e9739e
md"""
Can we generalise this to dimensions higher than 2?
Let's think about 3D. If we take columns of the first *three* rows of the original image, we have vectors in 3D.

If we plot these vectors in 3D, a rank-1 matrix will give a straight line in 3D, while a rank-2 matrix will give a **plane** in 3D. Rank-2 + noise gives a noisy cloud lying close to a plane.

Similarly to what we did above, we need to calculate the ellipsoid that best fits the data. The widths of the axes of the ellipsoid tell us how close to being a line or a plane (rank-1 or rank-2) the data is.
"""

# ╔═╡ 0bd9358e-f879-11ea-2c83-ed4e7bf9d903
md"In more than 3D we can no longer visualise the data, but the same idea applies. The calculations are done using the SVD.

If the widths of the ellipsoid in some directions are very small, we can ignore those directions and hence reduce the dimensionality of the data, by changing coordinates to the principal components."

# ╔═╡ 9960e1c2-85a0-11eb-3d68-cd3a07a9c0b5
md"""
## What is the Singular-Value Decomposition (SVD)?
"""

# ╔═╡ dc4cca88-85a0-11eb-2791-d7610a610e36
md"""
The **Singular-Value Decomposition (SVD)** is a way of writing any matrix in terms of simpler matrices. Thinking in terms of linear transformations, *any* linear transformation $T$ has the same effect as a sequence of three simple transformations:

T = rotation₂ ∘ stretch ∘ rotation₁

In terms of matrices, for *any* matrix $M$ it is possible to write

$$M = U \, \Sigma \, V^\text{T}$$

where $U$ and $V$ are **orthogonal** matrices, i.e. they satisfy $U U^\text{T} = I$ and $V V^\text{T} = I$, and $\Sigma$ is a diagonal matrix.

Orthogonal matrices have determinant $\pm 1$, so they leave areas unchanged. They are rotations, possibly combined with reflections.

There are algorithms from numerical linear algebra to calculate this decomposition. In Julia we call the `svd` function, e.g.
"""

# ╔═╡ 453689c2-85a2-11eb-2cbc-7d6476b42f2f
let
	M = [2 1
		 1 1]

	svd(M)
end

# ╔═╡ 91b77acc-85a2-11eb-19bb-bd2bdd0c1a68
md"""
Let's look at the action of the matrix on the unit disc. To generate points in the unit disc we generate points in $[-1, 1]^2$ and *reject* those lying outside:
"""

# ╔═╡ f92c75f6-85a3-11eb-1689-23aeaa3daeb7
begin
	unit_disc = [ (-1.0 .+ 2.0 .* rand(2)) for i in 1:2000 ]
	unit_disc = reduce(hcat, [x for x in unit_disc if x[1]^2 + x[2]^2 < 1])
end

# ╔═╡ 03069da6-85a4-11eb-2ac5-87b767846550
scatter(unit_disc[1, :], unit_disc[2, :], ratio=1, leg=false, alpha=0.5, ms=3)

# ╔═╡ 40b87cbe-85a4-11eb-30f8-cf7b5e79c19a
pp1 = begin
	scatter(unit_disc[1, :], unit_disc[2, :], ratio=1, leg=false, alpha=0.5, title="stretch + rotate")
	result =  [1 + tt  tt; tt  1] * unit_disc
	scatter!(result[1, :], result[2, :], alpha=0.2)

	ylims!(-3, 3)
	xlims!(-3, 3)
end;

# ╔═╡ 28a7d6dc-85a5-11eb-0e4b-b7e9b4c592ed
pp2 = begin
	UU, Sigma, VV = svd([1 + tt  tt; tt  1])
	scatter(unit_disc[1, :], unit_disc[2, :], ratio=1, leg=false, alpha=0.5, title="stretch")

	result2 = Diagonal(Sigma) * unit_disc

	scatter!(result2[1, :], result2[2, :], alpha=0.2)

	ylims!(-3, 3)
	xlims!(-3, 3)

end;

# ╔═╡ 1647a126-85a4-11eb-3923-5f5a6f703403
md"""
t = $(@bind tt Slider(0:0.01:1, show_value=true))
"""

# ╔═╡ 6ec7f980-85a5-11eb-12fc-cb132db28d83
plot(pp2, pp1)

# ╔═╡ 92a2827e-84e9-11eb-1e85-6f49b1da7277
md"""
## Rotations in 300 dimensions
"""

# ╔═╡ e84adec2-84e8-11eb-2157-dd491588ccf0
md"""
We have been thinking of 300 points in 2 dimensions. Instead, we could think about 2 points **in 300 dimensions**. In some sense, this is what
"transpose really does"!
"""

# ╔═╡ 571b88ac-85a6-11eb-2887-a368a19bce4d
md"""
The $V$ in the SVD is another rotation that we "cannot see" in the above pictures. In our case it is a rotation in 300 dimensions! But in fact we can visualise it as follows, where we multiply our data by a $300 \times 300$ orthogonal matrix!

We first take a random *anti-symmetric* matrix, i.e. one for which $M = -M^\text{t}$.
We then turn that into an orthogonal matrix by taking the so-called **matrix exponential**.
"""

# ╔═╡ 12010a58-84eb-11eb-106f-cb4e3e0c879b
begin
	dim = size(M, 2)
	anti_symmetric = randn(dim, dim)
	anti_symmetric -= anti_symmetric'
end

# ╔═╡ 696d2768-84eb-11eb-39e0-612e074a2c27
@bind t Slider(0:0.0002:1, show_value=true, default=0.0)

# ╔═╡ 7b7b5128-84eb-11eb-3974-9b4c08fab8bb
begin
	M_rotated = M * exp(t * anti_symmetric)
	scatter(M_rotated[1, :], M_rotated[2, :], leg=false, alpha=0.5, )
	scatter!(M[1, :], M[2, :], alpha=0.5)

	ylims!(-0.3, 0.3)
	xlims!(-0.6, 0.6)
end

# ╔═╡ 805b9616-85a7-11eb-22e8-db8ee67071ae
md"""
We see that the data rotates around in 300 dimensions, but always is projected to the *same* ellipse.
"""

# ╔═╡ 90656ce6-84fb-11eb-1aac-4bd7747613db
U, Σ, V = svd(M, full=true)

# ╔═╡ aec542a2-84fb-11eb-322c-27fc2c45f6ef
M18 = M * V

# ╔═╡ b55dcfd2-84fb-11eb-1766-17dc8b7a17d0
scatter(M18[1, :], M18[2, :], alpha=0.5, leg=false, ratio=1, xlim=(-5,5))

# ╔═╡ 1cf3e098-f864-11ea-3f3a-c53017b73490
md"#### Appendix"

# ╔═╡ 2917943c-f864-11ea-3ee6-db952ca7cd67
begin
	show_image(M) = get.(Ref(ColorSchemes.rainbow), M ./ maximum(M))
	show_image(x::AbstractVector) = show_image(x')
end

# ╔═╡ 72bb11b0-f88f-11ea-0e55-b1108300f854
loss(M1, M2) = sum( (M1[i] - M2[i])^2 for i in 1:length(M1) if !ismissing(M2[i]) )

# ╔═╡ feeeb24a-f88f-11ea-287f-219e53615f32
function split_up(v, m, n)
	return v[1:m], v[m+1:m+n], v[m+n+1:2m+n], v[2m+n+1:2m+2n]
end

# ╔═╡ 0bcc8852-f890-11ea-3715-11cbead7f636
function ff(v, m, n)
	v1, w1, v2, w2 = split_up(v, m, n)

	loss(outer(v1, w1) + outer(v2, w2), M3)
end

# ╔═╡ 7040dc72-f893-11ea-3d22-4fbd452faa41
ff2(v) = ff(v, m, n)

# ╔═╡ 1dbcf15a-f890-11ea-008c-8935edfbdb1c
ff(rand(total), m, n)

# ╔═╡ Cell order:
# ╟─4c1ebac8-81b7-11eb-19fa-f704b4d84a21
# ╠═cf82077a-81c2-11eb-1de2-09ed6c35d810
# ╠═c593a748-81b6-11eb-295a-a9800f9dec6d
# ╟─deb2af50-8524-11eb-0dd4-9d799ff6d3e2
# ╟─2e50a070-853f-11eb-2045-b1cc43c29768
# ╟─ed7ff6b2-f863-11ea-1a59-eb242a8674e3
# ╟─fed5845e-f863-11ea-2f95-c331d3c62647
# ╠═0e1a6d80-f864-11ea-074a-5f7890180114
# ╠═2e497e30-f895-11ea-09f1-d7f2c1f61193
# ╟─cfdd04f8-815a-11eb-0409-79a2599c29ab
# ╟─ab3d55cc-f905-11ea-2f22-5398f3aca803
# ╠═13b6c108-f864-11ea-2447-2b0741f15c7b
# ╠═e66b30a6-f914-11ea-2c0f-35282d45a30a
# ╠═43bff19e-f864-11ea-2315-0f85b532a325
# ╠═71d1b12e-f895-11ea-39df-f5c18a7766c3
# ╠═79d2c6f4-f895-11ea-30c4-9d1102c99482
# ╟─356267fa-815b-11eb-1c57-ad14fd6e91a7
# ╟─cdbe1d8e-f905-11ea-3884-efeeef386dda
# ╟─d9aa9af0-f865-11ea-379e-f16b452bd94c
# ╟─2e8ae92a-f867-11ea-0219-1bdd9627c1ea
# ╠═9ad13804-815c-11eb-0253-8f8baf15eee3
# ╠═38adc490-f867-11ea-1de5-3b633aff7c97
# ╠═b183b6ca-f864-11ea-0b34-4dd3f4f5e69d
# ╟─946fde3c-815b-11eb-3039-db4105bc43ab
# ╟─ab924210-815b-11eb-07fe-411db58fbc3a
# ╠═74c04322-815b-11eb-2308-7b3d571cf613
# ╟─b5094384-815b-11eb-06fd-1f40134c6fd8
# ╟─cc4f3fee-815b-11eb-2982-9b797b806b45
# ╟─dc55775a-815b-11eb-15b7-7993190bffab
# ╟─9cf23f9a-f864-11ea-3a08-af448aceefd8
# ╠═a5b62530-f864-11ea-21e8-71ccfed487f8
# ╠═f6713bec-815b-11eb-2fc4-6b0326a64b16
# ╠═5471ddce-f867-11ea-2519-21981f5ea68b
# ╟─c41df86c-f865-11ea-1253-4942bbdbe9d2
# ╟─7fca33ac-f864-11ea-2a8b-933eb382c172
# ╟─283f5da4-f866-11ea-27d4-957ca2551b92
# ╠═1957f71c-f8eb-11ea-0dcf-339bfa7f96fc
# ╠═54977286-f908-11ea-166d-d1df33f38454
# ╠═7b4e90b4-f866-11ea-26b3-95efde6c650b
# ╟─f574ad7c-f866-11ea-0efa-d9d0602aa63b
# ╟─8775b3fe-f866-11ea-3e6f-9732e39a3525
# ╠═cb7e85e4-815c-11eb-3923-7b9537801bae
# ╠═1147cbda-f867-11ea-08fa-ef6ed2ae1e93
# ╟─8a611e36-f867-11ea-121f-317b7c145fe3
# ╟─f7371934-f867-11ea-3b53-d1566684585c
# ╟─987c1f2e-f868-11ea-1125-0d8c02843ae4
# ╟─9e78b048-f868-11ea-192e-d903265d1eb5
# ╟─24df1f32-ec90-11ea-1f6d-03c1bfa5df8e
# ╟─b264f724-81bf-11eb-1052-295b81cde5fb
# ╟─09991230-8526-11eb-04aa-fb904bd2036c
# ╠═aec46a9b-f743-4cbd-97a7-3ef3cac78b12
# ╠═1b8c743e-ec90-11ea-10aa-e3b94f768f82
# ╟─eb867f18-852e-11eb-005f-e15b6d0d0d95
# ╟─f7079016-852e-11eb-3bc9-53fa0846276f
# ╟─870d3efa-f8fc-11ea-1593-1552511dcf86
# ╠═4fb82f18-852f-11eb-278d-cf93571f4adc
# ╟─5fcf832c-852f-11eb-1354-792933a891a5
# ╟─ef4a2a54-81bf-11eb-358b-0da2072f20c8
# ╟─f5358ce4-f86a-11ea-2989-b1f37be89183
# ╠═2c3721da-f86b-11ea-36cf-3fe4c6622dc6
# ╟─03ab44c0-f8fd-11ea-2243-1f3580f98a65
# ╟─6dec0db8-ec93-11ea-24ad-e17870ee64c2
# ╟─5fab2c32-f86b-11ea-2f27-ed5feaac1fa5
# ╟─ae9a2900-ec93-11ea-1ae5-0748221328fc
# ╟─b81c9db2-ec93-11ea-0dbd-4bd0951cb2cc
# ╟─80722856-f86d-11ea-363d-53fc5f6b8152
# ╟─b8fa6a1c-f86d-11ea-3d6b-2959d737254b
# ╟─3547f296-f86f-11ea-1698-53d3c1a0bc30
# ╟─7a83101e-f871-11ea-1d87-4946162777b5
# ╠═e8276b4e-f86f-11ea-38be-218a72452b10
# ╠═1f373bd0-853f-11eb-0f8e-19cb7f376182
# ╠═31e4b138-84e8-11eb-36a8-8b90746fbb0f
# ╟─d71fdaea-f86f-11ea-1a1f-45e4d50926d3
# ╠═757c6808-f8fe-11ea-39bb-47e4da65113a
# ╠═1232e848-8540-11eb-089b-2185cc06f23a
# ╠═3b71142c-f86f-11ea-0d43-47011d00786c
# ╠═88bbe1bc-f86f-11ea-3b6b-29175ddbea04
# ╠═7cb04c9a-8358-11eb-1255-8d8c90916c37
# ╟─cd9e05ee-f86f-11ea-0422-25f8329c7ef2
# ╟─7eb51908-f906-11ea-19d2-e947d81cb743
# ╠═8b8e6b2e-8531-11eb-1ea6-637db25b28d5
# ╟─4f1980ea-f86f-11ea-3df2-35cca6c961f3
# ╟─c9da6e64-8540-11eb-3984-47fdf8be0dac
# ╠═f70065aa-835a-11eb-00cb-ffa27bcb486e
# ╠═2ffe7ed0-f870-11ea-06aa-390581500ca1
# ╟─a5cdad52-f906-11ea-0486-755a6403a367
# ╟─0115c974-f871-11ea-1204-054510848849
# ╠═0935c870-f871-11ea-2a0b-b1b824379350
# ╠═6646abe0-835b-11eb-328a-55ca22f89c7d
# ╠═ef850e8e-84e7-11eb-1cb0-870c3000841d
# ╠═031a2894-84e8-11eb-1381-9b9e86f2fa0a
# ╟─e4af4d26-f877-11ea-1de3-a9f8d389138e
# ╟─bf57f674-f906-11ea-08eb-9b50818a025b
# ╠═17e015fe-f8ff-11ea-17b4-a3aa072cd7b3
# ╟─045b9b98-f8ff-11ea-0d49-5b209319e951
# ╟─cfec1ec4-f8ff-11ea-265d-ab4844f0f739
# ╠═e6e900b8-f904-11ea-2a0d-953b99785553
# ╟─301f4d06-8162-11eb-1cd6-31dd8da164b6
# ╟─aaff88e8-f877-11ea-1527-ff4d3db663db
# ╟─aefa84de-f877-11ea-3e26-678008e9739e
# ╟─0bd9358e-f879-11ea-2c83-ed4e7bf9d903
# ╟─9960e1c2-85a0-11eb-3d68-cd3a07a9c0b5
# ╟─dc4cca88-85a0-11eb-2791-d7610a610e36
# ╠═453689c2-85a2-11eb-2cbc-7d6476b42f2f
# ╟─91b77acc-85a2-11eb-19bb-bd2bdd0c1a68
# ╠═f92c75f6-85a3-11eb-1689-23aeaa3daeb7
# ╠═03069da6-85a4-11eb-2ac5-87b767846550
# ╠═40b87cbe-85a4-11eb-30f8-cf7b5e79c19a
# ╠═28a7d6dc-85a5-11eb-0e4b-b7e9b4c592ed
# ╟─1647a126-85a4-11eb-3923-5f5a6f703403
# ╠═6ec7f980-85a5-11eb-12fc-cb132db28d83
# ╟─92a2827e-84e9-11eb-1e85-6f49b1da7277
# ╟─e84adec2-84e8-11eb-2157-dd491588ccf0
# ╟─571b88ac-85a6-11eb-2887-a368a19bce4d
# ╠═12010a58-84eb-11eb-106f-cb4e3e0c879b
# ╠═696d2768-84eb-11eb-39e0-612e074a2c27
# ╠═7b7b5128-84eb-11eb-3974-9b4c08fab8bb
# ╟─805b9616-85a7-11eb-22e8-db8ee67071ae
# ╠═90656ce6-84fb-11eb-1aac-4bd7747613db
# ╠═aec542a2-84fb-11eb-322c-27fc2c45f6ef
# ╠═b55dcfd2-84fb-11eb-1766-17dc8b7a17d0
# ╟─1cf3e098-f864-11ea-3f3a-c53017b73490
# ╠═2917943c-f864-11ea-3ee6-db952ca7cd67
# ╠═72bb11b0-f88f-11ea-0e55-b1108300f854
# ╠═feeeb24a-f88f-11ea-287f-219e53615f32
# ╠═0bcc8852-f890-11ea-3715-11cbead7f636
# ╠═7040dc72-f893-11ea-3d22-4fbd452faa41
# ╠═1dbcf15a-f890-11ea-008c-8935edfbdb1c


function getDelta2Kernel{T <: FloatingPoint}(:: Type{T})
        nType = T == Float64 ? "double" : "float"

        return "
        #if defined(cl_khr_fp64)  // Khronos extension available?
        #pragma OPENCL EXTENSION cl_khr_fp64 : enable
        #elif defined(cl_amd_fp64)  // AMD extension available?
        #pragma OPENCL EXTENSION cl_amd_fp64 : enable
        #endif
        #define number $nType

        #define delta(x,y)  d[y*D2 + x]
        #define AField(x,y) a[y*D2 + x]
        #define BField(x,y) b[y*D2 + x]
        #define Lap(x,y)    l[y*D2 + x]

        #define Out(x,y)    out[y*D2 + x]

        __kernel void delta2(
                      __global const number *d,
                      __global const number *a,
                      __global const number *b,
                      __global const number *l,
                      __global number *out,
                      const int D2,
                      const number decay) {

            int i = get_global_id(0);
            int j = get_global_id(1);

            Out(i,j) = delta(i,j) / (1 + AField(i,j)) - decay * BField(i,j) + Lap(i,j);
        }
"
end

function delta2CL!{T <: FloatingPoint}(
    delta_buff, af_buff, bf_buff, lap_buff,
    out_buff,
    decay :: Real, d1 :: Int64, d2 :: Int64,
    ctx, queue, program, :: Type{T})

    k = Kernel(program, "delta2")

    call(queue, k, (d1,d2), nothing, delta_buff, af_buff, bf_buff, lap_buff, out_buff, int32(d2), convert(T, decay))
end


"""

    py95ProcessEigenVector(v)

Process eigen vectors of EDHDE matrix as defined in the Pena & Yohai (1995) algorithm.

# Arguments
- `v::Array{Float64, 1}`: Eigen vector of EDHDE matrix.

# References
Peña, Daniel, and Victor J. Yohai. "The detection of influential subsets in linear
regression by using an influence matrix." Journal of the Royal Statistical Society:
Series B (Methodological) 57.1 (1995): 145-156.
"""
function py95ProcessEigenVector(v::Array{Float64,1})
    eps = 0.0001
    n = length(v)
    k = 2.5
    c1 = Int(floor(n / 4.0))
    c2 = Int(floor(n / 4.0))
    ordered_coordinates = sortperm(v)
    a = zeros(Float64, n)
    b = zeros(Float64, n)
    @inbounds for i in n:(-1):(n - c1)
        a[i] = v[i] / v[i - 1]
    end
    @inbounds for i in 1:c2
        b[i] = v[i] / v[i + 1]
    end
    set_of_a = filter(i -> abs(a[i]) > k, ordered_coordinates)
    set_of_b = filter(i -> abs(b[i]) > k, ordered_coordinates)
    outliersetJ = Int[]
    outliersetI = Int[]

    if length(set_of_a) == 0 || length(set_of_b) == 0
        return (nothing, nothing)
    end
    i0 = set_of_b[1]
    j0 = set_of_a[1]
    if i0 > 1
        outliersetI = ordered_coordinates[1:(j0 - 1)]
    end
    if j0 > 1
        outliersetJ = ordered_coordinates[n:(-1):(n - i0 + 1)]
    end
    (outliersetI, outliersetJ)
end

"""

    py95SuspectedObservations(setting)

Determine suspected observations (outliers) as defined in the Pena & Yohai (1995) algorithm.

# Arguments
- `setting::RegressionSetting`: RegressionSetting object with a formula and dataset.

# References
Peña, Daniel, and Victor J. Yohai. "The detection of influential subsets in linear
regression by using an influence matrix." Journal of the Royal Statistical Society:
Series B (Methodological) 57.1 (1995): 145-156.
"""
function py95SuspectedObservations(setting::RegressionSetting)
    X, y = @extractRegressionSetting setting
    return py95SuspectedObservations(X, y)
end

function py95SuspectedObservations(X::Array{Float64,2}, y::Array{Float64,1})
    n, p = size(X)
    nhalf = Int(floor(n / 2.0))
    olsreg = ols(X, y)
    resids = residuals(olsreg)
    H = hatmatrix(X)
    s2 = sum(resids.^2.0) / (n - p)
    D = zeros(Float64, (n, n))
    E = zeros(Float64, (n, n))
    @inbounds for i in 1:n
        D[i, i] = 1 / (1 - H[i, i])
        E[i, i] = resids[i]
    end
    M = (1.0 / (p * s2)) * E * D * H * D * E
    eig = eigen(M)
    eig_values = eig.values
    eig_vectors = eig.vectors
    nonzero_eigen_indices = filter(i -> imag(eig_values[i]) == 0, 1:n)
    nonzero_eigen_vectors = eig_vectors[:, nonzero_eigen_indices]
    real_vectors = convert(Array{Float64,2}, nonzero_eigen_vectors)
    s1, s2 = size(real_vectors)
    suspected_observation_sets = Set{Array{Int,1}}()
    for i in 1:s2
        set1, set2 = py95ProcessEigenVector(real_vectors[:,i])
        if !(set1 isa Nothing)
            if length(set1) < nhalf
                push!(suspected_observation_sets, set1)
            end
        end
        if !(set2 isa Nothing)
            if length(set2) < nhalf
                push!(suspected_observation_sets, set2)
            end
        end
    end
    return suspected_observation_sets
end

"""

    jacknifedS(setting, omittedIndices)

Calculate Jacknife standard error in which the given indices are omitted from the data.

# Arguments
- `setting::RegressionSetting`: RegressionSetting object with a formula and dataset.
- `omittedIndices::Array{Int, 1}`: Indices of omitted variables.

# References
Peña, Daniel, and Victor J. Yohai. "The detection of influential subsets in linear
regression by using an influence matrix." Journal of the Royal Statistical Society:
Series B (Methodological) 57.1 (1995): 145-156.
"""
function jacknifedS(setting::RegressionSetting, omittedIndices::Array{Int,1})::Float64
    X = designMatrix(setting)
    y = responseVector(setting)
    return jacknifedS(X, y, omittedIndices)
end

function jacknifedS(X::Array{Float64,2}, y::Array{Float64,1}, omittedIndices::Array{Int,1})::Float64
    n, p = size(X)
    indices = [i for i in 1:n if !(i in omittedIndices)]
    olsreg = ols(X[indices,:], y[indices])
    e = residuals(olsreg)
    s = sqrt(sum(e.^2.0) / (n - p - length(omittedIndices)))
    return s
end

"""

    py95(setting)

Perform the Pena & Yohai (1995) algorithm for the given regression setting.

# Arguments
- `setting::RegressionSetting`: RegressionSetting object with a formula and dataset.

# Examples
```julia-repl
julia> reg0001 = createRegressionSetting(@formula(y ~ x1 + x2 + x3), hbk);
julia> py95(reg0001)
ict{Any,Any} with 2 entries:
  "outliers"       => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
  "suspected.sets" => Set([[14, 13], [43, 54, 24, 38, 22], [6, 10], [14, 7, 8, 3, 10, 2, 5, 6, 1, 9, 4…
```

# References
Peña, Daniel, and Victor J. Yohai. "The detection of influential subsets in linear
regression by using an influence matrix." Journal of the Royal Statistical Society:
Series B (Methodological) 57.1 (1995): 145-156.
"""
function py95(setting::RegressionSetting)
    X = designMatrix(setting)
    y = responseVector(setting)
    return py95(X, y)
end

function py95(X::Array{Float64,2}, y::Array{Float64,1})
    n, p = size(X)
    all_indices = collect(1:n)
    suspicious_sets = py95SuspectedObservations(X, y)
    outlierset = Set{Int}()
    for aset in suspicious_sets
        clean_indices = setdiff(all_indices, aset)
        olsreg = ols(X[clean_indices,:], y[clean_indices])
        betas = coef(olsreg)
        e = [y[i] - sum(X[i,:] .* betas) for i in 1:n]
        jks = jacknifedS(X, y, aset)
        stds = e / jks
        outlier_indices = filter(i -> abs(stds[i]) > 2.5, 1:n)
        for element in outlier_indices
            push!(outlierset, element)
        end
    end
    result = Dict()
    result["suspected.sets"] = suspicious_sets
    result["outliers"] = sort(collect(outlierset))
    return result
end


"""
      predictEnsemble( predictions::Matrix{Float64}, weights::Vector{Float64} )

      perform bayesian ensemble prediction.
      #Arguments
      - `predictions::Matrix{Float64}`: each column is the prediction of one hypothesis.
      - `weights::Vector{Float64}`:     mixing weights.
      #Return
      - `Vector{Float64}`:              prediction y.
  """
  function predictEnsemble( predictions::Matrix{Float64}, weights::Vector{Float64} )
    return predictions * weights;
  end

  """
  predictEnsemble( predictions::Vector{Matrix{Float64}}, weights::Vector{Float64} )

  perform bayesian ensemble prediction.
  #Arguments
  - `predictions::Vector{Matrix{Float64}}`: each matrix is the prediction of one hypothesis.
  - `weights::Vector{Float64}`:             mixing weights.
  #Return
  - `Vector{Float64}`:                      prediction y.
  """
  function predictEnsemble( predictions::Vector{Matrix{Float64}}, weights::Vector{Float64} )
    len = size( weights, 1 );
    res = zeros( Float64, size( predictions[1], 1 ), size( predictions[1], 2 ) );
    for i in 1:1:len
      @inbounds res .+= weights[i] .* predictions[i];
    end
    return res
  end


"""
Test script for dirBin0Rec0 model: one parameter for total number of links and one for reciprocity.
Investigate the existence of regions of the θ η plane where the latters cannot be estimated for finite N
"""

using ScoreDrivenERGM
using ScoreDrivenERGM.Utilities

import ScoreDrivenERGM.StaticNets: ErgmDirBin0Rec0(), ergm_par_from_mean_vals,diadProbFromPars , samplSingMatCan, stats_from_mat, get_one_mple, estimate,ErgmDirBin0Rec0, exp_val_stats, sample_ergm, estimate

using PyPlot
pygui(true)
using RCall
using Statistics

ErgmRcall.clean_start_RCall()
ergmTermsString = "edges +  mutual"
model = ErgmDirBin0Rec0()

function sample_ergm_sequence_static_estimate_mle(model::ErgmDirBin0Rec0, N, parDgpSeq, nSample)

    T = size(parDgpSeq)[2]
    parMle = zeros(T, 2, nSample)
    obs = zeros(T, 3, nSample)

    for i =1:T

        A_S = sample_ergm(model, N, parDgpSeq[:,i], nSample )

        obs[i,:,:] = reduce(hcat,[StaticNets.stats_from_mat(model, A) for A in eachslice(A_S, dims=3)])
        parMle[i,:,:] = reduce(hcat,[StaticNets.estimate(model, A) for A in eachslice(A_S, dims=3)])
    end

    return parMle, obs
end

N=30
begin
gix, ax = subplots(2,1)
Nvals = [30, 50, 70, 90]
θ_0 = -5
for N in Nvals
    parDgpSeq = reduce(hcat, [ [θ_0, η ] for η = 2:0.05:7])
    parMle, obs = sample_ergm_sequence_static_estimate_mle(model, N, parDgpSeq, 150 )
    fracZeroRec = mean(obs[:,2,:].==0;dims=2)
    fracUb = mean(obs[:,2,:].==obs[:,1,:]/2;dims=2)
    xplot = parDgpSeq[2,:]
    ax[1].plot(xplot, fracZeroRec)
    ax[2].plot(xplot, fracUb)

end
    ax[1].set_title("Fraction of samples with zero reciprocal pairs θ_dgp = $θ_0 ")
    ax[1].set_xlabel( "η_dgp")
    ax[1].legend(Nvals)
    ax[1].grid()
    ax[2].set_title("Fraction of samples with max reciprocal pairs θ_dgp = $θ_0 ")
    ax[2].set_xlabel( "η_dgp")
    ax[2].legend(Nvals)
    ax[2].grid()
    tight_layout()
end


mutable struct PlaneWaveMW{T,P}
  direction::P
  polarisation::P
  wavenumber::T
  amplitude::T
end

function PlaneWaveMW(d,p,k,a = 1)
    T = promote_type(eltype(d), eltype(p), typeof(k), typeof(a))
    P = similar_type(typeof(d), T)
    PlaneWaveMW{T,P}(d,p,k,a)
end

"""
    planewavemw3d(;direction, polarization, wavenumber[, amplitude=1])

Create a plane wave solution to Maxwell's equations.
"""
planewavemw3d(;
    direction    = error("missing arguement `direction`"),
    polarization = error("missing arguement `polarization`"),
    wavenumber   = error("missing arguement `wavenumber`"),
    amplitude    = 1,
    ) = PlaneWaveMW(direction, polarization, wavenumber, amplitude)

function (e::PlaneWaveMW)(x)
  k = e.wavenumber
  d = e.direction
  u = e.polarisation
  a = e.amplitude
  a * exp(-im * k * dot(d, x)) * u
end

function curl(field::PlaneWaveMW)
  k = field.wavenumber
  d = field.direction
  u = field.polarisation
  a = field.amplitude
  v = d × u
  b = -a * im * k
  PlaneWaveMW(d, v, k, b)
end

*(a::Number, e::PlaneWaveMW) = PlaneWaveMW(e.direction, e.polarisation, e.wavenumber, a*e.amplitude)

abstract type Dipole end

mutable struct DipoleMW{T,P} <: Dipole
    location::P
    orientation::P
    wavenumber::T
end

function DipoleMW(l,o,k)
    T = promote_type(eltype(l), eltype(o), typeof(k))
    P = similar_type(typeof(l), T)
    DipoleMW{T,P}(l,o,k)
end

mutable struct curlDipoleMW{T,P} <: Dipole
    location::P
    orientation::P
    wavenumber::T
end

function curlDipoleMW(l,o,k)
    T = promote_type(eltype(l), eltype(o), typeof(k))
    P = similar_type(typeof(l), T)
    curlDipoleMW{T,P}(l,o,k)
end

"""
    dipolemw3d(;location, orientation, wavenumber)

Create an electric dipole solution to Maxwell's equations representing the electric
field part. Implementation is based on (9.18) of Jackson's “Classical electrodynamics”,
with the notable difference that the ``\exp(ikr)`` is used.
"""
dipolemw3d(;
    location    = error("missing arguement `location`"),
    orientation = error("missing arguement `orientation`"),
    wavenumber   = error("missing arguement `wavenumber`"),
    ) = DipoleMW(location, orientation, wavenumber)

function (d::DipoleMW)(x; isfarfield=false)
    k = d.wavenumber
    x_0 = d.location
    p = d.orientation
    r = norm(x-x_0)
    n = (x - x_0)/r
    if isfarfield
      # postfactor (4*π*im)/k to be consistent with BEAST far field computation
      # and, of course, omitted exp(-im*k*r)/r factor in (9.19)
      # of Jackson's Classical Electrodynamics
      return k^2/(4*π)*cross(cross(n,p),n)*(4*π*im)/k
    else
      return 1/(4*π)*exp(-im*k*r)*(k^2/r*cross(cross(n,p),n) +
              (1/r^3 + im*k/r^2)*(3*n*dot(n,p) - p))
    end
end

function (d::curlDipoleMW)(x; isfarfield=false)
    k = d.wavenumber
    x_0 = d.location
    p = d.orientation
    r = norm(x-x_0)
    n =  (x - x_0)/r
    if isfarfield
      # postfactor (4*π*im)/k to be consistent with BEAST far field computation
      return (-im*k)*k^2/(4*π)*cross(n,p)*(4*π*im)/k
    else
      return -im*(k^3)/(4*π)*cross(n,p)*exp(-im*k*r)/r*(1 + 1/(im*k*r))
    end
end

function curl(d::DipoleMW)
    return curlDipoleMW(d.location, d.orientation, d.wavenumber)
end

*(a::Number, d::DipoleMW) = DipoleMW(d.location, a .* d.orientation, d.wavenumber)
*(a::Number, d::curlDipoleMW) = curlDipoleMW(d.location, a .* d.orientation, d.wavenumber)

mutable struct CrossTraceMW{F} <: Functional
  field::F
end

mutable struct TangTraceMW{F} <: Functional
  field::F
end

cross(::NormalVector, p::Function) = CrossTraceMW(p)
cross(::NormalVector, p::PlaneWaveMW) = CrossTraceMW(p)
cross(::NormalVector, p::Dipole) = CrossTraceMW(p)
cross(t::CrossTraceMW, ::NormalVector) = TangTraceMW(t.field)

function (ϕ::CrossTraceMW)(p)
  F = ϕ.field
  x = cartesian(p)
  n = normal(p)
  return n × F(x)
end

function (ϕ::TangTraceMW)(p)
  F = ϕ.field
  x = cartesian(p)
  n = normal(p)
  return (n × F(x)) × n
end

integrand(::TangTraceMW, gx, ϕx) = gx[1] ⋅ ϕx
integrand(::CrossTraceMW, test_vals, field_val) = test_vals[1] ⋅ field_val

struct NDotTrace{F} <: Functional
  field::F
end

(ϕ::NDotTrace)(p) = dot(normal(p), ϕ.field(cartesian(p)))
integrand(::NDotTrace, g, ϕ) = dot(g.value, ϕ)
LinearAlgebra.dot(::NormalVector, f) = NDotTrace(f)

mutable struct CurlGreen{T,U,V}
  wavenumber::T
  source::U
  position::V
end

function (f::CurlGreen)(x)
  γ = im * f.wavenumber
  r = x-f.position
  R = norm(r)
  g = exp(-γ*R)/(4π*R)
  j = f.source
  return -γ/R * (1 + 1/(γ*R)) * g * (r × j)
end

mutable struct CurlCurlGreen{T,U,V}
  wavenumber::T
  source::U
  position::V
end

cross(::NormalVector, p::CurlGreen) = CrossTraceMW(p)
cross(::NormalVector, p::CurlCurlGreen) = CrossTraceMW(p)

function (f::CurlCurlGreen)(x)
  γ = im * f.wavenumber
  r = x - f.position
  R = norm(r)
  g = exp(-γ*R)/(4π*R)
  j = f.source
  return (-γ^2/R^2 * (r × j) × r + (1/R^2 + γ/R)/R^2 * (3r * dot(r,j) - R^2 * j)) * g
end

curl(f::CurlGreen) = CurlCurlGreen(f.wavenumber, f.source, f.position)
function curl(f::CurlCurlGreen)
  κ = f.wavenumber
  j = κ^2 * f.source
  x = f.position
  return CurlGreen(κ, j, x)
end

Base.:*(a::Number, f::CurlGreen) = CurlGreen(f.wavenumber, a*f.source, f.position)
Base.:*(a::Number, f::CurlCurlGreen) = CurlCurlGreen(f.wavenumber, a*f.source, f.position)


uv = UncertainValue(rand(1000))

#######################
# TruncateLowerQuantile
#######################
uvc = constrain(uv, TruncateLowerQuantile(0.2))
@test minimum(resample(uvc, 1000)) >= quantile(uv, 0.2) - tol

#######################
# TruncateUpperQuantile
#######################
uvc = constrain(uv, TruncateUpperQuantile(0.8))
@test maximum(resample(uvc, 1000)) <= quantile(uv, 0.8) + tol

#######################
# TruncateQuantiles
#######################
uvc = constrain(uv, TruncateQuantiles(0.2, 0.8))
@test all(quantile(uv, 0.2) - tol .<= resample(uvc, 1000) .<= quantile(uv, 0.8) + tol)

#######################
# TruncateMinimum
#######################
uvc = constrain(uv, TruncateMinimum(0.5))
@test minimum(resample(uvc, 1000)) >= 0.5 - tol

#######################
# TruncateMaximum
#######################
uvc = constrain(uv, TruncateMaximum(1.5))
@test maximum(resample(uvc, 1000)) <= 1.5 + tol

#######################
# TruncateRange
#######################
uvc = constrain(uv, TruncateRange(0.5, 1.5))
@test all(0.5 - tol .<= resample(uvc, 1000) .<= 1.5 + tol)


include("../envs/RLEnvironments.jl")
using .RLEnvironments
using Plots

function act(Q, s, ϵ)
    rand() <= ϵ && return rand(1:size(Q, 1))
    a = @view Q[:, s]
    return rand(findall(a .== maximum(a)))
end

function episode!(e, env, maxt, Q, ϵ)
    reset!(env)
    done = false
    t = 0
    while !done && t < maxt
        t += 1
        s = env.current
        a = act(Q, s, ϵ)
        _, r, done = step!(env, a)
        e[t] = (s, a, r)
    end
    return t
end

function run!(Q, env; epochs=20, maxt=100, ϵ=0.1, α=0.5, γ=0.99)
    rewards = zeros(epochs)
    steps = zeros(Int64, epochs)
    e = Vector{Tuple{Int64, Int64, Float64}}(undef, maxt)

    for i in 1:epochs
        t = episode!(e, env, maxt, Q, ϵ)
        rdisc = 0.0
        for step in Iterators.reverse(view(e, 1:t))
            rdisc = step[3] + γ * rdisc
            Q[step[2], step[1]] += α * (rdisc - Q[step[2], step[1]])

            rewards[i] += step[3]
        end
        steps[i] = t
        ϵ = max(ϵ * 0.995, 0.01)
    end
    return rewards, steps
end

env = SimpleRooms{Int64, Int64, Float64}()
Q = zeros(length(env.actionspace), length(env.observationspace))

r, s = run!(Q, env)
plt = plot([r s], labels=["Reward" "Steps"], color=[:red :blue], layout=(2, 1))
display(plt)
#savefig(plt, "mccontrol")


## Step1EndGrid is an endogenous grid search to find the optimal policy function

#module Step1EndGrid

using LinearAlgebra, Random, Distributions, Statistics, Plots
using BenchmarkTools, Interpolations, Roots

"""
    Performs an endogenous search to find the optimal policy function.
    To do so, 3 functions are nested together:
    1. RHSEuler: computes the RHS of the Euler equation
    2. initialpolicy: for a guess value of consumption, finds a'
    3. EndGridSearch: solves for optimal consumption and a'
    Takes price q as given.
"""

function RHSEuler(A, E, Π, q, β, σ, e, avalue, aprime; tol = 0.01)

    # avalue is the actual value of assets
    # aprime is the actual value of the new assets

    """
    This function computes the RHS of the Euler Equation.
    Inputs:
        - A: assets today
        - E: exogenous shocks
        - Π: transtion matrix
        - q: price (guessed one)
        - β: discount factor
        - σ: preference parameter
        - e: position parameter for the shock (position, not value)
        - avalue: actual value of assets today (value, not position)
        - aprime: actual value of assets tomorrow
        - Aprimetol: provisional grid of future assets
    Output:
        - RHS: right hand side of the Euler Equation, for 2 exogenous state var.
    """

    a_size = length(A)
    e_size = size(E)[1]

    # Create a grid of A multiplied by the length of E
    Agrid = repeat(A, 1, e_size)

    Aprimetol = copy(Agrid) .+ 2*tol

    # Define the first derivative of utility function
    # (in the future, make it more general)
    lowerbound = 1E-7
    up(c::Real) = (max(c, lowerbound)).^(-σ)

    Uprime = zeros(e_size, 1)
    RHS = zeros(e_size, 1)

    aux = LinearInterpolation(Agrid[:, e], Aprimetol[:, e])
    adoubleprime = aux(aprime)
    Uprime = up(aprime .+ E[e] .- q .* adoubleprime)
    temp = Uprime .* ones(e_size, 1)
    temp2 = (Π[e, :]' * temp)[1,1]
    RHS = up(avalue + E[e] - aprime * q) - β * temp2

    return RHS

end

function initialpolicy(A, E, Π, β, σ, q; maxT = 600, tol = 0.01)

    """
    This function creates the first grid of assets in the following period.
    Inputs:
        - A: grid of assets today
        - E: exogenous shocks
        - Π: transtion matrix
        - β: discount factor
        - σ: preference parameter
        - q: price (guessed one)
    Output:
        - Aprime: grid of assets tomorrow
    """

    a_size = length(A)
    e_size = size(E)[1]

    # Create a grid of A multiplied by the length of E
    Agrid = repeat(A, 1, e_size)
    Aprime = copy(Agrid)
    Aprimetol = copy(Agrid) .+ 2*tol

    # Iteration
    normdiff = Inf
    t = 1

    while normdiff > tol && t <= maxT

        for a = 1:a_size
            for e = 1:e_size

                FOCfunction(aprime) = RHSEuler(A, E, Π, q, β, σ, e, Agrid[a, e], aprime)

                if FOCfunction(Agrid[1, e]) > 0.0
                    Aprime[a, e] = Agrid[1, e]
                elseif FOCfunction(Agrid[end, e]) < 0.0
                    Aprime[a, e] = Agrid[end, e]
                else
                    Aprime[a, e] = fzero(FOCfunction, Agrid[1, e], Agrid[end, e])
                end

            end
        end

        t += 1
        normdiff = maximum(abs.(Aprimetol - Aprime))
        Aprimetol = copy(Aprime)

    end

    return Aprime

end

function EndGridSearch(A, E, Π, β, σ, q; maxT = 600, tol = 0.10)

    """
    Function that finds the optimal policy function.
    Inputs:
        - A: assets today
        - E: exogenous shocks
        - Π: transition matrix
        - β: discount factor
        - σ: preference parameter
        - q: price (guess)
    Output:
        - pol_func: optimal policy function
    """

    # Check that parameters and price make sense
    @assert β < 1
    @assert σ > 1
    @assert q > β

    a_size = length(A)
    e_size = size(E)[1]

    # Check that transition matrix is well defined
    for e = 1:e_size
        @assert sum(Π[e,:]) == 1
    end

    # Define the first derivative of utility function and its inverse
    # (in the future, make it more general)
    lb = 1E-7
    lowerbound = lb .* ones(a_size, e_size)
    up(c) = max.(c, lowerbound).^(-σ)
    upinv(B) = B.^(-1/σ)

    # Compute Aprime: initial grid of next period's assets
    Aprime = initialpolicy(A, E, Π, β, σ, q)

    # Compute initial consumption (use it as guess for iteration)
    # Remember that it cannot be negative
    C = Aprime .+ E'
    for a = 1:a_size
        for e = 1:e_size
            C[a, e] = (C[a, e] <= 0) ? 0.0 : C[a,e]
        end
    end

    Cupdate = 0.0 .* similar(Aprime)
    Aupdate = similar(Aprime)

    # Iteration to find optimal consumption
    normdiff = Inf
    t = 1

    while normdiff > tol && t <= maxT

        # Find Ctilda from the Euler Equation (weakly positive)
        B = β .* up(C) * Π'
        Ctilda = upinv(B ./ q)
        for a = 1:a_size
            for e = 1:e_size
                Ctilda[a, e] = (Ctilda[a, e] <= 0) ? 0.0 : Ctilda[a,e]
            end
        end

        # Solve for A from the budget constraint
        A = Ctilda + Aprime .* q .- E'

        # Adjust for whether budget constraint binds (only low shock)
        binds = Aprime .< A[1,:]'
        Cupdate[binds] .= (Aprime .+ E' .- Aprime[1,:]' .* q)[binds]
        # Generate auxiliary matrix for when binding
        auxbinding = similar(A)
        for e = 1:e_size
            auxfunc = LinearInterpolation(A[:, e], Ctilda[:, e])
            auxbinding[:, e] = auxfunc.(Aprime[:, e])
        end
        # Fill Cupdate for binding
        Cupdate[.!binds] .= auxbinding[.!binds]

        # Replace negative consumption with 0, if exists
        for a = 1:a_size
            for e = 1:e_size
                Cupdate[a, e] = (Cupdate[a, e] <= 0) ? 0.0 : Cupdate[a,e]
            end
        end

        # values for next iteration
        t += 1
        normdiff = maximum(abs.(Cupdate - C))
        C = copy(Cupdate)

    end

    # Now find the optimal policy function in my original grid
    pol_func = zeros(a_size, e_size)
    for e = 1:e_size
        auxfunc = LinearInterpolation(A[:, e], Aprime[:, e])
        pol_func[:, e] = auxfunc.(Aprime[:, e])
    end

    return pol_func

end

#export RHSEuler, initialpolicy, EndGridSearch

#end


export ImaginaryFiber2D

"""
    ImaginaryFiber2D(f, y=(0., 0.))

Construct the fiber `ImaginaryFiber2D` of the bivariate polynomial `f` at `y`.
"""
function ImaginaryFiber2D(p::MP.AbstractPolynomialLike{S}, y=(0.0, 0.0)) where S
    vars = MP.variables(p)
    @assert (length(vars) == 2) "Expected a bivariate polynomial, but got $(p)."

    T = realified_coefficient_type(S)
    f_re, f_im = SP.Polynomial.( one(T) .* realify(p))

    ImaginaryFiber2D(f_re, f_im, y, zeros(T, 2, 4))
end

function update_fiber!(F::ImaginaryFiber2D{T}, y) where T
    F.y = (y[1], y[2])
    F
end

function evaluate(fiber::ImaginaryFiber2D{T}, x::SVector{2,T}) where T
    xy = SVector(x[1], x[2], fiber.y[1], fiber.y[2])
    out = SVector(SP.evaluate(fiber.f_re, xy), SP.evaluate(fiber.f_im, xy))
    out
end

function jacobian(fiber::ImaginaryFiber2D, x::AbstractVector, precomputed=false)
    U = fiber.U
    xy = SVector(x[1], x[2], fiber.y[1], fiber.y[2])
    @inbounds begin
        ∇f_re = SP.gradient(fiber.f_re, xy)
        for i=1:4
            U[1, i] = ∇f_re[i]
        end
        U[2, 3] = U[1, 1]
        U[2, 4] = U[1, 2]
        U[2, 1] = -U[1, 3]
        U[2, 2] = -U[1, 4]
    end

    # SMatrix has column-major storage
    SMatrix{2,2}(U[1,1], U[2,1], U[1, 2], U[2, 2])
end


using Bridge
 using Random
 using DataFrames
 using CSV

 SRC_DIR = joinpath(Base.source_dir(), "..", "src")
 include(joinpath(SRC_DIR, "auxiliary", "data_simulation_fns.jl"))
 OUT_DIR = joinpath(Base.source_dir(), "..", "output")
 mkpath(OUT_DIR)

 include(joinpath(SRC_DIR, "examples", "fitzHughNagumo.jl"))
 param = :simpleConjug
 FILENAME_OUT = joinpath(OUT_DIR,
                         "test_path_part_obs_"*String(param)*".csv")

 P = FitzhughDiffusion(param, 10.0, -8.0, 15.0, 0.0, 3.0)
 # starting point under :regular parametrisation
 x0 = ℝ{2}(-0.5, 0.6)
 # translate to conjugate parametrisation
 x0 = regularToConjug(x0, P.ϵ, 0.0)

 dt = 1/50000
 T = 10.0
 tt = 0.0:dt:T

 Random.seed!(4)
 XX, _ = simulateSegment(0.0, x0, P, tt)

 num_obs = 100
 skip = div(length(tt), num_obs)
 Time = collect(tt)[1:skip:end]
 df = DataFrame(time=Time, x1=[x[1] for x in XX.yy[1:skip:end]],
                x2=[(i==1 ? x0[2] : NaN) for (i,t) in enumerate(Time)])
 CSV.write(FILENAME_OUT, df)


module ManifoldLearning

    import Base: show, summary
    import SparseArrays: AbstractSparseArray, SparseMatrixCSC, spzeros, spdiagm, findnz
    import Statistics: mean, std
    import StatsBase: StatsBase, fit, standardize
    import MultivariateStats: outdim, projection,  KernelPCA, transform, transform!,
                              principalvars, dmat2gram, gram2dmat, pairwise
    import LinearAlgebra: eigvals, mul!, svd, qr, Symmetric, eigen, eigen!, tr, rmul!, I, norm, Diagonal
    import LightGraphs: neighbors, nv, add_edge!, connected_components, vertices,
                        dijkstra_shortest_paths, induced_subgraph, weights
    import SimpleWeightedGraphs: SimpleWeightedGraph

    export

    # Transformation types
    AbstractDimensionalityReduction,
    Isomap,             # Type: Isomap model
    HLLE,               # Type: Hessian Eigenmaps model
    LLE,                # Type: Locally Linear Embedding model
    LTSA,               # Type: Local Tangent Space Alignment model
    LEM,                # Type: Laplacian Eigenmaps model
    DiffMap,            # Type: Diffusion maps model

    ## common interface
    outdim,             # the output dimension of the transformation
    fit,                # perform the manifold learning
    transform,          # the projection matrix
    eigvals,            # eigenvalues from the spectral analysis
    neighbors,          # the number of nearest neighbors used for aproximate local subspace
    vertices,           # vertices of largest connected component
    projection          # the projection matrix (deprecated)

    include("interface.jl")
    include("utils.jl")
    include("nearestneighbors.jl")
    include("isomap.jl")
    include("hlle.jl")
    include("lle.jl")
    include("ltsa.jl")
    include("lem.jl")
    include("diffmaps.jl")

    # deprecated functions
    @deprecate transform(DimensionalityReduction, X; k=k, d=d) fit(DimensionalityReduction, X; k=k, maxoutdim=d)
    @deprecate projection(DimensionalityReductionModel) transform(DimensionalityReductionModel)
end


using CVChannel
using Test
using DelimitedFiles
"""
This script looks at the communication value of the
generalized Siddhu channel using SDP relaxations.
Specifically, it shows that 2-sym cv is effectively
the same as cv PPT, and that they are loose at least
some of the time. It also shows multiplicativity over PPT cone.
"""

@testset "Investigate generalized Siddhu channel" begin
    println("We calculate cvPPT, 2symCV, and cvPPT of the channel ran in parallel.")
    println("We then show that 2symCV provides no improvement to cvPPT and that cvPPT")
    println("is effectively multiplicative for the gen Siddhu with itself.")
    s_range = [0:0.1:0.5;]
    μ_range = [0:0.1:1;]
    cv_table = zeros(length(s_range),length(μ_range))
    par_cv_table = zeros(length(s_range),length(μ_range))
    twosym_cv_table = zeros(length(s_range),length(μ_range))
    non_mult_table = zeros(length(s_range),length(μ_range))
    s_ctr, μ_ctr = 1,1
    for s_id in s_range
        println("---Now scanning for s=",s_id,"---")
        for μ_id in μ_range
            println("μ=",μ_id)
            genSidChan(X) = generalizedSiddhu(X,s_id,μ_id)
            gensid_chan = Choi(genSidChan,3,3)
            #To save time we only calculate cv once since its same channel
            cv, = pptCV(gensid_chan, :dual)
            par_choi = parChoi(gensid_chan,gensid_chan)
            parcv, = pptCV(par_choi, :primal)
            cv_2sym, = twoSymCVPrimal(gensid_chan)
            cv_table[s_ctr,μ_ctr] =cv
            par_cv_table[s_ctr,μ_ctr] = parcv
            twosym_cv_table[s_ctr,μ_ctr] = cv_2sym
            non_mult_table[s_ctr,μ_ctr] = parcv - cv^2
            μ_ctr += 1
        end
        s_ctr += 1
        μ_ctr = 1
    end

    println("First we verify that cvPPT is effectively multiplicative over the whole range.")
    @test all(non_mult_table -> non_mult_table < 2e-5 , non_mult_table[:,:])

    println("Next we verify that cvPPT is approximately 2symCV over the whole range.")
    diff = twosym_cv_table - cv_table
    @test all(diff -> diff < 3e-6, diff[:,:])

    println("Given this we just look at the 2symCV of the channel.")

    info_vec = Vector{Union{Nothing,String}}(nothing, length(s_range)+1)
    label_vec = hcat("s↓ μ:",μ_range')
    info_vec[1] = "INFO:"
    info_vec[2] = "Generated by two-sym-CV-gen-Siddhu-channel.jl"
    info_vec[3] = "s_range = " * string(s_range)
    info_vec[4] = "μ_range = " * string(μ_range)
    data_to_save = hcat(vcat(label_vec,hcat(s_range,twosym_cv_table)),info_vec)

    println("Here is the 2symCV of the channel:")
    show(stdout, "text/plain", data_to_save)
    println("\nPlease name the file you'd like to write these results to: \n")
    file_name = readline()
    file_to_open = string(file_name,".csv")
    writedlm(file_to_open, data_to_save, ',')
end


using OscillatoryIntegralsODE
using OrdinaryDiffEq
using Test

@testset "bessel" begin
    nu = 100.
    r = 100.
    f(x) = exp(-x^2/16)
    bi = BesselJIntegral{Float64}(nu, r)
    result = levintegrate(bi, f, 1.0, 5.0, Vern9(); abstol=1e-6, reltol=1e-6)
    ref = 0.006311630277650583
    @test abs( (result - ref) / ref ) < 1e-4
end

@testset "spherical bessel" begin
    nu = 100.
    r = 100.
    f(x) = exp(-x^2/16)
    bi = SphericalBesselJIntegral{Float64}(nu, r)
    result = levintegrate(bi, f, 1.0, 5.0, Vern9(); abstol=1e-6, reltol=1e-6)
    ref = 0.0008322179291456167
    @test abs( (result - ref) / ref ) < 1e-4
end

@testset "harmonic" begin
    ω = 100.
    f(x) = exp(-x^2/16)
    hi = HarmonicIntegral{Complex{Float64}}(ω)
    result = levintegrate(hi, f, 1.0, 5.0, Vern9(); abstol=1e-6, reltol=1e-6)
    ref = 0.003798145404262353+0.009953606082753418im
    @test abs(result - ref) / abs(ref) < 1e-4
end


# ---
# title: 1539. Kth Missing Positive Number
# id: problem1539
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Array, Hash Table
# link: <https://leetcode.com/problems/kth-missing-positive-number/description/>
# hidden: true
# ---
#
# Given an array `arr` of positive integers sorted in a **strictly increasing
# order** , and an integer `k`.
#
# _Find the_`kth` _  positive integer that is missing from this array._
#
#
#
# **Example 1:**
#
#
#
#     Input: arr = [2,3,4,7,11], k = 5
#     Output: 9
#     Explanation: The missing positive integers are [1,5,6,8,9,10,12,13,...]. The 5th missing positive integer is 9.
#
#
# **Example 2:**
#
#
#
#     Input: arr = [1,2,3,4], k = 2
#     Output: 6
#     Explanation: The missing positive integers are [5,6,7,...]. The 2nd missing positive integer is 6.
#
#
#
#
# **Constraints:**
#
#   * `1 <= arr.length <= 1000`
#   * `1 <= arr[i] <= 1000`
#   * `1 <= k <= 1000`
#   * `arr[i] < arr[j]` for `1 <= i < j <= arr.length`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end